1	/*
2	* This file is part of the Chelsio T6 Crypto driver for Linux.
3	*
4	* Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5	*
6	* This software is available to you under a choice of one of two
7	* licenses. You may choose to be licensed under the terms of the GNU
8	* General Public License (GPL) Version 2, available from the file
9	* COPYING in the main directory of this source tree, or the
10	* OpenIB.org BSD license below:
11	*
12	* Redistribution and use in source and binary forms, with or
13	* without modification, are permitted provided that the following
14	* conditions are met:
15	*
16	* - Redistributions of source code must retain the above
17	* copyright notice, this list of conditions and the following
18	* disclaimer.
19	*
20	* - Redistributions in binary form must reproduce the above
21	* copyright notice, this list of conditions and the following
22	* disclaimer in the documentation and/or other materials
23	* provided with the distribution.
24	*
25	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32	* SOFTWARE.
33	*
34	* Written and Maintained by:
35	* Manoj Malviya (manojmalviya@chelsio.com)
36	* Atul Gupta (atul.gupta@chelsio.com)
37	* Jitendra Lulla (jlulla@chelsio.com)
38	* Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39	* Harsh Jain (harsh@chelsio.com)
40	*/
41
42	#define pr_fmt(fmt) "chcr:" fmt
43
44	#include <linux/kernel.h>
45	#include <linux/module.h>
46	#include <linux/crypto.h>
47	#include <linux/skbuff.h>
48	#include <linux/rtnetlink.h>
49	#include <linux/highmem.h>
50	#include <linux/scatterlist.h>
51
52	#include <crypto/aes.h>
53	#include <crypto/algapi.h>
54	#include <crypto/gcm.h>
55	#include <crypto/sha1.h>
56	#include <crypto/sha2.h>
57	#include <crypto/authenc.h>
58	#include <crypto/ctr.h>
59	#include <crypto/gf128mul.h>
60	#include <crypto/internal/aead.h>
61	#include <crypto/null.h>
62	#include <crypto/internal/skcipher.h>
63	#include <crypto/aead.h>
64	#include <crypto/scatterwalk.h>
65	#include <crypto/internal/hash.h>
66
67	#include "t4fw_api.h"
68	#include "t4_msg.h"
69	#include "chcr_core.h"
70	#include "chcr_algo.h"
71	#include "chcr_crypto.h"
72
73	#define IV AES_BLOCK_SIZE
74
75	static unsigned int sgl_ent_len[] = {
76	0, 0, 16, 24, 40, 48, 64, 72, 88,
77	96, 112, 120, 136, 144, 160, 168, 184,
78	192, 208, 216, 232, 240, 256, 264, 280,
79	288, 304, 312, 328, 336, 352, 360, 376
80	};
81
82	static unsigned int dsgl_ent_len[] = {
83	0, 32, 32, 48, 48, 64, 64, 80, 80,
84	112, 112, 128, 128, 144, 144, 160, 160,
85	192, 192, 208, 208, 224, 224, 240, 240,
86	272, 272, 288, 288, 304, 304, 320, 320
87	};
88
89	static u32 round_constant[11] = {
90	0x01000000, 0x02000000, 0x04000000, 0x08000000,
91	0x10000000, 0x20000000, 0x40000000, 0x80000000,
92	0x1B000000, 0x36000000, 0x6C000000
93	};
94
95	static int chcr_handle_cipher_resp(struct skcipher_request *req,
96	unsigned char *input, int err);
97
98	static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
99	{
100	return &ctx->crypto_ctx->aeadctx;
101	}
102
103	static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
104	{
105	return &ctx->crypto_ctx->ablkctx;
106	}
107
108	static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
109	{
110	return &ctx->crypto_ctx->hmacctx;
111	}
112
113	static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
114	{
115	return gctx->ctx->gcm;
116	}
117
118	static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
119	{
120	return gctx->ctx->authenc;
121	}
122
123	static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
124	{
125	return container_of(ctx->dev, struct uld_ctx, dev);
126	}
127
128	static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx)
129	{
130	memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr));
131	}
132
133	static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
134	unsigned int entlen,
135	unsigned int skip)
136	{
137	int nents = 0;
138	unsigned int less;
139	unsigned int skip_len = 0;
140
141	while (sg && skip) {
142	if (sg_dma_len(sg) <= skip) {
143	skip -= sg_dma_len(sg);
144	skip_len = 0;
145	sg = sg_next(sg);
146	} else {
147	skip_len = skip;
148	skip = 0;
149	}
150	}
151
152	while (sg && reqlen) {
153	less = min(reqlen, sg_dma_len(sg) - skip_len);
154	nents += DIV_ROUND_UP(less, entlen);
155	reqlen -= less;
156	skip_len = 0;
157	sg = sg_next(sg);
158	}
159	return nents;
160	}
161
162	static inline int get_aead_subtype(struct crypto_aead *aead)
163	{
164	struct aead_alg *alg = crypto_aead_alg(tfm: aead);
165	struct chcr_alg_template *chcr_crypto_alg =
166	container_of(alg, struct chcr_alg_template, alg.aead);
167	return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
168	}
169
170	void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
171	{
172	u8 temp[SHA512_DIGEST_SIZE];
173	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
174	int authsize = crypto_aead_authsize(tfm);
175	struct cpl_fw6_pld *fw6_pld;
176	int cmp = 0;
177
178	fw6_pld = (struct cpl_fw6_pld *)input;
179	if ((get_aead_subtype(aead: tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
180	(get_aead_subtype(aead: tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
181	cmp = crypto_memneq(a: &fw6_pld->data[2], b: (fw6_pld + 1), size: authsize);
182	} else {
183
184	sg_pcopy_to_buffer(sgl: req->src, nents: sg_nents(sg: req->src), buf: temp,
185	buflen: authsize, skip: req->assoclen +
186	req->cryptlen - authsize);
187	cmp = crypto_memneq(a: temp, b: (fw6_pld + 1), size: authsize);
188	}
189	if (cmp)
190	*err = -EBADMSG;
191	else
192	*err = 0;
193	}
194
195	static int chcr_inc_wrcount(struct chcr_dev *dev)
196	{
197	if (dev->state == CHCR_DETACH)
198	return 1;
199	atomic_inc(v: &dev->inflight);
200	return 0;
201	}
202
203	static inline void chcr_dec_wrcount(struct chcr_dev *dev)
204	{
205	atomic_dec(v: &dev->inflight);
206	}
207
208	static inline int chcr_handle_aead_resp(struct aead_request *req,
209	unsigned char *input,
210	int err)
211	{
212	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
213	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
214	struct chcr_dev *dev = a_ctx(tfm)->dev;
215
216	chcr_aead_common_exit(req);
217	if (reqctx->verify == VERIFY_SW) {
218	chcr_verify_tag(req, input, err: &err);
219	reqctx->verify = VERIFY_HW;
220	}
221	chcr_dec_wrcount(dev);
222	aead_request_complete(req, err);
223
224	return err;
225	}
226
227	static void get_aes_decrypt_key(unsigned char *dec_key,
228	const unsigned char *key,
229	unsigned int keylength)
230	{
231	u32 temp;
232	u32 w_ring[MAX_NK];
233	int i, j, k;
234	u8 nr, nk;
235
236	switch (keylength) {
237	case AES_KEYLENGTH_128BIT:
238	nk = KEYLENGTH_4BYTES;
239	nr = NUMBER_OF_ROUNDS_10;
240	break;
241	case AES_KEYLENGTH_192BIT:
242	nk = KEYLENGTH_6BYTES;
243	nr = NUMBER_OF_ROUNDS_12;
244	break;
245	case AES_KEYLENGTH_256BIT:
246	nk = KEYLENGTH_8BYTES;
247	nr = NUMBER_OF_ROUNDS_14;
248	break;
249	default:
250	return;
251	}
252	for (i = 0; i < nk; i++)
253	w_ring[i] = get_unaligned_be32(p: &key[i * 4]);
254
255	i = 0;
256	temp = w_ring[nk - 1];
257	while (i + nk < (nr + 1) * 4) {
258	if (!(i % nk)) {
259	/* RotWord(temp) */
260	temp = (temp << 8) | (temp >> 24);
261	temp = aes_ks_subword(w: temp);
262	temp ^= round_constant[i / nk];
263	} else if (nk == 8 && (i % 4 == 0)) {
264	temp = aes_ks_subword(w: temp);
265	}
266	w_ring[i % nk] ^= temp;
267	temp = w_ring[i % nk];
268	i++;
269	}
270	i--;
271	for (k = 0, j = i % nk; k < nk; k++) {
272	put_unaligned_be32(val: w_ring[j], p: &dec_key[k * 4]);
273	j--;
274	if (j < 0)
275	j += nk;
276	}
277	}
278
279	static int chcr_prepare_hmac_key(const u8 *raw_key, unsigned int raw_key_len,
280	int digestsize, void *istate, void *ostate)
281	{
282	__be32 *istate32 = istate, *ostate32 = ostate;
283	__be64 *istate64 = istate, *ostate64 = ostate;
284	union {
285	struct hmac_sha1_key sha1;
286	struct hmac_sha224_key sha224;
287	struct hmac_sha256_key sha256;
288	struct hmac_sha384_key sha384;
289	struct hmac_sha512_key sha512;
290	} k;
291
292	switch (digestsize) {
293	case SHA1_DIGEST_SIZE:
294	hmac_sha1_preparekey(key: &k.sha1, raw_key, raw_key_len);
295	for (int i = 0; i < ARRAY_SIZE(k.sha1.istate.h); i++) {
296	istate32[i] = cpu_to_be32(k.sha1.istate.h[i]);
297	ostate32[i] = cpu_to_be32(k.sha1.ostate.h[i]);
298	}
299	break;
300	case SHA224_DIGEST_SIZE:
301	hmac_sha224_preparekey(key: &k.sha224, raw_key, raw_key_len);
302	for (int i = 0; i < ARRAY_SIZE(k.sha224.key.istate.h); i++) {
303	istate32[i] = cpu_to_be32(k.sha224.key.istate.h[i]);
304	ostate32[i] = cpu_to_be32(k.sha224.key.ostate.h[i]);
305	}
306	break;
307	case SHA256_DIGEST_SIZE:
308	hmac_sha256_preparekey(key: &k.sha256, raw_key, raw_key_len);
309	for (int i = 0; i < ARRAY_SIZE(k.sha256.key.istate.h); i++) {
310	istate32[i] = cpu_to_be32(k.sha256.key.istate.h[i]);
311	ostate32[i] = cpu_to_be32(k.sha256.key.ostate.h[i]);
312	}
313	break;
314	case SHA384_DIGEST_SIZE:
315	hmac_sha384_preparekey(key: &k.sha384, raw_key, raw_key_len);
316	for (int i = 0; i < ARRAY_SIZE(k.sha384.key.istate.h); i++) {
317	istate64[i] = cpu_to_be64(k.sha384.key.istate.h[i]);
318	ostate64[i] = cpu_to_be64(k.sha384.key.ostate.h[i]);
319	}
320	break;
321	case SHA512_DIGEST_SIZE:
322	hmac_sha512_preparekey(key: &k.sha512, raw_key, raw_key_len);
323	for (int i = 0; i < ARRAY_SIZE(k.sha512.key.istate.h); i++) {
324	istate64[i] = cpu_to_be64(k.sha512.key.istate.h[i]);
325	ostate64[i] = cpu_to_be64(k.sha512.key.ostate.h[i]);
326	}
327	break;
328	default:
329	return -EINVAL;
330	}
331	memzero_explicit(s: &k, count: sizeof(k));
332	return 0;
333	}
334
335	static inline int is_hmac(struct crypto_tfm *tfm)
336	{
337	struct crypto_alg *alg = tfm->__crt_alg;
338	struct chcr_alg_template *chcr_crypto_alg =
339	container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
340	alg.hash);
341	if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
342	return 1;
343	return 0;
344	}
345
346	static inline void dsgl_walk_init(struct dsgl_walk *walk,
347	struct cpl_rx_phys_dsgl *dsgl)
348	{
349	walk->dsgl = dsgl;
350	walk->nents = 0;
351	walk->to = (struct phys_sge_pairs *)(dsgl + 1);
352	}
353
354	static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid,
355	int pci_chan_id)
356	{
357	struct cpl_rx_phys_dsgl *phys_cpl;
358
359	phys_cpl = walk->dsgl;
360
361	phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
362	| CPL_RX_PHYS_DSGL_ISRDMA_V(0));
363	phys_cpl->pcirlxorder_to_noofsgentr =
364	htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
365	CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
366	CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
367	CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
368	CPL_RX_PHYS_DSGL_DCAID_V(0) |
369	CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents));
370	phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
371	phys_cpl->rss_hdr_int.qid = htons(qid);
372	phys_cpl->rss_hdr_int.hash_val = 0;
373	phys_cpl->rss_hdr_int.channel = pci_chan_id;
374	}
375
376	static inline void dsgl_walk_add_page(struct dsgl_walk *walk,
377	size_t size,
378	dma_addr_t addr)
379	{
380	int j;
381
382	if (!size)
383	return;
384	j = walk->nents;
385	walk->to->len[j % 8] = htons(size);
386	walk->to->addr[j % 8] = cpu_to_be64(addr);
387	j++;
388	if ((j % 8) == 0)
389	walk->to++;
390	walk->nents = j;
391	}
392
393	static void dsgl_walk_add_sg(struct dsgl_walk *walk,
394	struct scatterlist *sg,
395	unsigned int slen,
396	unsigned int skip)
397	{
398	int skip_len = 0;
399	unsigned int left_size = slen, len = 0;
400	unsigned int j = walk->nents;
401	int offset, ent_len;
402
403	if (!slen)
404	return;
405	while (sg && skip) {
406	if (sg_dma_len(sg) <= skip) {
407	skip -= sg_dma_len(sg);
408	skip_len = 0;
409	sg = sg_next(sg);
410	} else {
411	skip_len = skip;
412	skip = 0;
413	}
414	}
415
416	while (left_size && sg) {
417	len = min_t(u32, left_size, sg_dma_len(sg) - skip_len);
418	offset = 0;
419	while (len) {
420	ent_len = min_t(u32, len, CHCR_DST_SG_SIZE);
421	walk->to->len[j % 8] = htons(ent_len);
422	walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
423	offset + skip_len);
424	offset += ent_len;
425	len -= ent_len;
426	j++;
427	if ((j % 8) == 0)
428	walk->to++;
429	}
430	walk->last_sg = sg;
431	walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) -
432	skip_len) + skip_len;
433	left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len);
434	skip_len = 0;
435	sg = sg_next(sg);
436	}
437	walk->nents = j;
438	}
439
440	static inline void ulptx_walk_init(struct ulptx_walk *walk,
441	struct ulptx_sgl *ulp)
442	{
443	walk->sgl = ulp;
444	walk->nents = 0;
445	walk->pair_idx = 0;
446	walk->pair = ulp->sge;
447	walk->last_sg = NULL;
448	walk->last_sg_len = 0;
449	}
450
451	static inline void ulptx_walk_end(struct ulptx_walk *walk)
452	{
453	walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
454	ULPTX_NSGE_V(walk->nents));
455	}
456
457
458	static inline void ulptx_walk_add_page(struct ulptx_walk *walk,
459	size_t size,
460	dma_addr_t addr)
461	{
462	if (!size)
463	return;
464
465	if (walk->nents == 0) {
466	walk->sgl->len0 = cpu_to_be32(size);
467	walk->sgl->addr0 = cpu_to_be64(addr);
468	} else {
469	walk->pair->addr[walk->pair_idx] = cpu_to_be64(addr);
470	walk->pair->len[walk->pair_idx] = cpu_to_be32(size);
471	walk->pair_idx = !walk->pair_idx;
472	if (!walk->pair_idx)
473	walk->pair++;
474	}
475	walk->nents++;
476	}
477
478	static void ulptx_walk_add_sg(struct ulptx_walk *walk,
479	struct scatterlist *sg,
480	unsigned int len,
481	unsigned int skip)
482	{
483	int small;
484	int skip_len = 0;
485	unsigned int sgmin;
486
487	if (!len)
488	return;
489	while (sg && skip) {
490	if (sg_dma_len(sg) <= skip) {
491	skip -= sg_dma_len(sg);
492	skip_len = 0;
493	sg = sg_next(sg);
494	} else {
495	skip_len = skip;
496	skip = 0;
497	}
498	}
499	WARN(!sg, "SG should not be null here\n");
500	if (sg && (walk->nents == 0)) {
501	small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
502	sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
503	walk->sgl->len0 = cpu_to_be32(sgmin);
504	walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len);
505	walk->nents++;
506	len -= sgmin;
507	walk->last_sg = sg;
508	walk->last_sg_len = sgmin + skip_len;
509	skip_len += sgmin;
510	if (sg_dma_len(sg) == skip_len) {
511	sg = sg_next(sg);
512	skip_len = 0;
513	}
514	}
515
516	while (sg && len) {
517	small = min(sg_dma_len(sg) - skip_len, len);
518	sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
519	walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin);
520	walk->pair->addr[walk->pair_idx] =
521	cpu_to_be64(sg_dma_address(sg) + skip_len);
522	walk->pair_idx = !walk->pair_idx;
523	walk->nents++;
524	if (!walk->pair_idx)
525	walk->pair++;
526	len -= sgmin;
527	skip_len += sgmin;
528	walk->last_sg = sg;
529	walk->last_sg_len = skip_len;
530	if (sg_dma_len(sg) == skip_len) {
531	sg = sg_next(sg);
532	skip_len = 0;
533	}
534	}
535	}
536
537	static inline int get_cryptoalg_subtype(struct crypto_skcipher *tfm)
538	{
539	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
540	struct chcr_alg_template *chcr_crypto_alg =
541	container_of(alg, struct chcr_alg_template, alg.skcipher);
542
543	return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
544	}
545
546	static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
547	{
548	struct adapter *adap = netdev2adap(dev);
549	struct sge_uld_txq_info *txq_info =
550	adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
551	struct sge_uld_txq *txq;
552	int ret = 0;
553
554	local_bh_disable();
555	txq = &txq_info->uldtxq[idx];
556	spin_lock(lock: &txq->sendq.lock);
557	if (txq->full)
558	ret = -1;
559	spin_unlock(lock: &txq->sendq.lock);
560	local_bh_enable();
561	return ret;
562	}
563
564	static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
565	struct _key_ctx *key_ctx)
566	{
567	if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
568	memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
569	} else {
570	memcpy(key_ctx->key,
571	ablkctx->key + (ablkctx->enckey_len >> 1),
572	ablkctx->enckey_len >> 1);
573	memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
574	ablkctx->rrkey, ablkctx->enckey_len >> 1);
575	}
576	return 0;
577	}
578
579	static int chcr_hash_ent_in_wr(struct scatterlist *src,
580	unsigned int minsg,
581	unsigned int space,
582	unsigned int srcskip)
583	{
584	int srclen = 0;
585	int srcsg = minsg;
586	int soffset = 0, sless;
587
588	if (sg_dma_len(src) == srcskip) {
589	src = sg_next(sg: src);
590	srcskip = 0;
591	}
592	while (src && space > (sgl_ent_len[srcsg + 1])) {
593	sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip,
594	CHCR_SRC_SG_SIZE);
595	srclen += sless;
596	soffset += sless;
597	srcsg++;
598	if (sg_dma_len(src) == (soffset + srcskip)) {
599	src = sg_next(sg: src);
600	soffset = 0;
601	srcskip = 0;
602	}
603	}
604	return srclen;
605	}
606
607	static int chcr_sg_ent_in_wr(struct scatterlist *src,
608	struct scatterlist *dst,
609	unsigned int minsg,
610	unsigned int space,
611	unsigned int srcskip,
612	unsigned int dstskip)
613	{
614	int srclen = 0, dstlen = 0;
615	int srcsg = minsg, dstsg = minsg;
616	int offset = 0, soffset = 0, less, sless = 0;
617
618	if (sg_dma_len(src) == srcskip) {
619	src = sg_next(sg: src);
620	srcskip = 0;
621	}
622	if (sg_dma_len(dst) == dstskip) {
623	dst = sg_next(sg: dst);
624	dstskip = 0;
625	}
626
627	while (src && dst &&
628	space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
629	sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset,
630	CHCR_SRC_SG_SIZE);
631	srclen += sless;
632	srcsg++;
633	offset = 0;
634	while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
635	space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
636	if (srclen <= dstlen)
637	break;
638	less = min_t(unsigned int, sg_dma_len(dst) - offset -
639	dstskip, CHCR_DST_SG_SIZE);
640	dstlen += less;
641	offset += less;
642	if ((offset + dstskip) == sg_dma_len(dst)) {
643	dst = sg_next(sg: dst);
644	offset = 0;
645	}
646	dstsg++;
647	dstskip = 0;
648	}
649	soffset += sless;
650	if ((soffset + srcskip) == sg_dma_len(src)) {
651	src = sg_next(sg: src);
652	srcskip = 0;
653	soffset = 0;
654	}
655
656	}
657	return min(srclen, dstlen);
658	}
659
660	static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
661	struct skcipher_request *req,
662	u8 *iv,
663	unsigned short op_type)
664	{
665	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
666	int err;
667
668	skcipher_request_set_tfm(req: &reqctx->fallback_req, tfm: cipher);
669	skcipher_request_set_callback(req: &reqctx->fallback_req, flags: req->base.flags,
670	compl: req->base.complete, data: req->base.data);
671	skcipher_request_set_crypt(req: &reqctx->fallback_req, src: req->src, dst: req->dst,
672	cryptlen: req->cryptlen, iv);
673
674	err = op_type ? crypto_skcipher_decrypt(req: &reqctx->fallback_req) :
675	crypto_skcipher_encrypt(req: &reqctx->fallback_req);
676
677	return err;
678
679	}
680
681	static inline int get_qidxs(struct crypto_async_request *req,
682	unsigned int *txqidx, unsigned int *rxqidx)
683	{
684	struct crypto_tfm *tfm = req->tfm;
685	int ret = 0;
686
687	switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
688	case CRYPTO_ALG_TYPE_AEAD:
689	{
690	struct aead_request *aead_req =
691	container_of(req, struct aead_request, base);
692	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req: aead_req);
693	*txqidx = reqctx->txqidx;
694	*rxqidx = reqctx->rxqidx;
695	break;
696	}
697	case CRYPTO_ALG_TYPE_SKCIPHER:
698	{
699	struct skcipher_request *sk_req =
700	container_of(req, struct skcipher_request, base);
701	struct chcr_skcipher_req_ctx *reqctx =
702	skcipher_request_ctx(req: sk_req);
703	*txqidx = reqctx->txqidx;
704	*rxqidx = reqctx->rxqidx;
705	break;
706	}
707	case CRYPTO_ALG_TYPE_AHASH:
708	{
709	struct ahash_request *ahash_req =
710	container_of(req, struct ahash_request, base);
711	struct chcr_ahash_req_ctx *reqctx =
712	ahash_request_ctx(req: ahash_req);
713	*txqidx = reqctx->txqidx;
714	*rxqidx = reqctx->rxqidx;
715	break;
716	}
717	default:
718	ret = -EINVAL;
719	/* should never get here */
720	BUG();
721	break;
722	}
723	return ret;
724	}
725
726	static inline void create_wreq(struct chcr_context *ctx,
727	struct chcr_wr *chcr_req,
728	struct crypto_async_request *req,
729	unsigned int imm,
730	int hash_sz,
731	unsigned int len16,
732	unsigned int sc_len,
733	unsigned int lcb)
734	{
735	struct uld_ctx *u_ctx = ULD_CTX(ctx);
736	unsigned int tx_channel_id, rx_channel_id;
737	unsigned int txqidx = 0, rxqidx = 0;
738	unsigned int qid, fid, portno;
739
740	get_qidxs(req, txqidx: &txqidx, rxqidx: &rxqidx);
741	qid = u_ctx->lldi.rxq_ids[rxqidx];
742	fid = u_ctx->lldi.rxq_ids[0];
743	portno = rxqidx / ctx->rxq_perchan;
744	tx_channel_id = txqidx / ctx->txq_perchan;
745	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[portno]);
746
747
748	chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
749	chcr_req->wreq.pld_size_hash_size =
750	htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
751	chcr_req->wreq.len16_pkd =
752	htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16)));
753	chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
754	chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(rx_channel_id, qid,
755	!!lcb, txqidx);
756
757	chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(tx_channel_id, fid);
758	chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) -
759	((sizeof(chcr_req->wreq)) >> 4)));
760	chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm);
761	chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
762	sizeof(chcr_req->key_ctx) + sc_len);
763	}
764
765	/**
766	* create_cipher_wr - form the WR for cipher operations
767	* @wrparam: Container for create_cipher_wr()'s parameters
768	*/
769	static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
770	{
771	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: wrparam->req);
772	struct chcr_context *ctx = c_ctx(tfm);
773	struct uld_ctx *u_ctx = ULD_CTX(ctx);
774	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
775	struct sk_buff *skb = NULL;
776	struct chcr_wr *chcr_req;
777	struct cpl_rx_phys_dsgl *phys_cpl;
778	struct ulptx_sgl *ulptx;
779	struct chcr_skcipher_req_ctx *reqctx =
780	skcipher_request_ctx(req: wrparam->req);
781	unsigned int temp = 0, transhdr_len, dst_size;
782	int error;
783	int nents;
784	unsigned int kctx_len;
785	gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
786	GFP_KERNEL : GFP_ATOMIC;
787	struct adapter *adap = padap(dev: ctx->dev);
788	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
789
790	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
791	nents = sg_nents_xlen(sg: reqctx->dstsg, reqlen: wrparam->bytes, CHCR_DST_SG_SIZE,
792	skip: reqctx->dst_ofst);
793	dst_size = get_space_for_phys_dsgl(sgl_entr: nents);
794	kctx_len = roundup(ablkctx->enckey_len, 16);
795	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
796	nents = sg_nents_xlen(sg: reqctx->srcsg, reqlen: wrparam->bytes,
797	CHCR_SRC_SG_SIZE, skip: reqctx->src_ofst);
798	temp = reqctx->imm ? roundup(wrparam->bytes, 16) :
799	(sgl_len(n: nents) * 8);
800	transhdr_len += temp;
801	transhdr_len = roundup(transhdr_len, 16);
802	skb = alloc_skb(size: SGE_MAX_WR_LEN, priority: flags);
803	if (!skb) {
804	error = -ENOMEM;
805	goto err;
806	}
807	chcr_req = __skb_put_zero(skb, len: transhdr_len);
808	chcr_req->sec_cpl.op_ivinsrtofst =
809	FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
810
811	chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes);
812	chcr_req->sec_cpl.aadstart_cipherstop_hi =
813	FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0);
814
815	chcr_req->sec_cpl.cipherstop_lo_authinsert =
816	FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
817	chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
818	ablkctx->ciph_mode,
819	0, 0, IV >> 1);
820	chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
821	0, 1, dst_size);
822
823	chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
824	if ((reqctx->op == CHCR_DECRYPT_OP) &&
825	(!(get_cryptoalg_subtype(tfm) ==
826	CRYPTO_ALG_SUB_TYPE_CTR)) &&
827	(!(get_cryptoalg_subtype(tfm) ==
828	CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
829	generate_copy_rrkey(ablkctx, key_ctx: &chcr_req->key_ctx);
830	} else {
831	if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
832	(ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
833	memcpy(chcr_req->key_ctx.key, ablkctx->key,
834	ablkctx->enckey_len);
835	} else {
836	memcpy(chcr_req->key_ctx.key, ablkctx->key +
837	(ablkctx->enckey_len >> 1),
838	ablkctx->enckey_len >> 1);
839	memcpy(chcr_req->key_ctx.key +
840	(ablkctx->enckey_len >> 1),
841	ablkctx->key,
842	ablkctx->enckey_len >> 1);
843	}
844	}
845	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
846	ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
847	chcr_add_cipher_src_ent(req: wrparam->req, ulptx, wrparam);
848	chcr_add_cipher_dst_ent(req: wrparam->req, phys_cpl, wrparam, qid: wrparam->qid);
849
850	atomic_inc(v: &adap->chcr_stats.cipher_rqst);
851	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV
852	+ (reqctx->imm ? (wrparam->bytes) : 0);
853	create_wreq(ctx: c_ctx(tfm), chcr_req, req: &(wrparam->req->base), imm: reqctx->imm, hash_sz: 0,
854	len16: transhdr_len, sc_len: temp,
855	lcb: ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
856	reqctx->skb = skb;
857
858	if (reqctx->op && (ablkctx->ciph_mode ==
859	CHCR_SCMD_CIPHER_MODE_AES_CBC))
860	sg_pcopy_to_buffer(sgl: wrparam->req->src,
861	nents: sg_nents(sg: wrparam->req->src), buf: wrparam->req->iv, buflen: 16,
862	skip: reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE);
863
864	return skb;
865	err:
866	return ERR_PTR(error);
867	}
868
869	static inline int chcr_keyctx_ck_size(unsigned int keylen)
870	{
871	int ck_size = 0;
872
873	if (keylen == AES_KEYSIZE_128)
874	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
875	else if (keylen == AES_KEYSIZE_192)
876	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
877	else if (keylen == AES_KEYSIZE_256)
878	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
879	else
880	ck_size = 0;
881
882	return ck_size;
883	}
884	static int chcr_cipher_fallback_setkey(struct crypto_skcipher *cipher,
885	const u8 *key,
886	unsigned int keylen)
887	{
888	struct ablk_ctx *ablkctx = ABLK_CTX(ctx: c_ctx(tfm: cipher));
889
890	crypto_skcipher_clear_flags(tfm: ablkctx->sw_cipher,
891	CRYPTO_TFM_REQ_MASK);
892	crypto_skcipher_set_flags(tfm: ablkctx->sw_cipher,
893	flags: cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK);
894	return crypto_skcipher_setkey(tfm: ablkctx->sw_cipher, key, keylen);
895	}
896
897	static int chcr_aes_cbc_setkey(struct crypto_skcipher *cipher,
898	const u8 *key,
899	unsigned int keylen)
900	{
901	struct ablk_ctx *ablkctx = ABLK_CTX(ctx: c_ctx(tfm: cipher));
902	unsigned int ck_size, context_size;
903	u16 alignment = 0;
904	int err;
905
906	err = chcr_cipher_fallback_setkey(cipher, key, keylen);
907	if (err)
908	goto badkey_err;
909
910	ck_size = chcr_keyctx_ck_size(keylen);
911	alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
912	memcpy(ablkctx->key, key, keylen);
913	ablkctx->enckey_len = keylen;
914	get_aes_decrypt_key(dec_key: ablkctx->rrkey, key: ablkctx->key, keylength: keylen << 3);
915	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
916	keylen + alignment) >> 4;
917
918	ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
919	0, 0, context_size);
920	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
921	return 0;
922	badkey_err:
923	ablkctx->enckey_len = 0;
924
925	return err;
926	}
927
928	static int chcr_aes_ctr_setkey(struct crypto_skcipher *cipher,
929	const u8 *key,
930	unsigned int keylen)
931	{
932	struct ablk_ctx *ablkctx = ABLK_CTX(ctx: c_ctx(tfm: cipher));
933	unsigned int ck_size, context_size;
934	u16 alignment = 0;
935	int err;
936
937	err = chcr_cipher_fallback_setkey(cipher, key, keylen);
938	if (err)
939	goto badkey_err;
940	ck_size = chcr_keyctx_ck_size(keylen);
941	alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
942	memcpy(ablkctx->key, key, keylen);
943	ablkctx->enckey_len = keylen;
944	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
945	keylen + alignment) >> 4;
946
947	ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
948	0, 0, context_size);
949	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
950
951	return 0;
952	badkey_err:
953	ablkctx->enckey_len = 0;
954
955	return err;
956	}
957
958	static int chcr_aes_rfc3686_setkey(struct crypto_skcipher *cipher,
959	const u8 *key,
960	unsigned int keylen)
961	{
962	struct ablk_ctx *ablkctx = ABLK_CTX(ctx: c_ctx(tfm: cipher));
963	unsigned int ck_size, context_size;
964	u16 alignment = 0;
965	int err;
966
967	if (keylen < CTR_RFC3686_NONCE_SIZE)
968	return -EINVAL;
969	memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
970	CTR_RFC3686_NONCE_SIZE);
971
972	keylen -= CTR_RFC3686_NONCE_SIZE;
973	err = chcr_cipher_fallback_setkey(cipher, key, keylen);
974	if (err)
975	goto badkey_err;
976
977	ck_size = chcr_keyctx_ck_size(keylen);
978	alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
979	memcpy(ablkctx->key, key, keylen);
980	ablkctx->enckey_len = keylen;
981	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
982	keylen + alignment) >> 4;
983
984	ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
985	0, 0, context_size);
986	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
987
988	return 0;
989	badkey_err:
990	ablkctx->enckey_len = 0;
991
992	return err;
993	}
994	static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
995	{
996	unsigned int size = AES_BLOCK_SIZE;
997	__be32 *b = (__be32 *)(dstiv + size);
998	u32 c, prev;
999
1000	memcpy(dstiv, srciv, AES_BLOCK_SIZE);
1001	for (; size >= 4; size -= 4) {
1002	prev = be32_to_cpu(*--b);
1003	c = prev + add;
1004	*b = cpu_to_be32(c);
1005	if (prev < c)
1006	break;
1007	add = 1;
1008	}
1009
1010	}
1011
1012	static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
1013	{
1014	__be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
1015	u64 c;
1016	u32 temp = be32_to_cpu(*--b);
1017
1018	temp = ~temp;
1019	c = (u64)temp + 1; // No of block can processed without overflow
1020	if ((bytes / AES_BLOCK_SIZE) >= c)
1021	bytes = c * AES_BLOCK_SIZE;
1022	return bytes;
1023	}
1024
1025	static int chcr_update_tweak(struct skcipher_request *req, u8 *iv,
1026	u32 isfinal)
1027	{
1028	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1029	struct ablk_ctx *ablkctx = ABLK_CTX(ctx: c_ctx(tfm));
1030	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1031	struct crypto_aes_ctx aes;
1032	int ret, i;
1033	u8 *key;
1034	unsigned int keylen;
1035	int round = reqctx->last_req_len / AES_BLOCK_SIZE;
1036	int round8 = round / 8;
1037
1038	memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1039
1040	keylen = ablkctx->enckey_len / 2;
1041	key = ablkctx->key + keylen;
1042	/* For a 192 bit key remove the padded zeroes which was
1043	* added in chcr_xts_setkey
1044	*/
1045	if (KEY_CONTEXT_CK_SIZE_G(ntohl(ablkctx->key_ctx_hdr))
1046	== CHCR_KEYCTX_CIPHER_KEY_SIZE_192)
1047	ret = aes_expandkey(ctx: &aes, in_key: key, key_len: keylen - 8);
1048	else
1049	ret = aes_expandkey(ctx: &aes, in_key: key, key_len: keylen);
1050	if (ret)
1051	return ret;
1052	aes_encrypt(ctx: &aes, out: iv, in: iv);
1053	for (i = 0; i < round8; i++)
1054	gf128mul_x8_ble(r: (le128 *)iv, x: (le128 *)iv);
1055
1056	for (i = 0; i < (round % 8); i++)
1057	gf128mul_x_ble(r: (le128 *)iv, x: (le128 *)iv);
1058
1059	if (!isfinal)
1060	aes_decrypt(ctx: &aes, out: iv, in: iv);
1061
1062	memzero_explicit(s: &aes, count: sizeof(aes));
1063	return 0;
1064	}
1065
1066	static int chcr_update_cipher_iv(struct skcipher_request *req,
1067	struct cpl_fw6_pld *fw6_pld, u8 *iv)
1068	{
1069	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1070	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1071	int subtype = get_cryptoalg_subtype(tfm);
1072	int ret = 0;
1073
1074	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1075	ctr_add_iv(dstiv: iv, srciv: req->iv, add: (reqctx->processed /
1076	AES_BLOCK_SIZE));
1077	else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
1078	*(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1079	CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
1080	AES_BLOCK_SIZE) + 1);
1081	else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1082	ret = chcr_update_tweak(req, iv, isfinal: 0);
1083	else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1084	if (reqctx->op)
1085	/*Updated before sending last WR*/
1086	memcpy(iv, req->iv, AES_BLOCK_SIZE);
1087	else
1088	memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1089	}
1090
1091	return ret;
1092
1093	}
1094
1095	/* We need separate function for final iv because in rfc3686 Initial counter
1096	* starts from 1 and buffer size of iv is 8 byte only which remains constant
1097	* for subsequent update requests
1098	*/
1099
1100	static int chcr_final_cipher_iv(struct skcipher_request *req,
1101	struct cpl_fw6_pld *fw6_pld, u8 *iv)
1102	{
1103	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1104	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1105	int subtype = get_cryptoalg_subtype(tfm);
1106	int ret = 0;
1107
1108	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1109	ctr_add_iv(dstiv: iv, srciv: req->iv, DIV_ROUND_UP(reqctx->processed,
1110	AES_BLOCK_SIZE));
1111	else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) {
1112	if (!reqctx->partial_req)
1113	memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1114	else
1115	ret = chcr_update_tweak(req, iv, isfinal: 1);
1116	}
1117	else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1118	/*Already updated for Decrypt*/
1119	if (!reqctx->op)
1120	memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1121
1122	}
1123	return ret;
1124
1125	}
1126
1127	static int chcr_handle_cipher_resp(struct skcipher_request *req,
1128	unsigned char *input, int err)
1129	{
1130	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1131	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1132	struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
1133	struct ablk_ctx *ablkctx = ABLK_CTX(ctx: c_ctx(tfm));
1134	struct uld_ctx *u_ctx = ULD_CTX(ctx: c_ctx(tfm));
1135	struct chcr_dev *dev = c_ctx(tfm)->dev;
1136	struct chcr_context *ctx = c_ctx(tfm);
1137	struct adapter *adap = padap(dev: ctx->dev);
1138	struct cipher_wr_param wrparam;
1139	struct sk_buff *skb;
1140	int bytes;
1141
1142	if (err)
1143	goto unmap;
1144	if (req->cryptlen == reqctx->processed) {
1145	chcr_cipher_dma_unmap(dev: &ULD_CTX(ctx: c_ctx(tfm))->lldi.pdev->dev,
1146	req);
1147	err = chcr_final_cipher_iv(req, fw6_pld, iv: req->iv);
1148	goto complete;
1149	}
1150
1151	if (!reqctx->imm) {
1152	bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0,
1153	CIP_SPACE_LEFT(ablkctx->enckey_len),
1154	reqctx->src_ofst, reqctx->dst_ofst);
1155	if ((bytes + reqctx->processed) >= req->cryptlen)
1156	bytes = req->cryptlen - reqctx->processed;
1157	else
1158	bytes = rounddown(bytes, 16);
1159	} else {
1160	/*CTR mode counter overflow*/
1161	bytes = req->cryptlen - reqctx->processed;
1162	}
1163	err = chcr_update_cipher_iv(req, fw6_pld, iv: reqctx->iv);
1164	if (err)
1165	goto unmap;
1166
1167	if (unlikely(bytes == 0)) {
1168	chcr_cipher_dma_unmap(dev: &ULD_CTX(ctx: c_ctx(tfm))->lldi.pdev->dev,
1169	req);
1170	memcpy(req->iv, reqctx->init_iv, IV);
1171	atomic_inc(v: &adap->chcr_stats.fallback);
1172	err = chcr_cipher_fallback(cipher: ablkctx->sw_cipher, req, iv: req->iv,
1173	op_type: reqctx->op);
1174	goto complete;
1175	}
1176
1177	if (get_cryptoalg_subtype(tfm) ==
1178	CRYPTO_ALG_SUB_TYPE_CTR)
1179	bytes = adjust_ctr_overflow(iv: reqctx->iv, bytes);
1180	wrparam.qid = u_ctx->lldi.rxq_ids[reqctx->rxqidx];
1181	wrparam.req = req;
1182	wrparam.bytes = bytes;
1183	skb = create_cipher_wr(wrparam: &wrparam);
1184	if (IS_ERR(ptr: skb)) {
1185	pr_err("%s : Failed to form WR. No memory\n", __func__);
1186	err = PTR_ERR(ptr: skb);
1187	goto unmap;
1188	}
1189	skb->dev = u_ctx->lldi.ports[0];
1190	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1191	chcr_send_wr(skb);
1192	reqctx->last_req_len = bytes;
1193	reqctx->processed += bytes;
1194	if (get_cryptoalg_subtype(tfm) ==
1195	CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1196	CRYPTO_TFM_REQ_MAY_SLEEP ) {
1197	complete(&ctx->cbc_aes_aio_done);
1198	}
1199	return 0;
1200	unmap:
1201	chcr_cipher_dma_unmap(dev: &ULD_CTX(ctx: c_ctx(tfm))->lldi.pdev->dev, req);
1202	complete:
1203	if (get_cryptoalg_subtype(tfm) ==
1204	CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1205	CRYPTO_TFM_REQ_MAY_SLEEP ) {
1206	complete(&ctx->cbc_aes_aio_done);
1207	}
1208	chcr_dec_wrcount(dev);
1209	skcipher_request_complete(req, err);
1210	return err;
1211	}
1212
1213	static int process_cipher(struct skcipher_request *req,
1214	unsigned short qid,
1215	struct sk_buff **skb,
1216	unsigned short op_type)
1217	{
1218	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1219	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1220	unsigned int ivsize = crypto_skcipher_ivsize(tfm);
1221	struct ablk_ctx *ablkctx = ABLK_CTX(ctx: c_ctx(tfm));
1222	struct adapter *adap = padap(dev: c_ctx(tfm)->dev);
1223	struct cipher_wr_param wrparam;
1224	int bytes, err = -EINVAL;
1225	int subtype;
1226
1227	reqctx->processed = 0;
1228	reqctx->partial_req = 0;
1229	if (!req->iv)
1230	goto error;
1231	subtype = get_cryptoalg_subtype(tfm);
1232	if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1233	(req->cryptlen == 0) ||
1234	(req->cryptlen % crypto_skcipher_blocksize(tfm))) {
1235	if (req->cryptlen == 0 && subtype != CRYPTO_ALG_SUB_TYPE_XTS)
1236	goto fallback;
1237	else if (req->cryptlen % crypto_skcipher_blocksize(tfm) &&
1238	subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1239	goto fallback;
1240	pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1241	ablkctx->enckey_len, req->cryptlen, ivsize);
1242	goto error;
1243	}
1244
1245	err = chcr_cipher_dma_map(dev: &ULD_CTX(ctx: c_ctx(tfm))->lldi.pdev->dev, req);
1246	if (err)
1247	goto error;
1248	if (req->cryptlen < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) +
1249	AES_MIN_KEY_SIZE +
1250	sizeof(struct cpl_rx_phys_dsgl) +
1251	/*Min dsgl size*/
1252	32))) {
1253	/* Can be sent as Imm*/
1254	unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len;
1255
1256	dnents = sg_nents_xlen(sg: req->dst, reqlen: req->cryptlen,
1257	CHCR_DST_SG_SIZE, skip: 0);
1258	phys_dsgl = get_space_for_phys_dsgl(sgl_entr: dnents);
1259	kctx_len = roundup(ablkctx->enckey_len, 16);
1260	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
1261	reqctx->imm = (transhdr_len + IV + req->cryptlen) <=
1262	SGE_MAX_WR_LEN;
1263	bytes = IV + req->cryptlen;
1264
1265	} else {
1266	reqctx->imm = 0;
1267	}
1268
1269	if (!reqctx->imm) {
1270	bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0,
1271	CIP_SPACE_LEFT(ablkctx->enckey_len),
1272	0, 0);
1273	if ((bytes + reqctx->processed) >= req->cryptlen)
1274	bytes = req->cryptlen - reqctx->processed;
1275	else
1276	bytes = rounddown(bytes, 16);
1277	} else {
1278	bytes = req->cryptlen;
1279	}
1280	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) {
1281	bytes = adjust_ctr_overflow(iv: req->iv, bytes);
1282	}
1283	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1284	memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1285	memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv,
1286	CTR_RFC3686_IV_SIZE);
1287
1288	/* initialize counter portion of counter block */
1289	*(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1290	CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1291	memcpy(reqctx->init_iv, reqctx->iv, IV);
1292
1293	} else {
1294
1295	memcpy(reqctx->iv, req->iv, IV);
1296	memcpy(reqctx->init_iv, req->iv, IV);
1297	}
1298	if (unlikely(bytes == 0)) {
1299	chcr_cipher_dma_unmap(dev: &ULD_CTX(ctx: c_ctx(tfm))->lldi.pdev->dev,
1300	req);
1301	fallback: atomic_inc(v: &adap->chcr_stats.fallback);
1302	err = chcr_cipher_fallback(cipher: ablkctx->sw_cipher, req,
1303	iv: subtype ==
1304	CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 ?
1305	reqctx->iv : req->iv,
1306	op_type);
1307	goto error;
1308	}
1309	reqctx->op = op_type;
1310	reqctx->srcsg = req->src;
1311	reqctx->dstsg = req->dst;
1312	reqctx->src_ofst = 0;
1313	reqctx->dst_ofst = 0;
1314	wrparam.qid = qid;
1315	wrparam.req = req;
1316	wrparam.bytes = bytes;
1317	*skb = create_cipher_wr(wrparam: &wrparam);
1318	if (IS_ERR(ptr: *skb)) {
1319	err = PTR_ERR(ptr: *skb);
1320	goto unmap;
1321	}
1322	reqctx->processed = bytes;
1323	reqctx->last_req_len = bytes;
1324	reqctx->partial_req = !!(req->cryptlen - reqctx->processed);
1325
1326	return 0;
1327	unmap:
1328	chcr_cipher_dma_unmap(dev: &ULD_CTX(ctx: c_ctx(tfm))->lldi.pdev->dev, req);
1329	error:
1330	return err;
1331	}
1332
1333	static int chcr_aes_encrypt(struct skcipher_request *req)
1334	{
1335	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1336	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1337	struct chcr_dev *dev = c_ctx(tfm)->dev;
1338	struct sk_buff *skb = NULL;
1339	int err;
1340	struct uld_ctx *u_ctx = ULD_CTX(ctx: c_ctx(tfm));
1341	struct chcr_context *ctx = c_ctx(tfm);
1342	unsigned int cpu;
1343
1344	cpu = get_cpu();
1345	reqctx->txqidx = cpu % ctx->ntxq;
1346	reqctx->rxqidx = cpu % ctx->nrxq;
1347	put_cpu();
1348
1349	err = chcr_inc_wrcount(dev);
1350	if (err)
1351	return -ENXIO;
1352	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1353	reqctx->txqidx) &&
1354	(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1355	err = -ENOSPC;
1356	goto error;
1357	}
1358
1359	err = process_cipher(req, qid: u_ctx->lldi.rxq_ids[reqctx->rxqidx],
1360	skb: &skb, CHCR_ENCRYPT_OP);
1361	if (err || !skb)
1362	return err;
1363	skb->dev = u_ctx->lldi.ports[0];
1364	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1365	chcr_send_wr(skb);
1366	if (get_cryptoalg_subtype(tfm) ==
1367	CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1368	CRYPTO_TFM_REQ_MAY_SLEEP ) {
1369	reqctx->partial_req = 1;
1370	wait_for_completion(&ctx->cbc_aes_aio_done);
1371	}
1372	return -EINPROGRESS;
1373	error:
1374	chcr_dec_wrcount(dev);
1375	return err;
1376	}
1377
1378	static int chcr_aes_decrypt(struct skcipher_request *req)
1379	{
1380	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1381	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1382	struct uld_ctx *u_ctx = ULD_CTX(ctx: c_ctx(tfm));
1383	struct chcr_dev *dev = c_ctx(tfm)->dev;
1384	struct sk_buff *skb = NULL;
1385	int err;
1386	struct chcr_context *ctx = c_ctx(tfm);
1387	unsigned int cpu;
1388
1389	cpu = get_cpu();
1390	reqctx->txqidx = cpu % ctx->ntxq;
1391	reqctx->rxqidx = cpu % ctx->nrxq;
1392	put_cpu();
1393
1394	err = chcr_inc_wrcount(dev);
1395	if (err)
1396	return -ENXIO;
1397
1398	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1399	reqctx->txqidx) &&
1400	(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))))
1401	return -ENOSPC;
1402	err = process_cipher(req, qid: u_ctx->lldi.rxq_ids[reqctx->rxqidx],
1403	skb: &skb, CHCR_DECRYPT_OP);
1404	if (err || !skb)
1405	return err;
1406	skb->dev = u_ctx->lldi.ports[0];
1407	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1408	chcr_send_wr(skb);
1409	return -EINPROGRESS;
1410	}
1411	static int chcr_device_init(struct chcr_context *ctx)
1412	{
1413	struct uld_ctx *u_ctx = NULL;
1414	int txq_perchan, ntxq;
1415	int err = 0, rxq_perchan;
1416
1417	if (!ctx->dev) {
1418	u_ctx = assign_chcr_device();
1419	if (!u_ctx) {
1420	err = -ENXIO;
1421	pr_err("chcr device assignment fails\n");
1422	goto out;
1423	}
1424	ctx->dev = &u_ctx->dev;
1425	ntxq = u_ctx->lldi.ntxq;
1426	rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
1427	txq_perchan = ntxq / u_ctx->lldi.nchan;
1428	ctx->ntxq = ntxq;
1429	ctx->nrxq = u_ctx->lldi.nrxq;
1430	ctx->rxq_perchan = rxq_perchan;
1431	ctx->txq_perchan = txq_perchan;
1432	}
1433	out:
1434	return err;
1435	}
1436
1437	static int chcr_init_tfm(struct crypto_skcipher *tfm)
1438	{
1439	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1440	struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1441	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1442
1443	ablkctx->sw_cipher = crypto_alloc_skcipher(alg_name: alg->base.cra_name, type: 0,
1444	CRYPTO_ALG_NEED_FALLBACK);
1445	if (IS_ERR(ptr: ablkctx->sw_cipher)) {
1446	pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
1447	return PTR_ERR(ptr: ablkctx->sw_cipher);
1448	}
1449	init_completion(x: &ctx->cbc_aes_aio_done);
1450	crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct chcr_skcipher_req_ctx) +
1451	crypto_skcipher_reqsize(tfm: ablkctx->sw_cipher));
1452
1453	return chcr_device_init(ctx);
1454	}
1455
1456	static int chcr_rfc3686_init(struct crypto_skcipher *tfm)
1457	{
1458	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1459	struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1460	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1461
1462	/*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1463	* cannot be used as fallback in chcr_handle_cipher_response
1464	*/
1465	ablkctx->sw_cipher = crypto_alloc_skcipher(alg_name: "ctr(aes)", type: 0,
1466	CRYPTO_ALG_NEED_FALLBACK);
1467	if (IS_ERR(ptr: ablkctx->sw_cipher)) {
1468	pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
1469	return PTR_ERR(ptr: ablkctx->sw_cipher);
1470	}
1471	crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct chcr_skcipher_req_ctx) +
1472	crypto_skcipher_reqsize(tfm: ablkctx->sw_cipher));
1473	return chcr_device_init(ctx);
1474	}
1475
1476
1477	static void chcr_exit_tfm(struct crypto_skcipher *tfm)
1478	{
1479	struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1480	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1481
1482	crypto_free_skcipher(tfm: ablkctx->sw_cipher);
1483	}
1484
1485	static int get_alg_config(struct algo_param *params,
1486	unsigned int auth_size)
1487	{
1488	switch (auth_size) {
1489	case SHA1_DIGEST_SIZE:
1490	params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1491	params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1492	params->result_size = SHA1_DIGEST_SIZE;
1493	break;
1494	case SHA224_DIGEST_SIZE:
1495	params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1496	params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1497	params->result_size = SHA256_DIGEST_SIZE;
1498	break;
1499	case SHA256_DIGEST_SIZE:
1500	params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1501	params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1502	params->result_size = SHA256_DIGEST_SIZE;
1503	break;
1504	case SHA384_DIGEST_SIZE:
1505	params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1506	params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1507	params->result_size = SHA512_DIGEST_SIZE;
1508	break;
1509	case SHA512_DIGEST_SIZE:
1510	params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1511	params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1512	params->result_size = SHA512_DIGEST_SIZE;
1513	break;
1514	default:
1515	pr_err("ERROR, unsupported digest size\n");
1516	return -EINVAL;
1517	}
1518	return 0;
1519	}
1520
1521	/**
1522	* create_hash_wr - Create hash work request
1523	* @req: Cipher req base
1524	* @param: Container for create_hash_wr()'s parameters
1525	*/
1526	static struct sk_buff *create_hash_wr(struct ahash_request *req,
1527	struct hash_wr_param *param)
1528	{
1529	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1530	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1531	struct chcr_context *ctx = h_ctx(tfm);
1532	struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1533	struct sk_buff *skb = NULL;
1534	struct uld_ctx *u_ctx = ULD_CTX(ctx);
1535	struct chcr_wr *chcr_req;
1536	struct ulptx_sgl *ulptx;
1537	unsigned int nents = 0, transhdr_len;
1538	unsigned int temp = 0;
1539	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1540	GFP_ATOMIC;
1541	struct adapter *adap = padap(dev: h_ctx(tfm)->dev);
1542	int error = 0;
1543	unsigned int rx_channel_id = req_ctx->rxqidx / ctx->rxq_perchan;
1544
1545	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
1546	transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len);
1547	req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len +
1548	param->sg_len) <= SGE_MAX_WR_LEN;
1549	nents = sg_nents_xlen(sg: req_ctx->hctx_wr.srcsg, reqlen: param->sg_len,
1550	CHCR_SRC_SG_SIZE, skip: req_ctx->hctx_wr.src_ofst);
1551	nents += param->bfr_len ? 1 : 0;
1552	transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len +
1553	param->sg_len, 16) : (sgl_len(n: nents) * 8);
1554	transhdr_len = roundup(transhdr_len, 16);
1555
1556	skb = alloc_skb(size: transhdr_len, priority: flags);
1557	if (!skb)
1558	return ERR_PTR(error: -ENOMEM);
1559	chcr_req = __skb_put_zero(skb, len: transhdr_len);
1560
1561	chcr_req->sec_cpl.op_ivinsrtofst =
1562	FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 0);
1563
1564	chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
1565
1566	chcr_req->sec_cpl.aadstart_cipherstop_hi =
1567	FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1568	chcr_req->sec_cpl.cipherstop_lo_authinsert =
1569	FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1570	chcr_req->sec_cpl.seqno_numivs =
1571	FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
1572	param->opad_needed, 0);
1573
1574	chcr_req->sec_cpl.ivgen_hdrlen =
1575	FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1576
1577	memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1578	param->alg_prm.result_size);
1579
1580	if (param->opad_needed)
1581	memcpy(chcr_req->key_ctx.key +
1582	((param->alg_prm.result_size <= 32) ? 32 :
1583	CHCR_HASH_MAX_DIGEST_SIZE),
1584	hmacctx->opad, param->alg_prm.result_size);
1585
1586	chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
1587	param->alg_prm.mk_size, 0,
1588	param->opad_needed,
1589	((param->kctx_len +
1590	sizeof(chcr_req->key_ctx)) >> 4));
1591	chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
1592	ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len +
1593	DUMMY_BYTES);
1594	if (param->bfr_len != 0) {
1595	req_ctx->hctx_wr.dma_addr =
1596	dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr,
1597	param->bfr_len, DMA_TO_DEVICE);
1598	if (dma_mapping_error(dev: &u_ctx->lldi.pdev->dev,
1599	dma_addr: req_ctx->hctx_wr. dma_addr)) {
1600	error = -ENOMEM;
1601	goto err;
1602	}
1603	req_ctx->hctx_wr.dma_len = param->bfr_len;
1604	} else {
1605	req_ctx->hctx_wr.dma_addr = 0;
1606	}
1607	chcr_add_hash_src_ent(req, ulptx, param);
1608	/* Request upto max wr size */
1609	temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ?
1610	(param->sg_len + param->bfr_len) : 0);
1611	atomic_inc(v: &adap->chcr_stats.digest_rqst);
1612	create_wreq(ctx: h_ctx(tfm), chcr_req, req: &req->base, imm: req_ctx->hctx_wr.imm,
1613	hash_sz: param->hash_size, len16: transhdr_len,
1614	sc_len: temp, lcb: 0);
1615	req_ctx->hctx_wr.skb = skb;
1616	return skb;
1617	err:
1618	kfree_skb(skb);
1619	return ERR_PTR(error);
1620	}
1621
1622	static int chcr_ahash_update(struct ahash_request *req)
1623	{
1624	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1625	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1626	struct uld_ctx *u_ctx = ULD_CTX(ctx: h_ctx(tfm: rtfm));
1627	struct chcr_context *ctx = h_ctx(tfm: rtfm);
1628	struct chcr_dev *dev = h_ctx(tfm: rtfm)->dev;
1629	struct sk_buff *skb;
1630	u8 remainder = 0, bs;
1631	unsigned int nbytes = req->nbytes;
1632	struct hash_wr_param params;
1633	int error;
1634	unsigned int cpu;
1635
1636	cpu = get_cpu();
1637	req_ctx->txqidx = cpu % ctx->ntxq;
1638	req_ctx->rxqidx = cpu % ctx->nrxq;
1639	put_cpu();
1640
1641	bs = crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm: rtfm));
1642
1643	if (nbytes + req_ctx->reqlen >= bs) {
1644	remainder = (nbytes + req_ctx->reqlen) % bs;
1645	nbytes = nbytes + req_ctx->reqlen - remainder;
1646	} else {
1647	sg_pcopy_to_buffer(sgl: req->src, nents: sg_nents(sg: req->src), buf: req_ctx->reqbfr
1648	+ req_ctx->reqlen, buflen: nbytes, skip: 0);
1649	req_ctx->reqlen += nbytes;
1650	return 0;
1651	}
1652	error = chcr_inc_wrcount(dev);
1653	if (error)
1654	return -ENXIO;
1655	/* Detach state for CHCR means lldi or padap is freed. Increasing
1656	* inflight count for dev guarantees that lldi and padap is valid
1657	*/
1658	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1659	req_ctx->txqidx) &&
1660	(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1661	error = -ENOSPC;
1662	goto err;
1663	}
1664
1665	chcr_init_hctx_per_wr(reqctx: req_ctx);
1666	error = chcr_hash_dma_map(dev: &u_ctx->lldi.pdev->dev, req);
1667	if (error) {
1668	error = -ENOMEM;
1669	goto err;
1670	}
1671	get_alg_config(params: &params.alg_prm, auth_size: crypto_ahash_digestsize(tfm: rtfm));
1672	params.kctx_len = roundup(params.alg_prm.result_size, 16);
1673	params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1674	HASH_SPACE_LEFT(params.kctx_len), 0);
1675	if (params.sg_len > req->nbytes)
1676	params.sg_len = req->nbytes;
1677	params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) -
1678	req_ctx->reqlen;
1679	params.opad_needed = 0;
1680	params.more = 1;
1681	params.last = 0;
1682	params.bfr_len = req_ctx->reqlen;
1683	params.scmd1 = 0;
1684	req_ctx->hctx_wr.srcsg = req->src;
1685
1686	params.hash_size = params.alg_prm.result_size;
1687	req_ctx->data_len += params.sg_len + params.bfr_len;
1688	skb = create_hash_wr(req, param: &params);
1689	if (IS_ERR(ptr: skb)) {
1690	error = PTR_ERR(ptr: skb);
1691	goto unmap;
1692	}
1693
1694	req_ctx->hctx_wr.processed += params.sg_len;
1695	if (remainder) {
1696	/* Swap buffers */
1697	swap(req_ctx->reqbfr, req_ctx->skbfr);
1698	sg_pcopy_to_buffer(sgl: req->src, nents: sg_nents(sg: req->src),
1699	buf: req_ctx->reqbfr, buflen: remainder, skip: req->nbytes -
1700	remainder);
1701	}
1702	req_ctx->reqlen = remainder;
1703	skb->dev = u_ctx->lldi.ports[0];
1704	set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1705	chcr_send_wr(skb);
1706	return -EINPROGRESS;
1707	unmap:
1708	chcr_hash_dma_unmap(dev: &u_ctx->lldi.pdev->dev, req);
1709	err:
1710	chcr_dec_wrcount(dev);
1711	return error;
1712	}
1713
1714	static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1715	{
1716	memset(bfr_ptr, 0, bs);
1717	*bfr_ptr = 0x80;
1718	if (bs == 64)
1719	*(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1720	else
1721	*(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1722	}
1723
1724	static int chcr_ahash_final(struct ahash_request *req)
1725	{
1726	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1727	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1728	struct chcr_dev *dev = h_ctx(tfm: rtfm)->dev;
1729	struct hash_wr_param params;
1730	struct sk_buff *skb;
1731	struct uld_ctx *u_ctx = ULD_CTX(ctx: h_ctx(tfm: rtfm));
1732	struct chcr_context *ctx = h_ctx(tfm: rtfm);
1733	u8 bs = crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm: rtfm));
1734	int error;
1735	unsigned int cpu;
1736
1737	cpu = get_cpu();
1738	req_ctx->txqidx = cpu % ctx->ntxq;
1739	req_ctx->rxqidx = cpu % ctx->nrxq;
1740	put_cpu();
1741
1742	error = chcr_inc_wrcount(dev);
1743	if (error)
1744	return -ENXIO;
1745
1746	chcr_init_hctx_per_wr(reqctx: req_ctx);
1747	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm)))
1748	params.opad_needed = 1;
1749	else
1750	params.opad_needed = 0;
1751	params.sg_len = 0;
1752	req_ctx->hctx_wr.isfinal = 1;
1753	get_alg_config(params: &params.alg_prm, auth_size: crypto_ahash_digestsize(tfm: rtfm));
1754	params.kctx_len = roundup(params.alg_prm.result_size, 16);
1755	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm))) {
1756	params.opad_needed = 1;
1757	params.kctx_len *= 2;
1758	} else {
1759	params.opad_needed = 0;
1760	}
1761
1762	req_ctx->hctx_wr.result = 1;
1763	params.bfr_len = req_ctx->reqlen;
1764	req_ctx->data_len += params.bfr_len + params.sg_len;
1765	req_ctx->hctx_wr.srcsg = req->src;
1766	if (req_ctx->reqlen == 0) {
1767	create_last_hash_block(bfr_ptr: req_ctx->reqbfr, bs, scmd1: req_ctx->data_len);
1768	params.last = 0;
1769	params.more = 1;
1770	params.scmd1 = 0;
1771	params.bfr_len = bs;
1772
1773	} else {
1774	params.scmd1 = req_ctx->data_len;
1775	params.last = 1;
1776	params.more = 0;
1777	}
1778	params.hash_size = crypto_ahash_digestsize(tfm: rtfm);
1779	skb = create_hash_wr(req, param: &params);
1780	if (IS_ERR(ptr: skb)) {
1781	error = PTR_ERR(ptr: skb);
1782	goto err;
1783	}
1784	req_ctx->reqlen = 0;
1785	skb->dev = u_ctx->lldi.ports[0];
1786	set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1787	chcr_send_wr(skb);
1788	return -EINPROGRESS;
1789	err:
1790	chcr_dec_wrcount(dev);
1791	return error;
1792	}
1793
1794	static int chcr_ahash_finup(struct ahash_request *req)
1795	{
1796	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1797	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1798	struct chcr_dev *dev = h_ctx(tfm: rtfm)->dev;
1799	struct uld_ctx *u_ctx = ULD_CTX(ctx: h_ctx(tfm: rtfm));
1800	struct chcr_context *ctx = h_ctx(tfm: rtfm);
1801	struct sk_buff *skb;
1802	struct hash_wr_param params;
1803	u8 bs;
1804	int error;
1805	unsigned int cpu;
1806
1807	cpu = get_cpu();
1808	req_ctx->txqidx = cpu % ctx->ntxq;
1809	req_ctx->rxqidx = cpu % ctx->nrxq;
1810	put_cpu();
1811
1812	bs = crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm: rtfm));
1813	error = chcr_inc_wrcount(dev);
1814	if (error)
1815	return -ENXIO;
1816
1817	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1818	req_ctx->txqidx) &&
1819	(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1820	error = -ENOSPC;
1821	goto err;
1822	}
1823	chcr_init_hctx_per_wr(reqctx: req_ctx);
1824	error = chcr_hash_dma_map(dev: &u_ctx->lldi.pdev->dev, req);
1825	if (error) {
1826	error = -ENOMEM;
1827	goto err;
1828	}
1829
1830	get_alg_config(params: &params.alg_prm, auth_size: crypto_ahash_digestsize(tfm: rtfm));
1831	params.kctx_len = roundup(params.alg_prm.result_size, 16);
1832	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm))) {
1833	params.kctx_len *= 2;
1834	params.opad_needed = 1;
1835	} else {
1836	params.opad_needed = 0;
1837	}
1838
1839	params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1840	HASH_SPACE_LEFT(params.kctx_len), 0);
1841	if (params.sg_len < req->nbytes) {
1842	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm))) {
1843	params.kctx_len /= 2;
1844	params.opad_needed = 0;
1845	}
1846	params.last = 0;
1847	params.more = 1;
1848	params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs)
1849	- req_ctx->reqlen;
1850	params.hash_size = params.alg_prm.result_size;
1851	params.scmd1 = 0;
1852	} else {
1853	params.last = 1;
1854	params.more = 0;
1855	params.sg_len = req->nbytes;
1856	params.hash_size = crypto_ahash_digestsize(tfm: rtfm);
1857	params.scmd1 = req_ctx->data_len + req_ctx->reqlen +
1858	params.sg_len;
1859	}
1860	params.bfr_len = req_ctx->reqlen;
1861	req_ctx->data_len += params.bfr_len + params.sg_len;
1862	req_ctx->hctx_wr.result = 1;
1863	req_ctx->hctx_wr.srcsg = req->src;
1864	if ((req_ctx->reqlen + req->nbytes) == 0) {
1865	create_last_hash_block(bfr_ptr: req_ctx->reqbfr, bs, scmd1: req_ctx->data_len);
1866	params.last = 0;
1867	params.more = 1;
1868	params.scmd1 = 0;
1869	params.bfr_len = bs;
1870	}
1871	skb = create_hash_wr(req, param: &params);
1872	if (IS_ERR(ptr: skb)) {
1873	error = PTR_ERR(ptr: skb);
1874	goto unmap;
1875	}
1876	req_ctx->reqlen = 0;
1877	req_ctx->hctx_wr.processed += params.sg_len;
1878	skb->dev = u_ctx->lldi.ports[0];
1879	set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1880	chcr_send_wr(skb);
1881	return -EINPROGRESS;
1882	unmap:
1883	chcr_hash_dma_unmap(dev: &u_ctx->lldi.pdev->dev, req);
1884	err:
1885	chcr_dec_wrcount(dev);
1886	return error;
1887	}
1888
1889	static int chcr_hmac_init(struct ahash_request *areq);
1890	static int chcr_sha_init(struct ahash_request *areq);
1891
1892	static int chcr_ahash_digest(struct ahash_request *req)
1893	{
1894	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1895	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1896	struct chcr_dev *dev = h_ctx(tfm: rtfm)->dev;
1897	struct uld_ctx *u_ctx = ULD_CTX(ctx: h_ctx(tfm: rtfm));
1898	struct chcr_context *ctx = h_ctx(tfm: rtfm);
1899	struct sk_buff *skb;
1900	struct hash_wr_param params;
1901	u8 bs;
1902	int error;
1903	unsigned int cpu;
1904
1905	cpu = get_cpu();
1906	req_ctx->txqidx = cpu % ctx->ntxq;
1907	req_ctx->rxqidx = cpu % ctx->nrxq;
1908	put_cpu();
1909
1910	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm)))
1911	chcr_hmac_init(areq: req);
1912	else
1913	chcr_sha_init(areq: req);
1914
1915	bs = crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm: rtfm));
1916	error = chcr_inc_wrcount(dev);
1917	if (error)
1918	return -ENXIO;
1919
1920	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1921	req_ctx->txqidx) &&
1922	(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1923	error = -ENOSPC;
1924	goto err;
1925	}
1926
1927	chcr_init_hctx_per_wr(reqctx: req_ctx);
1928	error = chcr_hash_dma_map(dev: &u_ctx->lldi.pdev->dev, req);
1929	if (error) {
1930	error = -ENOMEM;
1931	goto err;
1932	}
1933
1934	get_alg_config(params: &params.alg_prm, auth_size: crypto_ahash_digestsize(tfm: rtfm));
1935	params.kctx_len = roundup(params.alg_prm.result_size, 16);
1936	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm))) {
1937	params.kctx_len *= 2;
1938	params.opad_needed = 1;
1939	} else {
1940	params.opad_needed = 0;
1941	}
1942	params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1943	HASH_SPACE_LEFT(params.kctx_len), 0);
1944	if (params.sg_len < req->nbytes) {
1945	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm))) {
1946	params.kctx_len /= 2;
1947	params.opad_needed = 0;
1948	}
1949	params.last = 0;
1950	params.more = 1;
1951	params.scmd1 = 0;
1952	params.sg_len = rounddown(params.sg_len, bs);
1953	params.hash_size = params.alg_prm.result_size;
1954	} else {
1955	params.sg_len = req->nbytes;
1956	params.hash_size = crypto_ahash_digestsize(tfm: rtfm);
1957	params.last = 1;
1958	params.more = 0;
1959	params.scmd1 = req->nbytes + req_ctx->data_len;
1960
1961	}
1962	params.bfr_len = 0;
1963	req_ctx->hctx_wr.result = 1;
1964	req_ctx->hctx_wr.srcsg = req->src;
1965	req_ctx->data_len += params.bfr_len + params.sg_len;
1966
1967	if (req->nbytes == 0) {
1968	create_last_hash_block(bfr_ptr: req_ctx->reqbfr, bs, scmd1: req_ctx->data_len);
1969	params.more = 1;
1970	params.bfr_len = bs;
1971	}
1972
1973	skb = create_hash_wr(req, param: &params);
1974	if (IS_ERR(ptr: skb)) {
1975	error = PTR_ERR(ptr: skb);
1976	goto unmap;
1977	}
1978	req_ctx->hctx_wr.processed += params.sg_len;
1979	skb->dev = u_ctx->lldi.ports[0];
1980	set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1981	chcr_send_wr(skb);
1982	return -EINPROGRESS;
1983	unmap:
1984	chcr_hash_dma_unmap(dev: &u_ctx->lldi.pdev->dev, req);
1985	err:
1986	chcr_dec_wrcount(dev);
1987	return error;
1988	}
1989
1990	static int chcr_ahash_continue(struct ahash_request *req)
1991	{
1992	struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
1993	struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
1994	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1995	struct chcr_context *ctx = h_ctx(tfm: rtfm);
1996	struct uld_ctx *u_ctx = ULD_CTX(ctx);
1997	struct sk_buff *skb;
1998	struct hash_wr_param params;
1999	u8 bs;
2000	int error;
2001	unsigned int cpu;
2002
2003	cpu = get_cpu();
2004	reqctx->txqidx = cpu % ctx->ntxq;
2005	reqctx->rxqidx = cpu % ctx->nrxq;
2006	put_cpu();
2007
2008	bs = crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm: rtfm));
2009	get_alg_config(params: &params.alg_prm, auth_size: crypto_ahash_digestsize(tfm: rtfm));
2010	params.kctx_len = roundup(params.alg_prm.result_size, 16);
2011	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm))) {
2012	params.kctx_len *= 2;
2013	params.opad_needed = 1;
2014	} else {
2015	params.opad_needed = 0;
2016	}
2017	params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0,
2018	HASH_SPACE_LEFT(params.kctx_len),
2019	hctx_wr->src_ofst);
2020	if ((params.sg_len + hctx_wr->processed) > req->nbytes)
2021	params.sg_len = req->nbytes - hctx_wr->processed;
2022	if (!hctx_wr->result ||
2023	((params.sg_len + hctx_wr->processed) < req->nbytes)) {
2024	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm))) {
2025	params.kctx_len /= 2;
2026	params.opad_needed = 0;
2027	}
2028	params.last = 0;
2029	params.more = 1;
2030	params.sg_len = rounddown(params.sg_len, bs);
2031	params.hash_size = params.alg_prm.result_size;
2032	params.scmd1 = 0;
2033	} else {
2034	params.last = 1;
2035	params.more = 0;
2036	params.hash_size = crypto_ahash_digestsize(tfm: rtfm);
2037	params.scmd1 = reqctx->data_len + params.sg_len;
2038	}
2039	params.bfr_len = 0;
2040	reqctx->data_len += params.sg_len;
2041	skb = create_hash_wr(req, param: &params);
2042	if (IS_ERR(ptr: skb)) {
2043	error = PTR_ERR(ptr: skb);
2044	goto err;
2045	}
2046	hctx_wr->processed += params.sg_len;
2047	skb->dev = u_ctx->lldi.ports[0];
2048	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
2049	chcr_send_wr(skb);
2050	return 0;
2051	err:
2052	return error;
2053	}
2054
2055	static inline void chcr_handle_ahash_resp(struct ahash_request *req,
2056	unsigned char *input,
2057	int err)
2058	{
2059	struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2060	struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
2061	int digestsize, updated_digestsize;
2062	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2063	struct uld_ctx *u_ctx = ULD_CTX(ctx: h_ctx(tfm));
2064	struct chcr_dev *dev = h_ctx(tfm)->dev;
2065
2066	if (input == NULL)
2067	goto out;
2068	digestsize = crypto_ahash_digestsize(tfm: crypto_ahash_reqtfm(req));
2069	updated_digestsize = digestsize;
2070	if (digestsize == SHA224_DIGEST_SIZE)
2071	updated_digestsize = SHA256_DIGEST_SIZE;
2072	else if (digestsize == SHA384_DIGEST_SIZE)
2073	updated_digestsize = SHA512_DIGEST_SIZE;
2074
2075	if (hctx_wr->dma_addr) {
2076	dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr,
2077	hctx_wr->dma_len, DMA_TO_DEVICE);
2078	hctx_wr->dma_addr = 0;
2079	}
2080	if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) ==
2081	req->nbytes)) {
2082	if (hctx_wr->result == 1) {
2083	hctx_wr->result = 0;
2084	memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
2085	digestsize);
2086	} else {
2087	memcpy(reqctx->partial_hash,
2088	input + sizeof(struct cpl_fw6_pld),
2089	updated_digestsize);
2090
2091	}
2092	goto unmap;
2093	}
2094	memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
2095	updated_digestsize);
2096
2097	err = chcr_ahash_continue(req);
2098	if (err)
2099	goto unmap;
2100	return;
2101	unmap:
2102	if (hctx_wr->is_sg_map)
2103	chcr_hash_dma_unmap(dev: &u_ctx->lldi.pdev->dev, req);
2104
2105
2106	out:
2107	chcr_dec_wrcount(dev);
2108	ahash_request_complete(req, err);
2109	}
2110
2111	/*
2112	* chcr_handle_resp - Unmap the DMA buffers associated with the request
2113	* @req: crypto request
2114	*/
2115	int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
2116	int err)
2117	{
2118	struct crypto_tfm *tfm = req->tfm;
2119	struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2120	struct adapter *adap = padap(dev: ctx->dev);
2121
2122	switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
2123	case CRYPTO_ALG_TYPE_AEAD:
2124	err = chcr_handle_aead_resp(req: aead_request_cast(req), input, err);
2125	break;
2126
2127	case CRYPTO_ALG_TYPE_SKCIPHER:
2128	chcr_handle_cipher_resp(req: skcipher_request_cast(req),
2129	input, err);
2130	break;
2131	case CRYPTO_ALG_TYPE_AHASH:
2132	chcr_handle_ahash_resp(req: ahash_request_cast(req), input, err);
2133	}
2134	atomic_inc(v: &adap->chcr_stats.complete);
2135	return err;
2136	}
2137	static int chcr_ahash_export(struct ahash_request *areq, void *out)
2138	{
2139	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
2140	struct chcr_ahash_req_ctx *state = out;
2141
2142	state->reqlen = req_ctx->reqlen;
2143	state->data_len = req_ctx->data_len;
2144	memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
2145	memcpy(state->partial_hash, req_ctx->partial_hash,
2146	CHCR_HASH_MAX_DIGEST_SIZE);
2147	chcr_init_hctx_per_wr(reqctx: state);
2148	return 0;
2149	}
2150
2151	static int chcr_ahash_import(struct ahash_request *areq, const void *in)
2152	{
2153	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
2154	struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
2155
2156	req_ctx->reqlen = state->reqlen;
2157	req_ctx->data_len = state->data_len;
2158	req_ctx->reqbfr = req_ctx->bfr1;
2159	req_ctx->skbfr = req_ctx->bfr2;
2160	memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
2161	memcpy(req_ctx->partial_hash, state->partial_hash,
2162	CHCR_HASH_MAX_DIGEST_SIZE);
2163	chcr_init_hctx_per_wr(reqctx: req_ctx);
2164	return 0;
2165	}
2166
2167	static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2168	unsigned int keylen)
2169	{
2170	struct hmac_ctx *hmacctx = HMAC_CTX(ctx: h_ctx(tfm));
2171
2172	/* use the key to calculate the ipad and opad. ipad will sent with the
2173	* first request's data. opad will be sent with the final hash result
2174	* ipad in hmacctx->ipad and opad in hmacctx->opad location
2175	*/
2176	return chcr_prepare_hmac_key(raw_key: key, raw_key_len: keylen, digestsize: crypto_ahash_digestsize(tfm),
2177	istate: hmacctx->ipad, ostate: hmacctx->opad);
2178	}
2179
2180	static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key,
2181	unsigned int key_len)
2182	{
2183	struct ablk_ctx *ablkctx = ABLK_CTX(ctx: c_ctx(tfm: cipher));
2184	unsigned short context_size = 0;
2185	int err;
2186
2187	err = chcr_cipher_fallback_setkey(cipher, key, keylen: key_len);
2188	if (err)
2189	goto badkey_err;
2190
2191	memcpy(ablkctx->key, key, key_len);
2192	ablkctx->enckey_len = key_len;
2193	get_aes_decrypt_key(dec_key: ablkctx->rrkey, key: ablkctx->key, keylength: key_len << 2);
2194	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
2195	/* Both keys for xts must be aligned to 16 byte boundary
2196	* by padding with zeros. So for 24 byte keys padding 8 zeroes.
2197	*/
2198	if (key_len == 48) {
2199	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len
2200	+ 16) >> 4;
2201	memmove(ablkctx->key + 32, ablkctx->key + 24, 24);
2202	memset(ablkctx->key + 24, 0, 8);
2203	memset(ablkctx->key + 56, 0, 8);
2204	ablkctx->enckey_len = 64;
2205	ablkctx->key_ctx_hdr =
2206	FILL_KEY_CTX_HDR(CHCR_KEYCTX_CIPHER_KEY_SIZE_192,
2207	CHCR_KEYCTX_NO_KEY, 1,
2208	0, context_size);
2209	} else {
2210	ablkctx->key_ctx_hdr =
2211	FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
2212	CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
2213	CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
2214	CHCR_KEYCTX_NO_KEY, 1,
2215	0, context_size);
2216	}
2217	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
2218	return 0;
2219	badkey_err:
2220	ablkctx->enckey_len = 0;
2221
2222	return err;
2223	}
2224
2225	static int chcr_sha_init(struct ahash_request *areq)
2226	{
2227	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
2228	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq);
2229	int digestsize = crypto_ahash_digestsize(tfm);
2230
2231	req_ctx->data_len = 0;
2232	req_ctx->reqlen = 0;
2233	req_ctx->reqbfr = req_ctx->bfr1;
2234	req_ctx->skbfr = req_ctx->bfr2;
2235	copy_hash_init_values(key: req_ctx->partial_hash, digestsize);
2236
2237	return 0;
2238	}
2239
2240	static int chcr_sha_cra_init(struct crypto_tfm *tfm)
2241	{
2242	crypto_ahash_set_reqsize(tfm: __crypto_ahash_cast(tfm),
2243	reqsize: sizeof(struct chcr_ahash_req_ctx));
2244	return chcr_device_init(ctx: crypto_tfm_ctx(tfm));
2245	}
2246
2247	static int chcr_hmac_init(struct ahash_request *areq)
2248	{
2249	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
2250	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req: areq);
2251	struct hmac_ctx *hmacctx = HMAC_CTX(ctx: h_ctx(tfm: rtfm));
2252	unsigned int digestsize = crypto_ahash_digestsize(tfm: rtfm);
2253	unsigned int bs = crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm: rtfm));
2254
2255	chcr_sha_init(areq);
2256	req_ctx->data_len = bs;
2257	if (is_hmac(tfm: crypto_ahash_tfm(tfm: rtfm))) {
2258	if (digestsize == SHA224_DIGEST_SIZE)
2259	memcpy(req_ctx->partial_hash, hmacctx->ipad,
2260	SHA256_DIGEST_SIZE);
2261	else if (digestsize == SHA384_DIGEST_SIZE)
2262	memcpy(req_ctx->partial_hash, hmacctx->ipad,
2263	SHA512_DIGEST_SIZE);
2264	else
2265	memcpy(req_ctx->partial_hash, hmacctx->ipad,
2266	digestsize);
2267	}
2268	return 0;
2269	}
2270
2271	static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
2272	{
2273	crypto_ahash_set_reqsize(tfm: __crypto_ahash_cast(tfm),
2274	reqsize: sizeof(struct chcr_ahash_req_ctx));
2275	return chcr_device_init(ctx: crypto_tfm_ctx(tfm));
2276	}
2277
2278	inline void chcr_aead_common_exit(struct aead_request *req)
2279	{
2280	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2281	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2282	struct uld_ctx *u_ctx = ULD_CTX(ctx: a_ctx(tfm));
2283
2284	chcr_aead_dma_unmap(dev: &u_ctx->lldi.pdev->dev, req, op_type: reqctx->op);
2285	}
2286
2287	static int chcr_aead_common_init(struct aead_request *req)
2288	{
2289	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2290	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
2291	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2292	unsigned int authsize = crypto_aead_authsize(tfm);
2293	int error = -EINVAL;
2294
2295	/* validate key size */
2296	if (aeadctx->enckey_len == 0)
2297	goto err;
2298	if (reqctx->op && req->cryptlen < authsize)
2299	goto err;
2300	if (reqctx->b0_len)
2301	reqctx->scratch_pad = reqctx->iv + IV;
2302	else
2303	reqctx->scratch_pad = NULL;
2304
2305	error = chcr_aead_dma_map(dev: &ULD_CTX(ctx: a_ctx(tfm))->lldi.pdev->dev, req,
2306	op_type: reqctx->op);
2307	if (error) {
2308	error = -ENOMEM;
2309	goto err;
2310	}
2311
2312	return 0;
2313	err:
2314	return error;
2315	}
2316
2317	static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents,
2318	int aadmax, int wrlen,
2319	unsigned short op_type)
2320	{
2321	unsigned int authsize = crypto_aead_authsize(tfm: crypto_aead_reqtfm(req));
2322
2323	if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
2324	dst_nents > MAX_DSGL_ENT ||
2325	(req->assoclen > aadmax) ||
2326	(wrlen > SGE_MAX_WR_LEN))
2327	return 1;
2328	return 0;
2329	}
2330
2331	static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
2332	{
2333	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2334	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
2335	struct aead_request *subreq = aead_request_ctx_dma(req);
2336
2337	aead_request_set_tfm(req: subreq, tfm: aeadctx->sw_cipher);
2338	aead_request_set_callback(req: subreq, flags: req->base.flags,
2339	compl: req->base.complete, data: req->base.data);
2340	aead_request_set_crypt(req: subreq, src: req->src, dst: req->dst, cryptlen: req->cryptlen,
2341	iv: req->iv);
2342	aead_request_set_ad(req: subreq, assoclen: req->assoclen);
2343	return op_type ? crypto_aead_decrypt(req: subreq) :
2344	crypto_aead_encrypt(req: subreq);
2345	}
2346
2347	static struct sk_buff *create_authenc_wr(struct aead_request *req,
2348	unsigned short qid,
2349	int size)
2350	{
2351	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2352	struct chcr_context *ctx = a_ctx(tfm);
2353	struct uld_ctx *u_ctx = ULD_CTX(ctx);
2354	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2355	struct chcr_authenc_ctx *actx = AUTHENC_CTX(gctx: aeadctx);
2356	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2357	struct sk_buff *skb = NULL;
2358	struct chcr_wr *chcr_req;
2359	struct cpl_rx_phys_dsgl *phys_cpl;
2360	struct ulptx_sgl *ulptx;
2361	unsigned int transhdr_len;
2362	unsigned int dst_size = 0, temp, subtype = get_aead_subtype(aead: tfm);
2363	unsigned int kctx_len = 0, dnents, snents;
2364	unsigned int authsize = crypto_aead_authsize(tfm);
2365	int error = -EINVAL;
2366	u8 *ivptr;
2367	int null = 0;
2368	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2369	GFP_ATOMIC;
2370	struct adapter *adap = padap(dev: ctx->dev);
2371	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2372
2373	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
2374	if (req->cryptlen == 0)
2375	return NULL;
2376
2377	reqctx->b0_len = 0;
2378	error = chcr_aead_common_init(req);
2379	if (error)
2380	return ERR_PTR(error);
2381
2382	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL ||
2383	subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2384	null = 1;
2385	}
2386	dnents = sg_nents_xlen(sg: req->dst, reqlen: req->assoclen + req->cryptlen +
2387	(reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE, skip: 0);
2388	dnents += MIN_AUTH_SG; // For IV
2389	snents = sg_nents_xlen(sg: req->src, reqlen: req->assoclen + req->cryptlen,
2390	CHCR_SRC_SG_SIZE, skip: 0);
2391	dst_size = get_space_for_phys_dsgl(sgl_entr: dnents);
2392	kctx_len = (KEY_CONTEXT_CTX_LEN_G(ntohl(aeadctx->key_ctx_hdr)) << 4)
2393	- sizeof(chcr_req->key_ctx);
2394	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2395	reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <
2396	SGE_MAX_WR_LEN;
2397	temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16)
2398	: (sgl_len(n: snents) * 8);
2399	transhdr_len += temp;
2400	transhdr_len = roundup(transhdr_len, 16);
2401
2402	if (chcr_aead_need_fallback(req, dst_nents: dnents, T6_MAX_AAD_SIZE,
2403	wrlen: transhdr_len, op_type: reqctx->op)) {
2404	atomic_inc(v: &adap->chcr_stats.fallback);
2405	chcr_aead_common_exit(req);
2406	return ERR_PTR(error: chcr_aead_fallback(req, op_type: reqctx->op));
2407	}
2408	skb = alloc_skb(size: transhdr_len, priority: flags);
2409	if (!skb) {
2410	error = -ENOMEM;
2411	goto err;
2412	}
2413
2414	chcr_req = __skb_put_zero(skb, len: transhdr_len);
2415
2416	temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
2417
2418	/*
2419	* Input order is AAD,IV and Payload. where IV should be included as
2420	* the part of authdata. All other fields should be filled according
2421	* to the hardware spec
2422	*/
2423	chcr_req->sec_cpl.op_ivinsrtofst =
2424	FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
2425	chcr_req->sec_cpl.pldlen = htonl(req->assoclen + IV + req->cryptlen);
2426	chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2427	null ? 0 : 1 + IV,
2428	null ? 0 : IV + req->assoclen,
2429	req->assoclen + IV + 1,
2430	(temp & 0x1F0) >> 4);
2431	chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2432	temp & 0xF,
2433	null ? 0 : req->assoclen + IV + 1,
2434	temp, temp);
2435	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL ||
2436	subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA)
2437	temp = CHCR_SCMD_CIPHER_MODE_AES_CTR;
2438	else
2439	temp = CHCR_SCMD_CIPHER_MODE_AES_CBC;
2440	chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op,
2441	(reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0,
2442	temp,
2443	actx->auth_mode, aeadctx->hmac_ctrl,
2444	IV >> 1);
2445	chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2446	0, 0, dst_size);
2447
2448	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2449	if (reqctx->op == CHCR_ENCRYPT_OP ||
2450	subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2451	subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL)
2452	memcpy(chcr_req->key_ctx.key, aeadctx->key,
2453	aeadctx->enckey_len);
2454	else
2455	memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2456	aeadctx->enckey_len);
2457
2458	memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
2459	actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16));
2460	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2461	ivptr = (u8 *)(phys_cpl + 1) + dst_size;
2462	ulptx = (struct ulptx_sgl *)(ivptr + IV);
2463	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2464	subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2465	memcpy(ivptr, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE);
2466	memcpy(ivptr + CTR_RFC3686_NONCE_SIZE, req->iv,
2467	CTR_RFC3686_IV_SIZE);
2468	*(__be32 *)(ivptr + CTR_RFC3686_NONCE_SIZE +
2469	CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
2470	} else {
2471	memcpy(ivptr, req->iv, IV);
2472	}
2473	chcr_add_aead_dst_ent(req, phys_cpl, qid);
2474	chcr_add_aead_src_ent(req, ulptx);
2475	atomic_inc(v: &adap->chcr_stats.cipher_rqst);
2476	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
2477	kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
2478	create_wreq(ctx: a_ctx(tfm), chcr_req, req: &req->base, imm: reqctx->imm, hash_sz: size,
2479	len16: transhdr_len, sc_len: temp, lcb: 0);
2480	reqctx->skb = skb;
2481
2482	return skb;
2483	err:
2484	chcr_aead_common_exit(req);
2485
2486	return ERR_PTR(error);
2487	}
2488
2489	int chcr_aead_dma_map(struct device *dev,
2490	struct aead_request *req,
2491	unsigned short op_type)
2492	{
2493	int error;
2494	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2495	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2496	unsigned int authsize = crypto_aead_authsize(tfm);
2497	int src_len, dst_len;
2498
2499	/* calculate and handle src and dst sg length separately
2500	* for inplace and out-of place operations
2501	*/
2502	if (req->src == req->dst) {
2503	src_len = req->assoclen + req->cryptlen + (op_type ?
2504	0 : authsize);
2505	dst_len = src_len;
2506	} else {
2507	src_len = req->assoclen + req->cryptlen;
2508	dst_len = req->assoclen + req->cryptlen + (op_type ?
2509	-authsize : authsize);
2510	}
2511
2512	if (!req->cryptlen || !src_len || !dst_len)
2513	return 0;
2514	reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len),
2515	DMA_BIDIRECTIONAL);
2516	if (dma_mapping_error(dev, dma_addr: reqctx->iv_dma))
2517	return -ENOMEM;
2518	if (reqctx->b0_len)
2519	reqctx->b0_dma = reqctx->iv_dma + IV;
2520	else
2521	reqctx->b0_dma = 0;
2522	if (req->src == req->dst) {
2523	error = dma_map_sg(dev, req->src,
2524	sg_nents_for_len(req->src, src_len),
2525	DMA_BIDIRECTIONAL);
2526	if (!error)
2527	goto err;
2528	} else {
2529	error = dma_map_sg(dev, req->src,
2530	sg_nents_for_len(req->src, src_len),
2531	DMA_TO_DEVICE);
2532	if (!error)
2533	goto err;
2534	error = dma_map_sg(dev, req->dst,
2535	sg_nents_for_len(req->dst, dst_len),
2536	DMA_FROM_DEVICE);
2537	if (!error) {
2538	dma_unmap_sg(dev, req->src,
2539	sg_nents_for_len(req->src, src_len),
2540	DMA_TO_DEVICE);
2541	goto err;
2542	}
2543	}
2544
2545	return 0;
2546	err:
2547	dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2548	return -ENOMEM;
2549	}
2550
2551	void chcr_aead_dma_unmap(struct device *dev,
2552	struct aead_request *req,
2553	unsigned short op_type)
2554	{
2555	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2556	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2557	unsigned int authsize = crypto_aead_authsize(tfm);
2558	int src_len, dst_len;
2559
2560	/* calculate and handle src and dst sg length separately
2561	* for inplace and out-of place operations
2562	*/
2563	if (req->src == req->dst) {
2564	src_len = req->assoclen + req->cryptlen + (op_type ?
2565	0 : authsize);
2566	dst_len = src_len;
2567	} else {
2568	src_len = req->assoclen + req->cryptlen;
2569	dst_len = req->assoclen + req->cryptlen + (op_type ?
2570	-authsize : authsize);
2571	}
2572
2573	if (!req->cryptlen || !src_len || !dst_len)
2574	return;
2575
2576	dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len),
2577	DMA_BIDIRECTIONAL);
2578	if (req->src == req->dst) {
2579	dma_unmap_sg(dev, req->src,
2580	sg_nents_for_len(req->src, src_len),
2581	DMA_BIDIRECTIONAL);
2582	} else {
2583	dma_unmap_sg(dev, req->src,
2584	sg_nents_for_len(req->src, src_len),
2585	DMA_TO_DEVICE);
2586	dma_unmap_sg(dev, req->dst,
2587	sg_nents_for_len(req->dst, dst_len),
2588	DMA_FROM_DEVICE);
2589	}
2590	}
2591
2592	void chcr_add_aead_src_ent(struct aead_request *req,
2593	struct ulptx_sgl *ulptx)
2594	{
2595	struct ulptx_walk ulp_walk;
2596	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2597
2598	if (reqctx->imm) {
2599	u8 *buf = (u8 *)ulptx;
2600
2601	if (reqctx->b0_len) {
2602	memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
2603	buf += reqctx->b0_len;
2604	}
2605	sg_pcopy_to_buffer(sgl: req->src, nents: sg_nents(sg: req->src),
2606	buf, buflen: req->cryptlen + req->assoclen, skip: 0);
2607	} else {
2608	ulptx_walk_init(walk: &ulp_walk, ulp: ulptx);
2609	if (reqctx->b0_len)
2610	ulptx_walk_add_page(walk: &ulp_walk, size: reqctx->b0_len,
2611	addr: reqctx->b0_dma);
2612	ulptx_walk_add_sg(walk: &ulp_walk, sg: req->src, len: req->cryptlen +
2613	req->assoclen, skip: 0);
2614	ulptx_walk_end(walk: &ulp_walk);
2615	}
2616	}
2617
2618	void chcr_add_aead_dst_ent(struct aead_request *req,
2619	struct cpl_rx_phys_dsgl *phys_cpl,
2620	unsigned short qid)
2621	{
2622	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2623	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2624	struct dsgl_walk dsgl_walk;
2625	unsigned int authsize = crypto_aead_authsize(tfm);
2626	struct chcr_context *ctx = a_ctx(tfm);
2627	struct uld_ctx *u_ctx = ULD_CTX(ctx);
2628	u32 temp;
2629	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2630
2631	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
2632	dsgl_walk_init(walk: &dsgl_walk, dsgl: phys_cpl);
2633	dsgl_walk_add_page(walk: &dsgl_walk, IV + reqctx->b0_len, addr: reqctx->iv_dma);
2634	temp = req->assoclen + req->cryptlen +
2635	(reqctx->op ? -authsize : authsize);
2636	dsgl_walk_add_sg(walk: &dsgl_walk, sg: req->dst, slen: temp, skip: 0);
2637	dsgl_walk_end(walk: &dsgl_walk, qid, pci_chan_id: rx_channel_id);
2638	}
2639
2640	void chcr_add_cipher_src_ent(struct skcipher_request *req,
2641	void *ulptx,
2642	struct cipher_wr_param *wrparam)
2643	{
2644	struct ulptx_walk ulp_walk;
2645	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2646	u8 *buf = ulptx;
2647
2648	memcpy(buf, reqctx->iv, IV);
2649	buf += IV;
2650	if (reqctx->imm) {
2651	sg_pcopy_to_buffer(sgl: req->src, nents: sg_nents(sg: req->src),
2652	buf, buflen: wrparam->bytes, skip: reqctx->processed);
2653	} else {
2654	ulptx_walk_init(walk: &ulp_walk, ulp: (struct ulptx_sgl *)buf);
2655	ulptx_walk_add_sg(walk: &ulp_walk, sg: reqctx->srcsg, len: wrparam->bytes,
2656	skip: reqctx->src_ofst);
2657	reqctx->srcsg = ulp_walk.last_sg;
2658	reqctx->src_ofst = ulp_walk.last_sg_len;
2659	ulptx_walk_end(walk: &ulp_walk);
2660	}
2661	}
2662
2663	void chcr_add_cipher_dst_ent(struct skcipher_request *req,
2664	struct cpl_rx_phys_dsgl *phys_cpl,
2665	struct cipher_wr_param *wrparam,
2666	unsigned short qid)
2667	{
2668	struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2669	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: wrparam->req);
2670	struct chcr_context *ctx = c_ctx(tfm);
2671	struct uld_ctx *u_ctx = ULD_CTX(ctx);
2672	struct dsgl_walk dsgl_walk;
2673	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2674
2675	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
2676	dsgl_walk_init(walk: &dsgl_walk, dsgl: phys_cpl);
2677	dsgl_walk_add_sg(walk: &dsgl_walk, sg: reqctx->dstsg, slen: wrparam->bytes,
2678	skip: reqctx->dst_ofst);
2679	reqctx->dstsg = dsgl_walk.last_sg;
2680	reqctx->dst_ofst = dsgl_walk.last_sg_len;
2681	dsgl_walk_end(walk: &dsgl_walk, qid, pci_chan_id: rx_channel_id);
2682	}
2683
2684	void chcr_add_hash_src_ent(struct ahash_request *req,
2685	struct ulptx_sgl *ulptx,
2686	struct hash_wr_param *param)
2687	{
2688	struct ulptx_walk ulp_walk;
2689	struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2690
2691	if (reqctx->hctx_wr.imm) {
2692	u8 *buf = (u8 *)ulptx;
2693
2694	if (param->bfr_len) {
2695	memcpy(buf, reqctx->reqbfr, param->bfr_len);
2696	buf += param->bfr_len;
2697	}
2698
2699	sg_pcopy_to_buffer(sgl: reqctx->hctx_wr.srcsg,
2700	nents: sg_nents(sg: reqctx->hctx_wr.srcsg), buf,
2701	buflen: param->sg_len, skip: 0);
2702	} else {
2703	ulptx_walk_init(walk: &ulp_walk, ulp: ulptx);
2704	if (param->bfr_len)
2705	ulptx_walk_add_page(walk: &ulp_walk, size: param->bfr_len,
2706	addr: reqctx->hctx_wr.dma_addr);
2707	ulptx_walk_add_sg(walk: &ulp_walk, sg: reqctx->hctx_wr.srcsg,
2708	len: param->sg_len, skip: reqctx->hctx_wr.src_ofst);
2709	reqctx->hctx_wr.srcsg = ulp_walk.last_sg;
2710	reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len;
2711	ulptx_walk_end(walk: &ulp_walk);
2712	}
2713	}
2714
2715	int chcr_hash_dma_map(struct device *dev,
2716	struct ahash_request *req)
2717	{
2718	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2719	int error = 0;
2720
2721	if (!req->nbytes)
2722	return 0;
2723	error = dma_map_sg(dev, req->src, sg_nents(req->src),
2724	DMA_TO_DEVICE);
2725	if (!error)
2726	return -ENOMEM;
2727	req_ctx->hctx_wr.is_sg_map = 1;
2728	return 0;
2729	}
2730
2731	void chcr_hash_dma_unmap(struct device *dev,
2732	struct ahash_request *req)
2733	{
2734	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2735
2736	if (!req->nbytes)
2737	return;
2738
2739	dma_unmap_sg(dev, req->src, sg_nents(req->src),
2740	DMA_TO_DEVICE);
2741	req_ctx->hctx_wr.is_sg_map = 0;
2742
2743	}
2744
2745	int chcr_cipher_dma_map(struct device *dev,
2746	struct skcipher_request *req)
2747	{
2748	int error;
2749
2750	if (req->src == req->dst) {
2751	error = dma_map_sg(dev, req->src, sg_nents(req->src),
2752	DMA_BIDIRECTIONAL);
2753	if (!error)
2754	goto err;
2755	} else {
2756	error = dma_map_sg(dev, req->src, sg_nents(req->src),
2757	DMA_TO_DEVICE);
2758	if (!error)
2759	goto err;
2760	error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2761	DMA_FROM_DEVICE);
2762	if (!error) {
2763	dma_unmap_sg(dev, req->src, sg_nents(req->src),
2764	DMA_TO_DEVICE);
2765	goto err;
2766	}
2767	}
2768
2769	return 0;
2770	err:
2771	return -ENOMEM;
2772	}
2773
2774	void chcr_cipher_dma_unmap(struct device *dev,
2775	struct skcipher_request *req)
2776	{
2777	if (req->src == req->dst) {
2778	dma_unmap_sg(dev, req->src, sg_nents(req->src),
2779	DMA_BIDIRECTIONAL);
2780	} else {
2781	dma_unmap_sg(dev, req->src, sg_nents(req->src),
2782	DMA_TO_DEVICE);
2783	dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2784	DMA_FROM_DEVICE);
2785	}
2786	}
2787
2788	static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2789	{
2790	__be32 data;
2791
2792	memset(block, 0, csize);
2793	block += csize;
2794
2795	if (csize >= 4)
2796	csize = 4;
2797	else if (msglen > (unsigned int)(1 << (8 * csize)))
2798	return -EOVERFLOW;
2799
2800	data = cpu_to_be32(msglen);
2801	memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2802
2803	return 0;
2804	}
2805
2806	static int generate_b0(struct aead_request *req, u8 *ivptr,
2807	unsigned short op_type)
2808	{
2809	unsigned int l, lp, m;
2810	int rc;
2811	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2812	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2813	u8 *b0 = reqctx->scratch_pad;
2814
2815	m = crypto_aead_authsize(tfm: aead);
2816
2817	memcpy(b0, ivptr, 16);
2818
2819	lp = b0[0];
2820	l = lp + 1;
2821
2822	/* set m, bits 3-5 */
2823	*b0 |= (8 * ((m - 2) / 2));
2824
2825	/* set adata, bit 6, if associated data is used */
2826	if (req->assoclen)
2827	*b0 |= 64;
2828	rc = set_msg_len(block: b0 + 16 - l,
2829	msglen: (op_type == CHCR_DECRYPT_OP) ?
2830	req->cryptlen - m : req->cryptlen, csize: l);
2831
2832	return rc;
2833	}
2834
2835	static inline int crypto_ccm_check_iv(const u8 *iv)
2836	{
2837	/* 2 <= L <= 8, so 1 <= L' <= 7. */
2838	if (iv[0] < 1 || iv[0] > 7)
2839	return -EINVAL;
2840
2841	return 0;
2842	}
2843
2844	static int ccm_format_packet(struct aead_request *req,
2845	u8 *ivptr,
2846	unsigned int sub_type,
2847	unsigned short op_type,
2848	unsigned int assoclen)
2849	{
2850	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2851	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2852	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
2853	int rc = 0;
2854
2855	if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2856	ivptr[0] = 3;
2857	memcpy(ivptr + 1, &aeadctx->salt[0], 3);
2858	memcpy(ivptr + 4, req->iv, 8);
2859	memset(ivptr + 12, 0, 4);
2860	} else {
2861	memcpy(ivptr, req->iv, 16);
2862	}
2863	if (assoclen)
2864	put_unaligned_be16(val: assoclen, p: &reqctx->scratch_pad[16]);
2865
2866	rc = generate_b0(req, ivptr, op_type);
2867	/* zero the ctr value */
2868	memset(ivptr + 15 - ivptr[0], 0, ivptr[0] + 1);
2869	return rc;
2870	}
2871
2872	static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2873	unsigned int dst_size,
2874	struct aead_request *req,
2875	unsigned short op_type)
2876	{
2877	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2878	struct chcr_context *ctx = a_ctx(tfm);
2879	struct uld_ctx *u_ctx = ULD_CTX(ctx);
2880	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2881	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2882	unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2883	unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2884	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2885	unsigned int ccm_xtra;
2886	unsigned int tag_offset = 0, auth_offset = 0;
2887	unsigned int assoclen;
2888
2889	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
2890
2891	if (get_aead_subtype(aead: tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2892	assoclen = req->assoclen - 8;
2893	else
2894	assoclen = req->assoclen;
2895	ccm_xtra = CCM_B0_SIZE +
2896	((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2897
2898	auth_offset = req->cryptlen ?
2899	(req->assoclen + IV + 1 + ccm_xtra) : 0;
2900	if (op_type == CHCR_DECRYPT_OP) {
2901	if (crypto_aead_authsize(tfm) != req->cryptlen)
2902	tag_offset = crypto_aead_authsize(tfm);
2903	else
2904	auth_offset = 0;
2905	}
2906
2907	sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
2908	sec_cpl->pldlen =
2909	htonl(req->assoclen + IV + req->cryptlen + ccm_xtra);
2910	/* For CCM there wil be b0 always. So AAD start will be 1 always */
2911	sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2912	1 + IV, IV + assoclen + ccm_xtra,
2913	req->assoclen + IV + 1 + ccm_xtra, 0);
2914
2915	sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2916	auth_offset, tag_offset,
2917	(op_type == CHCR_ENCRYPT_OP) ? 0 :
2918	crypto_aead_authsize(tfm));
2919	sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2920	(op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
2921	cipher_mode, mac_mode,
2922	aeadctx->hmac_ctrl, IV >> 1);
2923
2924	sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2925	0, dst_size);
2926	}
2927
2928	static int aead_ccm_validate_input(unsigned short op_type,
2929	struct aead_request *req,
2930	struct chcr_aead_ctx *aeadctx,
2931	unsigned int sub_type)
2932	{
2933	if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2934	if (crypto_ccm_check_iv(iv: req->iv)) {
2935	pr_err("CCM: IV check fails\n");
2936	return -EINVAL;
2937	}
2938	} else {
2939	if (req->assoclen != 16 && req->assoclen != 20) {
2940	pr_err("RFC4309: Invalid AAD length %d\n",
2941	req->assoclen);
2942	return -EINVAL;
2943	}
2944	}
2945	return 0;
2946	}
2947
2948	static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2949	unsigned short qid,
2950	int size)
2951	{
2952	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2953	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
2954	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
2955	struct sk_buff *skb = NULL;
2956	struct chcr_wr *chcr_req;
2957	struct cpl_rx_phys_dsgl *phys_cpl;
2958	struct ulptx_sgl *ulptx;
2959	unsigned int transhdr_len;
2960	unsigned int dst_size = 0, kctx_len, dnents, temp, snents;
2961	unsigned int sub_type, assoclen = req->assoclen;
2962	unsigned int authsize = crypto_aead_authsize(tfm);
2963	int error = -EINVAL;
2964	u8 *ivptr;
2965	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2966	GFP_ATOMIC;
2967	struct adapter *adap = padap(dev: a_ctx(tfm)->dev);
2968
2969	sub_type = get_aead_subtype(aead: tfm);
2970	if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2971	assoclen -= 8;
2972	reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
2973	error = chcr_aead_common_init(req);
2974	if (error)
2975	return ERR_PTR(error);
2976
2977	error = aead_ccm_validate_input(op_type: reqctx->op, req, aeadctx, sub_type);
2978	if (error)
2979	goto err;
2980	dnents = sg_nents_xlen(sg: req->dst, reqlen: req->assoclen + req->cryptlen
2981	+ (reqctx->op ? -authsize : authsize),
2982	CHCR_DST_SG_SIZE, skip: 0);
2983	dnents += MIN_CCM_SG; // For IV and B0
2984	dst_size = get_space_for_phys_dsgl(sgl_entr: dnents);
2985	snents = sg_nents_xlen(sg: req->src, reqlen: req->assoclen + req->cryptlen,
2986	CHCR_SRC_SG_SIZE, skip: 0);
2987	snents += MIN_CCM_SG; //For B0
2988	kctx_len = roundup(aeadctx->enckey_len, 16) * 2;
2989	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2990	reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen +
2991	reqctx->b0_len) <= SGE_MAX_WR_LEN;
2992	temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen +
2993	reqctx->b0_len, 16) :
2994	(sgl_len(n: snents) * 8);
2995	transhdr_len += temp;
2996	transhdr_len = roundup(transhdr_len, 16);
2997
2998	if (chcr_aead_need_fallback(req, dst_nents: dnents, T6_MAX_AAD_SIZE -
2999	reqctx->b0_len, wrlen: transhdr_len, op_type: reqctx->op)) {
3000	atomic_inc(v: &adap->chcr_stats.fallback);
3001	chcr_aead_common_exit(req);
3002	return ERR_PTR(error: chcr_aead_fallback(req, op_type: reqctx->op));
3003	}
3004	skb = alloc_skb(size: transhdr_len, priority: flags);
3005
3006	if (!skb) {
3007	error = -ENOMEM;
3008	goto err;
3009	}
3010
3011	chcr_req = __skb_put_zero(skb, len: transhdr_len);
3012
3013	fill_sec_cpl_for_aead(sec_cpl: &chcr_req->sec_cpl, dst_size, req, op_type: reqctx->op);
3014
3015	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3016	memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
3017	memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3018	aeadctx->key, aeadctx->enckey_len);
3019
3020	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
3021	ivptr = (u8 *)(phys_cpl + 1) + dst_size;
3022	ulptx = (struct ulptx_sgl *)(ivptr + IV);
3023	error = ccm_format_packet(req, ivptr, sub_type, op_type: reqctx->op, assoclen);
3024	if (error)
3025	goto dstmap_fail;
3026	chcr_add_aead_dst_ent(req, phys_cpl, qid);
3027	chcr_add_aead_src_ent(req, ulptx);
3028
3029	atomic_inc(v: &adap->chcr_stats.aead_rqst);
3030	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3031	kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen +
3032	reqctx->b0_len) : 0);
3033	create_wreq(ctx: a_ctx(tfm), chcr_req, req: &req->base, imm: reqctx->imm, hash_sz: 0,
3034	len16: transhdr_len, sc_len: temp, lcb: 0);
3035	reqctx->skb = skb;
3036
3037	return skb;
3038	dstmap_fail:
3039	kfree_skb(skb);
3040	err:
3041	chcr_aead_common_exit(req);
3042	return ERR_PTR(error);
3043	}
3044
3045	static struct sk_buff *create_gcm_wr(struct aead_request *req,
3046	unsigned short qid,
3047	int size)
3048	{
3049	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3050	struct chcr_context *ctx = a_ctx(tfm);
3051	struct uld_ctx *u_ctx = ULD_CTX(ctx);
3052	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3053	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
3054	struct sk_buff *skb = NULL;
3055	struct chcr_wr *chcr_req;
3056	struct cpl_rx_phys_dsgl *phys_cpl;
3057	struct ulptx_sgl *ulptx;
3058	unsigned int transhdr_len, dnents = 0, snents;
3059	unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen;
3060	unsigned int authsize = crypto_aead_authsize(tfm);
3061	int error = -EINVAL;
3062	u8 *ivptr;
3063	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
3064	GFP_ATOMIC;
3065	struct adapter *adap = padap(dev: ctx->dev);
3066	unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
3067
3068	rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]);
3069	if (get_aead_subtype(aead: tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
3070	assoclen = req->assoclen - 8;
3071
3072	reqctx->b0_len = 0;
3073	error = chcr_aead_common_init(req);
3074	if (error)
3075	return ERR_PTR(error);
3076	dnents = sg_nents_xlen(sg: req->dst, reqlen: req->assoclen + req->cryptlen +
3077	(reqctx->op ? -authsize : authsize),
3078	CHCR_DST_SG_SIZE, skip: 0);
3079	snents = sg_nents_xlen(sg: req->src, reqlen: req->assoclen + req->cryptlen,
3080	CHCR_SRC_SG_SIZE, skip: 0);
3081	dnents += MIN_GCM_SG; // For IV
3082	dst_size = get_space_for_phys_dsgl(sgl_entr: dnents);
3083	kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE;
3084	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
3085	reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <=
3086	SGE_MAX_WR_LEN;
3087	temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) :
3088	(sgl_len(n: snents) * 8);
3089	transhdr_len += temp;
3090	transhdr_len = roundup(transhdr_len, 16);
3091	if (chcr_aead_need_fallback(req, dst_nents: dnents, T6_MAX_AAD_SIZE,
3092	wrlen: transhdr_len, op_type: reqctx->op)) {
3093
3094	atomic_inc(v: &adap->chcr_stats.fallback);
3095	chcr_aead_common_exit(req);
3096	return ERR_PTR(error: chcr_aead_fallback(req, op_type: reqctx->op));
3097	}
3098	skb = alloc_skb(size: transhdr_len, priority: flags);
3099	if (!skb) {
3100	error = -ENOMEM;
3101	goto err;
3102	}
3103
3104	chcr_req = __skb_put_zero(skb, len: transhdr_len);
3105
3106	//Offset of tag from end
3107	temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
3108	chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
3109	rx_channel_id, 2, 1);
3110	chcr_req->sec_cpl.pldlen =
3111	htonl(req->assoclen + IV + req->cryptlen);
3112	chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
3113	assoclen ? 1 + IV : 0,
3114	assoclen ? IV + assoclen : 0,
3115	req->assoclen + IV + 1, 0);
3116	chcr_req->sec_cpl.cipherstop_lo_authinsert =
3117	FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + IV + 1,
3118	temp, temp);
3119	chcr_req->sec_cpl.seqno_numivs =
3120	FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op ==
3121	CHCR_ENCRYPT_OP) ? 1 : 0,
3122	CHCR_SCMD_CIPHER_MODE_AES_GCM,
3123	CHCR_SCMD_AUTH_MODE_GHASH,
3124	aeadctx->hmac_ctrl, IV >> 1);
3125	chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
3126	0, 0, dst_size);
3127	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3128	memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
3129	memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3130	GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
3131
3132	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
3133	ivptr = (u8 *)(phys_cpl + 1) + dst_size;
3134	/* prepare a 16 byte iv */
3135	/* S A L T | IV | 0x00000001 */
3136	if (get_aead_subtype(aead: tfm) ==
3137	CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
3138	memcpy(ivptr, aeadctx->salt, 4);
3139	memcpy(ivptr + 4, req->iv, GCM_RFC4106_IV_SIZE);
3140	} else {
3141	memcpy(ivptr, req->iv, GCM_AES_IV_SIZE);
3142	}
3143	put_unaligned_be32(val: 0x01, p: &ivptr[12]);
3144	ulptx = (struct ulptx_sgl *)(ivptr + 16);
3145
3146	chcr_add_aead_dst_ent(req, phys_cpl, qid);
3147	chcr_add_aead_src_ent(req, ulptx);
3148	atomic_inc(v: &adap->chcr_stats.aead_rqst);
3149	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3150	kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
3151	create_wreq(ctx: a_ctx(tfm), chcr_req, req: &req->base, imm: reqctx->imm, hash_sz: size,
3152	len16: transhdr_len, sc_len: temp, lcb: reqctx->verify);
3153	reqctx->skb = skb;
3154	return skb;
3155
3156	err:
3157	chcr_aead_common_exit(req);
3158	return ERR_PTR(error);
3159	}
3160
3161
3162
3163	static int chcr_aead_cra_init(struct crypto_aead *tfm)
3164	{
3165	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
3166	struct aead_alg *alg = crypto_aead_alg(tfm);
3167
3168	aeadctx->sw_cipher = crypto_alloc_aead(alg_name: alg->base.cra_name, type: 0,
3169	CRYPTO_ALG_NEED_FALLBACK |
3170	CRYPTO_ALG_ASYNC);
3171	if (IS_ERR(ptr: aeadctx->sw_cipher))
3172	return PTR_ERR(ptr: aeadctx->sw_cipher);
3173	crypto_aead_set_reqsize_dma(
3174	aead: tfm, max(sizeof(struct chcr_aead_reqctx),
3175	sizeof(struct aead_request) +
3176	crypto_aead_reqsize(aeadctx->sw_cipher)));
3177	return chcr_device_init(ctx: a_ctx(tfm));
3178	}
3179
3180	static void chcr_aead_cra_exit(struct crypto_aead *tfm)
3181	{
3182	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
3183
3184	crypto_free_aead(tfm: aeadctx->sw_cipher);
3185	}
3186
3187	static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
3188	unsigned int authsize)
3189	{
3190	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
3191
3192	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
3193	aeadctx->mayverify = VERIFY_HW;
3194	return crypto_aead_setauthsize(tfm: aeadctx->sw_cipher, authsize);
3195	}
3196	static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
3197	unsigned int authsize)
3198	{
3199	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
3200	u32 maxauth = crypto_aead_maxauthsize(aead: tfm);
3201
3202	/*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
3203	* true for sha1. authsize == 12 condition should be before
3204	* authsize == (maxauth >> 1)
3205	*/
3206	if (authsize == ICV_4) {
3207	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3208	aeadctx->mayverify = VERIFY_HW;
3209	} else if (authsize == ICV_6) {
3210	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3211	aeadctx->mayverify = VERIFY_HW;
3212	} else if (authsize == ICV_10) {
3213	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3214	aeadctx->mayverify = VERIFY_HW;
3215	} else if (authsize == ICV_12) {
3216	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3217	aeadctx->mayverify = VERIFY_HW;
3218	} else if (authsize == ICV_14) {
3219	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3220	aeadctx->mayverify = VERIFY_HW;
3221	} else if (authsize == (maxauth >> 1)) {
3222	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3223	aeadctx->mayverify = VERIFY_HW;
3224	} else if (authsize == maxauth) {
3225	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3226	aeadctx->mayverify = VERIFY_HW;
3227	} else {
3228	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3229	aeadctx->mayverify = VERIFY_SW;
3230	}
3231	return crypto_aead_setauthsize(tfm: aeadctx->sw_cipher, authsize);
3232	}
3233
3234
3235	static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
3236	{
3237	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
3238
3239	switch (authsize) {
3240	case ICV_4:
3241	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3242	aeadctx->mayverify = VERIFY_HW;
3243	break;
3244	case ICV_8:
3245	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3246	aeadctx->mayverify = VERIFY_HW;
3247	break;
3248	case ICV_12:
3249	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3250	aeadctx->mayverify = VERIFY_HW;
3251	break;
3252	case ICV_14:
3253	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3254	aeadctx->mayverify = VERIFY_HW;
3255	break;
3256	case ICV_16:
3257	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3258	aeadctx->mayverify = VERIFY_HW;
3259	break;
3260	case ICV_13:
3261	case ICV_15:
3262	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3263	aeadctx->mayverify = VERIFY_SW;
3264	break;
3265	default:
3266	return -EINVAL;
3267	}
3268	return crypto_aead_setauthsize(tfm: aeadctx->sw_cipher, authsize);
3269	}
3270
3271	static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
3272	unsigned int authsize)
3273	{
3274	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
3275
3276	switch (authsize) {
3277	case ICV_8:
3278	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3279	aeadctx->mayverify = VERIFY_HW;
3280	break;
3281	case ICV_12:
3282	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3283	aeadctx->mayverify = VERIFY_HW;
3284	break;
3285	case ICV_16:
3286	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3287	aeadctx->mayverify = VERIFY_HW;
3288	break;
3289	default:
3290	return -EINVAL;
3291	}
3292	return crypto_aead_setauthsize(tfm: aeadctx->sw_cipher, authsize);
3293	}
3294
3295	static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
3296	unsigned int authsize)
3297	{
3298	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm));
3299
3300	switch (authsize) {
3301	case ICV_4:
3302	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3303	aeadctx->mayverify = VERIFY_HW;
3304	break;
3305	case ICV_6:
3306	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3307	aeadctx->mayverify = VERIFY_HW;
3308	break;
3309	case ICV_8:
3310	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3311	aeadctx->mayverify = VERIFY_HW;
3312	break;
3313	case ICV_10:
3314	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3315	aeadctx->mayverify = VERIFY_HW;
3316	break;
3317	case ICV_12:
3318	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3319	aeadctx->mayverify = VERIFY_HW;
3320	break;
3321	case ICV_14:
3322	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3323	aeadctx->mayverify = VERIFY_HW;
3324	break;
3325	case ICV_16:
3326	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3327	aeadctx->mayverify = VERIFY_HW;
3328	break;
3329	default:
3330	return -EINVAL;
3331	}
3332	return crypto_aead_setauthsize(tfm: aeadctx->sw_cipher, authsize);
3333	}
3334
3335	static int chcr_ccm_common_setkey(struct crypto_aead *aead,
3336	const u8 *key,
3337	unsigned int keylen)
3338	{
3339	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm: aead));
3340	unsigned char ck_size, mk_size;
3341	int key_ctx_size = 0;
3342
3343	key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2;
3344	if (keylen == AES_KEYSIZE_128) {
3345	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3346	mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
3347	} else if (keylen == AES_KEYSIZE_192) {
3348	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3349	mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
3350	} else if (keylen == AES_KEYSIZE_256) {
3351	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3352	mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
3353	} else {
3354	aeadctx->enckey_len = 0;
3355	return -EINVAL;
3356	}
3357	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
3358	key_ctx_size >> 4);
3359	memcpy(aeadctx->key, key, keylen);
3360	aeadctx->enckey_len = keylen;
3361
3362	return 0;
3363	}
3364
3365	static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
3366	const u8 *key,
3367	unsigned int keylen)
3368	{
3369	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm: aead));
3370	int error;
3371
3372	crypto_aead_clear_flags(tfm: aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3373	crypto_aead_set_flags(tfm: aeadctx->sw_cipher, flags: crypto_aead_get_flags(tfm: aead) &
3374	CRYPTO_TFM_REQ_MASK);
3375	error = crypto_aead_setkey(tfm: aeadctx->sw_cipher, key, keylen);
3376	if (error)
3377	return error;
3378	return chcr_ccm_common_setkey(aead, key, keylen);
3379	}
3380
3381	static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
3382	unsigned int keylen)
3383	{
3384	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm: aead));
3385	int error;
3386
3387	if (keylen < 3) {
3388	aeadctx->enckey_len = 0;
3389	return -EINVAL;
3390	}
3391	crypto_aead_clear_flags(tfm: aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3392	crypto_aead_set_flags(tfm: aeadctx->sw_cipher, flags: crypto_aead_get_flags(tfm: aead) &
3393	CRYPTO_TFM_REQ_MASK);
3394	error = crypto_aead_setkey(tfm: aeadctx->sw_cipher, key, keylen);
3395	if (error)
3396	return error;
3397	keylen -= 3;
3398	memcpy(aeadctx->salt, key + keylen, 3);
3399	return chcr_ccm_common_setkey(aead, key, keylen);
3400	}
3401
3402	static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
3403	unsigned int keylen)
3404	{
3405	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm: aead));
3406	struct chcr_gcm_ctx *gctx = GCM_CTX(gctx: aeadctx);
3407	unsigned int ck_size;
3408	int ret = 0, key_ctx_size = 0;
3409	struct crypto_aes_ctx aes;
3410
3411	aeadctx->enckey_len = 0;
3412	crypto_aead_clear_flags(tfm: aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3413	crypto_aead_set_flags(tfm: aeadctx->sw_cipher, flags: crypto_aead_get_flags(tfm: aead)
3414	& CRYPTO_TFM_REQ_MASK);
3415	ret = crypto_aead_setkey(tfm: aeadctx->sw_cipher, key, keylen);
3416	if (ret)
3417	goto out;
3418
3419	if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3420	keylen > 3) {
3421	keylen -= 4; /* nonce/salt is present in the last 4 bytes */
3422	memcpy(aeadctx->salt, key + keylen, 4);
3423	}
3424	if (keylen == AES_KEYSIZE_128) {
3425	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3426	} else if (keylen == AES_KEYSIZE_192) {
3427	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3428	} else if (keylen == AES_KEYSIZE_256) {
3429	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3430	} else {
3431	pr_err("GCM: Invalid key length %d\n", keylen);
3432	ret = -EINVAL;
3433	goto out;
3434	}
3435
3436	memcpy(aeadctx->key, key, keylen);
3437	aeadctx->enckey_len = keylen;
3438	key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) +
3439	AEAD_H_SIZE;
3440	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
3441	CHCR_KEYCTX_MAC_KEY_SIZE_128,
3442	0, 0,
3443	key_ctx_size >> 4);
3444	/* Calculate the H = CIPH(K, 0 repeated 16 times).
3445	* It will go in key context
3446	*/
3447	ret = aes_expandkey(ctx: &aes, in_key: key, key_len: keylen);
3448	if (ret) {
3449	aeadctx->enckey_len = 0;
3450	goto out;
3451	}
3452	memset(gctx->ghash_h, 0, AEAD_H_SIZE);
3453	aes_encrypt(ctx: &aes, out: gctx->ghash_h, in: gctx->ghash_h);
3454	memzero_explicit(s: &aes, count: sizeof(aes));
3455
3456	out:
3457	return ret;
3458	}
3459
3460	static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
3461	unsigned int keylen)
3462	{
3463	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm: authenc));
3464	struct chcr_authenc_ctx *actx = AUTHENC_CTX(gctx: aeadctx);
3465	/* it contains auth and cipher key both*/
3466	struct crypto_authenc_keys keys;
3467	unsigned int subtype;
3468	unsigned int max_authsize = crypto_aead_alg(tfm: authenc)->maxauthsize;
3469	int err = 0, key_ctx_len = 0;
3470	unsigned char ck_size = 0;
3471	struct algo_param param;
3472	int align;
3473
3474	crypto_aead_clear_flags(tfm: aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3475	crypto_aead_set_flags(tfm: aeadctx->sw_cipher, flags: crypto_aead_get_flags(tfm: authenc)
3476	& CRYPTO_TFM_REQ_MASK);
3477	err = crypto_aead_setkey(tfm: aeadctx->sw_cipher, key, keylen);
3478	if (err)
3479	goto out;
3480
3481	if (crypto_authenc_extractkeys(keys: &keys, key, keylen) != 0)
3482	goto out;
3483
3484	if (get_alg_config(params: &param, auth_size: max_authsize)) {
3485	pr_err("Unsupported digest size\n");
3486	goto out;
3487	}
3488	subtype = get_aead_subtype(aead: authenc);
3489	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3490	subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3491	if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3492	goto out;
3493	memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3494	- CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3495	keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3496	}
3497	if (keys.enckeylen == AES_KEYSIZE_128) {
3498	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3499	} else if (keys.enckeylen == AES_KEYSIZE_192) {
3500	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3501	} else if (keys.enckeylen == AES_KEYSIZE_256) {
3502	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3503	} else {
3504	pr_err("Unsupported cipher key\n");
3505	goto out;
3506	}
3507
3508	/* Copy only encryption key. We use authkey to generate h(ipad) and
3509	* h(opad) so authkey is not needed again. authkeylen size have the
3510	* size of the hash digest size.
3511	*/
3512	memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3513	aeadctx->enckey_len = keys.enckeylen;
3514	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3515	subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3516
3517	get_aes_decrypt_key(dec_key: actx->dec_rrkey, key: aeadctx->key,
3518	keylength: aeadctx->enckey_len << 3);
3519	}
3520
3521	align = KEYCTX_ALIGN_PAD(max_authsize);
3522	err = chcr_prepare_hmac_key(raw_key: keys.authkey, raw_key_len: keys.authkeylen, digestsize: max_authsize,
3523	istate: actx->h_iopad,
3524	ostate: actx->h_iopad + param.result_size + align);
3525	if (err)
3526	goto out;
3527
3528	key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16) +
3529	(param.result_size + align) * 2;
3530	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 0, 1,
3531	key_ctx_len >> 4);
3532	actx->auth_mode = param.auth_mode;
3533
3534	memzero_explicit(s: &keys, count: sizeof(keys));
3535	return 0;
3536
3537	out:
3538	aeadctx->enckey_len = 0;
3539	memzero_explicit(s: &keys, count: sizeof(keys));
3540	return -EINVAL;
3541	}
3542
3543	static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3544	const u8 *key, unsigned int keylen)
3545	{
3546	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx: a_ctx(tfm: authenc));
3547	struct chcr_authenc_ctx *actx = AUTHENC_CTX(gctx: aeadctx);
3548	struct crypto_authenc_keys keys;
3549	int err;
3550	/* it contains auth and cipher key both*/
3551	unsigned int subtype;
3552	int key_ctx_len = 0;
3553	unsigned char ck_size = 0;
3554
3555	crypto_aead_clear_flags(tfm: aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3556	crypto_aead_set_flags(tfm: aeadctx->sw_cipher, flags: crypto_aead_get_flags(tfm: authenc)
3557	& CRYPTO_TFM_REQ_MASK);
3558	err = crypto_aead_setkey(tfm: aeadctx->sw_cipher, key, keylen);
3559	if (err)
3560	goto out;
3561
3562	if (crypto_authenc_extractkeys(keys: &keys, key, keylen) != 0)
3563	goto out;
3564
3565	subtype = get_aead_subtype(aead: authenc);
3566	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3567	subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3568	if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3569	goto out;
3570	memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3571	- CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3572	keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3573	}
3574	if (keys.enckeylen == AES_KEYSIZE_128) {
3575	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3576	} else if (keys.enckeylen == AES_KEYSIZE_192) {
3577	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3578	} else if (keys.enckeylen == AES_KEYSIZE_256) {
3579	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3580	} else {
3581	pr_err("Unsupported cipher key %d\n", keys.enckeylen);
3582	goto out;
3583	}
3584	memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3585	aeadctx->enckey_len = keys.enckeylen;
3586	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3587	subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3588	get_aes_decrypt_key(dec_key: actx->dec_rrkey, key: aeadctx->key,
3589	keylength: aeadctx->enckey_len << 3);
3590	}
3591	key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16);
3592
3593	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3594	0, key_ctx_len >> 4);
3595	actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
3596	memzero_explicit(s: &keys, count: sizeof(keys));
3597	return 0;
3598	out:
3599	aeadctx->enckey_len = 0;
3600	memzero_explicit(s: &keys, count: sizeof(keys));
3601	return -EINVAL;
3602	}
3603
3604	static int chcr_aead_op(struct aead_request *req,
3605	int size,
3606	create_wr_t create_wr_fn)
3607	{
3608	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3609	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
3610	struct chcr_context *ctx = a_ctx(tfm);
3611	struct uld_ctx *u_ctx = ULD_CTX(ctx);
3612	struct sk_buff *skb;
3613	struct chcr_dev *cdev;
3614
3615	cdev = a_ctx(tfm)->dev;
3616	if (!cdev) {
3617	pr_err("%s : No crypto device.\n", __func__);
3618	return -ENXIO;
3619	}
3620
3621	if (chcr_inc_wrcount(dev: cdev)) {
3622	/* Detach state for CHCR means lldi or padap is freed.
3623	* We cannot increment fallback here.
3624	*/
3625	return chcr_aead_fallback(req, op_type: reqctx->op);
3626	}
3627
3628	if (cxgb4_is_crypto_q_full(dev: u_ctx->lldi.ports[0],
3629	idx: reqctx->txqidx) &&
3630	(!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) {
3631	chcr_dec_wrcount(dev: cdev);
3632	return -ENOSPC;
3633	}
3634
3635	if (get_aead_subtype(aead: tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3636	crypto_ipsec_check_assoclen(assoclen: req->assoclen) != 0) {
3637	pr_err("RFC4106: Invalid value of assoclen %d\n",
3638	req->assoclen);
3639	return -EINVAL;
3640	}
3641
3642	/* Form a WR from req */
3643	skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], size);
3644
3645	if (IS_ERR_OR_NULL(ptr: skb)) {
3646	chcr_dec_wrcount(dev: cdev);
3647	return PTR_ERR_OR_ZERO(ptr: skb);
3648	}
3649
3650	skb->dev = u_ctx->lldi.ports[0];
3651	set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
3652	chcr_send_wr(skb);
3653	return -EINPROGRESS;
3654	}
3655
3656	static int chcr_aead_encrypt(struct aead_request *req)
3657	{
3658	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3659	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
3660	struct chcr_context *ctx = a_ctx(tfm);
3661	unsigned int cpu;
3662
3663	cpu = get_cpu();
3664	reqctx->txqidx = cpu % ctx->ntxq;
3665	reqctx->rxqidx = cpu % ctx->nrxq;
3666	put_cpu();
3667
3668	reqctx->verify = VERIFY_HW;
3669	reqctx->op = CHCR_ENCRYPT_OP;
3670
3671	switch (get_aead_subtype(aead: tfm)) {
3672	case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3673	case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3674	case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3675	case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3676	return chcr_aead_op(req, size: 0, create_wr_fn: create_authenc_wr);
3677	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3678	case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3679	return chcr_aead_op(req, size: 0, create_wr_fn: create_aead_ccm_wr);
3680	default:
3681	return chcr_aead_op(req, size: 0, create_wr_fn: create_gcm_wr);
3682	}
3683	}
3684
3685	static int chcr_aead_decrypt(struct aead_request *req)
3686	{
3687	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3688	struct chcr_context *ctx = a_ctx(tfm);
3689	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3690	struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req);
3691	int size;
3692	unsigned int cpu;
3693
3694	cpu = get_cpu();
3695	reqctx->txqidx = cpu % ctx->ntxq;
3696	reqctx->rxqidx = cpu % ctx->nrxq;
3697	put_cpu();
3698
3699	if (aeadctx->mayverify == VERIFY_SW) {
3700	size = crypto_aead_maxauthsize(aead: tfm);
3701	reqctx->verify = VERIFY_SW;
3702	} else {
3703	size = 0;
3704	reqctx->verify = VERIFY_HW;
3705	}
3706	reqctx->op = CHCR_DECRYPT_OP;
3707	switch (get_aead_subtype(aead: tfm)) {
3708	case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3709	case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3710	case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3711	case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3712	return chcr_aead_op(req, size, create_wr_fn: create_authenc_wr);
3713	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3714	case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3715	return chcr_aead_op(req, size, create_wr_fn: create_aead_ccm_wr);
3716	default:
3717	return chcr_aead_op(req, size, create_wr_fn: create_gcm_wr);
3718	}
3719	}
3720
3721	static struct chcr_alg_template driver_algs[] = {
3722	/* AES-CBC */
3723	{
3724	.type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
3725	.is_registered = 0,
3726	.alg.skcipher = {
3727	.base.cra_name = "cbc(aes)",
3728	.base.cra_driver_name = "cbc-aes-chcr",
3729	.base.cra_blocksize = AES_BLOCK_SIZE,
3730
3731	.init = chcr_init_tfm,
3732	.exit = chcr_exit_tfm,
3733	.min_keysize = AES_MIN_KEY_SIZE,
3734	.max_keysize = AES_MAX_KEY_SIZE,
3735	.ivsize = AES_BLOCK_SIZE,
3736	.setkey = chcr_aes_cbc_setkey,
3737	.encrypt = chcr_aes_encrypt,
3738	.decrypt = chcr_aes_decrypt,
3739	}
3740	},
3741	{
3742	.type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
3743	.is_registered = 0,
3744	.alg.skcipher = {
3745	.base.cra_name = "xts(aes)",
3746	.base.cra_driver_name = "xts-aes-chcr",
3747	.base.cra_blocksize = AES_BLOCK_SIZE,
3748
3749	.init = chcr_init_tfm,
3750	.exit = chcr_exit_tfm,
3751	.min_keysize = 2 * AES_MIN_KEY_SIZE,
3752	.max_keysize = 2 * AES_MAX_KEY_SIZE,
3753	.ivsize = AES_BLOCK_SIZE,
3754	.setkey = chcr_aes_xts_setkey,
3755	.encrypt = chcr_aes_encrypt,
3756	.decrypt = chcr_aes_decrypt,
3757	}
3758	},
3759	{
3760	.type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3761	.is_registered = 0,
3762	.alg.skcipher = {
3763	.base.cra_name = "ctr(aes)",
3764	.base.cra_driver_name = "ctr-aes-chcr",
3765	.base.cra_blocksize = 1,
3766
3767	.init = chcr_init_tfm,
3768	.exit = chcr_exit_tfm,
3769	.min_keysize = AES_MIN_KEY_SIZE,
3770	.max_keysize = AES_MAX_KEY_SIZE,
3771	.ivsize = AES_BLOCK_SIZE,
3772	.setkey = chcr_aes_ctr_setkey,
3773	.encrypt = chcr_aes_encrypt,
3774	.decrypt = chcr_aes_decrypt,
3775	}
3776	},
3777	{
3778	.type = CRYPTO_ALG_TYPE_SKCIPHER |
3779	CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3780	.is_registered = 0,
3781	.alg.skcipher = {
3782	.base.cra_name = "rfc3686(ctr(aes))",
3783	.base.cra_driver_name = "rfc3686-ctr-aes-chcr",
3784	.base.cra_blocksize = 1,
3785
3786	.init = chcr_rfc3686_init,
3787	.exit = chcr_exit_tfm,
3788	.min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3789	.max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3790	.ivsize = CTR_RFC3686_IV_SIZE,
3791	.setkey = chcr_aes_rfc3686_setkey,
3792	.encrypt = chcr_aes_encrypt,
3793	.decrypt = chcr_aes_decrypt,
3794	}
3795	},
3796	/* SHA */
3797	{
3798	.type = CRYPTO_ALG_TYPE_AHASH,
3799	.is_registered = 0,
3800	.alg.hash = {
3801	.halg.digestsize = SHA1_DIGEST_SIZE,
3802	.halg.base = {
3803	.cra_name = "sha1",
3804	.cra_driver_name = "sha1-chcr",
3805	.cra_blocksize = SHA1_BLOCK_SIZE,
3806	}
3807	}
3808	},
3809	{
3810	.type = CRYPTO_ALG_TYPE_AHASH,
3811	.is_registered = 0,
3812	.alg.hash = {
3813	.halg.digestsize = SHA256_DIGEST_SIZE,
3814	.halg.base = {
3815	.cra_name = "sha256",
3816	.cra_driver_name = "sha256-chcr",
3817	.cra_blocksize = SHA256_BLOCK_SIZE,
3818	}
3819	}
3820	},
3821	{
3822	.type = CRYPTO_ALG_TYPE_AHASH,
3823	.is_registered = 0,
3824	.alg.hash = {
3825	.halg.digestsize = SHA224_DIGEST_SIZE,
3826	.halg.base = {
3827	.cra_name = "sha224",
3828	.cra_driver_name = "sha224-chcr",
3829	.cra_blocksize = SHA224_BLOCK_SIZE,
3830	}
3831	}
3832	},
3833	{
3834	.type = CRYPTO_ALG_TYPE_AHASH,
3835	.is_registered = 0,
3836	.alg.hash = {
3837	.halg.digestsize = SHA384_DIGEST_SIZE,
3838	.halg.base = {
3839	.cra_name = "sha384",
3840	.cra_driver_name = "sha384-chcr",
3841	.cra_blocksize = SHA384_BLOCK_SIZE,
3842	}
3843	}
3844	},
3845	{
3846	.type = CRYPTO_ALG_TYPE_AHASH,
3847	.is_registered = 0,
3848	.alg.hash = {
3849	.halg.digestsize = SHA512_DIGEST_SIZE,
3850	.halg.base = {
3851	.cra_name = "sha512",
3852	.cra_driver_name = "sha512-chcr",
3853	.cra_blocksize = SHA512_BLOCK_SIZE,
3854	}
3855	}
3856	},
3857	/* HMAC */
3858	{
3859	.type = CRYPTO_ALG_TYPE_HMAC,
3860	.is_registered = 0,
3861	.alg.hash = {
3862	.halg.digestsize = SHA1_DIGEST_SIZE,
3863	.halg.base = {
3864	.cra_name = "hmac(sha1)",
3865	.cra_driver_name = "hmac-sha1-chcr",
3866	.cra_blocksize = SHA1_BLOCK_SIZE,
3867	}
3868	}
3869	},
3870	{
3871	.type = CRYPTO_ALG_TYPE_HMAC,
3872	.is_registered = 0,
3873	.alg.hash = {
3874	.halg.digestsize = SHA224_DIGEST_SIZE,
3875	.halg.base = {
3876	.cra_name = "hmac(sha224)",
3877	.cra_driver_name = "hmac-sha224-chcr",
3878	.cra_blocksize = SHA224_BLOCK_SIZE,
3879	}
3880	}
3881	},
3882	{
3883	.type = CRYPTO_ALG_TYPE_HMAC,
3884	.is_registered = 0,
3885	.alg.hash = {
3886	.halg.digestsize = SHA256_DIGEST_SIZE,
3887	.halg.base = {
3888	.cra_name = "hmac(sha256)",
3889	.cra_driver_name = "hmac-sha256-chcr",
3890	.cra_blocksize = SHA256_BLOCK_SIZE,
3891	}
3892	}
3893	},
3894	{
3895	.type = CRYPTO_ALG_TYPE_HMAC,
3896	.is_registered = 0,
3897	.alg.hash = {
3898	.halg.digestsize = SHA384_DIGEST_SIZE,
3899	.halg.base = {
3900	.cra_name = "hmac(sha384)",
3901	.cra_driver_name = "hmac-sha384-chcr",
3902	.cra_blocksize = SHA384_BLOCK_SIZE,
3903	}
3904	}
3905	},
3906	{
3907	.type = CRYPTO_ALG_TYPE_HMAC,
3908	.is_registered = 0,
3909	.alg.hash = {
3910	.halg.digestsize = SHA512_DIGEST_SIZE,
3911	.halg.base = {
3912	.cra_name = "hmac(sha512)",
3913	.cra_driver_name = "hmac-sha512-chcr",
3914	.cra_blocksize = SHA512_BLOCK_SIZE,
3915	}
3916	}
3917	},
3918	/* Add AEAD Algorithms */
3919	{
3920	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3921	.is_registered = 0,
3922	.alg.aead = {
3923	.base = {
3924	.cra_name = "gcm(aes)",
3925	.cra_driver_name = "gcm-aes-chcr",
3926	.cra_blocksize = 1,
3927	.cra_priority = CHCR_AEAD_PRIORITY,
3928	.cra_ctxsize = sizeof(struct chcr_context) +
3929	sizeof(struct chcr_aead_ctx) +
3930	sizeof(struct chcr_gcm_ctx),
3931	},
3932	.ivsize = GCM_AES_IV_SIZE,
3933	.maxauthsize = GHASH_DIGEST_SIZE,
3934	.setkey = chcr_gcm_setkey,
3935	.setauthsize = chcr_gcm_setauthsize,
3936	}
3937	},
3938	{
3939	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
3940	.is_registered = 0,
3941	.alg.aead = {
3942	.base = {
3943	.cra_name = "rfc4106(gcm(aes))",
3944	.cra_driver_name = "rfc4106-gcm-aes-chcr",
3945	.cra_blocksize = 1,
3946	.cra_priority = CHCR_AEAD_PRIORITY + 1,
3947	.cra_ctxsize = sizeof(struct chcr_context) +
3948	sizeof(struct chcr_aead_ctx) +
3949	sizeof(struct chcr_gcm_ctx),
3950
3951	},
3952	.ivsize = GCM_RFC4106_IV_SIZE,
3953	.maxauthsize = GHASH_DIGEST_SIZE,
3954	.setkey = chcr_gcm_setkey,
3955	.setauthsize = chcr_4106_4309_setauthsize,
3956	}
3957	},
3958	{
3959	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
3960	.is_registered = 0,
3961	.alg.aead = {
3962	.base = {
3963	.cra_name = "ccm(aes)",
3964	.cra_driver_name = "ccm-aes-chcr",
3965	.cra_blocksize = 1,
3966	.cra_priority = CHCR_AEAD_PRIORITY,
3967	.cra_ctxsize = sizeof(struct chcr_context) +
3968	sizeof(struct chcr_aead_ctx),
3969
3970	},
3971	.ivsize = AES_BLOCK_SIZE,
3972	.maxauthsize = GHASH_DIGEST_SIZE,
3973	.setkey = chcr_aead_ccm_setkey,
3974	.setauthsize = chcr_ccm_setauthsize,
3975	}
3976	},
3977	{
3978	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
3979	.is_registered = 0,
3980	.alg.aead = {
3981	.base = {
3982	.cra_name = "rfc4309(ccm(aes))",
3983	.cra_driver_name = "rfc4309-ccm-aes-chcr",
3984	.cra_blocksize = 1,
3985	.cra_priority = CHCR_AEAD_PRIORITY + 1,
3986	.cra_ctxsize = sizeof(struct chcr_context) +
3987	sizeof(struct chcr_aead_ctx),
3988
3989	},
3990	.ivsize = 8,
3991	.maxauthsize = GHASH_DIGEST_SIZE,
3992	.setkey = chcr_aead_rfc4309_setkey,
3993	.setauthsize = chcr_4106_4309_setauthsize,
3994	}
3995	},
3996	{
3997	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
3998	.is_registered = 0,
3999	.alg.aead = {
4000	.base = {
4001	.cra_name = "authenc(hmac(sha1),cbc(aes))",
4002	.cra_driver_name =
4003	"authenc-hmac-sha1-cbc-aes-chcr",
4004	.cra_blocksize = AES_BLOCK_SIZE,
4005	.cra_priority = CHCR_AEAD_PRIORITY,
4006	.cra_ctxsize = sizeof(struct chcr_context) +
4007	sizeof(struct chcr_aead_ctx) +
4008	sizeof(struct chcr_authenc_ctx),
4009
4010	},
4011	.ivsize = AES_BLOCK_SIZE,
4012	.maxauthsize = SHA1_DIGEST_SIZE,
4013	.setkey = chcr_authenc_setkey,
4014	.setauthsize = chcr_authenc_setauthsize,
4015	}
4016	},
4017	{
4018	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4019	.is_registered = 0,
4020	.alg.aead = {
4021	.base = {
4022
4023	.cra_name = "authenc(hmac(sha256),cbc(aes))",
4024	.cra_driver_name =
4025	"authenc-hmac-sha256-cbc-aes-chcr",
4026	.cra_blocksize = AES_BLOCK_SIZE,
4027	.cra_priority = CHCR_AEAD_PRIORITY,
4028	.cra_ctxsize = sizeof(struct chcr_context) +
4029	sizeof(struct chcr_aead_ctx) +
4030	sizeof(struct chcr_authenc_ctx),
4031
4032	},
4033	.ivsize = AES_BLOCK_SIZE,
4034	.maxauthsize = SHA256_DIGEST_SIZE,
4035	.setkey = chcr_authenc_setkey,
4036	.setauthsize = chcr_authenc_setauthsize,
4037	}
4038	},
4039	{
4040	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4041	.is_registered = 0,
4042	.alg.aead = {
4043	.base = {
4044	.cra_name = "authenc(hmac(sha224),cbc(aes))",
4045	.cra_driver_name =
4046	"authenc-hmac-sha224-cbc-aes-chcr",
4047	.cra_blocksize = AES_BLOCK_SIZE,
4048	.cra_priority = CHCR_AEAD_PRIORITY,
4049	.cra_ctxsize = sizeof(struct chcr_context) +
4050	sizeof(struct chcr_aead_ctx) +
4051	sizeof(struct chcr_authenc_ctx),
4052	},
4053	.ivsize = AES_BLOCK_SIZE,
4054	.maxauthsize = SHA224_DIGEST_SIZE,
4055	.setkey = chcr_authenc_setkey,
4056	.setauthsize = chcr_authenc_setauthsize,
4057	}
4058	},
4059	{
4060	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4061	.is_registered = 0,
4062	.alg.aead = {
4063	.base = {
4064	.cra_name = "authenc(hmac(sha384),cbc(aes))",
4065	.cra_driver_name =
4066	"authenc-hmac-sha384-cbc-aes-chcr",
4067	.cra_blocksize = AES_BLOCK_SIZE,
4068	.cra_priority = CHCR_AEAD_PRIORITY,
4069	.cra_ctxsize = sizeof(struct chcr_context) +
4070	sizeof(struct chcr_aead_ctx) +
4071	sizeof(struct chcr_authenc_ctx),
4072
4073	},
4074	.ivsize = AES_BLOCK_SIZE,
4075	.maxauthsize = SHA384_DIGEST_SIZE,
4076	.setkey = chcr_authenc_setkey,
4077	.setauthsize = chcr_authenc_setauthsize,
4078	}
4079	},
4080	{
4081	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4082	.is_registered = 0,
4083	.alg.aead = {
4084	.base = {
4085	.cra_name = "authenc(hmac(sha512),cbc(aes))",
4086	.cra_driver_name =
4087	"authenc-hmac-sha512-cbc-aes-chcr",
4088	.cra_blocksize = AES_BLOCK_SIZE,
4089	.cra_priority = CHCR_AEAD_PRIORITY,
4090	.cra_ctxsize = sizeof(struct chcr_context) +
4091	sizeof(struct chcr_aead_ctx) +
4092	sizeof(struct chcr_authenc_ctx),
4093
4094	},
4095	.ivsize = AES_BLOCK_SIZE,
4096	.maxauthsize = SHA512_DIGEST_SIZE,
4097	.setkey = chcr_authenc_setkey,
4098	.setauthsize = chcr_authenc_setauthsize,
4099	}
4100	},
4101	{
4102	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL,
4103	.is_registered = 0,
4104	.alg.aead = {
4105	.base = {
4106	.cra_name = "authenc(digest_null,cbc(aes))",
4107	.cra_driver_name =
4108	"authenc-digest_null-cbc-aes-chcr",
4109	.cra_blocksize = AES_BLOCK_SIZE,
4110	.cra_priority = CHCR_AEAD_PRIORITY,
4111	.cra_ctxsize = sizeof(struct chcr_context) +
4112	sizeof(struct chcr_aead_ctx) +
4113	sizeof(struct chcr_authenc_ctx),
4114
4115	},
4116	.ivsize = AES_BLOCK_SIZE,
4117	.maxauthsize = 0,
4118	.setkey = chcr_aead_digest_null_setkey,
4119	.setauthsize = chcr_authenc_null_setauthsize,
4120	}
4121	},
4122	{
4123	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4124	.is_registered = 0,
4125	.alg.aead = {
4126	.base = {
4127	.cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4128	.cra_driver_name =
4129	"authenc-hmac-sha1-rfc3686-ctr-aes-chcr",
4130	.cra_blocksize = 1,
4131	.cra_priority = CHCR_AEAD_PRIORITY,
4132	.cra_ctxsize = sizeof(struct chcr_context) +
4133	sizeof(struct chcr_aead_ctx) +
4134	sizeof(struct chcr_authenc_ctx),
4135
4136	},
4137	.ivsize = CTR_RFC3686_IV_SIZE,
4138	.maxauthsize = SHA1_DIGEST_SIZE,
4139	.setkey = chcr_authenc_setkey,
4140	.setauthsize = chcr_authenc_setauthsize,
4141	}
4142	},
4143	{
4144	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4145	.is_registered = 0,
4146	.alg.aead = {
4147	.base = {
4148
4149	.cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4150	.cra_driver_name =
4151	"authenc-hmac-sha256-rfc3686-ctr-aes-chcr",
4152	.cra_blocksize = 1,
4153	.cra_priority = CHCR_AEAD_PRIORITY,
4154	.cra_ctxsize = sizeof(struct chcr_context) +
4155	sizeof(struct chcr_aead_ctx) +
4156	sizeof(struct chcr_authenc_ctx),
4157
4158	},
4159	.ivsize = CTR_RFC3686_IV_SIZE,
4160	.maxauthsize = SHA256_DIGEST_SIZE,
4161	.setkey = chcr_authenc_setkey,
4162	.setauthsize = chcr_authenc_setauthsize,
4163	}
4164	},
4165	{
4166	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4167	.is_registered = 0,
4168	.alg.aead = {
4169	.base = {
4170	.cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
4171	.cra_driver_name =
4172	"authenc-hmac-sha224-rfc3686-ctr-aes-chcr",
4173	.cra_blocksize = 1,
4174	.cra_priority = CHCR_AEAD_PRIORITY,
4175	.cra_ctxsize = sizeof(struct chcr_context) +
4176	sizeof(struct chcr_aead_ctx) +
4177	sizeof(struct chcr_authenc_ctx),
4178	},
4179	.ivsize = CTR_RFC3686_IV_SIZE,
4180	.maxauthsize = SHA224_DIGEST_SIZE,
4181	.setkey = chcr_authenc_setkey,
4182	.setauthsize = chcr_authenc_setauthsize,
4183	}
4184	},
4185	{
4186	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4187	.is_registered = 0,
4188	.alg.aead = {
4189	.base = {
4190	.cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4191	.cra_driver_name =
4192	"authenc-hmac-sha384-rfc3686-ctr-aes-chcr",
4193	.cra_blocksize = 1,
4194	.cra_priority = CHCR_AEAD_PRIORITY,
4195	.cra_ctxsize = sizeof(struct chcr_context) +
4196	sizeof(struct chcr_aead_ctx) +
4197	sizeof(struct chcr_authenc_ctx),
4198
4199	},
4200	.ivsize = CTR_RFC3686_IV_SIZE,
4201	.maxauthsize = SHA384_DIGEST_SIZE,
4202	.setkey = chcr_authenc_setkey,
4203	.setauthsize = chcr_authenc_setauthsize,
4204	}
4205	},
4206	{
4207	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4208	.is_registered = 0,
4209	.alg.aead = {
4210	.base = {
4211	.cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4212	.cra_driver_name =
4213	"authenc-hmac-sha512-rfc3686-ctr-aes-chcr",
4214	.cra_blocksize = 1,
4215	.cra_priority = CHCR_AEAD_PRIORITY,
4216	.cra_ctxsize = sizeof(struct chcr_context) +
4217	sizeof(struct chcr_aead_ctx) +
4218	sizeof(struct chcr_authenc_ctx),
4219
4220	},
4221	.ivsize = CTR_RFC3686_IV_SIZE,
4222	.maxauthsize = SHA512_DIGEST_SIZE,
4223	.setkey = chcr_authenc_setkey,
4224	.setauthsize = chcr_authenc_setauthsize,
4225	}
4226	},
4227	{
4228	.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL,
4229	.is_registered = 0,
4230	.alg.aead = {
4231	.base = {
4232	.cra_name = "authenc(digest_null,rfc3686(ctr(aes)))",
4233	.cra_driver_name =
4234	"authenc-digest_null-rfc3686-ctr-aes-chcr",
4235	.cra_blocksize = 1,
4236	.cra_priority = CHCR_AEAD_PRIORITY,
4237	.cra_ctxsize = sizeof(struct chcr_context) +
4238	sizeof(struct chcr_aead_ctx) +
4239	sizeof(struct chcr_authenc_ctx),
4240
4241	},
4242	.ivsize = CTR_RFC3686_IV_SIZE,
4243	.maxauthsize = 0,
4244	.setkey = chcr_aead_digest_null_setkey,
4245	.setauthsize = chcr_authenc_null_setauthsize,
4246	}
4247	},
4248	};
4249
4250	/*
4251	* chcr_unregister_alg - Deregister crypto algorithms with
4252	* kernel framework.
4253	*/
4254	static int chcr_unregister_alg(void)
4255	{
4256	int i;
4257
4258	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4259	switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4260	case CRYPTO_ALG_TYPE_SKCIPHER:
4261	if (driver_algs[i].is_registered && refcount_read(
4262	r: &driver_algs[i].alg.skcipher.base.cra_refcnt)
4263	== 1) {
4264	crypto_unregister_skcipher(
4265	alg: &driver_algs[i].alg.skcipher);
4266	driver_algs[i].is_registered = 0;
4267	}
4268	break;
4269	case CRYPTO_ALG_TYPE_AEAD:
4270	if (driver_algs[i].is_registered && refcount_read(
4271	r: &driver_algs[i].alg.aead.base.cra_refcnt) == 1) {
4272	crypto_unregister_aead(
4273	alg: &driver_algs[i].alg.aead);
4274	driver_algs[i].is_registered = 0;
4275	}
4276	break;
4277	case CRYPTO_ALG_TYPE_AHASH:
4278	if (driver_algs[i].is_registered && refcount_read(
4279	r: &driver_algs[i].alg.hash.halg.base.cra_refcnt)
4280	== 1) {
4281	crypto_unregister_ahash(
4282	alg: &driver_algs[i].alg.hash);
4283	driver_algs[i].is_registered = 0;
4284	}
4285	break;
4286	}
4287	}
4288	return 0;
4289	}
4290
4291	#define SZ_AHASH_CTX sizeof(struct chcr_context)
4292	#define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
4293	#define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
4294
4295	/*
4296	* chcr_register_alg - Register crypto algorithms with kernel framework.
4297	*/
4298	static int chcr_register_alg(void)
4299	{
4300	struct crypto_alg ai;
4301	struct ahash_alg *a_hash;
4302	int err = 0, i;
4303	char *name = NULL;
4304
4305	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4306	if (driver_algs[i].is_registered)
4307	continue;
4308	switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4309	case CRYPTO_ALG_TYPE_SKCIPHER:
4310	driver_algs[i].alg.skcipher.base.cra_priority =
4311	CHCR_CRA_PRIORITY;
4312	driver_algs[i].alg.skcipher.base.cra_module = THIS_MODULE;
4313	driver_algs[i].alg.skcipher.base.cra_flags =
4314	CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
4315	CRYPTO_ALG_ALLOCATES_MEMORY |
4316	CRYPTO_ALG_NEED_FALLBACK;
4317	driver_algs[i].alg.skcipher.base.cra_ctxsize =
4318	sizeof(struct chcr_context) +
4319	sizeof(struct ablk_ctx);
4320	driver_algs[i].alg.skcipher.base.cra_alignmask = 0;
4321
4322	err = crypto_register_skcipher(alg: &driver_algs[i].alg.skcipher);
4323	name = driver_algs[i].alg.skcipher.base.cra_driver_name;
4324	break;
4325	case CRYPTO_ALG_TYPE_AEAD:
4326	driver_algs[i].alg.aead.base.cra_flags =
4327	CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |
4328	CRYPTO_ALG_ALLOCATES_MEMORY;
4329	driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
4330	driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
4331	driver_algs[i].alg.aead.init = chcr_aead_cra_init;
4332	driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
4333	driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
4334	err = crypto_register_aead(alg: &driver_algs[i].alg.aead);
4335	name = driver_algs[i].alg.aead.base.cra_driver_name;
4336	break;
4337	case CRYPTO_ALG_TYPE_AHASH:
4338	a_hash = &driver_algs[i].alg.hash;
4339	a_hash->update = chcr_ahash_update;
4340	a_hash->final = chcr_ahash_final;
4341	a_hash->finup = chcr_ahash_finup;
4342	a_hash->digest = chcr_ahash_digest;
4343	a_hash->export = chcr_ahash_export;
4344	a_hash->import = chcr_ahash_import;
4345	a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
4346	a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
4347	a_hash->halg.base.cra_module = THIS_MODULE;
4348	a_hash->halg.base.cra_flags =
4349	CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4350	a_hash->halg.base.cra_alignmask = 0;
4351	a_hash->halg.base.cra_exit = NULL;
4352
4353	if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
4354	a_hash->halg.base.cra_init = chcr_hmac_cra_init;
4355	a_hash->init = chcr_hmac_init;
4356	a_hash->setkey = chcr_ahash_setkey;
4357	a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
4358	} else {
4359	a_hash->init = chcr_sha_init;
4360	a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
4361	a_hash->halg.base.cra_init = chcr_sha_cra_init;
4362	}
4363	err = crypto_register_ahash(alg: &driver_algs[i].alg.hash);
4364	ai = driver_algs[i].alg.hash.halg.base;
4365	name = ai.cra_driver_name;
4366	break;
4367	}
4368	if (err) {
4369	pr_err("%s : Algorithm registration failed\n", name);
4370	goto register_err;
4371	} else {
4372	driver_algs[i].is_registered = 1;
4373	}
4374	}
4375	return 0;
4376
4377	register_err:
4378	chcr_unregister_alg();
4379	return err;
4380	}
4381
4382	/*
4383	* start_crypto - Register the crypto algorithms.
4384	* This should called once when the first device comesup. After this
4385	* kernel will start calling driver APIs for crypto operations.
4386	*/
4387	int start_crypto(void)
4388	{
4389	return chcr_register_alg();
4390	}
4391
4392	/*
4393	* stop_crypto - Deregister all the crypto algorithms with kernel.
4394	* This should be called once when the last device goes down. After this
4395	* kernel will not call the driver API for crypto operations.
4396	*/
4397	int stop_crypto(void)
4398	{
4399	chcr_unregister_alg();
4400	return 0;
4401	}
4402