1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2/* Copyright(c) 2014 - 2020 Intel Corporation */
3#include <linux/slab.h>
4#include <linux/delay.h>
5#include <linux/pci_ids.h>
6
7#include "adf_accel_devices.h"
8#include "adf_common_drv.h"
9#include "icp_qat_hal.h"
10#include "icp_qat_uclo.h"
11
12#define BAD_REGADDR 0xffff
13#define MAX_RETRY_TIMES 10000
14#define INIT_CTX_ARB_VALUE 0x0
15#define INIT_CTX_ENABLE_VALUE 0x0
16#define INIT_PC_VALUE 0x0
17#define INIT_WAKEUP_EVENTS_VALUE 0x1
18#define INIT_SIG_EVENTS_VALUE 0x1
19#define INIT_CCENABLE_VALUE 0x2000
20#define RST_CSR_QAT_LSB 20
21#define RST_CSR_AE_LSB 0
22#define MC_TIMESTAMP_ENABLE (0x1 << 7)
23
24#define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
25 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
26 (~(1 << CE_REG_PAR_ERR_BITPOS)))
27#define INSERT_IMMED_GPRA_CONST(inst, const_val) \
28 (inst = ((inst & 0xFFFF00C03FFull) | \
29 ((((const_val) << 12) & 0x0FF00000ull) | \
30 (((const_val) << 10) & 0x0003FC00ull))))
31#define INSERT_IMMED_GPRB_CONST(inst, const_val) \
32 (inst = ((inst & 0xFFFF00FFF00ull) | \
33 ((((const_val) << 12) & 0x0FF00000ull) | \
34 (((const_val) << 0) & 0x000000FFull))))
35
36#define AE(handle, ae) ((handle)->hal_handle->aes[ae])
37
38static const u64 inst_4b[] = {
39 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
40 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
41 0x0A021000000ull
42};
43
44static const u64 inst[] = {
45 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
46 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
47 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
48 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
49 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
50 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
51 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
52 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
53 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
54 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
55 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
56 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
57 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
58 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
59 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
60 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
61 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
62 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
63 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
64 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
65 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
66 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
67};
68
69void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
70 unsigned char ae, unsigned int ctx_mask)
71{
72 AE(handle, ae).live_ctx_mask = ctx_mask;
73}
74
75#define CSR_RETRY_TIMES 500
76static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
77 unsigned char ae, unsigned int csr)
78{
79 unsigned int iterations = CSR_RETRY_TIMES;
80 int value;
81
82 do {
83 value = GET_AE_CSR(handle, ae, csr);
84 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
85 return value;
86 } while (iterations--);
87
88 pr_err("QAT: Read CSR timeout\n");
89 return 0;
90}
91
92static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
93 unsigned char ae, unsigned int csr,
94 unsigned int value)
95{
96 unsigned int iterations = CSR_RETRY_TIMES;
97
98 do {
99 SET_AE_CSR(handle, ae, csr, value);
100 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
101 return 0;
102 } while (iterations--);
103
104 pr_err("QAT: Write CSR Timeout\n");
105 return -EFAULT;
106}
107
108static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
109 unsigned char ae, unsigned char ctx,
110 unsigned int *events)
111{
112 unsigned int cur_ctx;
113
114 cur_ctx = qat_hal_rd_ae_csr(handle, ae, csr: CSR_CTX_POINTER);
115 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: ctx);
116 *events = qat_hal_rd_ae_csr(handle, ae, csr: CTX_WAKEUP_EVENTS_INDIRECT);
117 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: cur_ctx);
118}
119
120static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
121 unsigned char ae, unsigned int cycles,
122 int chk_inactive)
123{
124 unsigned int base_cnt = 0, cur_cnt = 0;
125 unsigned int csr = (1 << ACS_ABO_BITPOS);
126 int times = MAX_RETRY_TIMES;
127 int elapsed_cycles = 0;
128
129 base_cnt = qat_hal_rd_ae_csr(handle, ae, csr: PROFILE_COUNT);
130 base_cnt &= 0xffff;
131 while ((int)cycles > elapsed_cycles && times--) {
132 if (chk_inactive)
133 csr = qat_hal_rd_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS);
134
135 cur_cnt = qat_hal_rd_ae_csr(handle, ae, csr: PROFILE_COUNT);
136 cur_cnt &= 0xffff;
137 elapsed_cycles = cur_cnt - base_cnt;
138
139 if (elapsed_cycles < 0)
140 elapsed_cycles += 0x10000;
141
142 /* ensure at least 8 time cycles elapsed in wait_cycles */
143 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
144 return 0;
145 }
146 if (times < 0) {
147 pr_err("QAT: wait_num_cycles time out\n");
148 return -EFAULT;
149 }
150 return 0;
151}
152
153#define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit)))
154#define SET_BIT(wrd, bit) ((wrd) | 1 << (bit))
155
156int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
157 unsigned char ae, unsigned char mode)
158{
159 unsigned int csr, new_csr;
160
161 if (mode != 4 && mode != 8) {
162 pr_err("QAT: bad ctx mode=%d\n", mode);
163 return -EINVAL;
164 }
165
166 /* Sets the acceleration engine context mode to either four or eight */
167 csr = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
168 csr = IGNORE_W1C_MASK & csr;
169 new_csr = (mode == 4) ?
170 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
171 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
172 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: new_csr);
173 return 0;
174}
175
176int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
177 unsigned char ae, unsigned char mode)
178{
179 unsigned int csr, new_csr;
180
181 csr = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
182 csr &= IGNORE_W1C_MASK;
183
184 new_csr = (mode) ?
185 SET_BIT(csr, CE_NN_MODE_BITPOS) :
186 CLR_BIT(csr, CE_NN_MODE_BITPOS);
187
188 if (new_csr != csr)
189 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: new_csr);
190
191 return 0;
192}
193
194int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
195 unsigned char ae, enum icp_qat_uof_regtype lm_type,
196 unsigned char mode)
197{
198 unsigned int csr, new_csr;
199
200 csr = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
201 csr &= IGNORE_W1C_MASK;
202 switch (lm_type) {
203 case ICP_LMEM0:
204 new_csr = (mode) ?
205 SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
206 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
207 break;
208 case ICP_LMEM1:
209 new_csr = (mode) ?
210 SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
211 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
212 break;
213 case ICP_LMEM2:
214 new_csr = (mode) ?
215 SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) :
216 CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS);
217 break;
218 case ICP_LMEM3:
219 new_csr = (mode) ?
220 SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) :
221 CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS);
222 break;
223 default:
224 pr_err("QAT: lmType = 0x%x\n", lm_type);
225 return -EINVAL;
226 }
227
228 if (new_csr != csr)
229 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: new_csr);
230 return 0;
231}
232
233void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle,
234 unsigned char ae, unsigned char mode)
235{
236 unsigned int csr, new_csr;
237
238 csr = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
239 csr &= IGNORE_W1C_MASK;
240 new_csr = (mode) ?
241 SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) :
242 CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS);
243 if (new_csr != csr)
244 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: new_csr);
245}
246
247static unsigned short qat_hal_get_reg_addr(unsigned int type,
248 unsigned short reg_num)
249{
250 unsigned short reg_addr;
251
252 switch (type) {
253 case ICP_GPA_ABS:
254 case ICP_GPB_ABS:
255 reg_addr = 0x80 | (reg_num & 0x7f);
256 break;
257 case ICP_GPA_REL:
258 case ICP_GPB_REL:
259 reg_addr = reg_num & 0x1f;
260 break;
261 case ICP_SR_RD_REL:
262 case ICP_SR_WR_REL:
263 case ICP_SR_REL:
264 reg_addr = 0x180 | (reg_num & 0x1f);
265 break;
266 case ICP_SR_ABS:
267 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
268 break;
269 case ICP_DR_RD_REL:
270 case ICP_DR_WR_REL:
271 case ICP_DR_REL:
272 reg_addr = 0x1c0 | (reg_num & 0x1f);
273 break;
274 case ICP_DR_ABS:
275 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
276 break;
277 case ICP_NEIGH_REL:
278 reg_addr = 0x280 | (reg_num & 0x1f);
279 break;
280 case ICP_LMEM0:
281 reg_addr = 0x200;
282 break;
283 case ICP_LMEM1:
284 reg_addr = 0x220;
285 break;
286 case ICP_LMEM2:
287 reg_addr = 0x2c0;
288 break;
289 case ICP_LMEM3:
290 reg_addr = 0x2e0;
291 break;
292 case ICP_NO_DEST:
293 reg_addr = 0x300 | (reg_num & 0xff);
294 break;
295 default:
296 reg_addr = BAD_REGADDR;
297 break;
298 }
299 return reg_addr;
300}
301
302void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
303{
304 unsigned int reset_mask = handle->chip_info->icp_rst_mask;
305 unsigned int reset_csr = handle->chip_info->icp_rst_csr;
306 unsigned int csr_val;
307
308 csr_val = GET_CAP_CSR(handle, reset_csr);
309 csr_val |= reset_mask;
310 SET_CAP_CSR(handle, reset_csr, csr_val);
311}
312
313static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
314 unsigned char ae, unsigned int ctx_mask,
315 unsigned int ae_csr, unsigned int csr_val)
316{
317 unsigned int ctx, cur_ctx;
318
319 cur_ctx = qat_hal_rd_ae_csr(handle, ae, csr: CSR_CTX_POINTER);
320
321 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
322 if (!(ctx_mask & (1 << ctx)))
323 continue;
324 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: ctx);
325 qat_hal_wr_ae_csr(handle, ae, csr: ae_csr, value: csr_val);
326 }
327
328 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: cur_ctx);
329}
330
331static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
332 unsigned char ae, unsigned char ctx,
333 unsigned int ae_csr)
334{
335 unsigned int cur_ctx, csr_val;
336
337 cur_ctx = qat_hal_rd_ae_csr(handle, ae, csr: CSR_CTX_POINTER);
338 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: ctx);
339 csr_val = qat_hal_rd_ae_csr(handle, ae, csr: ae_csr);
340 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: cur_ctx);
341
342 return csr_val;
343}
344
345static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
346 unsigned char ae, unsigned int ctx_mask,
347 unsigned int events)
348{
349 unsigned int ctx, cur_ctx;
350
351 cur_ctx = qat_hal_rd_ae_csr(handle, ae, csr: CSR_CTX_POINTER);
352 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
353 if (!(ctx_mask & (1 << ctx)))
354 continue;
355 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: ctx);
356 qat_hal_wr_ae_csr(handle, ae, csr: CTX_SIG_EVENTS_INDIRECT, value: events);
357 }
358 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: cur_ctx);
359}
360
361static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
362 unsigned char ae, unsigned int ctx_mask,
363 unsigned int events)
364{
365 unsigned int ctx, cur_ctx;
366
367 cur_ctx = qat_hal_rd_ae_csr(handle, ae, csr: CSR_CTX_POINTER);
368 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
369 if (!(ctx_mask & (1 << ctx)))
370 continue;
371 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: ctx);
372 qat_hal_wr_ae_csr(handle, ae, csr: CTX_WAKEUP_EVENTS_INDIRECT,
373 value: events);
374 }
375 qat_hal_wr_ae_csr(handle, ae, csr: CSR_CTX_POINTER, value: cur_ctx);
376}
377
378static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
379{
380 unsigned long ae_mask = handle->hal_handle->ae_mask;
381 unsigned int base_cnt, cur_cnt;
382 unsigned char ae;
383 int times = MAX_RETRY_TIMES;
384
385 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
386 base_cnt = qat_hal_rd_ae_csr(handle, ae, csr: PROFILE_COUNT);
387 base_cnt &= 0xffff;
388
389 do {
390 cur_cnt = qat_hal_rd_ae_csr(handle, ae, csr: PROFILE_COUNT);
391 cur_cnt &= 0xffff;
392 } while (times-- && (cur_cnt == base_cnt));
393
394 if (times < 0) {
395 pr_err("QAT: AE%d is inactive!!\n", ae);
396 return -EFAULT;
397 }
398 }
399
400 return 0;
401}
402
403int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
404 unsigned int ae)
405{
406 unsigned int enable = 0, active = 0;
407
408 enable = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
409 active = qat_hal_rd_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS);
410 if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
411 (active & (1 << ACS_ABO_BITPOS)))
412 return 1;
413 else
414 return 0;
415}
416
417static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
418{
419 unsigned long ae_mask = handle->hal_handle->ae_mask;
420 unsigned int misc_ctl_csr, misc_ctl;
421 unsigned char ae;
422
423 misc_ctl_csr = handle->chip_info->misc_ctl_csr;
424 /* stop the timestamp timers */
425 misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr);
426 if (misc_ctl & MC_TIMESTAMP_ENABLE)
427 SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl &
428 (~MC_TIMESTAMP_ENABLE));
429
430 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
431 qat_hal_wr_ae_csr(handle, ae, csr: TIMESTAMP_LOW, value: 0);
432 qat_hal_wr_ae_csr(handle, ae, csr: TIMESTAMP_HIGH, value: 0);
433 }
434 /* start timestamp timers */
435 SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE);
436}
437
438#define ESRAM_AUTO_TINIT BIT(2)
439#define ESRAM_AUTO_TINIT_DONE BIT(3)
440#define ESRAM_AUTO_INIT_USED_CYCLES (1640)
441#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
442static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
443{
444 void __iomem *csr_addr =
445 (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
446 ESRAM_AUTO_INIT_CSR_OFFSET);
447 unsigned int csr_val;
448 int times = 30;
449
450 if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)
451 return 0;
452
453 csr_val = ADF_CSR_RD(csr_addr, 0);
454 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
455 return 0;
456
457 csr_val = ADF_CSR_RD(csr_addr, 0);
458 csr_val |= ESRAM_AUTO_TINIT;
459 ADF_CSR_WR(csr_addr, 0, csr_val);
460
461 do {
462 qat_hal_wait_cycles(handle, ae: 0, ESRAM_AUTO_INIT_USED_CYCLES, chk_inactive: 0);
463 csr_val = ADF_CSR_RD(csr_addr, 0);
464 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
465 if (times < 0) {
466 pr_err("QAT: Fail to init eSram!\n");
467 return -EFAULT;
468 }
469 return 0;
470}
471
472#define SHRAM_INIT_CYCLES 2060
473int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
474{
475 unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr;
476 unsigned int reset_mask = handle->chip_info->icp_rst_mask;
477 unsigned int reset_csr = handle->chip_info->icp_rst_csr;
478 unsigned long ae_mask = handle->hal_handle->ae_mask;
479 unsigned char ae = 0;
480 unsigned int times = 100;
481 unsigned int csr_val;
482
483 /* write to the reset csr */
484 csr_val = GET_CAP_CSR(handle, reset_csr);
485 csr_val &= ~reset_mask;
486 do {
487 SET_CAP_CSR(handle, reset_csr, csr_val);
488 if (!(times--))
489 goto out_err;
490 csr_val = GET_CAP_CSR(handle, reset_csr);
491 csr_val &= reset_mask;
492 } while (csr_val);
493 /* enable clock */
494 csr_val = GET_CAP_CSR(handle, clk_csr);
495 csr_val |= reset_mask;
496 SET_CAP_CSR(handle, clk_csr, csr_val);
497 if (qat_hal_check_ae_alive(handle))
498 goto out_err;
499
500 /* Set undefined power-up/reset states to reasonable default values */
501 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
502 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES,
503 INIT_CTX_ENABLE_VALUE);
504 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
505 ae_csr: CTX_STS_INDIRECT,
506 csr_val: handle->hal_handle->upc_mask &
507 INIT_PC_VALUE);
508 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
509 qat_hal_wr_ae_csr(handle, ae, csr: CC_ENABLE, INIT_CCENABLE_VALUE);
510 qat_hal_put_wakeup_event(handle, ae,
511 ICP_QAT_UCLO_AE_ALL_CTX,
512 INIT_WAKEUP_EVENTS_VALUE);
513 qat_hal_put_sig_event(handle, ae,
514 ICP_QAT_UCLO_AE_ALL_CTX,
515 INIT_SIG_EVENTS_VALUE);
516 }
517 if (qat_hal_init_esram(handle))
518 goto out_err;
519 if (qat_hal_wait_cycles(handle, ae: 0, SHRAM_INIT_CYCLES, chk_inactive: 0))
520 goto out_err;
521 qat_hal_reset_timestamp(handle);
522
523 return 0;
524out_err:
525 pr_err("QAT: failed to get device out of reset\n");
526 return -EFAULT;
527}
528
529static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
530 unsigned char ae, unsigned int ctx_mask)
531{
532 unsigned int ctx;
533
534 ctx = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
535 ctx &= IGNORE_W1C_MASK &
536 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
537 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: ctx);
538}
539
540static u64 qat_hal_parity_64bit(u64 word)
541{
542 word ^= word >> 1;
543 word ^= word >> 2;
544 word ^= word >> 4;
545 word ^= word >> 8;
546 word ^= word >> 16;
547 word ^= word >> 32;
548 return word & 1;
549}
550
551static u64 qat_hal_set_uword_ecc(u64 uword)
552{
553 u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
554 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
555 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
556 bit6_mask = 0xdaf69a46910ULL;
557
558 /* clear the ecc bits */
559 uword &= ~(0x7fULL << 0x2C);
560 uword |= qat_hal_parity_64bit(word: bit0_mask & uword) << 0x2C;
561 uword |= qat_hal_parity_64bit(word: bit1_mask & uword) << 0x2D;
562 uword |= qat_hal_parity_64bit(word: bit2_mask & uword) << 0x2E;
563 uword |= qat_hal_parity_64bit(word: bit3_mask & uword) << 0x2F;
564 uword |= qat_hal_parity_64bit(word: bit4_mask & uword) << 0x30;
565 uword |= qat_hal_parity_64bit(word: bit5_mask & uword) << 0x31;
566 uword |= qat_hal_parity_64bit(word: bit6_mask & uword) << 0x32;
567 return uword;
568}
569
570void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
571 unsigned char ae, unsigned int uaddr,
572 unsigned int words_num, u64 *uword)
573{
574 unsigned int ustore_addr;
575 unsigned int i;
576
577 ustore_addr = qat_hal_rd_ae_csr(handle, ae, csr: USTORE_ADDRESS);
578 uaddr |= UA_ECS;
579 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_ADDRESS, value: uaddr);
580 for (i = 0; i < words_num; i++) {
581 unsigned int uwrd_lo, uwrd_hi;
582 u64 tmp;
583
584 tmp = qat_hal_set_uword_ecc(uword: uword[i]);
585 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
586 uwrd_hi = (unsigned int)(tmp >> 0x20);
587 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_DATA_LOWER, value: uwrd_lo);
588 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_DATA_UPPER, value: uwrd_hi);
589 }
590 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_ADDRESS, value: ustore_addr);
591}
592
593static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
594 unsigned char ae, unsigned int ctx_mask)
595{
596 unsigned int ctx;
597
598 ctx = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
599 ctx &= IGNORE_W1C_MASK;
600 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
601 ctx |= (ctx_mask << CE_ENABLE_BITPOS);
602 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: ctx);
603}
604
605static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
606{
607 unsigned long ae_mask = handle->hal_handle->ae_mask;
608 unsigned char ae;
609 unsigned short reg;
610
611 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
612 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
613 qat_hal_init_rd_xfer(handle, ae, ctx_mask: 0, reg_type: ICP_SR_RD_ABS,
614 reg_num: reg, regdata: 0);
615 qat_hal_init_rd_xfer(handle, ae, ctx_mask: 0, reg_type: ICP_DR_RD_ABS,
616 reg_num: reg, regdata: 0);
617 }
618 }
619}
620
621static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
622{
623 unsigned long ae_mask = handle->hal_handle->ae_mask;
624 unsigned char ae;
625 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
626 int times = MAX_RETRY_TIMES;
627 unsigned int csr_val = 0;
628 unsigned int savctx = 0;
629 int ret = 0;
630
631 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
632 csr_val = qat_hal_rd_ae_csr(handle, ae, csr: AE_MISC_CONTROL);
633 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
634 qat_hal_wr_ae_csr(handle, ae, csr: AE_MISC_CONTROL, value: csr_val);
635 csr_val = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
636 csr_val &= IGNORE_W1C_MASK;
637 if (handle->chip_info->nn)
638 csr_val |= CE_NN_MODE;
639
640 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: csr_val);
641 qat_hal_wr_uwords(handle, ae, uaddr: 0, ARRAY_SIZE(inst),
642 uword: (u64 *)inst);
643 qat_hal_wr_indr_csr(handle, ae, ctx_mask, ae_csr: CTX_STS_INDIRECT,
644 csr_val: handle->hal_handle->upc_mask &
645 INIT_PC_VALUE);
646 savctx = qat_hal_rd_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS);
647 qat_hal_wr_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS, value: 0);
648 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
649 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
650 ae_csr: CTX_SIG_EVENTS_INDIRECT, csr_val: 0);
651 qat_hal_wr_ae_csr(handle, ae, csr: CTX_SIG_EVENTS_ACTIVE, value: 0);
652 qat_hal_enable_ctx(handle, ae, ctx_mask);
653 }
654 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
655 /* wait for AE to finish */
656 do {
657 ret = qat_hal_wait_cycles(handle, ae, cycles: 20, chk_inactive: 1);
658 } while (ret && times--);
659
660 if (times < 0) {
661 pr_err("QAT: clear GPR of AE %d failed", ae);
662 return -EINVAL;
663 }
664 qat_hal_disable_ctx(handle, ae, ctx_mask);
665 qat_hal_wr_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS,
666 value: savctx & ACS_ACNO);
667 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES,
668 INIT_CTX_ENABLE_VALUE);
669 qat_hal_wr_indr_csr(handle, ae, ctx_mask, ae_csr: CTX_STS_INDIRECT,
670 csr_val: handle->hal_handle->upc_mask &
671 INIT_PC_VALUE);
672 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
673 qat_hal_wr_ae_csr(handle, ae, csr: CC_ENABLE, INIT_CCENABLE_VALUE);
674 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
675 INIT_WAKEUP_EVENTS_VALUE);
676 qat_hal_put_sig_event(handle, ae, ctx_mask,
677 INIT_SIG_EVENTS_VALUE);
678 }
679 return 0;
680}
681
682static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
683 struct adf_accel_dev *accel_dev)
684{
685 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
686 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
687 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
688 unsigned int max_en_ae_id = 0;
689 struct adf_bar *sram_bar;
690 unsigned int csr_val = 0;
691 unsigned long ae_mask;
692 unsigned char ae = 0;
693 int ret = 0;
694
695 handle->pci_dev = pci_info->pci_dev;
696 switch (handle->pci_dev->device) {
697 case PCI_DEVICE_ID_INTEL_QAT_4XXX:
698 case PCI_DEVICE_ID_INTEL_QAT_401XX:
699 case PCI_DEVICE_ID_INTEL_QAT_402XX:
700 case PCI_DEVICE_ID_INTEL_QAT_420XX:
701 case PCI_DEVICE_ID_INTEL_QAT_6XXX:
702 handle->chip_info->mmp_sram_size = 0;
703 handle->chip_info->nn = false;
704 handle->chip_info->lm2lm3 = true;
705 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X;
706 handle->chip_info->icp_rst_csr = ICP_RESET_CPP0;
707 if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_420XX)
708 handle->chip_info->icp_rst_mask = 0x100155;
709 else
710 handle->chip_info->icp_rst_mask = 0x100015;
711 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0;
712 handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX;
713 handle->chip_info->wakeup_event_val = 0x80000000;
714 handle->chip_info->fw_auth = true;
715 handle->chip_info->css_3k = true;
716 if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX)
717 handle->chip_info->dual_sign = true;
718 handle->chip_info->tgroup_share_ustore = true;
719 handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX;
720 handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX;
721 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX;
722 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX;
723 handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX;
724 handle->chip_info->fcu_loaded_ae_pos = 0;
725
726 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX;
727 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX;
728 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX;
729 handle->hal_cap_ae_local_csr_addr_v =
730 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
731 + LOCAL_TO_XFER_REG_OFFSET);
732 break;
733 case PCI_DEVICE_ID_INTEL_QAT_C62X:
734 case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
735 handle->chip_info->mmp_sram_size = 0;
736 handle->chip_info->nn = true;
737 handle->chip_info->lm2lm3 = false;
738 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
739 handle->chip_info->icp_rst_csr = ICP_RESET;
740 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
741 (hw_data->accel_mask << RST_CSR_QAT_LSB);
742 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
743 handle->chip_info->misc_ctl_csr = MISC_CONTROL;
744 handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
745 handle->chip_info->fw_auth = true;
746 handle->chip_info->css_3k = false;
747 handle->chip_info->tgroup_share_ustore = false;
748 handle->chip_info->fcu_ctl_csr = FCU_CONTROL;
749 handle->chip_info->fcu_sts_csr = FCU_STATUS;
750 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI;
751 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO;
752 handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS;
753 handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS;
754 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
755 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
756 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
757 handle->hal_cap_ae_local_csr_addr_v =
758 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
759 + LOCAL_TO_XFER_REG_OFFSET);
760 break;
761 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
762 handle->chip_info->mmp_sram_size = 0x40000;
763 handle->chip_info->nn = true;
764 handle->chip_info->lm2lm3 = false;
765 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
766 handle->chip_info->icp_rst_csr = ICP_RESET;
767 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
768 (hw_data->accel_mask << RST_CSR_QAT_LSB);
769 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
770 handle->chip_info->misc_ctl_csr = MISC_CONTROL;
771 handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
772 handle->chip_info->fw_auth = false;
773 handle->chip_info->css_3k = false;
774 handle->chip_info->tgroup_share_ustore = false;
775 handle->chip_info->fcu_ctl_csr = 0;
776 handle->chip_info->fcu_sts_csr = 0;
777 handle->chip_info->fcu_dram_addr_hi = 0;
778 handle->chip_info->fcu_dram_addr_lo = 0;
779 handle->chip_info->fcu_loaded_ae_csr = 0;
780 handle->chip_info->fcu_loaded_ae_pos = 0;
781 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
782 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
783 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
784 handle->hal_cap_ae_local_csr_addr_v =
785 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
786 + LOCAL_TO_XFER_REG_OFFSET);
787 break;
788 default:
789 ret = -EINVAL;
790 goto out_err;
791 }
792
793 if (handle->chip_info->mmp_sram_size > 0) {
794 sram_bar =
795 &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
796 handle->hal_sram_addr_v = sram_bar->virt_addr;
797 }
798 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
799 handle->hal_handle->ae_mask = hw_data->ae_mask;
800 handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask;
801 handle->hal_handle->slice_mask = hw_data->accel_mask;
802 handle->cfg_ae_mask = ALL_AE_MASK;
803 /* create AE objects */
804 handle->hal_handle->upc_mask = 0x1ffff;
805 handle->hal_handle->max_ustore = 0x4000;
806
807 ae_mask = handle->hal_handle->ae_mask;
808 for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) {
809 handle->hal_handle->aes[ae].free_addr = 0;
810 handle->hal_handle->aes[ae].free_size =
811 handle->hal_handle->max_ustore;
812 handle->hal_handle->aes[ae].ustore_size =
813 handle->hal_handle->max_ustore;
814 handle->hal_handle->aes[ae].live_ctx_mask =
815 ICP_QAT_UCLO_AE_ALL_CTX;
816 max_en_ae_id = ae;
817 }
818 handle->hal_handle->ae_max_num = max_en_ae_id + 1;
819
820 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
821 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
822 csr_val = qat_hal_rd_ae_csr(handle, ae, csr: SIGNATURE_ENABLE);
823 csr_val |= 0x1;
824 qat_hal_wr_ae_csr(handle, ae, csr: SIGNATURE_ENABLE, value: csr_val);
825 }
826out_err:
827 return ret;
828}
829
830int qat_hal_init(struct adf_accel_dev *accel_dev)
831{
832 struct icp_qat_fw_loader_handle *handle;
833 int ret = 0;
834
835 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
836 if (!handle)
837 return -ENOMEM;
838
839 handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
840 if (!handle->hal_handle) {
841 ret = -ENOMEM;
842 goto out_hal_handle;
843 }
844
845 handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL);
846 if (!handle->chip_info) {
847 ret = -ENOMEM;
848 goto out_chip_info;
849 }
850
851 ret = qat_hal_chip_init(handle, accel_dev);
852 if (ret) {
853 dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n");
854 goto out_err;
855 }
856
857 /* take all AEs out of reset */
858 ret = qat_hal_clr_reset(handle);
859 if (ret) {
860 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
861 goto out_err;
862 }
863
864 qat_hal_clear_xfer(handle);
865 if (!handle->chip_info->fw_auth) {
866 ret = qat_hal_clear_gpr(handle);
867 if (ret)
868 goto out_err;
869 }
870
871 accel_dev->fw_loader->fw_loader = handle;
872 return 0;
873
874out_err:
875 kfree(objp: handle->chip_info);
876out_chip_info:
877 kfree(objp: handle->hal_handle);
878out_hal_handle:
879 kfree(objp: handle);
880 return ret;
881}
882
883void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
884{
885 if (!handle)
886 return;
887 kfree(objp: handle->chip_info);
888 kfree(objp: handle->hal_handle);
889 kfree(objp: handle);
890}
891
892int qat_hal_start(struct icp_qat_fw_loader_handle *handle)
893{
894 unsigned long ae_mask = handle->hal_handle->ae_mask;
895 u32 wakeup_val = handle->chip_info->wakeup_event_val;
896 u32 fcu_ctl_csr, fcu_sts_csr;
897 unsigned int fcu_sts;
898 unsigned char ae;
899 u32 ae_ctr = 0;
900 int retry = 0;
901
902 if (handle->chip_info->fw_auth) {
903 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
904 fcu_sts_csr = handle->chip_info->fcu_sts_csr;
905 ae_ctr = hweight32(ae_mask);
906 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START);
907 do {
908 msleep(FW_AUTH_WAIT_PERIOD);
909 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
910 if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
911 return ae_ctr;
912 } while (retry++ < FW_AUTH_MAX_RETRY);
913 pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts);
914 return 0;
915 } else {
916 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
917 qat_hal_put_wakeup_event(handle, ae, ctx_mask: 0, events: wakeup_val);
918 qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX);
919 ae_ctr++;
920 }
921 return ae_ctr;
922 }
923}
924
925void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
926 unsigned int ctx_mask)
927{
928 if (!handle->chip_info->fw_auth)
929 qat_hal_disable_ctx(handle, ae, ctx_mask);
930}
931
932void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
933 unsigned char ae, unsigned int ctx_mask, unsigned int upc)
934{
935 qat_hal_wr_indr_csr(handle, ae, ctx_mask, ae_csr: CTX_STS_INDIRECT,
936 csr_val: handle->hal_handle->upc_mask & upc);
937}
938
939static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
940 unsigned char ae, unsigned int uaddr,
941 unsigned int words_num, u64 *uword)
942{
943 unsigned int i, uwrd_lo, uwrd_hi;
944 unsigned int ustore_addr, misc_control;
945
946 misc_control = qat_hal_rd_ae_csr(handle, ae, csr: AE_MISC_CONTROL);
947 qat_hal_wr_ae_csr(handle, ae, csr: AE_MISC_CONTROL,
948 value: misc_control & 0xfffffffb);
949 ustore_addr = qat_hal_rd_ae_csr(handle, ae, csr: USTORE_ADDRESS);
950 uaddr |= UA_ECS;
951 for (i = 0; i < words_num; i++) {
952 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_ADDRESS, value: uaddr);
953 uaddr++;
954 uwrd_lo = qat_hal_rd_ae_csr(handle, ae, csr: USTORE_DATA_LOWER);
955 uwrd_hi = qat_hal_rd_ae_csr(handle, ae, csr: USTORE_DATA_UPPER);
956 uword[i] = uwrd_hi;
957 uword[i] = (uword[i] << 0x20) | uwrd_lo;
958 }
959 qat_hal_wr_ae_csr(handle, ae, csr: AE_MISC_CONTROL, value: misc_control);
960 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_ADDRESS, value: ustore_addr);
961}
962
963void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
964 unsigned char ae, unsigned int uaddr,
965 unsigned int words_num, unsigned int *data)
966{
967 unsigned int i, ustore_addr;
968
969 ustore_addr = qat_hal_rd_ae_csr(handle, ae, csr: USTORE_ADDRESS);
970 uaddr |= UA_ECS;
971 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_ADDRESS, value: uaddr);
972 for (i = 0; i < words_num; i++) {
973 unsigned int uwrd_lo, uwrd_hi, tmp;
974
975 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
976 ((data[i] & 0xff00) << 2) |
977 (0x3 << 8) | (data[i] & 0xff);
978 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
979 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
980 tmp = ((data[i] >> 0x10) & 0xffff);
981 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
982 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_DATA_LOWER, value: uwrd_lo);
983 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_DATA_UPPER, value: uwrd_hi);
984 }
985 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_ADDRESS, value: ustore_addr);
986}
987
988#define MAX_EXEC_INST 100
989static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
990 unsigned char ae, unsigned char ctx,
991 u64 *micro_inst, unsigned int inst_num,
992 int code_off, unsigned int max_cycle,
993 unsigned int *endpc)
994{
995 unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0;
996 unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0;
997 unsigned int ind_t_index = 0, ind_t_index_byte = 0;
998 unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0;
999 unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0;
1000 u64 savuwords[MAX_EXEC_INST];
1001 unsigned int ind_cnt_sig;
1002 unsigned int ind_sig, act_sig;
1003 unsigned int csr_val = 0, newcsr_val;
1004 unsigned int savctx;
1005 unsigned int savcc, wakeup_events, savpc;
1006 unsigned int ctxarb_ctl, ctx_enables;
1007
1008 if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
1009 pr_err("QAT: invalid instruction num %d\n", inst_num);
1010 return -EINVAL;
1011 }
1012 /* save current context */
1013 ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, ae_csr: LM_ADDR_0_INDIRECT);
1014 ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, ae_csr: LM_ADDR_1_INDIRECT);
1015 ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
1016 ae_csr: INDIRECT_LM_ADDR_0_BYTE_INDEX);
1017 ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
1018 ae_csr: INDIRECT_LM_ADDR_1_BYTE_INDEX);
1019 if (handle->chip_info->lm2lm3) {
1020 ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1021 ae_csr: LM_ADDR_2_INDIRECT);
1022 ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1023 ae_csr: LM_ADDR_3_INDIRECT);
1024 ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1025 ae_csr: INDIRECT_LM_ADDR_2_BYTE_INDEX);
1026 ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1027 ae_csr: INDIRECT_LM_ADDR_3_BYTE_INDEX);
1028 ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx,
1029 ae_csr: INDIRECT_T_INDEX);
1030 ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx,
1031 ae_csr: INDIRECT_T_INDEX_BYTE_INDEX);
1032 }
1033 if (inst_num <= MAX_EXEC_INST)
1034 qat_hal_get_uwords(handle, ae, uaddr: 0, words_num: inst_num, uword: savuwords);
1035 qat_hal_get_wakeup_event(handle, ae, ctx, events: &wakeup_events);
1036 savpc = qat_hal_rd_indr_csr(handle, ae, ctx, ae_csr: CTX_STS_INDIRECT);
1037 savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
1038 ctx_enables = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
1039 ctx_enables &= IGNORE_W1C_MASK;
1040 savcc = qat_hal_rd_ae_csr(handle, ae, csr: CC_ENABLE);
1041 savctx = qat_hal_rd_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS);
1042 ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ARB_CNTL);
1043 ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1044 ae_csr: FUTURE_COUNT_SIGNAL_INDIRECT);
1045 ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1046 ae_csr: CTX_SIG_EVENTS_INDIRECT);
1047 act_sig = qat_hal_rd_ae_csr(handle, ae, csr: CTX_SIG_EVENTS_ACTIVE);
1048 /* execute micro codes */
1049 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: ctx_enables);
1050 qat_hal_wr_uwords(handle, ae, uaddr: 0, words_num: inst_num, uword: micro_inst);
1051 qat_hal_wr_indr_csr(handle, ae, ctx_mask: (1 << ctx), ae_csr: CTX_STS_INDIRECT, csr_val: 0);
1052 qat_hal_wr_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS, value: ctx & ACS_ACNO);
1053 if (code_off)
1054 qat_hal_wr_ae_csr(handle, ae, csr: CC_ENABLE, value: savcc & 0xffffdfff);
1055 qat_hal_put_wakeup_event(handle, ae, ctx_mask: (1 << ctx), XCWE_VOLUNTARY);
1056 qat_hal_wr_indr_csr(handle, ae, ctx_mask: (1 << ctx), ae_csr: CTX_SIG_EVENTS_INDIRECT, csr_val: 0);
1057 qat_hal_wr_ae_csr(handle, ae, csr: CTX_SIG_EVENTS_ACTIVE, value: 0);
1058 qat_hal_enable_ctx(handle, ae, ctx_mask: (1 << ctx));
1059 /* wait for micro codes to finish */
1060 if (qat_hal_wait_cycles(handle, ae, cycles: max_cycle, chk_inactive: 1) != 0)
1061 return -EFAULT;
1062 if (endpc) {
1063 unsigned int ctx_status;
1064
1065 ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
1066 ae_csr: CTX_STS_INDIRECT);
1067 *endpc = ctx_status & handle->hal_handle->upc_mask;
1068 }
1069 /* retore to saved context */
1070 qat_hal_disable_ctx(handle, ae, ctx_mask: (1 << ctx));
1071 if (inst_num <= MAX_EXEC_INST)
1072 qat_hal_wr_uwords(handle, ae, uaddr: 0, words_num: inst_num, uword: savuwords);
1073 qat_hal_put_wakeup_event(handle, ae, ctx_mask: (1 << ctx), events: wakeup_events);
1074 qat_hal_wr_indr_csr(handle, ae, ctx_mask: (1 << ctx), ae_csr: CTX_STS_INDIRECT,
1075 csr_val: handle->hal_handle->upc_mask & savpc);
1076 csr_val = qat_hal_rd_ae_csr(handle, ae, csr: AE_MISC_CONTROL);
1077 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
1078 qat_hal_wr_ae_csr(handle, ae, csr: AE_MISC_CONTROL, value: newcsr_val);
1079 qat_hal_wr_ae_csr(handle, ae, csr: CC_ENABLE, value: savcc);
1080 qat_hal_wr_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS, value: savctx & ACS_ACNO);
1081 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ARB_CNTL, value: ctxarb_ctl);
1082 qat_hal_wr_indr_csr(handle, ae, ctx_mask: (1 << ctx),
1083 ae_csr: LM_ADDR_0_INDIRECT, csr_val: ind_lm_addr0);
1084 qat_hal_wr_indr_csr(handle, ae, ctx_mask: (1 << ctx),
1085 ae_csr: LM_ADDR_1_INDIRECT, csr_val: ind_lm_addr1);
1086 qat_hal_wr_indr_csr(handle, ae, ctx_mask: (1 << ctx),
1087 ae_csr: INDIRECT_LM_ADDR_0_BYTE_INDEX, csr_val: ind_lm_addr_byte0);
1088 qat_hal_wr_indr_csr(handle, ae, ctx_mask: (1 << ctx),
1089 ae_csr: INDIRECT_LM_ADDR_1_BYTE_INDEX, csr_val: ind_lm_addr_byte1);
1090 if (handle->chip_info->lm2lm3) {
1091 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), ae_csr: LM_ADDR_2_INDIRECT,
1092 csr_val: ind_lm_addr2);
1093 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), ae_csr: LM_ADDR_3_INDIRECT,
1094 csr_val: ind_lm_addr3);
1095 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1096 ae_csr: INDIRECT_LM_ADDR_2_BYTE_INDEX,
1097 csr_val: ind_lm_addr_byte2);
1098 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1099 ae_csr: INDIRECT_LM_ADDR_3_BYTE_INDEX,
1100 csr_val: ind_lm_addr_byte3);
1101 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1102 ae_csr: INDIRECT_T_INDEX, csr_val: ind_t_index);
1103 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1104 ae_csr: INDIRECT_T_INDEX_BYTE_INDEX,
1105 csr_val: ind_t_index_byte);
1106 }
1107 qat_hal_wr_indr_csr(handle, ae, ctx_mask: (1 << ctx),
1108 ae_csr: FUTURE_COUNT_SIGNAL_INDIRECT, csr_val: ind_cnt_sig);
1109 qat_hal_wr_indr_csr(handle, ae, ctx_mask: (1 << ctx),
1110 ae_csr: CTX_SIG_EVENTS_INDIRECT, csr_val: ind_sig);
1111 qat_hal_wr_ae_csr(handle, ae, csr: CTX_SIG_EVENTS_ACTIVE, value: act_sig);
1112 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: ctx_enables);
1113
1114 return 0;
1115}
1116
1117static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
1118 unsigned char ae, unsigned char ctx,
1119 enum icp_qat_uof_regtype reg_type,
1120 unsigned short reg_num, unsigned int *data)
1121{
1122 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
1123 unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
1124 unsigned short reg_addr;
1125 int status = 0;
1126 u64 insts, savuword;
1127
1128 reg_addr = qat_hal_get_reg_addr(type: reg_type, reg_num);
1129 if (reg_addr == BAD_REGADDR) {
1130 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
1131 return -EINVAL;
1132 }
1133 switch (reg_type) {
1134 case ICP_GPA_REL:
1135 insts = 0xA070000000ull | (reg_addr & 0x3ff);
1136 break;
1137 default:
1138 insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
1139 break;
1140 }
1141 savctx = qat_hal_rd_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS);
1142 ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ARB_CNTL);
1143 ctx_enables = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
1144 ctx_enables &= IGNORE_W1C_MASK;
1145 if (ctx != (savctx & ACS_ACNO))
1146 qat_hal_wr_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS,
1147 value: ctx & ACS_ACNO);
1148 qat_hal_get_uwords(handle, ae, uaddr: 0, words_num: 1, uword: &savuword);
1149 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: ctx_enables);
1150 ustore_addr = qat_hal_rd_ae_csr(handle, ae, csr: USTORE_ADDRESS);
1151 uaddr = UA_ECS;
1152 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_ADDRESS, value: uaddr);
1153 insts = qat_hal_set_uword_ecc(uword: insts);
1154 uwrd_lo = (unsigned int)(insts & 0xffffffff);
1155 uwrd_hi = (unsigned int)(insts >> 0x20);
1156 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_DATA_LOWER, value: uwrd_lo);
1157 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_DATA_UPPER, value: uwrd_hi);
1158 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_ADDRESS, value: uaddr);
1159 /* delay for at least 8 cycles */
1160 qat_hal_wait_cycles(handle, ae, cycles: 0x8, chk_inactive: 0);
1161 /*
1162 * read ALU output
1163 * the instruction should have been executed
1164 * prior to clearing the ECS in putUwords
1165 */
1166 *data = qat_hal_rd_ae_csr(handle, ae, csr: ALU_OUT);
1167 qat_hal_wr_ae_csr(handle, ae, csr: USTORE_ADDRESS, value: ustore_addr);
1168 qat_hal_wr_uwords(handle, ae, uaddr: 0, words_num: 1, uword: &savuword);
1169 if (ctx != (savctx & ACS_ACNO))
1170 qat_hal_wr_ae_csr(handle, ae, csr: ACTIVE_CTX_STATUS,
1171 value: savctx & ACS_ACNO);
1172 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ARB_CNTL, value: ctxarb_cntl);
1173 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: ctx_enables);
1174
1175 return status;
1176}
1177
1178static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
1179 unsigned char ae, unsigned char ctx,
1180 enum icp_qat_uof_regtype reg_type,
1181 unsigned short reg_num, unsigned int data)
1182{
1183 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
1184 u64 insts[] = {
1185 0x0F440000000ull,
1186 0x0F040000000ull,
1187 0x0F0000C0300ull,
1188 0x0E000010000ull
1189 };
1190 const int num_inst = ARRAY_SIZE(insts), code_off = 1;
1191 const int imm_w1 = 0, imm_w0 = 1;
1192
1193 dest_addr = qat_hal_get_reg_addr(type: reg_type, reg_num);
1194 if (dest_addr == BAD_REGADDR) {
1195 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1196 return -EINVAL;
1197 }
1198
1199 data16lo = 0xffff & data;
1200 data16hi = 0xffff & (data >> 0x10);
1201 src_hiaddr = qat_hal_get_reg_addr(type: ICP_NO_DEST, reg_num: (unsigned short)
1202 (0xff & data16hi));
1203 src_lowaddr = qat_hal_get_reg_addr(type: ICP_NO_DEST, reg_num: (unsigned short)
1204 (0xff & data16lo));
1205 switch (reg_type) {
1206 case ICP_GPA_REL:
1207 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1208 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1209 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1210 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1211 break;
1212 default:
1213 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1214 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1215
1216 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1217 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1218 break;
1219 }
1220
1221 return qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst: insts, inst_num: num_inst,
1222 code_off, max_cycle: num_inst * 0x5, NULL);
1223}
1224
1225int qat_hal_get_ins_num(void)
1226{
1227 return ARRAY_SIZE(inst_4b);
1228}
1229
1230static int qat_hal_concat_micro_code(u64 *micro_inst,
1231 unsigned int inst_num, unsigned int size,
1232 unsigned int addr, unsigned int *value)
1233{
1234 int i;
1235 unsigned int cur_value;
1236 const u64 *inst_arr;
1237 int fixup_offset;
1238 int usize = 0;
1239 int orig_num;
1240
1241 orig_num = inst_num;
1242 cur_value = value[0];
1243 inst_arr = inst_4b;
1244 usize = ARRAY_SIZE(inst_4b);
1245 fixup_offset = inst_num;
1246 for (i = 0; i < usize; i++)
1247 micro_inst[inst_num++] = inst_arr[i];
1248 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1249 fixup_offset++;
1250 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1251 fixup_offset++;
1252 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1253 fixup_offset++;
1254 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1255
1256 return inst_num - orig_num;
1257}
1258
1259static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1260 unsigned char ae, unsigned char ctx,
1261 int *pfirst_exec, u64 *micro_inst,
1262 unsigned int inst_num)
1263{
1264 int stat = 0;
1265 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1266 unsigned int gprb0 = 0, gprb1 = 0;
1267
1268 if (*pfirst_exec) {
1269 qat_hal_rd_rel_reg(handle, ae, ctx, reg_type: ICP_GPA_REL, reg_num: 0, data: &gpra0);
1270 qat_hal_rd_rel_reg(handle, ae, ctx, reg_type: ICP_GPA_REL, reg_num: 0x1, data: &gpra1);
1271 qat_hal_rd_rel_reg(handle, ae, ctx, reg_type: ICP_GPA_REL, reg_num: 0x2, data: &gpra2);
1272 qat_hal_rd_rel_reg(handle, ae, ctx, reg_type: ICP_GPB_REL, reg_num: 0, data: &gprb0);
1273 qat_hal_rd_rel_reg(handle, ae, ctx, reg_type: ICP_GPB_REL, reg_num: 0x1, data: &gprb1);
1274 *pfirst_exec = 0;
1275 }
1276 stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, code_off: 1,
1277 max_cycle: inst_num * 0x5, NULL);
1278 if (stat != 0)
1279 return -EFAULT;
1280 qat_hal_wr_rel_reg(handle, ae, ctx, reg_type: ICP_GPA_REL, reg_num: 0, data: gpra0);
1281 qat_hal_wr_rel_reg(handle, ae, ctx, reg_type: ICP_GPA_REL, reg_num: 0x1, data: gpra1);
1282 qat_hal_wr_rel_reg(handle, ae, ctx, reg_type: ICP_GPA_REL, reg_num: 0x2, data: gpra2);
1283 qat_hal_wr_rel_reg(handle, ae, ctx, reg_type: ICP_GPB_REL, reg_num: 0, data: gprb0);
1284 qat_hal_wr_rel_reg(handle, ae, ctx, reg_type: ICP_GPB_REL, reg_num: 0x1, data: gprb1);
1285
1286 return 0;
1287}
1288
1289int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1290 unsigned char ae,
1291 struct icp_qat_uof_batch_init *lm_init_header)
1292{
1293 struct icp_qat_uof_batch_init *plm_init;
1294 u64 *micro_inst_arry;
1295 int micro_inst_num;
1296 int alloc_inst_size;
1297 int first_exec = 1;
1298 int stat = 0;
1299
1300 plm_init = lm_init_header->next;
1301 alloc_inst_size = lm_init_header->size;
1302 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1303 alloc_inst_size = handle->hal_handle->max_ustore;
1304 micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64),
1305 GFP_KERNEL);
1306 if (!micro_inst_arry)
1307 return -ENOMEM;
1308 micro_inst_num = 0;
1309 while (plm_init) {
1310 unsigned int addr, *value, size;
1311
1312 ae = plm_init->ae;
1313 addr = plm_init->addr;
1314 value = plm_init->value;
1315 size = plm_init->size;
1316 micro_inst_num += qat_hal_concat_micro_code(micro_inst: micro_inst_arry,
1317 inst_num: micro_inst_num,
1318 size, addr, value);
1319 plm_init = plm_init->next;
1320 }
1321 /* exec micro codes */
1322 if (micro_inst_arry && micro_inst_num > 0) {
1323 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1324 stat = qat_hal_exec_micro_init_lm(handle, ae, ctx: 0, pfirst_exec: &first_exec,
1325 micro_inst: micro_inst_arry,
1326 inst_num: micro_inst_num);
1327 }
1328 kfree(objp: micro_inst_arry);
1329 return stat;
1330}
1331
1332static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1333 unsigned char ae, unsigned char ctx,
1334 enum icp_qat_uof_regtype reg_type,
1335 unsigned short reg_num, unsigned int val)
1336{
1337 int status = 0;
1338 unsigned int reg_addr;
1339 unsigned int ctx_enables;
1340 unsigned short mask;
1341 unsigned short dr_offset = 0x10;
1342
1343 ctx_enables = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
1344 if (CE_INUSE_CONTEXTS & ctx_enables) {
1345 if (ctx & 0x1) {
1346 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1347 return -EINVAL;
1348 }
1349 mask = 0x1f;
1350 dr_offset = 0x20;
1351 } else {
1352 mask = 0x0f;
1353 }
1354 if (reg_num & ~mask)
1355 return -EINVAL;
1356 reg_addr = reg_num + (ctx << 0x5);
1357 switch (reg_type) {
1358 case ICP_SR_RD_REL:
1359 case ICP_SR_REL:
1360 SET_AE_XFER(handle, ae, reg_addr, val);
1361 break;
1362 case ICP_DR_RD_REL:
1363 case ICP_DR_REL:
1364 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1365 break;
1366 default:
1367 status = -EINVAL;
1368 break;
1369 }
1370 return status;
1371}
1372
1373static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1374 unsigned char ae, unsigned char ctx,
1375 enum icp_qat_uof_regtype reg_type,
1376 unsigned short reg_num, unsigned int data)
1377{
1378 unsigned int gprval, ctx_enables;
1379 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1380 data16low;
1381 unsigned short reg_mask;
1382 int status = 0;
1383 u64 micro_inst[] = {
1384 0x0F440000000ull,
1385 0x0F040000000ull,
1386 0x0A000000000ull,
1387 0x0F0000C0300ull,
1388 0x0E000010000ull
1389 };
1390 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1391 const unsigned short gprnum = 0, dly = num_inst * 0x5;
1392
1393 ctx_enables = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
1394 if (CE_INUSE_CONTEXTS & ctx_enables) {
1395 if (ctx & 0x1) {
1396 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1397 return -EINVAL;
1398 }
1399 reg_mask = (unsigned short)~0x1f;
1400 } else {
1401 reg_mask = (unsigned short)~0xf;
1402 }
1403 if (reg_num & reg_mask)
1404 return -EINVAL;
1405 xfr_addr = qat_hal_get_reg_addr(type: reg_type, reg_num);
1406 if (xfr_addr == BAD_REGADDR) {
1407 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1408 return -EINVAL;
1409 }
1410 status = qat_hal_rd_rel_reg(handle, ae, ctx, reg_type: ICP_GPB_REL, reg_num: gprnum, data: &gprval);
1411 if (status) {
1412 pr_err("QAT: failed to read register");
1413 return status;
1414 }
1415 gpr_addr = qat_hal_get_reg_addr(type: ICP_GPB_REL, reg_num: gprnum);
1416 data16low = 0xffff & data;
1417 data16hi = 0xffff & (data >> 0x10);
1418 src_hiaddr = qat_hal_get_reg_addr(type: ICP_NO_DEST,
1419 reg_num: (unsigned short)(0xff & data16hi));
1420 src_lowaddr = qat_hal_get_reg_addr(type: ICP_NO_DEST,
1421 reg_num: (unsigned short)(0xff & data16low));
1422 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1423 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1424 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1425 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1426 micro_inst[0x2] = micro_inst[0x2] |
1427 ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1428 status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num: num_inst,
1429 code_off, max_cycle: dly, NULL);
1430 qat_hal_wr_rel_reg(handle, ae, ctx, reg_type: ICP_GPB_REL, reg_num: gprnum, data: gprval);
1431 return status;
1432}
1433
1434static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1435 unsigned char ae, unsigned char ctx,
1436 unsigned short nn, unsigned int val)
1437{
1438 unsigned int ctx_enables;
1439 int stat = 0;
1440
1441 ctx_enables = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
1442 ctx_enables &= IGNORE_W1C_MASK;
1443 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: ctx_enables | CE_NN_MODE);
1444
1445 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, reg_type: ICP_NEIGH_REL, reg_num: nn, data: val);
1446 qat_hal_wr_ae_csr(handle, ae, csr: CTX_ENABLES, value: ctx_enables);
1447 return stat;
1448}
1449
1450static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1451 *handle, unsigned char ae,
1452 unsigned short absreg_num,
1453 unsigned short *relreg,
1454 unsigned char *ctx)
1455{
1456 unsigned int ctx_enables;
1457
1458 ctx_enables = qat_hal_rd_ae_csr(handle, ae, csr: CTX_ENABLES);
1459 if (ctx_enables & CE_INUSE_CONTEXTS) {
1460 /* 4-ctx mode */
1461 *relreg = absreg_num & 0x1F;
1462 *ctx = (absreg_num >> 0x4) & 0x6;
1463 } else {
1464 /* 8-ctx mode */
1465 *relreg = absreg_num & 0x0F;
1466 *ctx = (absreg_num >> 0x4) & 0x7;
1467 }
1468 return 0;
1469}
1470
1471int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1472 unsigned char ae, unsigned long ctx_mask,
1473 enum icp_qat_uof_regtype reg_type,
1474 unsigned short reg_num, unsigned int regdata)
1475{
1476 int stat = 0;
1477 unsigned short reg;
1478 unsigned char ctx = 0;
1479 enum icp_qat_uof_regtype type;
1480
1481 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1482 return -EINVAL;
1483
1484 do {
1485 if (ctx_mask == 0) {
1486 qat_hal_convert_abs_to_rel(handle, ae, absreg_num: reg_num, relreg: &reg,
1487 ctx: &ctx);
1488 type = reg_type - 1;
1489 } else {
1490 reg = reg_num;
1491 type = reg_type;
1492 if (!test_bit(ctx, &ctx_mask))
1493 continue;
1494 }
1495 stat = qat_hal_wr_rel_reg(handle, ae, ctx, reg_type: type, reg_num: reg, data: regdata);
1496 if (stat) {
1497 pr_err("QAT: write gpr fail\n");
1498 return -EINVAL;
1499 }
1500 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1501
1502 return 0;
1503}
1504
1505int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1506 unsigned char ae, unsigned long ctx_mask,
1507 enum icp_qat_uof_regtype reg_type,
1508 unsigned short reg_num, unsigned int regdata)
1509{
1510 int stat = 0;
1511 unsigned short reg;
1512 unsigned char ctx = 0;
1513 enum icp_qat_uof_regtype type;
1514
1515 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1516 return -EINVAL;
1517
1518 do {
1519 if (ctx_mask == 0) {
1520 qat_hal_convert_abs_to_rel(handle, ae, absreg_num: reg_num, relreg: &reg,
1521 ctx: &ctx);
1522 type = reg_type - 3;
1523 } else {
1524 reg = reg_num;
1525 type = reg_type;
1526 if (!test_bit(ctx, &ctx_mask))
1527 continue;
1528 }
1529 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, reg_type: type, reg_num: reg,
1530 data: regdata);
1531 if (stat) {
1532 pr_err("QAT: write wr xfer fail\n");
1533 return -EINVAL;
1534 }
1535 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1536
1537 return 0;
1538}
1539
1540int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1541 unsigned char ae, unsigned long ctx_mask,
1542 enum icp_qat_uof_regtype reg_type,
1543 unsigned short reg_num, unsigned int regdata)
1544{
1545 int stat = 0;
1546 unsigned short reg;
1547 unsigned char ctx = 0;
1548 enum icp_qat_uof_regtype type;
1549
1550 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1551 return -EINVAL;
1552
1553 do {
1554 if (ctx_mask == 0) {
1555 qat_hal_convert_abs_to_rel(handle, ae, absreg_num: reg_num, relreg: &reg,
1556 ctx: &ctx);
1557 type = reg_type - 3;
1558 } else {
1559 reg = reg_num;
1560 type = reg_type;
1561 if (!test_bit(ctx, &ctx_mask))
1562 continue;
1563 }
1564 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, reg_type: type, reg_num: reg,
1565 val: regdata);
1566 if (stat) {
1567 pr_err("QAT: write rd xfer fail\n");
1568 return -EINVAL;
1569 }
1570 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1571
1572 return 0;
1573}
1574
1575int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1576 unsigned char ae, unsigned long ctx_mask,
1577 unsigned short reg_num, unsigned int regdata)
1578{
1579 int stat = 0;
1580 unsigned char ctx;
1581 if (!handle->chip_info->nn) {
1582 dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n",
1583 handle->pci_dev->device);
1584 return -EINVAL;
1585 }
1586
1587 if (ctx_mask == 0)
1588 return -EINVAL;
1589
1590 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1591 if (!test_bit(ctx, &ctx_mask))
1592 continue;
1593 stat = qat_hal_put_rel_nn(handle, ae, ctx, nn: reg_num, val: regdata);
1594 if (stat) {
1595 pr_err("QAT: write neigh error\n");
1596 return -EINVAL;
1597 }
1598 }
1599
1600 return 0;
1601}
1602

source code of linux/drivers/crypto/intel/qat/qat_common/qat_hal.c