1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2023 Intel Corporation */
3
4#define dev_fmt(fmt) "RateLimiting: " fmt
5
6#include <asm/errno.h>
7#include <asm/div64.h>
8
9#include <linux/dev_printk.h>
10#include <linux/kernel.h>
11#include <linux/pci.h>
12#include <linux/slab.h>
13#include <linux/units.h>
14
15#include "adf_accel_devices.h"
16#include "adf_cfg_services.h"
17#include "adf_common_drv.h"
18#include "adf_rl_admin.h"
19#include "adf_rl.h"
20#include "adf_sysfs_rl.h"
21
22#define RL_TOKEN_GRANULARITY_PCIEIN_BUCKET 0U
23#define RL_TOKEN_GRANULARITY_PCIEOUT_BUCKET 0U
24#define RL_TOKEN_PCIE_SIZE 64
25#define RL_TOKEN_ASYM_SIZE 1024
26#define RL_CSR_SIZE 4U
27#define RL_CAPABILITY_MASK GENMASK(6, 4)
28#define RL_CAPABILITY_VALUE 0x70
29#define RL_VALIDATE_NON_ZERO(input) ((input) == 0)
30#define ROOT_MASK GENMASK(1, 0)
31#define CLUSTER_MASK GENMASK(3, 0)
32#define LEAF_MASK GENMASK(5, 0)
33
34static int validate_user_input(struct adf_accel_dev *accel_dev,
35 struct adf_rl_sla_input_data *sla_in,
36 bool is_update)
37{
38 const unsigned long rp_mask = sla_in->rp_mask;
39 size_t rp_mask_size;
40 int i, cnt;
41
42 if (sla_in->pir < sla_in->cir) {
43 dev_notice(&GET_DEV(accel_dev),
44 "PIR must be >= CIR, setting PIR to CIR\n");
45 sla_in->pir = sla_in->cir;
46 }
47
48 if (!is_update) {
49 cnt = 0;
50 rp_mask_size = sizeof(sla_in->rp_mask) * BITS_PER_BYTE;
51 for_each_set_bit(i, &rp_mask, rp_mask_size) {
52 if (++cnt > RL_RP_CNT_PER_LEAF_MAX) {
53 dev_notice(&GET_DEV(accel_dev),
54 "Too many ring pairs selected for this SLA\n");
55 return -EINVAL;
56 }
57 }
58
59 if (sla_in->srv >= SVC_BASE_COUNT) {
60 dev_notice(&GET_DEV(accel_dev),
61 "Wrong service type\n");
62 return -EINVAL;
63 }
64
65 if (sla_in->type > RL_LEAF) {
66 dev_notice(&GET_DEV(accel_dev),
67 "Wrong node type\n");
68 return -EINVAL;
69 }
70
71 if (sla_in->parent_id < RL_PARENT_DEFAULT_ID ||
72 sla_in->parent_id >= RL_NODES_CNT_MAX) {
73 dev_notice(&GET_DEV(accel_dev),
74 "Wrong parent ID\n");
75 return -EINVAL;
76 }
77 }
78
79 return 0;
80}
81
82static int validate_sla_id(struct adf_accel_dev *accel_dev, int sla_id)
83{
84 struct rl_sla *sla;
85
86 if (sla_id <= RL_SLA_EMPTY_ID || sla_id >= RL_NODES_CNT_MAX) {
87 dev_notice(&GET_DEV(accel_dev), "Provided ID is out of bounds\n");
88 return -EINVAL;
89 }
90
91 sla = accel_dev->rate_limiting->sla[sla_id];
92
93 if (!sla) {
94 dev_notice(&GET_DEV(accel_dev), "SLA with provided ID does not exist\n");
95 return -EINVAL;
96 }
97
98 if (sla->type != RL_LEAF) {
99 dev_notice(&GET_DEV(accel_dev), "This ID is reserved for internal use\n");
100 return -EINVAL;
101 }
102
103 return 0;
104}
105
106/**
107 * find_parent() - Find the parent for a new SLA
108 * @rl_data: pointer to ratelimiting data
109 * @sla_in: pointer to user input data for a new SLA
110 *
111 * Function returns a pointer to the parent SLA. If the parent ID is provided
112 * as input in the user data, then such ID is validated and the parent SLA
113 * is returned.
114 * Otherwise, it returns the default parent SLA (root or cluster) for
115 * the new object.
116 *
117 * Return:
118 * * Pointer to the parent SLA object
119 * * NULL - when parent cannot be found
120 */
121static struct rl_sla *find_parent(struct adf_rl *rl_data,
122 struct adf_rl_sla_input_data *sla_in)
123{
124 int input_parent_id = sla_in->parent_id;
125 struct rl_sla *root = NULL;
126 struct rl_sla *parent_sla;
127 int i;
128
129 if (sla_in->type == RL_ROOT)
130 return NULL;
131
132 if (input_parent_id > RL_PARENT_DEFAULT_ID) {
133 parent_sla = rl_data->sla[input_parent_id];
134 /*
135 * SLA can be a parent if it has the same service as the child
136 * and its type is higher in the hierarchy,
137 * for example the parent type of a LEAF must be a CLUSTER.
138 */
139 if (parent_sla && parent_sla->srv == sla_in->srv &&
140 parent_sla->type == sla_in->type - 1)
141 return parent_sla;
142
143 return NULL;
144 }
145
146 /* If input_parent_id is not valid, get root for this service type. */
147 for (i = 0; i < RL_ROOT_MAX; i++) {
148 if (rl_data->root[i] && rl_data->root[i]->srv == sla_in->srv) {
149 root = rl_data->root[i];
150 break;
151 }
152 }
153
154 if (!root)
155 return NULL;
156
157 /*
158 * If the type of this SLA is cluster, then return the root.
159 * Otherwise, find the default (i.e. first) cluster for this service.
160 */
161 if (sla_in->type == RL_CLUSTER)
162 return root;
163
164 for (i = 0; i < RL_CLUSTER_MAX; i++) {
165 if (rl_data->cluster[i] && rl_data->cluster[i]->parent == root)
166 return rl_data->cluster[i];
167 }
168
169 return NULL;
170}
171
172/**
173 * adf_rl_get_sla_arr_of_type() - Returns a pointer to SLA type specific array
174 * @rl_data: pointer to ratelimiting data
175 * @type: SLA type
176 * @sla_arr: pointer to variable where requested pointer will be stored
177 *
178 * Return: Max number of elements allowed for the returned array
179 */
180u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type,
181 struct rl_sla ***sla_arr)
182{
183 switch (type) {
184 case RL_LEAF:
185 *sla_arr = rl_data->leaf;
186 return RL_LEAF_MAX;
187 case RL_CLUSTER:
188 *sla_arr = rl_data->cluster;
189 return RL_CLUSTER_MAX;
190 case RL_ROOT:
191 *sla_arr = rl_data->root;
192 return RL_ROOT_MAX;
193 default:
194 *sla_arr = NULL;
195 return 0;
196 }
197}
198
199/**
200 * prepare_rp_ids() - Creates an array of ring pair IDs from bitmask
201 * @accel_dev: pointer to acceleration device structure
202 * @sla: SLA object data where result will be written
203 * @rp_mask: bitmask of ring pair IDs
204 *
205 * Function tries to convert provided bitmap to an array of IDs. It checks if
206 * RPs aren't in use, are assigned to SLA service or if a number of provided
207 * IDs is not too big. If successful, writes the result into the field
208 * sla->ring_pairs_cnt.
209 *
210 * Return:
211 * * 0 - ok
212 * * -EINVAL - ring pairs array cannot be created from provided mask
213 */
214static int prepare_rp_ids(struct adf_accel_dev *accel_dev, struct rl_sla *sla,
215 const unsigned long rp_mask)
216{
217 enum adf_cfg_service_type arb_srv = adf_srv_to_cfg_svc_type(svc: sla->srv);
218 u16 rps_per_bundle = GET_HW_DATA(accel_dev)->num_banks_per_vf;
219 bool *rp_in_use = accel_dev->rate_limiting->rp_in_use;
220 size_t rp_cnt_max = ARRAY_SIZE(sla->ring_pairs_ids);
221 u16 rp_id_max = GET_HW_DATA(accel_dev)->num_banks;
222 u16 cnt = 0;
223 u16 rp_id;
224
225 for_each_set_bit(rp_id, &rp_mask, rp_id_max) {
226 if (cnt >= rp_cnt_max) {
227 dev_notice(&GET_DEV(accel_dev),
228 "Assigned more ring pairs than supported");
229 return -EINVAL;
230 }
231
232 if (rp_in_use[rp_id]) {
233 dev_notice(&GET_DEV(accel_dev),
234 "RP %u already assigned to other SLA", rp_id);
235 return -EINVAL;
236 }
237
238 if (GET_SRV_TYPE(accel_dev, rp_id % rps_per_bundle) != arb_srv) {
239 dev_notice(&GET_DEV(accel_dev),
240 "RP %u does not support SLA service", rp_id);
241 return -EINVAL;
242 }
243
244 sla->ring_pairs_ids[cnt++] = rp_id;
245 }
246
247 sla->ring_pairs_cnt = cnt;
248
249 return 0;
250}
251
252static void mark_rps_usage(struct rl_sla *sla, bool *rp_in_use, bool used)
253{
254 u16 rp_id;
255 int i;
256
257 for (i = 0; i < sla->ring_pairs_cnt; i++) {
258 rp_id = sla->ring_pairs_ids[i];
259 rp_in_use[rp_id] = used;
260 }
261}
262
263static void assign_rps_to_leaf(struct adf_accel_dev *accel_dev,
264 struct rl_sla *sla, bool clear)
265{
266 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
267 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
268 u32 base_offset = hw_data->rl_data.r2l_offset;
269 u32 node_id = clear ? 0U : (sla->node_id & LEAF_MASK);
270 u32 offset;
271 int i;
272
273 for (i = 0; i < sla->ring_pairs_cnt; i++) {
274 offset = base_offset + (RL_CSR_SIZE * sla->ring_pairs_ids[i]);
275 ADF_CSR_WR(pmisc_addr, offset, node_id);
276 }
277}
278
279static void assign_leaf_to_cluster(struct adf_accel_dev *accel_dev,
280 struct rl_sla *sla, bool clear)
281{
282 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
283 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
284 u32 base_offset = hw_data->rl_data.l2c_offset;
285 u32 node_id = sla->node_id & LEAF_MASK;
286 u32 parent_id = clear ? 0U : (sla->parent->node_id & CLUSTER_MASK);
287 u32 offset;
288
289 offset = base_offset + (RL_CSR_SIZE * node_id);
290 ADF_CSR_WR(pmisc_addr, offset, parent_id);
291}
292
293static void assign_cluster_to_root(struct adf_accel_dev *accel_dev,
294 struct rl_sla *sla, bool clear)
295{
296 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
297 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
298 u32 base_offset = hw_data->rl_data.c2s_offset;
299 u32 node_id = sla->node_id & CLUSTER_MASK;
300 u32 parent_id = clear ? 0U : (sla->parent->node_id & ROOT_MASK);
301 u32 offset;
302
303 offset = base_offset + (RL_CSR_SIZE * node_id);
304 ADF_CSR_WR(pmisc_addr, offset, parent_id);
305}
306
307static void assign_node_to_parent(struct adf_accel_dev *accel_dev,
308 struct rl_sla *sla, bool clear_assignment)
309{
310 switch (sla->type) {
311 case RL_LEAF:
312 assign_rps_to_leaf(accel_dev, sla, clear: clear_assignment);
313 assign_leaf_to_cluster(accel_dev, sla, clear: clear_assignment);
314 break;
315 case RL_CLUSTER:
316 assign_cluster_to_root(accel_dev, sla, clear: clear_assignment);
317 break;
318 default:
319 break;
320 }
321}
322
323/**
324 * can_parent_afford_sla() - Verifies if parent allows to create an SLA
325 * @sla_in: pointer to user input data for a new SLA
326 * @sla_parent: pointer to parent SLA object
327 * @sla_cir: current child CIR value (only for update)
328 * @is_update: request is a update
329 *
330 * Algorithm verifies if parent has enough remaining budget to take assignment
331 * of a child with provided parameters. In update case current CIR value must be
332 * returned to budget first.
333 * PIR value cannot exceed the PIR assigned to parent.
334 *
335 * Return:
336 * * true - SLA can be created
337 * * false - SLA cannot be created
338 */
339static bool can_parent_afford_sla(struct adf_rl_sla_input_data *sla_in,
340 struct rl_sla *sla_parent, u32 sla_cir,
341 bool is_update)
342{
343 u32 rem_cir = sla_parent->rem_cir;
344
345 if (is_update)
346 rem_cir += sla_cir;
347
348 if (sla_in->cir > rem_cir || sla_in->pir > sla_parent->pir)
349 return false;
350
351 return true;
352}
353
354/**
355 * can_node_afford_update() - Verifies if SLA can be updated with input data
356 * @sla_in: pointer to user input data for a new SLA
357 * @sla: pointer to SLA object selected for update
358 *
359 * Algorithm verifies if a new CIR value is big enough to satisfy currently
360 * assigned child SLAs and if PIR can be updated
361 *
362 * Return:
363 * * true - SLA can be updated
364 * * false - SLA cannot be updated
365 */
366static bool can_node_afford_update(struct adf_rl_sla_input_data *sla_in,
367 struct rl_sla *sla)
368{
369 u32 cir_in_use = sla->cir - sla->rem_cir;
370
371 /* new CIR cannot be smaller then currently consumed value */
372 if (cir_in_use > sla_in->cir)
373 return false;
374
375 /* PIR of root/cluster cannot be reduced in node with assigned children */
376 if (sla_in->pir < sla->pir && sla->type != RL_LEAF && cir_in_use > 0)
377 return false;
378
379 return true;
380}
381
382static bool is_enough_budget(struct adf_rl *rl_data, struct rl_sla *sla,
383 struct adf_rl_sla_input_data *sla_in,
384 bool is_update)
385{
386 u32 max_val = rl_data->device_data->scale_ref;
387 struct rl_sla *parent = sla->parent;
388 bool ret = true;
389
390 if (sla_in->cir > max_val || sla_in->pir > max_val)
391 ret = false;
392
393 switch (sla->type) {
394 case RL_LEAF:
395 ret &= can_parent_afford_sla(sla_in, sla_parent: parent, sla_cir: sla->cir,
396 is_update);
397 break;
398 case RL_CLUSTER:
399 ret &= can_parent_afford_sla(sla_in, sla_parent: parent, sla_cir: sla->cir,
400 is_update);
401
402 if (is_update)
403 ret &= can_node_afford_update(sla_in, sla);
404
405 break;
406 case RL_ROOT:
407 if (is_update)
408 ret &= can_node_afford_update(sla_in, sla);
409
410 break;
411 default:
412 ret = false;
413 break;
414 }
415
416 return ret;
417}
418
419static void update_budget(struct rl_sla *sla, u32 old_cir, bool is_update)
420{
421 switch (sla->type) {
422 case RL_LEAF:
423 if (is_update)
424 sla->parent->rem_cir += old_cir;
425
426 sla->parent->rem_cir -= sla->cir;
427 sla->rem_cir = 0;
428 break;
429 case RL_CLUSTER:
430 if (is_update) {
431 sla->parent->rem_cir += old_cir;
432 sla->rem_cir = sla->cir - (old_cir - sla->rem_cir);
433 } else {
434 sla->rem_cir = sla->cir;
435 }
436
437 sla->parent->rem_cir -= sla->cir;
438 break;
439 case RL_ROOT:
440 if (is_update)
441 sla->rem_cir = sla->cir - (old_cir - sla->rem_cir);
442 else
443 sla->rem_cir = sla->cir;
444 break;
445 default:
446 break;
447 }
448}
449
450/**
451 * get_next_free_sla_id() - finds next free ID in the SLA array
452 * @rl_data: Pointer to ratelimiting data structure
453 *
454 * Return:
455 * * 0 : RL_NODES_CNT_MAX - correct ID
456 * * -ENOSPC - all SLA slots are in use
457 */
458static int get_next_free_sla_id(struct adf_rl *rl_data)
459{
460 int i = 0;
461
462 while (i < RL_NODES_CNT_MAX && rl_data->sla[i++])
463 ;
464
465 if (i == RL_NODES_CNT_MAX)
466 return -ENOSPC;
467
468 return i - 1;
469}
470
471/**
472 * get_next_free_node_id() - finds next free ID in the array of that node type
473 * @rl_data: Pointer to ratelimiting data structure
474 * @sla: Pointer to SLA object for which the ID is searched
475 *
476 * Return:
477 * * 0 : RL_[NODE_TYPE]_MAX - correct ID
478 * * -ENOSPC - all slots of that type are in use
479 */
480static int get_next_free_node_id(struct adf_rl *rl_data, struct rl_sla *sla)
481{
482 struct adf_hw_device_data *hw_device = GET_HW_DATA(rl_data->accel_dev);
483 int max_id, i, step, rp_per_leaf;
484 struct rl_sla **sla_list;
485
486 rp_per_leaf = hw_device->num_banks / hw_device->num_banks_per_vf;
487
488 /*
489 * Static nodes mapping:
490 * root0 - cluster[0,4,8,12] - leaf[0-15]
491 * root1 - cluster[1,5,9,13] - leaf[16-31]
492 * root2 - cluster[2,6,10,14] - leaf[32-47]
493 */
494 switch (sla->type) {
495 case RL_LEAF:
496 i = sla->srv * rp_per_leaf;
497 step = 1;
498 max_id = i + rp_per_leaf;
499 sla_list = rl_data->leaf;
500 break;
501 case RL_CLUSTER:
502 i = sla->srv;
503 step = 4;
504 max_id = RL_CLUSTER_MAX;
505 sla_list = rl_data->cluster;
506 break;
507 case RL_ROOT:
508 return sla->srv;
509 default:
510 return -EINVAL;
511 }
512
513 while (i < max_id && sla_list[i])
514 i += step;
515
516 if (i >= max_id)
517 return -ENOSPC;
518
519 return i;
520}
521
522u32 adf_rl_calculate_slice_tokens(struct adf_accel_dev *accel_dev, u32 sla_val,
523 enum adf_base_services svc_type)
524{
525 struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;
526 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
527 u64 avail_slice_cycles, allocated_tokens;
528
529 if (!sla_val)
530 return 0;
531
532 /* Handle generation specific slice count adjustment */
533 avail_slice_cycles = hw_data->clock_frequency;
534 avail_slice_cycles *= hw_data->get_svc_slice_cnt(accel_dev, svc_type);
535
536 do_div(avail_slice_cycles, device_data->scan_interval);
537 allocated_tokens = avail_slice_cycles * sla_val;
538 do_div(allocated_tokens, device_data->scale_ref);
539
540 return allocated_tokens;
541}
542
543static u32 adf_rl_get_num_svc_aes(struct adf_accel_dev *accel_dev,
544 enum adf_base_services svc)
545{
546 struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;
547
548 if (svc >= SVC_BASE_COUNT)
549 return 0;
550
551 return device_data->svc_ae_mask[svc];
552}
553
554u32 adf_rl_calculate_ae_cycles(struct adf_accel_dev *accel_dev, u32 sla_val,
555 enum adf_base_services svc_type)
556{
557 struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;
558 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
559 u64 allocated_ae_cycles, avail_ae_cycles;
560
561 if (!sla_val)
562 return 0;
563
564 avail_ae_cycles = hw_data->clock_frequency;
565 avail_ae_cycles *= adf_rl_get_num_svc_aes(accel_dev, svc: svc_type);
566 do_div(avail_ae_cycles, device_data->scan_interval);
567
568 sla_val *= device_data->max_tp[svc_type];
569 sla_val /= device_data->scale_ref;
570
571 allocated_ae_cycles = (sla_val * avail_ae_cycles);
572 do_div(allocated_ae_cycles, device_data->max_tp[svc_type]);
573
574 return allocated_ae_cycles;
575}
576
577u32 adf_rl_calculate_pci_bw(struct adf_accel_dev *accel_dev, u32 sla_val,
578 enum adf_base_services svc_type, bool is_bw_out)
579{
580 struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;
581 u64 sla_to_bytes, allocated_bw, sla_scaled;
582
583 if (!sla_val)
584 return 0;
585
586 sla_to_bytes = sla_val;
587 sla_to_bytes *= device_data->max_tp[svc_type];
588 do_div(sla_to_bytes, device_data->scale_ref);
589
590 sla_to_bytes *= (svc_type == SVC_ASYM) ? RL_TOKEN_ASYM_SIZE : BYTES_PER_MBIT;
591 if (svc_type == SVC_DC && is_bw_out)
592 sla_to_bytes *= device_data->slices.dcpr_cnt -
593 device_data->dcpr_correction;
594
595 sla_scaled = sla_to_bytes * device_data->pcie_scale_mul;
596 do_div(sla_scaled, device_data->pcie_scale_div);
597 allocated_bw = sla_scaled;
598 do_div(allocated_bw, RL_TOKEN_PCIE_SIZE);
599 do_div(allocated_bw, device_data->scan_interval);
600
601 return allocated_bw;
602}
603
604/**
605 * add_new_sla_entry() - creates a new SLA object and fills it with user data
606 * @accel_dev: pointer to acceleration device structure
607 * @sla_in: pointer to user input data for a new SLA
608 * @sla_out: Pointer to variable that will contain the address of a new
609 * SLA object if the operation succeeds
610 *
611 * Return:
612 * * 0 - ok
613 * * -ENOMEM - memory allocation failed
614 * * -EINVAL - invalid user input
615 * * -ENOSPC - all available SLAs are in use
616 */
617static int add_new_sla_entry(struct adf_accel_dev *accel_dev,
618 struct adf_rl_sla_input_data *sla_in,
619 struct rl_sla **sla_out)
620{
621 struct adf_rl *rl_data = accel_dev->rate_limiting;
622 struct rl_sla *sla;
623 int ret = 0;
624
625 sla = kzalloc(sizeof(*sla), GFP_KERNEL);
626 if (!sla) {
627 ret = -ENOMEM;
628 goto ret_err;
629 }
630 *sla_out = sla;
631
632 if (!adf_is_service_enabled(accel_dev, svc: sla_in->srv)) {
633 dev_notice(&GET_DEV(accel_dev),
634 "Provided service is not enabled\n");
635 ret = -EINVAL;
636 goto ret_err;
637 }
638
639 sla->srv = sla_in->srv;
640 sla->type = sla_in->type;
641 ret = get_next_free_node_id(rl_data, sla);
642 if (ret < 0) {
643 dev_notice(&GET_DEV(accel_dev),
644 "Exceeded number of available nodes for that service\n");
645 goto ret_err;
646 }
647 sla->node_id = ret;
648
649 ret = get_next_free_sla_id(rl_data);
650 if (ret < 0) {
651 dev_notice(&GET_DEV(accel_dev),
652 "Allocated maximum SLAs number\n");
653 goto ret_err;
654 }
655 sla->sla_id = ret;
656
657 sla->parent = find_parent(rl_data, sla_in);
658 if (!sla->parent && sla->type != RL_ROOT) {
659 if (sla_in->parent_id != RL_PARENT_DEFAULT_ID)
660 dev_notice(&GET_DEV(accel_dev),
661 "Provided parent ID does not exist or cannot be parent for this SLA.");
662 else
663 dev_notice(&GET_DEV(accel_dev),
664 "Unable to find parent node for this service. Is service enabled?");
665 ret = -EINVAL;
666 goto ret_err;
667 }
668
669 if (sla->type == RL_LEAF) {
670 ret = prepare_rp_ids(accel_dev, sla, rp_mask: sla_in->rp_mask);
671 if (!sla->ring_pairs_cnt || ret) {
672 dev_notice(&GET_DEV(accel_dev),
673 "Unable to find ring pairs to assign to the leaf");
674 if (!ret)
675 ret = -EINVAL;
676
677 goto ret_err;
678 }
679 }
680
681 return 0;
682
683ret_err:
684 kfree(objp: sla);
685 *sla_out = NULL;
686
687 return ret;
688}
689
690static int initialize_default_nodes(struct adf_accel_dev *accel_dev)
691{
692 struct adf_rl *rl_data = accel_dev->rate_limiting;
693 struct adf_rl_hw_data *device_data = rl_data->device_data;
694 struct adf_rl_sla_input_data sla_in = { };
695 int ret = 0;
696 int i;
697
698 /* Init root for each enabled service */
699 sla_in.type = RL_ROOT;
700 sla_in.parent_id = RL_PARENT_DEFAULT_ID;
701
702 for (i = 0; i < SVC_BASE_COUNT; i++) {
703 if (!adf_is_service_enabled(accel_dev, svc: i))
704 continue;
705
706 sla_in.cir = device_data->scale_ref;
707 sla_in.pir = sla_in.cir;
708 sla_in.srv = i;
709
710 ret = adf_rl_add_sla(accel_dev, sla_in: &sla_in);
711 if (ret)
712 return ret;
713 }
714
715 /* Init default cluster for each root */
716 sla_in.type = RL_CLUSTER;
717 for (i = 0; i < SVC_BASE_COUNT; i++) {
718 if (!rl_data->root[i])
719 continue;
720 sla_in.cir = rl_data->root[i]->cir;
721 sla_in.pir = sla_in.cir;
722 sla_in.srv = rl_data->root[i]->srv;
723
724 ret = adf_rl_add_sla(accel_dev, sla_in: &sla_in);
725 if (ret)
726 return ret;
727 }
728
729 return 0;
730}
731
732static void clear_sla(struct adf_rl *rl_data, struct rl_sla *sla)
733{
734 bool *rp_in_use = rl_data->rp_in_use;
735 struct rl_sla **sla_type_arr = NULL;
736 int i, sla_id, node_id;
737 u32 old_cir;
738
739 sla_id = sla->sla_id;
740 node_id = sla->node_id;
741 old_cir = sla->cir;
742 sla->cir = 0;
743 sla->pir = 0;
744
745 for (i = 0; i < sla->ring_pairs_cnt; i++)
746 rp_in_use[sla->ring_pairs_ids[i]] = false;
747
748 update_budget(sla, old_cir, is_update: true);
749 adf_rl_get_sla_arr_of_type(rl_data, type: sla->type, sla_arr: &sla_type_arr);
750 assign_node_to_parent(accel_dev: rl_data->accel_dev, sla, clear_assignment: true);
751 adf_rl_send_admin_delete_msg(accel_dev: rl_data->accel_dev, node_id, node_type: sla->type);
752 mark_rps_usage(sla, rp_in_use: rl_data->rp_in_use, used: false);
753
754 kfree(objp: sla);
755 rl_data->sla[sla_id] = NULL;
756 sla_type_arr[node_id] = NULL;
757}
758
759static void free_all_sla(struct adf_accel_dev *accel_dev)
760{
761 struct adf_rl *rl_data = accel_dev->rate_limiting;
762 int sla_id;
763
764 mutex_lock(&rl_data->rl_lock);
765
766 for (sla_id = 0; sla_id < RL_NODES_CNT_MAX; sla_id++) {
767 if (!rl_data->sla[sla_id])
768 continue;
769
770 kfree(objp: rl_data->sla[sla_id]);
771 rl_data->sla[sla_id] = NULL;
772 }
773
774 mutex_unlock(lock: &rl_data->rl_lock);
775}
776
777/**
778 * add_update_sla() - handles the creation and the update of an SLA
779 * @accel_dev: pointer to acceleration device structure
780 * @sla_in: pointer to user input data for a new/updated SLA
781 * @is_update: flag to indicate if this is an update or an add operation
782 *
783 * Return:
784 * * 0 - ok
785 * * -ENOMEM - memory allocation failed
786 * * -EINVAL - user input data cannot be used to create SLA
787 * * -ENOSPC - all available SLAs are in use
788 */
789static int add_update_sla(struct adf_accel_dev *accel_dev,
790 struct adf_rl_sla_input_data *sla_in, bool is_update)
791{
792 struct adf_rl *rl_data = accel_dev->rate_limiting;
793 struct rl_sla **sla_type_arr = NULL;
794 struct rl_sla *sla = NULL;
795 u32 old_cir = 0;
796 int ret;
797
798 if (!sla_in) {
799 dev_warn(&GET_DEV(accel_dev),
800 "SLA input data pointer is missing\n");
801 return -EFAULT;
802 }
803
804 mutex_lock(&rl_data->rl_lock);
805
806 /* Input validation */
807 ret = validate_user_input(accel_dev, sla_in, is_update);
808 if (ret)
809 goto ret_err;
810
811 if (is_update) {
812 ret = validate_sla_id(accel_dev, sla_id: sla_in->sla_id);
813 if (ret)
814 goto ret_err;
815
816 sla = rl_data->sla[sla_in->sla_id];
817 old_cir = sla->cir;
818 } else {
819 ret = add_new_sla_entry(accel_dev, sla_in, sla_out: &sla);
820 if (ret)
821 goto ret_err;
822 }
823
824 if (!is_enough_budget(rl_data, sla, sla_in, is_update)) {
825 dev_notice(&GET_DEV(accel_dev),
826 "Input value exceeds the remaining budget%s\n",
827 is_update ? " or more budget is already in use" : "");
828 ret = -EINVAL;
829 goto ret_err;
830 }
831 sla->cir = sla_in->cir;
832 sla->pir = sla_in->pir;
833
834 /* Apply SLA */
835 assign_node_to_parent(accel_dev, sla, clear_assignment: false);
836 ret = adf_rl_send_admin_add_update_msg(accel_dev, sla, is_update);
837 if (ret) {
838 dev_notice(&GET_DEV(accel_dev),
839 "Failed to apply an SLA\n");
840 goto ret_err;
841 }
842 update_budget(sla, old_cir, is_update);
843
844 if (!is_update) {
845 mark_rps_usage(sla, rp_in_use: rl_data->rp_in_use, used: true);
846 adf_rl_get_sla_arr_of_type(rl_data, type: sla->type, sla_arr: &sla_type_arr);
847 sla_type_arr[sla->node_id] = sla;
848 rl_data->sla[sla->sla_id] = sla;
849 }
850
851 sla_in->sla_id = sla->sla_id;
852 goto ret_ok;
853
854ret_err:
855 if (!is_update) {
856 sla_in->sla_id = -1;
857 kfree(objp: sla);
858 }
859ret_ok:
860 mutex_unlock(lock: &rl_data->rl_lock);
861 return ret;
862}
863
864/**
865 * adf_rl_add_sla() - handles the creation of an SLA
866 * @accel_dev: pointer to acceleration device structure
867 * @sla_in: pointer to user input data required to add an SLA
868 *
869 * Return:
870 * * 0 - ok
871 * * -ENOMEM - memory allocation failed
872 * * -EINVAL - invalid user input
873 * * -ENOSPC - all available SLAs are in use
874 */
875int adf_rl_add_sla(struct adf_accel_dev *accel_dev,
876 struct adf_rl_sla_input_data *sla_in)
877{
878 return add_update_sla(accel_dev, sla_in, is_update: false);
879}
880
881/**
882 * adf_rl_update_sla() - handles the update of an SLA
883 * @accel_dev: pointer to acceleration device structure
884 * @sla_in: pointer to user input data required to update an SLA
885 *
886 * Return:
887 * * 0 - ok
888 * * -EINVAL - user input data cannot be used to update SLA
889 */
890int adf_rl_update_sla(struct adf_accel_dev *accel_dev,
891 struct adf_rl_sla_input_data *sla_in)
892{
893 return add_update_sla(accel_dev, sla_in, is_update: true);
894}
895
896/**
897 * adf_rl_get_sla() - returns an existing SLA data
898 * @accel_dev: pointer to acceleration device structure
899 * @sla_in: pointer to user data where SLA info will be stored
900 *
901 * The sla_id for which data are requested should be set in sla_id structure
902 *
903 * Return:
904 * * 0 - ok
905 * * -EINVAL - provided sla_id does not exist
906 */
907int adf_rl_get_sla(struct adf_accel_dev *accel_dev,
908 struct adf_rl_sla_input_data *sla_in)
909{
910 struct rl_sla *sla;
911 int ret, i;
912
913 ret = validate_sla_id(accel_dev, sla_id: sla_in->sla_id);
914 if (ret)
915 return ret;
916
917 sla = accel_dev->rate_limiting->sla[sla_in->sla_id];
918 sla_in->type = sla->type;
919 sla_in->srv = sla->srv;
920 sla_in->cir = sla->cir;
921 sla_in->pir = sla->pir;
922 sla_in->rp_mask = 0U;
923 if (sla->parent)
924 sla_in->parent_id = sla->parent->sla_id;
925 else
926 sla_in->parent_id = RL_PARENT_DEFAULT_ID;
927
928 for (i = 0; i < sla->ring_pairs_cnt; i++)
929 sla_in->rp_mask |= BIT(sla->ring_pairs_ids[i]);
930
931 return 0;
932}
933
934/**
935 * adf_rl_get_capability_remaining() - returns the remaining SLA value (CIR) for
936 * selected service or provided sla_id
937 * @accel_dev: pointer to acceleration device structure
938 * @srv: service ID for which capability is requested
939 * @sla_id: ID of the cluster or root to which we want assign a new SLA
940 *
941 * Check if the provided SLA id is valid. If it is and the service matches
942 * the requested service and the type is cluster or root, return the remaining
943 * capability.
944 * If the provided ID does not match the service or type, return the remaining
945 * capacity of the default cluster for that service.
946 *
947 * Return:
948 * * Positive value - correct remaining value
949 * * -EINVAL - algorithm cannot find a remaining value for provided data
950 */
951int adf_rl_get_capability_remaining(struct adf_accel_dev *accel_dev,
952 enum adf_base_services srv, int sla_id)
953{
954 struct adf_rl *rl_data = accel_dev->rate_limiting;
955 struct rl_sla *sla = NULL;
956 int i;
957
958 if (srv >= SVC_BASE_COUNT)
959 return -EINVAL;
960
961 if (sla_id > RL_SLA_EMPTY_ID && !validate_sla_id(accel_dev, sla_id)) {
962 sla = rl_data->sla[sla_id];
963
964 if (sla->srv == srv && sla->type <= RL_CLUSTER)
965 goto ret_ok;
966 }
967
968 for (i = 0; i < RL_CLUSTER_MAX; i++) {
969 if (!rl_data->cluster[i])
970 continue;
971
972 if (rl_data->cluster[i]->srv == srv) {
973 sla = rl_data->cluster[i];
974 goto ret_ok;
975 }
976 }
977
978 return -EINVAL;
979ret_ok:
980 return sla->rem_cir;
981}
982
983/**
984 * adf_rl_remove_sla() - removes provided sla_id
985 * @accel_dev: pointer to acceleration device structure
986 * @sla_id: ID of the cluster or root to which we want assign an new SLA
987 *
988 * Return:
989 * * 0 - ok
990 * * -EINVAL - wrong sla_id or it still have assigned children
991 */
992int adf_rl_remove_sla(struct adf_accel_dev *accel_dev, u32 sla_id)
993{
994 struct adf_rl *rl_data = accel_dev->rate_limiting;
995 struct rl_sla *sla;
996 int ret = 0;
997
998 mutex_lock(&rl_data->rl_lock);
999 ret = validate_sla_id(accel_dev, sla_id);
1000 if (ret)
1001 goto err_ret;
1002
1003 sla = rl_data->sla[sla_id];
1004
1005 if (sla->type < RL_LEAF && sla->rem_cir != sla->cir) {
1006 dev_notice(&GET_DEV(accel_dev),
1007 "To remove parent SLA all its children must be removed first");
1008 ret = -EINVAL;
1009 goto err_ret;
1010 }
1011
1012 clear_sla(rl_data, sla);
1013
1014err_ret:
1015 mutex_unlock(lock: &rl_data->rl_lock);
1016 return ret;
1017}
1018
1019/**
1020 * adf_rl_remove_sla_all() - removes all SLAs from device
1021 * @accel_dev: pointer to acceleration device structure
1022 * @incl_default: set to true if default SLAs also should be removed
1023 */
1024void adf_rl_remove_sla_all(struct adf_accel_dev *accel_dev, bool incl_default)
1025{
1026 struct adf_rl *rl_data = accel_dev->rate_limiting;
1027 int end_type = incl_default ? RL_ROOT : RL_LEAF;
1028 struct rl_sla **sla_type_arr = NULL;
1029 u32 max_id;
1030 int i, j;
1031
1032 mutex_lock(&rl_data->rl_lock);
1033
1034 /* Unregister and remove all SLAs */
1035 for (j = RL_LEAF; j >= end_type; j--) {
1036 max_id = adf_rl_get_sla_arr_of_type(rl_data, type: j, sla_arr: &sla_type_arr);
1037
1038 for (i = 0; i < max_id; i++) {
1039 if (!sla_type_arr[i])
1040 continue;
1041
1042 clear_sla(rl_data, sla: sla_type_arr[i]);
1043 }
1044 }
1045
1046 mutex_unlock(lock: &rl_data->rl_lock);
1047}
1048
1049int adf_rl_init(struct adf_accel_dev *accel_dev)
1050{
1051 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
1052 struct adf_rl_hw_data *rl_hw_data = &hw_data->rl_data;
1053 struct adf_rl *rl;
1054 int ret = 0;
1055
1056 /* Validate device parameters */
1057 if (RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_ASYM]) ||
1058 RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_SYM]) ||
1059 RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_DC]) ||
1060 RL_VALIDATE_NON_ZERO(rl_hw_data->scan_interval) ||
1061 RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_div) ||
1062 RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_mul) ||
1063 RL_VALIDATE_NON_ZERO(rl_hw_data->scale_ref)) {
1064 ret = -EOPNOTSUPP;
1065 goto err_ret;
1066 }
1067
1068 rl = kzalloc(sizeof(*rl), GFP_KERNEL);
1069 if (!rl) {
1070 ret = -ENOMEM;
1071 goto err_ret;
1072 }
1073
1074 mutex_init(&rl->rl_lock);
1075 rl->device_data = &accel_dev->hw_device->rl_data;
1076 rl->accel_dev = accel_dev;
1077 init_rwsem(&rl->user_input.lock);
1078 accel_dev->rate_limiting = rl;
1079
1080err_ret:
1081 return ret;
1082}
1083
1084int adf_rl_start(struct adf_accel_dev *accel_dev)
1085{
1086 struct adf_rl_hw_data *rl_hw_data = &GET_HW_DATA(accel_dev)->rl_data;
1087 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
1088 u16 fw_caps = GET_HW_DATA(accel_dev)->fw_capabilities;
1089 int ret;
1090
1091 if (!accel_dev->rate_limiting) {
1092 ret = -EOPNOTSUPP;
1093 goto ret_err;
1094 }
1095
1096 if ((fw_caps & RL_CAPABILITY_MASK) != RL_CAPABILITY_VALUE) {
1097 dev_info(&GET_DEV(accel_dev), "feature not supported by FW\n");
1098 ret = -EOPNOTSUPP;
1099 goto ret_free;
1100 }
1101
1102 ADF_CSR_WR(pmisc_addr, rl_hw_data->pciin_tb_offset,
1103 RL_TOKEN_GRANULARITY_PCIEIN_BUCKET);
1104 ADF_CSR_WR(pmisc_addr, rl_hw_data->pciout_tb_offset,
1105 RL_TOKEN_GRANULARITY_PCIEOUT_BUCKET);
1106
1107 ret = adf_rl_send_admin_init_msg(accel_dev, slices_int: &rl_hw_data->slices);
1108 if (ret) {
1109 dev_err(&GET_DEV(accel_dev), "initialization failed\n");
1110 goto ret_free;
1111 }
1112
1113 ret = initialize_default_nodes(accel_dev);
1114 if (ret) {
1115 dev_err(&GET_DEV(accel_dev),
1116 "failed to initialize default SLAs\n");
1117 goto ret_sla_rm;
1118 }
1119
1120 ret = adf_sysfs_rl_add(accel_dev);
1121 if (ret) {
1122 dev_err(&GET_DEV(accel_dev), "failed to add sysfs interface\n");
1123 goto ret_sysfs_rm;
1124 }
1125
1126 return 0;
1127
1128ret_sysfs_rm:
1129 adf_sysfs_rl_rm(accel_dev);
1130ret_sla_rm:
1131 adf_rl_remove_sla_all(accel_dev, incl_default: true);
1132ret_free:
1133 kfree(objp: accel_dev->rate_limiting);
1134 accel_dev->rate_limiting = NULL;
1135ret_err:
1136 return ret;
1137}
1138
1139void adf_rl_stop(struct adf_accel_dev *accel_dev)
1140{
1141 if (!accel_dev->rate_limiting)
1142 return;
1143
1144 adf_sysfs_rl_rm(accel_dev);
1145 free_all_sla(accel_dev);
1146}
1147
1148void adf_rl_exit(struct adf_accel_dev *accel_dev)
1149{
1150 if (!accel_dev->rate_limiting)
1151 return;
1152
1153 kfree(objp: accel_dev->rate_limiting);
1154 accel_dev->rate_limiting = NULL;
1155}
1156

source code of linux/drivers/crypto/intel/qat/qat_common/adf_rl.c