1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* trace_events_hist - trace event hist triggers
4	*
5	* Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6	*/
7
8	#include <linux/module.h>
9	#include <linux/kallsyms.h>
10	#include <linux/security.h>
11	#include <linux/mutex.h>
12	#include <linux/slab.h>
13	#include <linux/stacktrace.h>
14	#include <linux/rculist.h>
15	#include <linux/tracefs.h>
16
17	/* for gfp flag names */
18	#include <linux/trace_events.h>
19	#include <trace/events/mmflags.h>
20
21	#include "tracing_map.h"
22	#include "trace_synth.h"
23
24	#define ERRORS \
25	C(NONE, "No error"), \
26	C(DUPLICATE_VAR, "Variable already defined"), \
27	C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
28	C(TOO_MANY_VARS, "Too many variables defined"), \
29	C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \
30	C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \
31	C(TRIGGER_EEXIST, "Hist trigger already exists"), \
32	C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
33	C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \
34	C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \
35	C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \
36	C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \
37	C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \
38	C(EVENT_FILE_NOT_FOUND, "Event file not found"), \
39	C(HIST_NOT_FOUND, "Matching event histogram not found"), \
40	C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \
41	C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \
42	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \
43	C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \
44	C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
45	C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \
46	C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \
47	C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \
48	C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \
49	C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \
50	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \
51	C(TOO_MANY_PARAMS, "Too many action params"), \
52	C(PARAM_NOT_FOUND, "Couldn't find param"), \
53	C(INVALID_PARAM, "Invalid action param"), \
54	C(ACTION_NOT_FOUND, "No action found"), \
55	C(NO_SAVE_PARAMS, "No params found for save()"), \
56	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
57	C(ACTION_MISMATCH, "Handler doesn't support action"), \
58	C(NO_CLOSING_PAREN, "No closing paren found"), \
59	C(SUBSYS_NOT_FOUND, "Missing subsystem"), \
60	C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \
61	C(INVALID_REF_KEY, "Using variable references in keys not supported"), \
62	C(VAR_NOT_FOUND, "Couldn't find variable"), \
63	C(FIELD_NOT_FOUND, "Couldn't find field"), \
64	C(EMPTY_ASSIGNMENT, "Empty assignment"), \
65	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \
66	C(EMPTY_SORT_FIELD, "Empty sort field"), \
67	C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \
68	C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), \
69	C(INVALID_STR_OPERAND, "String type can not be an operand in expression"), \
70	C(EXPECT_NUMBER, "Expecting numeric literal"), \
71	C(UNARY_MINUS_SUBEXPR, "Unary minus not supported in sub-expressions"), \
72	C(DIVISION_BY_ZERO, "Division by zero"), \
73	C(NEED_NOHC_VAL, "Non-hitcount value is required for 'nohitcount'"),
74
75	#undef C
76	#define C(a, b) HIST_ERR_##a
77
78	enum { ERRORS };
79
80	#undef C
81	#define C(a, b) b
82
83	static const char *err_text[] = { ERRORS };
84
85	struct hist_field;
86
87	typedef u64 (*hist_field_fn_t) (struct hist_field *field,
88	struct tracing_map_elt *elt,
89	struct trace_buffer *buffer,
90	struct ring_buffer_event *rbe,
91	void *event);
92
93	#define HIST_FIELD_OPERANDS_MAX 2
94	#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
95	#define HIST_ACTIONS_MAX 8
96	#define HIST_CONST_DIGITS_MAX 21
97	#define HIST_DIV_SHIFT 20 /* For optimizing division by constants */
98
99	enum field_op_id {
100	FIELD_OP_NONE,
101	FIELD_OP_PLUS,
102	FIELD_OP_MINUS,
103	FIELD_OP_UNARY_MINUS,
104	FIELD_OP_DIV,
105	FIELD_OP_MULT,
106	};
107
108	enum hist_field_fn {
109	HIST_FIELD_FN_NOP,
110	HIST_FIELD_FN_VAR_REF,
111	HIST_FIELD_FN_COUNTER,
112	HIST_FIELD_FN_CONST,
113	HIST_FIELD_FN_LOG2,
114	HIST_FIELD_FN_BUCKET,
115	HIST_FIELD_FN_TIMESTAMP,
116	HIST_FIELD_FN_CPU,
117	HIST_FIELD_FN_COMM,
118	HIST_FIELD_FN_STRING,
119	HIST_FIELD_FN_DYNSTRING,
120	HIST_FIELD_FN_RELDYNSTRING,
121	HIST_FIELD_FN_PSTRING,
122	HIST_FIELD_FN_S64,
123	HIST_FIELD_FN_U64,
124	HIST_FIELD_FN_S32,
125	HIST_FIELD_FN_U32,
126	HIST_FIELD_FN_S16,
127	HIST_FIELD_FN_U16,
128	HIST_FIELD_FN_S8,
129	HIST_FIELD_FN_U8,
130	HIST_FIELD_FN_UMINUS,
131	HIST_FIELD_FN_MINUS,
132	HIST_FIELD_FN_PLUS,
133	HIST_FIELD_FN_DIV,
134	HIST_FIELD_FN_MULT,
135	HIST_FIELD_FN_DIV_POWER2,
136	HIST_FIELD_FN_DIV_NOT_POWER2,
137	HIST_FIELD_FN_DIV_MULT_SHIFT,
138	HIST_FIELD_FN_EXECNAME,
139	HIST_FIELD_FN_STACK,
140	};
141
142	/*
143	* A hist_var (histogram variable) contains variable information for
144	* hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
145	* flag set. A hist_var has a variable name e.g. ts0, and is
146	* associated with a given histogram trigger, as specified by
147	* hist_data. The hist_var idx is the unique index assigned to the
148	* variable by the hist trigger's tracing_map. The idx is what is
149	* used to set a variable's value and, by a variable reference, to
150	* retrieve it.
151	*/
152	struct hist_var {
153	char *name;
154	struct hist_trigger_data *hist_data;
155	unsigned int idx;
156	};
157
158	struct hist_field {
159	struct ftrace_event_field *field;
160	unsigned long flags;
161	unsigned long buckets;
162	const char *type;
163	struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
164	struct hist_trigger_data *hist_data;
165	enum hist_field_fn fn_num;
166	unsigned int ref;
167	unsigned int size;
168	unsigned int offset;
169	unsigned int is_signed;
170
171	/*
172	* Variable fields contain variable-specific info in var.
173	*/
174	struct hist_var var;
175	enum field_op_id operator;
176	char *system;
177	char *event_name;
178
179	/*
180	* The name field is used for EXPR and VAR_REF fields. VAR
181	* fields contain the variable name in var.name.
182	*/
183	char *name;
184
185	/*
186	* When a histogram trigger is hit, if it has any references
187	* to variables, the values of those variables are collected
188	* into a var_ref_vals array by resolve_var_refs(). The
189	* current value of each variable is read from the tracing_map
190	* using the hist field's hist_var.idx and entered into the
191	* var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
192	*/
193	unsigned int var_ref_idx;
194	bool read_once;
195
196	unsigned int var_str_idx;
197
198	/* Numeric literals are represented as u64 */
199	u64 constant;
200	/* Used to optimize division by constants */
201	u64 div_multiplier;
202	};
203
204	static u64 hist_fn_call(struct hist_field *hist_field,
205	struct tracing_map_elt *elt,
206	struct trace_buffer *buffer,
207	struct ring_buffer_event *rbe,
208	void *event);
209
210	static u64 hist_field_const(struct hist_field *field,
211	struct tracing_map_elt *elt,
212	struct trace_buffer *buffer,
213	struct ring_buffer_event *rbe,
214	void *event)
215	{
216	return field->constant;
217	}
218
219	static u64 hist_field_counter(struct hist_field *field,
220	struct tracing_map_elt *elt,
221	struct trace_buffer *buffer,
222	struct ring_buffer_event *rbe,
223	void *event)
224	{
225	return 1;
226	}
227
228	static u64 hist_field_string(struct hist_field *hist_field,
229	struct tracing_map_elt *elt,
230	struct trace_buffer *buffer,
231	struct ring_buffer_event *rbe,
232	void *event)
233	{
234	char *addr = (char *)(event + hist_field->field->offset);
235
236	return (u64)(unsigned long)addr;
237	}
238
239	static u64 hist_field_dynstring(struct hist_field *hist_field,
240	struct tracing_map_elt *elt,
241	struct trace_buffer *buffer,
242	struct ring_buffer_event *rbe,
243	void *event)
244	{
245	u32 str_item = *(u32 *)(event + hist_field->field->offset);
246	int str_loc = str_item & 0xffff;
247	char *addr = (char *)(event + str_loc);
248
249	return (u64)(unsigned long)addr;
250	}
251
252	static u64 hist_field_reldynstring(struct hist_field *hist_field,
253	struct tracing_map_elt *elt,
254	struct trace_buffer *buffer,
255	struct ring_buffer_event *rbe,
256	void *event)
257	{
258	u32 *item = event + hist_field->field->offset;
259	u32 str_item = *item;
260	int str_loc = str_item & 0xffff;
261	char *addr = (char *)&item[1] + str_loc;
262
263	return (u64)(unsigned long)addr;
264	}
265
266	static u64 hist_field_pstring(struct hist_field *hist_field,
267	struct tracing_map_elt *elt,
268	struct trace_buffer *buffer,
269	struct ring_buffer_event *rbe,
270	void *event)
271	{
272	char **addr = (char **)(event + hist_field->field->offset);
273
274	return (u64)(unsigned long)*addr;
275	}
276
277	static u64 hist_field_log2(struct hist_field *hist_field,
278	struct tracing_map_elt *elt,
279	struct trace_buffer *buffer,
280	struct ring_buffer_event *rbe,
281	void *event)
282	{
283	struct hist_field *operand = hist_field->operands[0];
284
285	u64 val = hist_fn_call(hist_field: operand, elt, buffer, rbe, event);
286
287	return (u64) ilog2(roundup_pow_of_two(val));
288	}
289
290	static u64 hist_field_bucket(struct hist_field *hist_field,
291	struct tracing_map_elt *elt,
292	struct trace_buffer *buffer,
293	struct ring_buffer_event *rbe,
294	void *event)
295	{
296	struct hist_field *operand = hist_field->operands[0];
297	unsigned long buckets = hist_field->buckets;
298
299	u64 val = hist_fn_call(hist_field: operand, elt, buffer, rbe, event);
300
301	if (WARN_ON_ONCE(!buckets))
302	return val;
303
304	if (val >= LONG_MAX)
305	val = div64_ul(val, buckets);
306	else
307	val = (u64)((unsigned long)val / buckets);
308	return val * buckets;
309	}
310
311	static u64 hist_field_plus(struct hist_field *hist_field,
312	struct tracing_map_elt *elt,
313	struct trace_buffer *buffer,
314	struct ring_buffer_event *rbe,
315	void *event)
316	{
317	struct hist_field *operand1 = hist_field->operands[0];
318	struct hist_field *operand2 = hist_field->operands[1];
319
320	u64 val1 = hist_fn_call(hist_field: operand1, elt, buffer, rbe, event);
321	u64 val2 = hist_fn_call(hist_field: operand2, elt, buffer, rbe, event);
322
323	return val1 + val2;
324	}
325
326	static u64 hist_field_minus(struct hist_field *hist_field,
327	struct tracing_map_elt *elt,
328	struct trace_buffer *buffer,
329	struct ring_buffer_event *rbe,
330	void *event)
331	{
332	struct hist_field *operand1 = hist_field->operands[0];
333	struct hist_field *operand2 = hist_field->operands[1];
334
335	u64 val1 = hist_fn_call(hist_field: operand1, elt, buffer, rbe, event);
336	u64 val2 = hist_fn_call(hist_field: operand2, elt, buffer, rbe, event);
337
338	return val1 - val2;
339	}
340
341	static u64 hist_field_div(struct hist_field *hist_field,
342	struct tracing_map_elt *elt,
343	struct trace_buffer *buffer,
344	struct ring_buffer_event *rbe,
345	void *event)
346	{
347	struct hist_field *operand1 = hist_field->operands[0];
348	struct hist_field *operand2 = hist_field->operands[1];
349
350	u64 val1 = hist_fn_call(hist_field: operand1, elt, buffer, rbe, event);
351	u64 val2 = hist_fn_call(hist_field: operand2, elt, buffer, rbe, event);
352
353	/* Return -1 for the undefined case */
354	if (!val2)
355	return -1;
356
357	/* Use shift if the divisor is a power of 2 */
358	if (!(val2 & (val2 - 1)))
359	return val1 >> __ffs64(word: val2);
360
361	return div64_u64(dividend: val1, divisor: val2);
362	}
363
364	static u64 div_by_power_of_two(struct hist_field *hist_field,
365	struct tracing_map_elt *elt,
366	struct trace_buffer *buffer,
367	struct ring_buffer_event *rbe,
368	void *event)
369	{
370	struct hist_field *operand1 = hist_field->operands[0];
371	struct hist_field *operand2 = hist_field->operands[1];
372
373	u64 val1 = hist_fn_call(hist_field: operand1, elt, buffer, rbe, event);
374
375	return val1 >> __ffs64(word: operand2->constant);
376	}
377
378	static u64 div_by_not_power_of_two(struct hist_field *hist_field,
379	struct tracing_map_elt *elt,
380	struct trace_buffer *buffer,
381	struct ring_buffer_event *rbe,
382	void *event)
383	{
384	struct hist_field *operand1 = hist_field->operands[0];
385	struct hist_field *operand2 = hist_field->operands[1];
386
387	u64 val1 = hist_fn_call(hist_field: operand1, elt, buffer, rbe, event);
388
389	return div64_u64(dividend: val1, divisor: operand2->constant);
390	}
391
392	static u64 div_by_mult_and_shift(struct hist_field *hist_field,
393	struct tracing_map_elt *elt,
394	struct trace_buffer *buffer,
395	struct ring_buffer_event *rbe,
396	void *event)
397	{
398	struct hist_field *operand1 = hist_field->operands[0];
399	struct hist_field *operand2 = hist_field->operands[1];
400
401	u64 val1 = hist_fn_call(hist_field: operand1, elt, buffer, rbe, event);
402
403	/*
404	* If the divisor is a constant, do a multiplication and shift instead.
405	*
406	* Choose Z = some power of 2. If Y <= Z, then:
407	* X / Y = (X * (Z / Y)) / Z
408	*
409	* (Z / Y) is a constant (mult) which is calculated at parse time, so:
410	* X / Y = (X * mult) / Z
411	*
412	* The division by Z can be replaced by a shift since Z is a power of 2:
413	* X / Y = (X * mult) >> HIST_DIV_SHIFT
414	*
415	* As long, as X < Z the results will not be off by more than 1.
416	*/
417	if (val1 < (1 << HIST_DIV_SHIFT)) {
418	u64 mult = operand2->div_multiplier;
419
420	return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
421	}
422
423	return div64_u64(dividend: val1, divisor: operand2->constant);
424	}
425
426	static u64 hist_field_mult(struct hist_field *hist_field,
427	struct tracing_map_elt *elt,
428	struct trace_buffer *buffer,
429	struct ring_buffer_event *rbe,
430	void *event)
431	{
432	struct hist_field *operand1 = hist_field->operands[0];
433	struct hist_field *operand2 = hist_field->operands[1];
434
435	u64 val1 = hist_fn_call(hist_field: operand1, elt, buffer, rbe, event);
436	u64 val2 = hist_fn_call(hist_field: operand2, elt, buffer, rbe, event);
437
438	return val1 * val2;
439	}
440
441	static u64 hist_field_unary_minus(struct hist_field *hist_field,
442	struct tracing_map_elt *elt,
443	struct trace_buffer *buffer,
444	struct ring_buffer_event *rbe,
445	void *event)
446	{
447	struct hist_field *operand = hist_field->operands[0];
448
449	s64 sval = (s64)hist_fn_call(hist_field: operand, elt, buffer, rbe, event);
450	u64 val = (u64)-sval;
451
452	return val;
453	}
454
455	#define DEFINE_HIST_FIELD_FN(type) \
456	static u64 hist_field_##type(struct hist_field *hist_field, \
457	struct tracing_map_elt *elt, \
458	struct trace_buffer *buffer, \
459	struct ring_buffer_event *rbe, \
460	void *event) \
461	{ \
462	type *addr = (type *)(event + hist_field->field->offset); \
463	\
464	return (u64)(unsigned long)*addr; \
465	}
466
467	DEFINE_HIST_FIELD_FN(s64);
468	DEFINE_HIST_FIELD_FN(u64);
469	DEFINE_HIST_FIELD_FN(s32);
470	DEFINE_HIST_FIELD_FN(u32);
471	DEFINE_HIST_FIELD_FN(s16);
472	DEFINE_HIST_FIELD_FN(u16);
473	DEFINE_HIST_FIELD_FN(s8);
474	DEFINE_HIST_FIELD_FN(u8);
475
476	#define for_each_hist_field(i, hist_data) \
477	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
478
479	#define for_each_hist_val_field(i, hist_data) \
480	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
481
482	#define for_each_hist_key_field(i, hist_data) \
483	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
484
485	#define HITCOUNT_IDX 0
486	#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
487
488	enum hist_field_flags {
489	HIST_FIELD_FL_HITCOUNT = 1 << 0,
490	HIST_FIELD_FL_KEY = 1 << 1,
491	HIST_FIELD_FL_STRING = 1 << 2,
492	HIST_FIELD_FL_HEX = 1 << 3,
493	HIST_FIELD_FL_SYM = 1 << 4,
494	HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
495	HIST_FIELD_FL_EXECNAME = 1 << 6,
496	HIST_FIELD_FL_SYSCALL = 1 << 7,
497	HIST_FIELD_FL_STACKTRACE = 1 << 8,
498	HIST_FIELD_FL_LOG2 = 1 << 9,
499	HIST_FIELD_FL_TIMESTAMP = 1 << 10,
500	HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
501	HIST_FIELD_FL_VAR = 1 << 12,
502	HIST_FIELD_FL_EXPR = 1 << 13,
503	HIST_FIELD_FL_VAR_REF = 1 << 14,
504	HIST_FIELD_FL_CPU = 1 << 15,
505	HIST_FIELD_FL_ALIAS = 1 << 16,
506	HIST_FIELD_FL_BUCKET = 1 << 17,
507	HIST_FIELD_FL_CONST = 1 << 18,
508	HIST_FIELD_FL_PERCENT = 1 << 19,
509	HIST_FIELD_FL_GRAPH = 1 << 20,
510	HIST_FIELD_FL_COMM = 1 << 21,
511	};
512
513	struct var_defs {
514	unsigned int n_vars;
515	char *name[TRACING_MAP_VARS_MAX];
516	char *expr[TRACING_MAP_VARS_MAX];
517	};
518
519	struct hist_trigger_attrs {
520	char *keys_str;
521	char *vals_str;
522	char *sort_key_str;
523	char *name;
524	char *clock;
525	bool pause;
526	bool cont;
527	bool clear;
528	bool ts_in_usecs;
529	bool no_hitcount;
530	unsigned int map_bits;
531
532	char *assignment_str[TRACING_MAP_VARS_MAX];
533	unsigned int n_assignments;
534
535	char *action_str[HIST_ACTIONS_MAX];
536	unsigned int n_actions;
537
538	struct var_defs var_defs;
539	};
540
541	struct field_var {
542	struct hist_field *var;
543	struct hist_field *val;
544	};
545
546	struct field_var_hist {
547	struct hist_trigger_data *hist_data;
548	char *cmd;
549	};
550
551	struct hist_trigger_data {
552	struct hist_field *fields[HIST_FIELDS_MAX];
553	unsigned int n_vals;
554	unsigned int n_keys;
555	unsigned int n_fields;
556	unsigned int n_vars;
557	unsigned int n_var_str;
558	unsigned int key_size;
559	struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
560	unsigned int n_sort_keys;
561	struct trace_event_file *event_file;
562	struct hist_trigger_attrs *attrs;
563	struct tracing_map *map;
564	bool enable_timestamps;
565	bool remove;
566	struct hist_field *var_refs[TRACING_MAP_VARS_MAX];
567	unsigned int n_var_refs;
568
569	struct action_data *actions[HIST_ACTIONS_MAX];
570	unsigned int n_actions;
571
572	struct field_var *field_vars[SYNTH_FIELDS_MAX];
573	unsigned int n_field_vars;
574	unsigned int n_field_var_str;
575	struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX];
576	unsigned int n_field_var_hists;
577
578	struct field_var *save_vars[SYNTH_FIELDS_MAX];
579	unsigned int n_save_vars;
580	unsigned int n_save_var_str;
581	};
582
583	struct action_data;
584
585	typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
586	struct tracing_map_elt *elt,
587	struct trace_buffer *buffer, void *rec,
588	struct ring_buffer_event *rbe, void *key,
589	struct action_data *data, u64 *var_ref_vals);
590
591	typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
592
593	enum handler_id {
594	HANDLER_ONMATCH = 1,
595	HANDLER_ONMAX,
596	HANDLER_ONCHANGE,
597	};
598
599	enum action_id {
600	ACTION_SAVE = 1,
601	ACTION_TRACE,
602	ACTION_SNAPSHOT,
603	};
604
605	struct action_data {
606	enum handler_id handler;
607	enum action_id action;
608	char *action_name;
609	action_fn_t fn;
610
611	unsigned int n_params;
612	char *params[SYNTH_FIELDS_MAX];
613
614	/*
615	* When a histogram trigger is hit, the values of any
616	* references to variables, including variables being passed
617	* as parameters to synthetic events, are collected into a
618	* var_ref_vals array. This var_ref_idx array is an array of
619	* indices into the var_ref_vals array, one for each synthetic
620	* event param, and is passed to the synthetic event
621	* invocation.
622	*/
623	unsigned int var_ref_idx[SYNTH_FIELDS_MAX];
624	struct synth_event *synth_event;
625	bool use_trace_keyword;
626	char *synth_event_name;
627
628	union {
629	struct {
630	char *event;
631	char *event_system;
632	} match_data;
633
634	struct {
635	/*
636	* var_str contains the $-unstripped variable
637	* name referenced by var_ref, and used when
638	* printing the action. Because var_ref
639	* creation is deferred to create_actions(),
640	* we need a per-action way to save it until
641	* then, thus var_str.
642	*/
643	char *var_str;
644
645	/*
646	* var_ref refers to the variable being
647	* tracked e.g onmax($var).
648	*/
649	struct hist_field *var_ref;
650
651	/*
652	* track_var contains the 'invisible' tracking
653	* variable created to keep the current
654	* e.g. max value.
655	*/
656	struct hist_field *track_var;
657
658	check_track_val_fn_t check_val;
659	action_fn_t save_data;
660	} track_data;
661	};
662	};
663
664	struct track_data {
665	u64 track_val;
666	bool updated;
667
668	unsigned int key_len;
669	void *key;
670	struct tracing_map_elt elt;
671
672	struct action_data *action_data;
673	struct hist_trigger_data *hist_data;
674	};
675
676	struct hist_elt_data {
677	char *comm;
678	u64 *var_ref_vals;
679	char **field_var_str;
680	int n_field_var_str;
681	};
682
683	struct snapshot_context {
684	struct tracing_map_elt *elt;
685	void *key;
686	};
687
688	/*
689	* Returns the specific division function to use if the divisor
690	* is constant. This avoids extra branches when the trigger is hit.
691	*/
692	static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
693	{
694	u64 div = divisor->constant;
695
696	if (!(div & (div - 1)))
697	return HIST_FIELD_FN_DIV_POWER2;
698
699	/* If the divisor is too large, do a regular division */
700	if (div > (1 << HIST_DIV_SHIFT))
701	return HIST_FIELD_FN_DIV_NOT_POWER2;
702
703	divisor->div_multiplier = div64_u64(dividend: (u64)(1 << HIST_DIV_SHIFT), divisor: div);
704	return HIST_FIELD_FN_DIV_MULT_SHIFT;
705	}
706
707	static void track_data_free(struct track_data *track_data)
708	{
709	struct hist_elt_data *elt_data;
710
711	if (!track_data)
712	return;
713
714	kfree(objp: track_data->key);
715
716	elt_data = track_data->elt.private_data;
717	if (elt_data) {
718	kfree(objp: elt_data->comm);
719	kfree(objp: elt_data);
720	}
721
722	kfree(objp: track_data);
723	}
724
725	static struct track_data *track_data_alloc(unsigned int key_len,
726	struct action_data *action_data,
727	struct hist_trigger_data *hist_data)
728	{
729	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
730	struct hist_elt_data *elt_data;
731
732	if (!data)
733	return ERR_PTR(error: -ENOMEM);
734
735	data->key = kzalloc(key_len, GFP_KERNEL);
736	if (!data->key) {
737	track_data_free(track_data: data);
738	return ERR_PTR(error: -ENOMEM);
739	}
740
741	data->key_len = key_len;
742	data->action_data = action_data;
743	data->hist_data = hist_data;
744
745	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
746	if (!elt_data) {
747	track_data_free(track_data: data);
748	return ERR_PTR(error: -ENOMEM);
749	}
750
751	data->elt.private_data = elt_data;
752
753	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
754	if (!elt_data->comm) {
755	track_data_free(track_data: data);
756	return ERR_PTR(error: -ENOMEM);
757	}
758
759	return data;
760	}
761
762	#define HIST_PREFIX "hist:"
763
764	static char *last_cmd;
765	static char last_cmd_loc[MAX_FILTER_STR_VAL];
766
767	static int errpos(char *str)
768	{
769	if (!str || !last_cmd)
770	return 0;
771
772	return err_pos(cmd: last_cmd, str);
773	}
774
775	static void last_cmd_set(struct trace_event_file *file, char *str)
776	{
777	const char *system = NULL, *name = NULL;
778	struct trace_event_call *call;
779
780	if (!str)
781	return;
782
783	kfree(objp: last_cmd);
784
785	last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str);
786	if (!last_cmd)
787	return;
788
789	if (file) {
790	call = file->event_call;
791	system = call->class->system;
792	if (system) {
793	name = trace_event_name(call);
794	if (!name)
795	system = NULL;
796	}
797	}
798
799	if (system)
800	snprintf(buf: last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
801	}
802
803	static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
804	{
805	if (!last_cmd)
806	return;
807
808	tracing_log_err(tr, loc: last_cmd_loc, cmd: last_cmd, errs: err_text,
809	type: err_type, pos: err_pos);
810	}
811
812	static void hist_err_clear(void)
813	{
814	if (last_cmd)
815	last_cmd[0] = '\0';
816	last_cmd_loc[0] = '\0';
817	}
818
819	typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
820	unsigned int *var_ref_idx);
821
822	static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
823	unsigned int *var_ref_idx)
824	{
825	struct tracepoint *tp = event->tp;
826
827	if (unlikely(static_key_enabled(&tp->key))) {
828	struct tracepoint_func *probe_func_ptr;
829	synth_probe_func_t probe_func;
830	void *__data;
831
832	if (!(cpu_online(raw_smp_processor_id())))
833	return;
834
835	probe_func_ptr = rcu_dereference_sched((tp)->funcs);
836	if (probe_func_ptr) {
837	do {
838	probe_func = probe_func_ptr->func;
839	__data = probe_func_ptr->data;
840	probe_func(__data, var_ref_vals, var_ref_idx);
841	} while ((++probe_func_ptr)->func);
842	}
843	}
844	}
845
846	static void action_trace(struct hist_trigger_data *hist_data,
847	struct tracing_map_elt *elt,
848	struct trace_buffer *buffer, void *rec,
849	struct ring_buffer_event *rbe, void *key,
850	struct action_data *data, u64 *var_ref_vals)
851	{
852	struct synth_event *event = data->synth_event;
853
854	trace_synth(event, var_ref_vals, var_ref_idx: data->var_ref_idx);
855	}
856
857	struct hist_var_data {
858	struct list_head list;
859	struct hist_trigger_data *hist_data;
860	};
861
862	static u64 hist_field_timestamp(struct hist_field *hist_field,
863	struct tracing_map_elt *elt,
864	struct trace_buffer *buffer,
865	struct ring_buffer_event *rbe,
866	void *event)
867	{
868	struct hist_trigger_data *hist_data = hist_field->hist_data;
869	struct trace_array *tr = hist_data->event_file->tr;
870
871	u64 ts = ring_buffer_event_time_stamp(buffer, event: rbe);
872
873	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
874	ts = ns2usecs(nsec: ts);
875
876	return ts;
877	}
878
879	static u64 hist_field_cpu(struct hist_field *hist_field,
880	struct tracing_map_elt *elt,
881	struct trace_buffer *buffer,
882	struct ring_buffer_event *rbe,
883	void *event)
884	{
885	int cpu = smp_processor_id();
886
887	return cpu;
888	}
889
890	static u64 hist_field_comm(struct hist_field *hist_field,
891	struct tracing_map_elt *elt,
892	struct trace_buffer *buffer,
893	struct ring_buffer_event *rbe,
894	void *event)
895	{
896	return (u64)(unsigned long)current->comm;
897	}
898
899	/**
900	* check_field_for_var_ref - Check if a VAR_REF field references a variable
901	* @hist_field: The VAR_REF field to check
902	* @var_data: The hist trigger that owns the variable
903	* @var_idx: The trigger variable identifier
904	*
905	* Check the given VAR_REF field to see whether or not it references
906	* the given variable associated with the given trigger.
907	*
908	* Return: The VAR_REF field if it does reference the variable, NULL if not
909	*/
910	static struct hist_field *
911	check_field_for_var_ref(struct hist_field *hist_field,
912	struct hist_trigger_data *var_data,
913	unsigned int var_idx)
914	{
915	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
916
917	if (hist_field && hist_field->var.idx == var_idx &&
918	hist_field->var.hist_data == var_data)
919	return hist_field;
920
921	return NULL;
922	}
923
924	/**
925	* find_var_ref - Check if a trigger has a reference to a trigger variable
926	* @hist_data: The hist trigger that might have a reference to the variable
927	* @var_data: The hist trigger that owns the variable
928	* @var_idx: The trigger variable identifier
929	*
930	* Check the list of var_refs[] on the first hist trigger to see
931	* whether any of them are references to the variable on the second
932	* trigger.
933	*
934	* Return: The VAR_REF field referencing the variable if so, NULL if not
935	*/
936	static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
937	struct hist_trigger_data *var_data,
938	unsigned int var_idx)
939	{
940	struct hist_field *hist_field;
941	unsigned int i;
942
943	for (i = 0; i < hist_data->n_var_refs; i++) {
944	hist_field = hist_data->var_refs[i];
945	if (check_field_for_var_ref(hist_field, var_data, var_idx))
946	return hist_field;
947	}
948
949	return NULL;
950	}
951
952	/**
953	* find_any_var_ref - Check if there is a reference to a given trigger variable
954	* @hist_data: The hist trigger
955	* @var_idx: The trigger variable identifier
956	*
957	* Check to see whether the given variable is currently referenced by
958	* any other trigger.
959	*
960	* The trigger the variable is defined on is explicitly excluded - the
961	* assumption being that a self-reference doesn't prevent a trigger
962	* from being removed.
963	*
964	* Return: The VAR_REF field referencing the variable if so, NULL if not
965	*/
966	static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
967	unsigned int var_idx)
968	{
969	struct trace_array *tr = hist_data->event_file->tr;
970	struct hist_field *found = NULL;
971	struct hist_var_data *var_data;
972
973	list_for_each_entry(var_data, &tr->hist_vars, list) {
974	if (var_data->hist_data == hist_data)
975	continue;
976	found = find_var_ref(hist_data: var_data->hist_data, var_data: hist_data, var_idx);
977	if (found)
978	break;
979	}
980
981	return found;
982	}
983
984	/**
985	* check_var_refs - Check if there is a reference to any of trigger's variables
986	* @hist_data: The hist trigger
987	*
988	* A trigger can define one or more variables. If any one of them is
989	* currently referenced by any other trigger, this function will
990	* determine that.
991	*
992	* Typically used to determine whether or not a trigger can be removed
993	* - if there are any references to a trigger's variables, it cannot.
994	*
995	* Return: True if there is a reference to any of trigger's variables
996	*/
997	static bool check_var_refs(struct hist_trigger_data *hist_data)
998	{
999	struct hist_field *field;
1000	bool found = false;
1001	int i;
1002
1003	for_each_hist_field(i, hist_data) {
1004	field = hist_data->fields[i];
1005	if (field && field->flags & HIST_FIELD_FL_VAR) {
1006	if (find_any_var_ref(hist_data, var_idx: field->var.idx)) {
1007	found = true;
1008	break;
1009	}
1010	}
1011	}
1012
1013	return found;
1014	}
1015
1016	static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1017	{
1018	struct trace_array *tr = hist_data->event_file->tr;
1019	struct hist_var_data *var_data, *found = NULL;
1020
1021	list_for_each_entry(var_data, &tr->hist_vars, list) {
1022	if (var_data->hist_data == hist_data) {
1023	found = var_data;
1024	break;
1025	}
1026	}
1027
1028	return found;
1029	}
1030
1031	static bool field_has_hist_vars(struct hist_field *hist_field,
1032	unsigned int level)
1033	{
1034	int i;
1035
1036	if (level > 3)
1037	return false;
1038
1039	if (!hist_field)
1040	return false;
1041
1042	if (hist_field->flags & HIST_FIELD_FL_VAR ||
1043	hist_field->flags & HIST_FIELD_FL_VAR_REF)
1044	return true;
1045
1046	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1047	struct hist_field *operand;
1048
1049	operand = hist_field->operands[i];
1050	if (field_has_hist_vars(hist_field: operand, level: level + 1))
1051	return true;
1052	}
1053
1054	return false;
1055	}
1056
1057	static bool has_hist_vars(struct hist_trigger_data *hist_data)
1058	{
1059	struct hist_field *hist_field;
1060	int i;
1061
1062	for_each_hist_field(i, hist_data) {
1063	hist_field = hist_data->fields[i];
1064	if (field_has_hist_vars(hist_field, level: 0))
1065	return true;
1066	}
1067
1068	return false;
1069	}
1070
1071	static int save_hist_vars(struct hist_trigger_data *hist_data)
1072	{
1073	struct trace_array *tr = hist_data->event_file->tr;
1074	struct hist_var_data *var_data;
1075
1076	var_data = find_hist_vars(hist_data);
1077	if (var_data)
1078	return 0;
1079
1080	if (tracing_check_open_get_tr(tr))
1081	return -ENODEV;
1082
1083	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1084	if (!var_data) {
1085	trace_array_put(tr);
1086	return -ENOMEM;
1087	}
1088
1089	var_data->hist_data = hist_data;
1090	list_add(new: &var_data->list, head: &tr->hist_vars);
1091
1092	return 0;
1093	}
1094
1095	static void remove_hist_vars(struct hist_trigger_data *hist_data)
1096	{
1097	struct trace_array *tr = hist_data->event_file->tr;
1098	struct hist_var_data *var_data;
1099
1100	var_data = find_hist_vars(hist_data);
1101	if (!var_data)
1102	return;
1103
1104	if (WARN_ON(check_var_refs(hist_data)))
1105	return;
1106
1107	list_del(entry: &var_data->list);
1108
1109	kfree(objp: var_data);
1110
1111	trace_array_put(tr);
1112	}
1113
1114	static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1115	const char *var_name)
1116	{
1117	struct hist_field *hist_field, *found = NULL;
1118	int i;
1119
1120	for_each_hist_field(i, hist_data) {
1121	hist_field = hist_data->fields[i];
1122	if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1123	strcmp(hist_field->var.name, var_name) == 0) {
1124	found = hist_field;
1125	break;
1126	}
1127	}
1128
1129	return found;
1130	}
1131
1132	static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1133	struct trace_event_file *file,
1134	const char *var_name)
1135	{
1136	struct hist_trigger_data *test_data;
1137	struct event_trigger_data *test;
1138	struct hist_field *hist_field;
1139
1140	lockdep_assert_held(&event_mutex);
1141
1142	hist_field = find_var_field(hist_data, var_name);
1143	if (hist_field)
1144	return hist_field;
1145
1146	list_for_each_entry(test, &file->triggers, list) {
1147	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1148	test_data = test->private_data;
1149	hist_field = find_var_field(hist_data: test_data, var_name);
1150	if (hist_field)
1151	return hist_field;
1152	}
1153	}
1154
1155	return NULL;
1156	}
1157
1158	static struct trace_event_file *find_var_file(struct trace_array *tr,
1159	char *system,
1160	char *event_name,
1161	char *var_name)
1162	{
1163	struct hist_trigger_data *var_hist_data;
1164	struct hist_var_data *var_data;
1165	struct trace_event_file *file, *found = NULL;
1166
1167	if (system)
1168	return find_event_file(tr, system, event: event_name);
1169
1170	list_for_each_entry(var_data, &tr->hist_vars, list) {
1171	var_hist_data = var_data->hist_data;
1172	file = var_hist_data->event_file;
1173	if (file == found)
1174	continue;
1175
1176	if (find_var_field(hist_data: var_hist_data, var_name)) {
1177	if (found) {
1178	hist_err(tr, err_type: HIST_ERR_VAR_NOT_UNIQUE, err_pos: errpos(str: var_name));
1179	return NULL;
1180	}
1181
1182	found = file;
1183	}
1184	}
1185
1186	return found;
1187	}
1188
1189	static struct hist_field *find_file_var(struct trace_event_file *file,
1190	const char *var_name)
1191	{
1192	struct hist_trigger_data *test_data;
1193	struct event_trigger_data *test;
1194	struct hist_field *hist_field;
1195
1196	lockdep_assert_held(&event_mutex);
1197
1198	list_for_each_entry(test, &file->triggers, list) {
1199	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1200	test_data = test->private_data;
1201	hist_field = find_var_field(hist_data: test_data, var_name);
1202	if (hist_field)
1203	return hist_field;
1204	}
1205	}
1206
1207	return NULL;
1208	}
1209
1210	static struct hist_field *
1211	find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1212	{
1213	struct trace_array *tr = hist_data->event_file->tr;
1214	struct hist_field *hist_field, *found = NULL;
1215	struct trace_event_file *file;
1216	unsigned int i;
1217
1218	for (i = 0; i < hist_data->n_actions; i++) {
1219	struct action_data *data = hist_data->actions[i];
1220
1221	if (data->handler == HANDLER_ONMATCH) {
1222	char *system = data->match_data.event_system;
1223	char *event_name = data->match_data.event;
1224
1225	file = find_var_file(tr, system, event_name, var_name);
1226	if (!file)
1227	continue;
1228	hist_field = find_file_var(file, var_name);
1229	if (hist_field) {
1230	if (found) {
1231	hist_err(tr, err_type: HIST_ERR_VAR_NOT_UNIQUE,
1232	err_pos: errpos(str: var_name));
1233	return ERR_PTR(error: -EINVAL);
1234	}
1235
1236	found = hist_field;
1237	}
1238	}
1239	}
1240	return found;
1241	}
1242
1243	static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1244	char *system,
1245	char *event_name,
1246	char *var_name)
1247	{
1248	struct trace_array *tr = hist_data->event_file->tr;
1249	struct hist_field *hist_field = NULL;
1250	struct trace_event_file *file;
1251
1252	if (!system || !event_name) {
1253	hist_field = find_match_var(hist_data, var_name);
1254	if (IS_ERR(ptr: hist_field))
1255	return NULL;
1256	if (hist_field)
1257	return hist_field;
1258	}
1259
1260	file = find_var_file(tr, system, event_name, var_name);
1261	if (!file)
1262	return NULL;
1263
1264	hist_field = find_file_var(file, var_name);
1265
1266	return hist_field;
1267	}
1268
1269	static u64 hist_field_var_ref(struct hist_field *hist_field,
1270	struct tracing_map_elt *elt,
1271	struct trace_buffer *buffer,
1272	struct ring_buffer_event *rbe,
1273	void *event)
1274	{
1275	struct hist_elt_data *elt_data;
1276	u64 var_val = 0;
1277
1278	if (WARN_ON_ONCE(!elt))
1279	return var_val;
1280
1281	elt_data = elt->private_data;
1282	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1283
1284	return var_val;
1285	}
1286
1287	static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1288	u64 *var_ref_vals, bool self)
1289	{
1290	struct hist_trigger_data *var_data;
1291	struct tracing_map_elt *var_elt;
1292	struct hist_field *hist_field;
1293	unsigned int i, var_idx;
1294	bool resolved = true;
1295	u64 var_val = 0;
1296
1297	for (i = 0; i < hist_data->n_var_refs; i++) {
1298	hist_field = hist_data->var_refs[i];
1299	var_idx = hist_field->var.idx;
1300	var_data = hist_field->var.hist_data;
1301
1302	if (var_data == NULL) {
1303	resolved = false;
1304	break;
1305	}
1306
1307	if ((self && var_data != hist_data) ||
1308	(!self && var_data == hist_data))
1309	continue;
1310
1311	var_elt = tracing_map_lookup(map: var_data->map, key);
1312	if (!var_elt) {
1313	resolved = false;
1314	break;
1315	}
1316
1317	if (!tracing_map_var_set(elt: var_elt, i: var_idx)) {
1318	resolved = false;
1319	break;
1320	}
1321
1322	if (self || !hist_field->read_once)
1323	var_val = tracing_map_read_var(elt: var_elt, i: var_idx);
1324	else
1325	var_val = tracing_map_read_var_once(elt: var_elt, i: var_idx);
1326
1327	var_ref_vals[i] = var_val;
1328	}
1329
1330	return resolved;
1331	}
1332
1333	static const char *hist_field_name(struct hist_field *field,
1334	unsigned int level)
1335	{
1336	const char *field_name = "";
1337
1338	if (WARN_ON_ONCE(!field))
1339	return field_name;
1340
1341	if (level > 1)
1342	return field_name;
1343
1344	if (field->field)
1345	field_name = field->field->name;
1346	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1347	field->flags & HIST_FIELD_FL_ALIAS ||
1348	field->flags & HIST_FIELD_FL_BUCKET)
1349	field_name = hist_field_name(field: field->operands[0], level: ++level);
1350	else if (field->flags & HIST_FIELD_FL_CPU)
1351	field_name = "common_cpu";
1352	else if (field->flags & HIST_FIELD_FL_COMM)
1353	field_name = "common_comm";
1354	else if (field->flags & HIST_FIELD_FL_EXPR ||
1355	field->flags & HIST_FIELD_FL_VAR_REF) {
1356	if (field->system) {
1357	static char full_name[MAX_FILTER_STR_VAL];
1358
1359	strcat(p: full_name, q: field->system);
1360	strcat(p: full_name, q: ".");
1361	strcat(p: full_name, q: field->event_name);
1362	strcat(p: full_name, q: ".");
1363	strcat(p: full_name, q: field->name);
1364	field_name = full_name;
1365	} else
1366	field_name = field->name;
1367	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1368	field_name = "common_timestamp";
1369	else if (field->flags & HIST_FIELD_FL_STACKTRACE) {
1370	field_name = "common_stacktrace";
1371	} else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1372	field_name = "hitcount";
1373
1374	if (field_name == NULL)
1375	field_name = "";
1376
1377	return field_name;
1378	}
1379
1380	static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1381	{
1382	switch (field_size) {
1383	case 8:
1384	if (field_is_signed)
1385	return HIST_FIELD_FN_S64;
1386	else
1387	return HIST_FIELD_FN_U64;
1388	case 4:
1389	if (field_is_signed)
1390	return HIST_FIELD_FN_S32;
1391	else
1392	return HIST_FIELD_FN_U32;
1393	case 2:
1394	if (field_is_signed)
1395	return HIST_FIELD_FN_S16;
1396	else
1397	return HIST_FIELD_FN_U16;
1398	case 1:
1399	if (field_is_signed)
1400	return HIST_FIELD_FN_S8;
1401	else
1402	return HIST_FIELD_FN_U8;
1403	}
1404
1405	return HIST_FIELD_FN_NOP;
1406	}
1407
1408	static int parse_map_size(char *str)
1409	{
1410	unsigned long size, map_bits;
1411	int ret;
1412
1413	ret = kstrtoul(s: str, base: 0, res: &size);
1414	if (ret)
1415	goto out;
1416
1417	map_bits = ilog2(roundup_pow_of_two(size));
1418	if (map_bits < TRACING_MAP_BITS_MIN ||
1419	map_bits > TRACING_MAP_BITS_MAX)
1420	ret = -EINVAL;
1421	else
1422	ret = map_bits;
1423	out:
1424	return ret;
1425	}
1426
1427	static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1428	{
1429	unsigned int i;
1430
1431	if (!attrs)
1432	return;
1433
1434	for (i = 0; i < attrs->n_assignments; i++)
1435	kfree(objp: attrs->assignment_str[i]);
1436
1437	for (i = 0; i < attrs->n_actions; i++)
1438	kfree(objp: attrs->action_str[i]);
1439
1440	kfree(objp: attrs->name);
1441	kfree(objp: attrs->sort_key_str);
1442	kfree(objp: attrs->keys_str);
1443	kfree(objp: attrs->vals_str);
1444	kfree(objp: attrs->clock);
1445	kfree(objp: attrs);
1446	}
1447
1448	static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1449	{
1450	int ret = -EINVAL;
1451
1452	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1453	return ret;
1454
1455	if ((str_has_prefix(str, prefix: "onmatch(")) ||
1456	(str_has_prefix(str, prefix: "onmax(")) ||
1457	(str_has_prefix(str, prefix: "onchange("))) {
1458	attrs->action_str[attrs->n_actions] = kstrdup(s: str, GFP_KERNEL);
1459	if (!attrs->action_str[attrs->n_actions]) {
1460	ret = -ENOMEM;
1461	return ret;
1462	}
1463	attrs->n_actions++;
1464	ret = 0;
1465	}
1466	return ret;
1467	}
1468
1469	static int parse_assignment(struct trace_array *tr,
1470	char *str, struct hist_trigger_attrs *attrs)
1471	{
1472	int len, ret = 0;
1473
1474	if ((len = str_has_prefix(str, prefix: "key=")) ||
1475	(len = str_has_prefix(str, prefix: "keys="))) {
1476	attrs->keys_str = kstrdup(s: str + len, GFP_KERNEL);
1477	if (!attrs->keys_str) {
1478	ret = -ENOMEM;
1479	goto out;
1480	}
1481	} else if ((len = str_has_prefix(str, prefix: "val=")) ||
1482	(len = str_has_prefix(str, prefix: "vals=")) ||
1483	(len = str_has_prefix(str, prefix: "values="))) {
1484	attrs->vals_str = kstrdup(s: str + len, GFP_KERNEL);
1485	if (!attrs->vals_str) {
1486	ret = -ENOMEM;
1487	goto out;
1488	}
1489	} else if ((len = str_has_prefix(str, prefix: "sort="))) {
1490	attrs->sort_key_str = kstrdup(s: str + len, GFP_KERNEL);
1491	if (!attrs->sort_key_str) {
1492	ret = -ENOMEM;
1493	goto out;
1494	}
1495	} else if (str_has_prefix(str, prefix: "name=")) {
1496	attrs->name = kstrdup(s: str, GFP_KERNEL);
1497	if (!attrs->name) {
1498	ret = -ENOMEM;
1499	goto out;
1500	}
1501	} else if ((len = str_has_prefix(str, prefix: "clock="))) {
1502	str += len;
1503
1504	str = strstrip(str);
1505	attrs->clock = kstrdup(s: str, GFP_KERNEL);
1506	if (!attrs->clock) {
1507	ret = -ENOMEM;
1508	goto out;
1509	}
1510	} else if ((len = str_has_prefix(str, prefix: "size="))) {
1511	int map_bits = parse_map_size(str: str + len);
1512
1513	if (map_bits < 0) {
1514	ret = map_bits;
1515	goto out;
1516	}
1517	attrs->map_bits = map_bits;
1518	} else {
1519	char *assignment;
1520
1521	if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1522	hist_err(tr, err_type: HIST_ERR_TOO_MANY_VARS, err_pos: errpos(str));
1523	ret = -EINVAL;
1524	goto out;
1525	}
1526
1527	assignment = kstrdup(s: str, GFP_KERNEL);
1528	if (!assignment) {
1529	ret = -ENOMEM;
1530	goto out;
1531	}
1532
1533	attrs->assignment_str[attrs->n_assignments++] = assignment;
1534	}
1535	out:
1536	return ret;
1537	}
1538
1539	static struct hist_trigger_attrs *
1540	parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1541	{
1542	struct hist_trigger_attrs *attrs;
1543	int ret = 0;
1544
1545	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1546	if (!attrs)
1547	return ERR_PTR(error: -ENOMEM);
1548
1549	while (trigger_str) {
1550	char *str = strsep(&trigger_str, ":");
1551	char *rhs;
1552
1553	rhs = strchr(str, '=');
1554	if (rhs) {
1555	if (!strlen(++rhs)) {
1556	ret = -EINVAL;
1557	hist_err(tr, err_type: HIST_ERR_EMPTY_ASSIGNMENT, err_pos: errpos(str));
1558	goto free;
1559	}
1560	ret = parse_assignment(tr, str, attrs);
1561	if (ret)
1562	goto free;
1563	} else if (strcmp(str, "nohitcount") == 0 ||
1564	strcmp(str, "NOHC") == 0)
1565	attrs->no_hitcount = true;
1566	else if (strcmp(str, "pause") == 0)
1567	attrs->pause = true;
1568	else if ((strcmp(str, "cont") == 0) ||
1569	(strcmp(str, "continue") == 0))
1570	attrs->cont = true;
1571	else if (strcmp(str, "clear") == 0)
1572	attrs->clear = true;
1573	else {
1574	ret = parse_action(str, attrs);
1575	if (ret)
1576	goto free;
1577	}
1578	}
1579
1580	if (!attrs->keys_str) {
1581	ret = -EINVAL;
1582	goto free;
1583	}
1584
1585	if (!attrs->clock) {
1586	attrs->clock = kstrdup(s: "global", GFP_KERNEL);
1587	if (!attrs->clock) {
1588	ret = -ENOMEM;
1589	goto free;
1590	}
1591	}
1592
1593	return attrs;
1594	free:
1595	destroy_hist_trigger_attrs(attrs);
1596
1597	return ERR_PTR(error: ret);
1598	}
1599
1600	static inline void save_comm(char *comm, struct task_struct *task)
1601	{
1602	if (!task->pid) {
1603	strcpy(p: comm, q: "<idle>");
1604	return;
1605	}
1606
1607	if (WARN_ON_ONCE(task->pid < 0)) {
1608	strcpy(p: comm, q: "<XXX>");
1609	return;
1610	}
1611
1612	strscpy(comm, task->comm, TASK_COMM_LEN);
1613	}
1614
1615	static void hist_elt_data_free(struct hist_elt_data *elt_data)
1616	{
1617	unsigned int i;
1618
1619	for (i = 0; i < elt_data->n_field_var_str; i++)
1620	kfree(objp: elt_data->field_var_str[i]);
1621
1622	kfree(objp: elt_data->field_var_str);
1623
1624	kfree(objp: elt_data->comm);
1625	kfree(objp: elt_data);
1626	}
1627
1628	static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1629	{
1630	struct hist_elt_data *elt_data = elt->private_data;
1631
1632	hist_elt_data_free(elt_data);
1633	}
1634
1635	static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1636	{
1637	struct hist_trigger_data *hist_data = elt->map->private_data;
1638	unsigned int size = TASK_COMM_LEN;
1639	struct hist_elt_data *elt_data;
1640	struct hist_field *hist_field;
1641	unsigned int i, n_str;
1642
1643	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1644	if (!elt_data)
1645	return -ENOMEM;
1646
1647	for_each_hist_field(i, hist_data) {
1648	hist_field = hist_data->fields[i];
1649
1650	if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1651	elt_data->comm = kzalloc(size, GFP_KERNEL);
1652	if (!elt_data->comm) {
1653	kfree(objp: elt_data);
1654	return -ENOMEM;
1655	}
1656	break;
1657	}
1658	}
1659
1660	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1661	hist_data->n_var_str;
1662	if (n_str > SYNTH_FIELDS_MAX) {
1663	hist_elt_data_free(elt_data);
1664	return -EINVAL;
1665	}
1666
1667	BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1668
1669	size = STR_VAR_LEN_MAX;
1670
1671	elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1672	if (!elt_data->field_var_str) {
1673	hist_elt_data_free(elt_data);
1674	return -EINVAL;
1675	}
1676	elt_data->n_field_var_str = n_str;
1677
1678	for (i = 0; i < n_str; i++) {
1679	elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1680	if (!elt_data->field_var_str[i]) {
1681	hist_elt_data_free(elt_data);
1682	return -ENOMEM;
1683	}
1684	}
1685
1686	elt->private_data = elt_data;
1687
1688	return 0;
1689	}
1690
1691	static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1692	{
1693	struct hist_elt_data *elt_data = elt->private_data;
1694
1695	if (elt_data->comm)
1696	save_comm(comm: elt_data->comm, current);
1697	}
1698
1699	static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1700	.elt_alloc = hist_trigger_elt_data_alloc,
1701	.elt_free = hist_trigger_elt_data_free,
1702	.elt_init = hist_trigger_elt_data_init,
1703	};
1704
1705	static const char *get_hist_field_flags(struct hist_field *hist_field)
1706	{
1707	const char *flags_str = NULL;
1708
1709	if (hist_field->flags & HIST_FIELD_FL_HEX)
1710	flags_str = "hex";
1711	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1712	flags_str = "sym";
1713	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1714	flags_str = "sym-offset";
1715	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1716	flags_str = "execname";
1717	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1718	flags_str = "syscall";
1719	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1720	flags_str = "log2";
1721	else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1722	flags_str = "buckets";
1723	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1724	flags_str = "usecs";
1725	else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1726	flags_str = "percent";
1727	else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1728	flags_str = "graph";
1729	else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1730	flags_str = "stacktrace";
1731
1732	return flags_str;
1733	}
1734
1735	static void expr_field_str(struct hist_field *field, char *expr)
1736	{
1737	if (field->flags & HIST_FIELD_FL_VAR_REF)
1738	strcat(p: expr, q: "$");
1739	else if (field->flags & HIST_FIELD_FL_CONST) {
1740	char str[HIST_CONST_DIGITS_MAX];
1741
1742	snprintf(buf: str, HIST_CONST_DIGITS_MAX, fmt: "%llu", field->constant);
1743	strcat(p: expr, q: str);
1744	}
1745
1746	strcat(p: expr, q: hist_field_name(field, level: 0));
1747
1748	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1749	const char *flags_str = get_hist_field_flags(hist_field: field);
1750
1751	if (flags_str) {
1752	strcat(p: expr, q: ".");
1753	strcat(p: expr, q: flags_str);
1754	}
1755	}
1756	}
1757
1758	static char *expr_str(struct hist_field *field, unsigned int level)
1759	{
1760	char *expr;
1761
1762	if (level > 1)
1763	return NULL;
1764
1765	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1766	if (!expr)
1767	return NULL;
1768
1769	if (!field->operands[0]) {
1770	expr_field_str(field, expr);
1771	return expr;
1772	}
1773
1774	if (field->operator == FIELD_OP_UNARY_MINUS) {
1775	char *subexpr;
1776
1777	strcat(p: expr, q: "-(");
1778	subexpr = expr_str(field: field->operands[0], level: ++level);
1779	if (!subexpr) {
1780	kfree(objp: expr);
1781	return NULL;
1782	}
1783	strcat(p: expr, q: subexpr);
1784	strcat(p: expr, q: ")");
1785
1786	kfree(objp: subexpr);
1787
1788	return expr;
1789	}
1790
1791	expr_field_str(field: field->operands[0], expr);
1792
1793	switch (field->operator) {
1794	case FIELD_OP_MINUS:
1795	strcat(p: expr, q: "-");
1796	break;
1797	case FIELD_OP_PLUS:
1798	strcat(p: expr, q: "+");
1799	break;
1800	case FIELD_OP_DIV:
1801	strcat(p: expr, q: "/");
1802	break;
1803	case FIELD_OP_MULT:
1804	strcat(p: expr, q: "*");
1805	break;
1806	default:
1807	kfree(objp: expr);
1808	return NULL;
1809	}
1810
1811	expr_field_str(field: field->operands[1], expr);
1812
1813	return expr;
1814	}
1815
1816	/*
1817	* If field_op != FIELD_OP_NONE, *sep points to the root operator
1818	* of the expression tree to be evaluated.
1819	*/
1820	static int contains_operator(char *str, char **sep)
1821	{
1822	enum field_op_id field_op = FIELD_OP_NONE;
1823	char *minus_op, *plus_op, *div_op, *mult_op;
1824
1825
1826	/*
1827	* Report the last occurrence of the operators first, so that the
1828	* expression is evaluated left to right. This is important since
1829	* subtraction and division are not associative.
1830	*
1831	* e.g
1832	* 64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1833	* 14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1834	*/
1835
1836	/*
1837	* First, find lower precedence addition and subtraction
1838	* since the expression will be evaluated recursively.
1839	*/
1840	minus_op = strrchr(str, '-');
1841	if (minus_op) {
1842	/*
1843	* Unary minus is not supported in sub-expressions. If
1844	* present, it is always the next root operator.
1845	*/
1846	if (minus_op == str) {
1847	field_op = FIELD_OP_UNARY_MINUS;
1848	goto out;
1849	}
1850
1851	field_op = FIELD_OP_MINUS;
1852	}
1853
1854	plus_op = strrchr(str, '+');
1855	if (plus_op || minus_op) {
1856	/*
1857	* For operators of the same precedence use to rightmost as the
1858	* root, so that the expression is evaluated left to right.
1859	*/
1860	if (plus_op > minus_op)
1861	field_op = FIELD_OP_PLUS;
1862	goto out;
1863	}
1864
1865	/*
1866	* Multiplication and division have higher precedence than addition and
1867	* subtraction.
1868	*/
1869	div_op = strrchr(str, '/');
1870	if (div_op)
1871	field_op = FIELD_OP_DIV;
1872
1873	mult_op = strrchr(str, '*');
1874	/*
1875	* For operators of the same precedence use to rightmost as the
1876	* root, so that the expression is evaluated left to right.
1877	*/
1878	if (mult_op > div_op)
1879	field_op = FIELD_OP_MULT;
1880
1881	out:
1882	if (sep) {
1883	switch (field_op) {
1884	case FIELD_OP_UNARY_MINUS:
1885	case FIELD_OP_MINUS:
1886	*sep = minus_op;
1887	break;
1888	case FIELD_OP_PLUS:
1889	*sep = plus_op;
1890	break;
1891	case FIELD_OP_DIV:
1892	*sep = div_op;
1893	break;
1894	case FIELD_OP_MULT:
1895	*sep = mult_op;
1896	break;
1897	case FIELD_OP_NONE:
1898	default:
1899	*sep = NULL;
1900	break;
1901	}
1902	}
1903
1904	return field_op;
1905	}
1906
1907	static void get_hist_field(struct hist_field *hist_field)
1908	{
1909	hist_field->ref++;
1910	}
1911
1912	static void __destroy_hist_field(struct hist_field *hist_field)
1913	{
1914	if (--hist_field->ref > 1)
1915	return;
1916
1917	kfree(objp: hist_field->var.name);
1918	kfree(objp: hist_field->name);
1919
1920	/* Can likely be a const */
1921	kfree_const(x: hist_field->type);
1922
1923	kfree(objp: hist_field->system);
1924	kfree(objp: hist_field->event_name);
1925
1926	kfree(objp: hist_field);
1927	}
1928
1929	static void destroy_hist_field(struct hist_field *hist_field,
1930	unsigned int level)
1931	{
1932	unsigned int i;
1933
1934	if (level > 3)
1935	return;
1936
1937	if (!hist_field)
1938	return;
1939
1940	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1941	return; /* var refs will be destroyed separately */
1942
1943	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1944	destroy_hist_field(hist_field: hist_field->operands[i], level: level + 1);
1945
1946	__destroy_hist_field(hist_field);
1947	}
1948
1949	static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1950	struct ftrace_event_field *field,
1951	unsigned long flags,
1952	char *var_name)
1953	{
1954	struct hist_field *hist_field;
1955
1956	if (field && is_function_field(field))
1957	return NULL;
1958
1959	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1960	if (!hist_field)
1961	return NULL;
1962
1963	hist_field->ref = 1;
1964
1965	hist_field->hist_data = hist_data;
1966
1967	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1968	goto out; /* caller will populate */
1969
1970	if (flags & HIST_FIELD_FL_VAR_REF) {
1971	hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1972	goto out;
1973	}
1974
1975	if (flags & HIST_FIELD_FL_HITCOUNT) {
1976	hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1977	hist_field->size = sizeof(u64);
1978	hist_field->type = "u64";
1979	goto out;
1980	}
1981
1982	if (flags & HIST_FIELD_FL_CONST) {
1983	hist_field->fn_num = HIST_FIELD_FN_CONST;
1984	hist_field->size = sizeof(u64);
1985	hist_field->type = kstrdup(s: "u64", GFP_KERNEL);
1986	if (!hist_field->type)
1987	goto free;
1988	goto out;
1989	}
1990
1991	if (flags & HIST_FIELD_FL_STACKTRACE) {
1992	if (field)
1993	hist_field->fn_num = HIST_FIELD_FN_STACK;
1994	else
1995	hist_field->fn_num = HIST_FIELD_FN_NOP;
1996	hist_field->size = HIST_STACKTRACE_SIZE;
1997	hist_field->type = kstrdup_const(s: "unsigned long[]", GFP_KERNEL);
1998	if (!hist_field->type)
1999	goto free;
2000	goto out;
2001	}
2002
2003	if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
2004	unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
2005	hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
2006	HIST_FIELD_FN_BUCKET;
2007	hist_field->operands[0] = create_hist_field(hist_data, field, flags: fl, NULL);
2008	if (!hist_field->operands[0])
2009	goto free;
2010	hist_field->size = hist_field->operands[0]->size;
2011	hist_field->type = kstrdup_const(s: hist_field->operands[0]->type, GFP_KERNEL);
2012	if (!hist_field->type)
2013	goto free;
2014	goto out;
2015	}
2016
2017	if (flags & HIST_FIELD_FL_TIMESTAMP) {
2018	hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2019	hist_field->size = sizeof(u64);
2020	hist_field->type = "u64";
2021	goto out;
2022	}
2023
2024	if (flags & HIST_FIELD_FL_CPU) {
2025	hist_field->fn_num = HIST_FIELD_FN_CPU;
2026	hist_field->size = sizeof(int);
2027	hist_field->type = "unsigned int";
2028	goto out;
2029	}
2030
2031	if (flags & HIST_FIELD_FL_COMM) {
2032	hist_field->fn_num = HIST_FIELD_FN_COMM;
2033	hist_field->size = MAX_FILTER_STR_VAL;
2034	hist_field->type = "char[]";
2035	goto out;
2036	}
2037
2038	if (WARN_ON_ONCE(!field))
2039	goto out;
2040
2041	/* Pointers to strings are just pointers and dangerous to dereference */
2042	if (is_string_field(field) &&
2043	(field->filter_type != FILTER_PTR_STRING)) {
2044	flags |= HIST_FIELD_FL_STRING;
2045
2046	hist_field->size = MAX_FILTER_STR_VAL;
2047	hist_field->type = kstrdup_const(s: field->type, GFP_KERNEL);
2048	if (!hist_field->type)
2049	goto free;
2050
2051	if (field->filter_type == FILTER_STATIC_STRING) {
2052	hist_field->fn_num = HIST_FIELD_FN_STRING;
2053	hist_field->size = field->size;
2054	} else if (field->filter_type == FILTER_DYN_STRING) {
2055	hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2056	} else if (field->filter_type == FILTER_RDYN_STRING)
2057	hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2058	else
2059	hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2060	} else if (field->filter_type == FILTER_STACKTRACE) {
2061	flags |= HIST_FIELD_FL_STACKTRACE;
2062
2063	hist_field->size = MAX_FILTER_STR_VAL;
2064	hist_field->type = kstrdup_const(s: field->type, GFP_KERNEL);
2065	if (!hist_field->type)
2066	goto free;
2067
2068	hist_field->fn_num = HIST_FIELD_FN_STACK;
2069	} else {
2070	hist_field->size = field->size;
2071	hist_field->is_signed = field->is_signed;
2072	hist_field->type = kstrdup_const(s: field->type, GFP_KERNEL);
2073	if (!hist_field->type)
2074	goto free;
2075
2076	hist_field->fn_num = select_value_fn(field_size: field->size,
2077	field_is_signed: field->is_signed);
2078	if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2079	destroy_hist_field(hist_field, level: 0);
2080	return NULL;
2081	}
2082	}
2083	out:
2084	hist_field->field = field;
2085	hist_field->flags = flags;
2086
2087	if (var_name) {
2088	hist_field->var.name = kstrdup(s: var_name, GFP_KERNEL);
2089	if (!hist_field->var.name)
2090	goto free;
2091	}
2092
2093	return hist_field;
2094	free:
2095	destroy_hist_field(hist_field, level: 0);
2096	return NULL;
2097	}
2098
2099	static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2100	{
2101	unsigned int i;
2102
2103	for (i = 0; i < HIST_FIELDS_MAX; i++) {
2104	if (hist_data->fields[i]) {
2105	destroy_hist_field(hist_field: hist_data->fields[i], level: 0);
2106	hist_data->fields[i] = NULL;
2107	}
2108	}
2109
2110	for (i = 0; i < hist_data->n_var_refs; i++) {
2111	WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2112	__destroy_hist_field(hist_field: hist_data->var_refs[i]);
2113	hist_data->var_refs[i] = NULL;
2114	}
2115	}
2116
2117	static int init_var_ref(struct hist_field *ref_field,
2118	struct hist_field *var_field,
2119	char *system, char *event_name)
2120	{
2121	int err = 0;
2122
2123	ref_field->var.idx = var_field->var.idx;
2124	ref_field->var.hist_data = var_field->hist_data;
2125	ref_field->size = var_field->size;
2126	ref_field->is_signed = var_field->is_signed;
2127	ref_field->flags |= var_field->flags &
2128	(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2129
2130	if (system) {
2131	ref_field->system = kstrdup(s: system, GFP_KERNEL);
2132	if (!ref_field->system)
2133	return -ENOMEM;
2134	}
2135
2136	if (event_name) {
2137	ref_field->event_name = kstrdup(s: event_name, GFP_KERNEL);
2138	if (!ref_field->event_name) {
2139	err = -ENOMEM;
2140	goto free;
2141	}
2142	}
2143
2144	if (var_field->var.name) {
2145	ref_field->name = kstrdup(s: var_field->var.name, GFP_KERNEL);
2146	if (!ref_field->name) {
2147	err = -ENOMEM;
2148	goto free;
2149	}
2150	} else if (var_field->name) {
2151	ref_field->name = kstrdup(s: var_field->name, GFP_KERNEL);
2152	if (!ref_field->name) {
2153	err = -ENOMEM;
2154	goto free;
2155	}
2156	}
2157
2158	ref_field->type = kstrdup_const(s: var_field->type, GFP_KERNEL);
2159	if (!ref_field->type) {
2160	err = -ENOMEM;
2161	goto free;
2162	}
2163	out:
2164	return err;
2165	free:
2166	kfree(objp: ref_field->system);
2167	ref_field->system = NULL;
2168	kfree(objp: ref_field->event_name);
2169	ref_field->event_name = NULL;
2170	kfree(objp: ref_field->name);
2171	ref_field->name = NULL;
2172
2173	goto out;
2174	}
2175
2176	static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2177	struct hist_field *var_field)
2178	{
2179	struct hist_field *ref_field;
2180	int i;
2181
2182	for (i = 0; i < hist_data->n_var_refs; i++) {
2183	ref_field = hist_data->var_refs[i];
2184	if (ref_field->var.idx == var_field->var.idx &&
2185	ref_field->var.hist_data == var_field->hist_data)
2186	return i;
2187	}
2188
2189	return -ENOENT;
2190	}
2191
2192	/**
2193	* create_var_ref - Create a variable reference and attach it to trigger
2194	* @hist_data: The trigger that will be referencing the variable
2195	* @var_field: The VAR field to create a reference to
2196	* @system: The optional system string
2197	* @event_name: The optional event_name string
2198	*
2199	* Given a variable hist_field, create a VAR_REF hist_field that
2200	* represents a reference to it.
2201	*
2202	* This function also adds the reference to the trigger that
2203	* now references the variable.
2204	*
2205	* Return: The VAR_REF field if successful, NULL if not
2206	*/
2207	static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2208	struct hist_field *var_field,
2209	char *system, char *event_name)
2210	{
2211	unsigned long flags = HIST_FIELD_FL_VAR_REF;
2212	struct hist_field *ref_field;
2213	int i;
2214
2215	/* Check if the variable already exists */
2216	for (i = 0; i < hist_data->n_var_refs; i++) {
2217	ref_field = hist_data->var_refs[i];
2218	if (ref_field->var.idx == var_field->var.idx &&
2219	ref_field->var.hist_data == var_field->hist_data) {
2220	get_hist_field(hist_field: ref_field);
2221	return ref_field;
2222	}
2223	}
2224	/* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2225	if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2226	return NULL;
2227	ref_field = create_hist_field(hist_data: var_field->hist_data, NULL, flags, NULL);
2228	if (ref_field) {
2229	if (init_var_ref(ref_field, var_field, system, event_name)) {
2230	destroy_hist_field(hist_field: ref_field, level: 0);
2231	return NULL;
2232	}
2233
2234	hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2235	ref_field->var_ref_idx = hist_data->n_var_refs++;
2236	}
2237
2238	return ref_field;
2239	}
2240
2241	static bool is_var_ref(char *var_name)
2242	{
2243	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2244	return false;
2245
2246	return true;
2247	}
2248
2249	static char *field_name_from_var(struct hist_trigger_data *hist_data,
2250	char *var_name)
2251	{
2252	char *name, *field;
2253	unsigned int i;
2254
2255	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2256	name = hist_data->attrs->var_defs.name[i];
2257
2258	if (strcmp(var_name, name) == 0) {
2259	field = hist_data->attrs->var_defs.expr[i];
2260	if (contains_operator(str: field, NULL) || is_var_ref(var_name: field))
2261	continue;
2262	return field;
2263	}
2264	}
2265
2266	return NULL;
2267	}
2268
2269	static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2270	char *system, char *event_name,
2271	char *var_name)
2272	{
2273	struct trace_event_call *call;
2274
2275	if (system && event_name) {
2276	call = hist_data->event_file->event_call;
2277
2278	if (strcmp(system, call->class->system) != 0)
2279	return NULL;
2280
2281	if (strcmp(event_name, trace_event_name(call)) != 0)
2282	return NULL;
2283	}
2284
2285	if (!!system != !!event_name)
2286	return NULL;
2287
2288	if (!is_var_ref(var_name))
2289	return NULL;
2290
2291	var_name++;
2292
2293	return field_name_from_var(hist_data, var_name);
2294	}
2295
2296	static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2297	char *system, char *event_name,
2298	char *var_name)
2299	{
2300	struct hist_field *var_field = NULL, *ref_field = NULL;
2301	struct trace_array *tr = hist_data->event_file->tr;
2302
2303	if (!is_var_ref(var_name))
2304	return NULL;
2305
2306	var_name++;
2307
2308	var_field = find_event_var(hist_data, system, event_name, var_name);
2309	if (var_field)
2310	ref_field = create_var_ref(hist_data, var_field,
2311	system, event_name);
2312
2313	if (!ref_field)
2314	hist_err(tr, err_type: HIST_ERR_VAR_NOT_FOUND, err_pos: errpos(str: var_name));
2315
2316	return ref_field;
2317	}
2318
2319	static struct ftrace_event_field *
2320	parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2321	char *field_str, unsigned long *flags, unsigned long *buckets)
2322	{
2323	struct ftrace_event_field *field = NULL;
2324	char *field_name, *modifier, *str;
2325	struct trace_array *tr = file->tr;
2326
2327	modifier = str = kstrdup(s: field_str, GFP_KERNEL);
2328	if (!modifier)
2329	return ERR_PTR(error: -ENOMEM);
2330
2331	field_name = strsep(&modifier, ".");
2332	if (modifier) {
2333	if (strcmp(modifier, "hex") == 0)
2334	*flags |= HIST_FIELD_FL_HEX;
2335	else if (strcmp(modifier, "sym") == 0)
2336	*flags |= HIST_FIELD_FL_SYM;
2337	/*
2338	* 'sym-offset' occurrences in the trigger string are modified
2339	* to 'symXoffset' to simplify arithmetic expression parsing.
2340	*/
2341	else if (strcmp(modifier, "symXoffset") == 0)
2342	*flags |= HIST_FIELD_FL_SYM_OFFSET;
2343	else if ((strcmp(modifier, "execname") == 0) &&
2344	(strcmp(field_name, "common_pid") == 0))
2345	*flags |= HIST_FIELD_FL_EXECNAME;
2346	else if (strcmp(modifier, "syscall") == 0)
2347	*flags |= HIST_FIELD_FL_SYSCALL;
2348	else if (strcmp(modifier, "stacktrace") == 0)
2349	*flags |= HIST_FIELD_FL_STACKTRACE;
2350	else if (strcmp(modifier, "log2") == 0)
2351	*flags |= HIST_FIELD_FL_LOG2;
2352	else if (strcmp(modifier, "usecs") == 0)
2353	*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2354	else if (strncmp(modifier, "bucket", 6) == 0) {
2355	int ret;
2356
2357	modifier += 6;
2358
2359	if (*modifier == 's')
2360	modifier++;
2361	if (*modifier != '=')
2362	goto error;
2363	modifier++;
2364	ret = kstrtoul(s: modifier, base: 0, res: buckets);
2365	if (ret || !(*buckets))
2366	goto error;
2367	*flags |= HIST_FIELD_FL_BUCKET;
2368	} else if (strncmp(modifier, "percent", 7) == 0) {
2369	if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2370	goto error;
2371	*flags |= HIST_FIELD_FL_PERCENT;
2372	} else if (strncmp(modifier, "graph", 5) == 0) {
2373	if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2374	goto error;
2375	*flags |= HIST_FIELD_FL_GRAPH;
2376	} else {
2377	error:
2378	hist_err(tr, err_type: HIST_ERR_BAD_FIELD_MODIFIER, err_pos: errpos(str: modifier));
2379	field = ERR_PTR(error: -EINVAL);
2380	goto out;
2381	}
2382	}
2383
2384	if (strcmp(field_name, "common_timestamp") == 0) {
2385	*flags |= HIST_FIELD_FL_TIMESTAMP;
2386	hist_data->enable_timestamps = true;
2387	if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2388	hist_data->attrs->ts_in_usecs = true;
2389	} else if (strcmp(field_name, "common_stacktrace") == 0) {
2390	*flags |= HIST_FIELD_FL_STACKTRACE;
2391	} else if (strcmp(field_name, "common_cpu") == 0) {
2392	*flags |= HIST_FIELD_FL_CPU;
2393	} else if (strcmp(field_name, "common_comm") == 0) {
2394	*flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING;
2395	} else if (strcmp(field_name, "hitcount") == 0)
2396	*flags |= HIST_FIELD_FL_HITCOUNT;
2397	else {
2398	field = trace_find_event_field(call: file->event_call, name: field_name);
2399	if (!field || !field->size) {
2400	/*
2401	* For backward compatibility, if field_name
2402	* was "cpu" or "stacktrace", then we treat this
2403	* the same as common_cpu and common_stacktrace
2404	* respectively. This also works for "CPU", and
2405	* "STACKTRACE".
2406	*/
2407	if (field && field->filter_type == FILTER_CPU) {
2408	*flags |= HIST_FIELD_FL_CPU;
2409	} else if (field && field->filter_type == FILTER_STACKTRACE) {
2410	*flags |= HIST_FIELD_FL_STACKTRACE;
2411	} else if (field && field->filter_type == FILTER_COMM) {
2412	*flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING;
2413	} else {
2414	hist_err(tr, err_type: HIST_ERR_FIELD_NOT_FOUND,
2415	err_pos: errpos(str: field_name));
2416	field = ERR_PTR(error: -EINVAL);
2417	goto out;
2418	}
2419	}
2420	}
2421	out:
2422	kfree(objp: str);
2423
2424	return field;
2425	}
2426
2427	static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2428	struct hist_field *var_ref,
2429	char *var_name)
2430	{
2431	struct hist_field *alias = NULL;
2432	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2433
2434	alias = create_hist_field(hist_data, NULL, flags, var_name);
2435	if (!alias)
2436	return NULL;
2437
2438	alias->fn_num = var_ref->fn_num;
2439	alias->operands[0] = var_ref;
2440
2441	if (init_var_ref(ref_field: alias, var_field: var_ref, system: var_ref->system, event_name: var_ref->event_name)) {
2442	destroy_hist_field(hist_field: alias, level: 0);
2443	return NULL;
2444	}
2445
2446	alias->var_ref_idx = var_ref->var_ref_idx;
2447
2448	return alias;
2449	}
2450
2451	static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2452	char *str, char *var_name,
2453	unsigned long *flags)
2454	{
2455	struct trace_array *tr = hist_data->event_file->tr;
2456	struct hist_field *field = NULL;
2457	u64 constant;
2458
2459	if (kstrtoull(s: str, base: 0, res: &constant)) {
2460	hist_err(tr, err_type: HIST_ERR_EXPECT_NUMBER, err_pos: errpos(str));
2461	return NULL;
2462	}
2463
2464	*flags |= HIST_FIELD_FL_CONST;
2465	field = create_hist_field(hist_data, NULL, flags: *flags, var_name);
2466	if (!field)
2467	return NULL;
2468
2469	field->constant = constant;
2470
2471	return field;
2472	}
2473
2474	static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2475	struct trace_event_file *file, char *str,
2476	unsigned long *flags, char *var_name)
2477	{
2478	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2479	struct ftrace_event_field *field = NULL;
2480	struct hist_field *hist_field = NULL;
2481	unsigned long buckets = 0;
2482	int ret = 0;
2483
2484	if (isdigit(c: str[0])) {
2485	hist_field = parse_const(hist_data, str, var_name, flags);
2486	if (!hist_field) {
2487	ret = -EINVAL;
2488	goto out;
2489	}
2490	return hist_field;
2491	}
2492
2493	s = strchr(str, '.');
2494	if (s) {
2495	s = strchr(++s, '.');
2496	if (s) {
2497	ref_system = strsep(&str, ".");
2498	if (!str) {
2499	ret = -EINVAL;
2500	goto out;
2501	}
2502	ref_event = strsep(&str, ".");
2503	if (!str) {
2504	ret = -EINVAL;
2505	goto out;
2506	}
2507	ref_var = str;
2508	}
2509	}
2510
2511	s = local_field_var_ref(hist_data, system: ref_system, event_name: ref_event, var_name: ref_var);
2512	if (!s) {
2513	hist_field = parse_var_ref(hist_data, system: ref_system,
2514	event_name: ref_event, var_name: ref_var);
2515	if (hist_field) {
2516	if (var_name) {
2517	hist_field = create_alias(hist_data, var_ref: hist_field, var_name);
2518	if (!hist_field) {
2519	ret = -ENOMEM;
2520	goto out;
2521	}
2522	}
2523	return hist_field;
2524	}
2525	} else
2526	str = s;
2527
2528	field = parse_field(hist_data, file, field_str: str, flags, buckets: &buckets);
2529	if (IS_ERR(ptr: field)) {
2530	ret = PTR_ERR(ptr: field);
2531	goto out;
2532	}
2533
2534	hist_field = create_hist_field(hist_data, field, flags: *flags, var_name);
2535	if (!hist_field) {
2536	ret = -ENOMEM;
2537	goto out;
2538	}
2539	hist_field->buckets = buckets;
2540
2541	return hist_field;
2542	out:
2543	return ERR_PTR(error: ret);
2544	}
2545
2546	static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2547	struct trace_event_file *file,
2548	char *str, unsigned long flags,
2549	char *var_name, unsigned int *n_subexprs);
2550
2551	static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2552	struct trace_event_file *file,
2553	char *str, unsigned long flags,
2554	char *var_name, unsigned int *n_subexprs)
2555	{
2556	struct hist_field *operand1, *expr = NULL;
2557	unsigned long operand_flags;
2558	int ret = 0;
2559	char *s;
2560
2561	/* Unary minus operator, increment n_subexprs */
2562	++*n_subexprs;
2563
2564	/* we support only -(xxx) i.e. explicit parens required */
2565
2566	if (*n_subexprs > 3) {
2567	hist_err(tr: file->tr, err_type: HIST_ERR_TOO_MANY_SUBEXPR, err_pos: errpos(str));
2568	ret = -EINVAL;
2569	goto free;
2570	}
2571
2572	str++; /* skip leading '-' */
2573
2574	s = strchr(str, '(');
2575	if (s)
2576	str++;
2577	else {
2578	ret = -EINVAL;
2579	goto free;
2580	}
2581
2582	s = strrchr(str, ')');
2583	if (s) {
2584	/* unary minus not supported in sub-expressions */
2585	if (*(s+1) != '\0') {
2586	hist_err(tr: file->tr, err_type: HIST_ERR_UNARY_MINUS_SUBEXPR,
2587	err_pos: errpos(str));
2588	ret = -EINVAL;
2589	goto free;
2590	}
2591	*s = '\0';
2592	}
2593	else {
2594	ret = -EINVAL; /* no closing ')' */
2595	goto free;
2596	}
2597
2598	flags |= HIST_FIELD_FL_EXPR;
2599	expr = create_hist_field(hist_data, NULL, flags, var_name);
2600	if (!expr) {
2601	ret = -ENOMEM;
2602	goto free;
2603	}
2604
2605	operand_flags = 0;
2606	operand1 = parse_expr(hist_data, file, str, flags: operand_flags, NULL, n_subexprs);
2607	if (IS_ERR(ptr: operand1)) {
2608	ret = PTR_ERR(ptr: operand1);
2609	goto free;
2610	}
2611	if (operand1->flags & HIST_FIELD_FL_STRING) {
2612	/* String type can not be the operand of unary operator. */
2613	hist_err(tr: file->tr, err_type: HIST_ERR_INVALID_STR_OPERAND, err_pos: errpos(str));
2614	destroy_hist_field(hist_field: operand1, level: 0);
2615	ret = -EINVAL;
2616	goto free;
2617	}
2618
2619	expr->flags |= operand1->flags &
2620	(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2621	expr->fn_num = HIST_FIELD_FN_UMINUS;
2622	expr->operands[0] = operand1;
2623	expr->size = operand1->size;
2624	expr->is_signed = operand1->is_signed;
2625	expr->operator = FIELD_OP_UNARY_MINUS;
2626	expr->name = expr_str(field: expr, level: 0);
2627	expr->type = kstrdup_const(s: operand1->type, GFP_KERNEL);
2628	if (!expr->type) {
2629	ret = -ENOMEM;
2630	goto free;
2631	}
2632
2633	return expr;
2634	free:
2635	destroy_hist_field(hist_field: expr, level: 0);
2636	return ERR_PTR(error: ret);
2637	}
2638
2639	/*
2640	* If the operands are var refs, return pointers the
2641	* variable(s) referenced in var1 and var2, else NULL.
2642	*/
2643	static int check_expr_operands(struct trace_array *tr,
2644	struct hist_field *operand1,
2645	struct hist_field *operand2,
2646	struct hist_field **var1,
2647	struct hist_field **var2)
2648	{
2649	unsigned long operand1_flags = operand1->flags;
2650	unsigned long operand2_flags = operand2->flags;
2651
2652	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2653	(operand1_flags & HIST_FIELD_FL_ALIAS)) {
2654	struct hist_field *var;
2655
2656	var = find_var_field(hist_data: operand1->var.hist_data, var_name: operand1->name);
2657	if (!var)
2658	return -EINVAL;
2659	operand1_flags = var->flags;
2660	*var1 = var;
2661	}
2662
2663	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2664	(operand2_flags & HIST_FIELD_FL_ALIAS)) {
2665	struct hist_field *var;
2666
2667	var = find_var_field(hist_data: operand2->var.hist_data, var_name: operand2->name);
2668	if (!var)
2669	return -EINVAL;
2670	operand2_flags = var->flags;
2671	*var2 = var;
2672	}
2673
2674	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2675	(operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2676	hist_err(tr, err_type: HIST_ERR_TIMESTAMP_MISMATCH, err_pos: 0);
2677	return -EINVAL;
2678	}
2679
2680	return 0;
2681	}
2682
2683	static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2684	struct trace_event_file *file,
2685	char *str, unsigned long flags,
2686	char *var_name, unsigned int *n_subexprs)
2687	{
2688	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2689	struct hist_field *var1 = NULL, *var2 = NULL;
2690	unsigned long operand_flags, operand2_flags;
2691	int field_op, ret = -EINVAL;
2692	char *sep, *operand1_str;
2693	enum hist_field_fn op_fn;
2694	bool combine_consts;
2695
2696	if (*n_subexprs > 3) {
2697	hist_err(tr: file->tr, err_type: HIST_ERR_TOO_MANY_SUBEXPR, err_pos: errpos(str));
2698	return ERR_PTR(error: -EINVAL);
2699	}
2700
2701	field_op = contains_operator(str, sep: &sep);
2702
2703	if (field_op == FIELD_OP_NONE)
2704	return parse_atom(hist_data, file, str, flags: &flags, var_name);
2705
2706	if (field_op == FIELD_OP_UNARY_MINUS)
2707	return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2708
2709	/* Binary operator found, increment n_subexprs */
2710	++*n_subexprs;
2711
2712	/* Split the expression string at the root operator */
2713	if (!sep)
2714	return ERR_PTR(error: -EINVAL);
2715
2716	*sep = '\0';
2717	operand1_str = str;
2718	str = sep+1;
2719
2720	/* Binary operator requires both operands */
2721	if (*operand1_str == '\0' || *str == '\0')
2722	return ERR_PTR(error: -EINVAL);
2723
2724	operand_flags = 0;
2725
2726	/* LHS of string is an expression e.g. a+b in a+b+c */
2727	operand1 = parse_expr(hist_data, file, str: operand1_str, flags: operand_flags, NULL, n_subexprs);
2728	if (IS_ERR(ptr: operand1))
2729	return ERR_CAST(ptr: operand1);
2730
2731	if (operand1->flags & HIST_FIELD_FL_STRING) {
2732	hist_err(tr: file->tr, err_type: HIST_ERR_INVALID_STR_OPERAND, err_pos: errpos(str: operand1_str));
2733	ret = -EINVAL;
2734	goto free_op1;
2735	}
2736
2737	/* RHS of string is another expression e.g. c in a+b+c */
2738	operand_flags = 0;
2739	operand2 = parse_expr(hist_data, file, str, flags: operand_flags, NULL, n_subexprs);
2740	if (IS_ERR(ptr: operand2)) {
2741	ret = PTR_ERR(ptr: operand2);
2742	goto free_op1;
2743	}
2744	if (operand2->flags & HIST_FIELD_FL_STRING) {
2745	hist_err(tr: file->tr, err_type: HIST_ERR_INVALID_STR_OPERAND, err_pos: errpos(str));
2746	ret = -EINVAL;
2747	goto free_operands;
2748	}
2749
2750	switch (field_op) {
2751	case FIELD_OP_MINUS:
2752	op_fn = HIST_FIELD_FN_MINUS;
2753	break;
2754	case FIELD_OP_PLUS:
2755	op_fn = HIST_FIELD_FN_PLUS;
2756	break;
2757	case FIELD_OP_DIV:
2758	op_fn = HIST_FIELD_FN_DIV;
2759	break;
2760	case FIELD_OP_MULT:
2761	op_fn = HIST_FIELD_FN_MULT;
2762	break;
2763	default:
2764	ret = -EINVAL;
2765	goto free_operands;
2766	}
2767
2768	ret = check_expr_operands(tr: file->tr, operand1, operand2, var1: &var1, var2: &var2);
2769	if (ret)
2770	goto free_operands;
2771
2772	operand_flags = var1 ? var1->flags : operand1->flags;
2773	operand2_flags = var2 ? var2->flags : operand2->flags;
2774
2775	/*
2776	* If both operands are constant, the expression can be
2777	* collapsed to a single constant.
2778	*/
2779	combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2780
2781	flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2782
2783	flags |= operand1->flags &
2784	(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2785
2786	expr = create_hist_field(hist_data, NULL, flags, var_name);
2787	if (!expr) {
2788	ret = -ENOMEM;
2789	goto free_operands;
2790	}
2791
2792	operand1->read_once = true;
2793	operand2->read_once = true;
2794
2795	/* The operands are now owned and free'd by 'expr' */
2796	expr->operands[0] = operand1;
2797	expr->operands[1] = operand2;
2798
2799	if (field_op == FIELD_OP_DIV &&
2800	operand2_flags & HIST_FIELD_FL_CONST) {
2801	u64 divisor = var2 ? var2->constant : operand2->constant;
2802
2803	if (!divisor) {
2804	hist_err(tr: file->tr, err_type: HIST_ERR_DIVISION_BY_ZERO, err_pos: errpos(str));
2805	ret = -EDOM;
2806	goto free_expr;
2807	}
2808
2809	/*
2810	* Copy the divisor here so we don't have to look it up
2811	* later if this is a var ref
2812	*/
2813	operand2->constant = divisor;
2814	op_fn = hist_field_get_div_fn(divisor: operand2);
2815	}
2816
2817	expr->fn_num = op_fn;
2818
2819	if (combine_consts) {
2820	if (var1)
2821	expr->operands[0] = var1;
2822	if (var2)
2823	expr->operands[1] = var2;
2824
2825	expr->constant = hist_fn_call(hist_field: expr, NULL, NULL, NULL, NULL);
2826	expr->fn_num = HIST_FIELD_FN_CONST;
2827
2828	expr->operands[0] = NULL;
2829	expr->operands[1] = NULL;
2830
2831	/*
2832	* var refs won't be destroyed immediately
2833	* See: destroy_hist_field()
2834	*/
2835	destroy_hist_field(hist_field: operand2, level: 0);
2836	destroy_hist_field(hist_field: operand1, level: 0);
2837
2838	expr->name = expr_str(field: expr, level: 0);
2839	} else {
2840	/* The operand sizes should be the same, so just pick one */
2841	expr->size = operand1->size;
2842	expr->is_signed = operand1->is_signed;
2843
2844	expr->operator = field_op;
2845	expr->type = kstrdup_const(s: operand1->type, GFP_KERNEL);
2846	if (!expr->type) {
2847	ret = -ENOMEM;
2848	goto free_expr;
2849	}
2850
2851	expr->name = expr_str(field: expr, level: 0);
2852	}
2853
2854	return expr;
2855
2856	free_operands:
2857	destroy_hist_field(hist_field: operand2, level: 0);
2858	free_op1:
2859	destroy_hist_field(hist_field: operand1, level: 0);
2860	return ERR_PTR(error: ret);
2861
2862	free_expr:
2863	destroy_hist_field(hist_field: expr, level: 0);
2864	return ERR_PTR(error: ret);
2865	}
2866
2867	static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2868	struct trace_event_file *file)
2869	{
2870	struct event_trigger_data *test;
2871
2872	lockdep_assert_held(&event_mutex);
2873
2874	list_for_each_entry(test, &file->triggers, list) {
2875	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2876	if (test->private_data == hist_data)
2877	return test->filter_str;
2878	}
2879	}
2880
2881	return NULL;
2882	}
2883
2884	static struct event_command trigger_hist_cmd;
2885	static int event_hist_trigger_parse(struct event_command *cmd_ops,
2886	struct trace_event_file *file,
2887	char *glob, char *cmd,
2888	char *param_and_filter);
2889
2890	static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2891	struct hist_trigger_data *hist_data,
2892	unsigned int n_keys)
2893	{
2894	struct hist_field *target_hist_field, *hist_field;
2895	unsigned int n, i, j;
2896
2897	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2898	return false;
2899
2900	i = hist_data->n_vals;
2901	j = target_hist_data->n_vals;
2902
2903	for (n = 0; n < n_keys; n++) {
2904	hist_field = hist_data->fields[i + n];
2905	target_hist_field = target_hist_data->fields[j + n];
2906
2907	if (strcmp(hist_field->type, target_hist_field->type) != 0)
2908	return false;
2909	if (hist_field->size != target_hist_field->size)
2910	return false;
2911	if (hist_field->is_signed != target_hist_field->is_signed)
2912	return false;
2913	}
2914
2915	return true;
2916	}
2917
2918	static struct hist_trigger_data *
2919	find_compatible_hist(struct hist_trigger_data *target_hist_data,
2920	struct trace_event_file *file)
2921	{
2922	struct hist_trigger_data *hist_data;
2923	struct event_trigger_data *test;
2924	unsigned int n_keys;
2925
2926	lockdep_assert_held(&event_mutex);
2927
2928	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2929
2930	list_for_each_entry(test, &file->triggers, list) {
2931	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2932	hist_data = test->private_data;
2933
2934	if (compatible_keys(target_hist_data, hist_data, n_keys))
2935	return hist_data;
2936	}
2937	}
2938
2939	return NULL;
2940	}
2941
2942	static struct trace_event_file *event_file(struct trace_array *tr,
2943	char *system, char *event_name)
2944	{
2945	struct trace_event_file *file;
2946
2947	file = __find_event_file(tr, system, event: event_name);
2948	if (!file)
2949	return ERR_PTR(error: -EINVAL);
2950
2951	return file;
2952	}
2953
2954	static struct hist_field *
2955	find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2956	char *system, char *event_name, char *field_name)
2957	{
2958	struct hist_field *event_var;
2959	char *synthetic_name;
2960
2961	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2962	if (!synthetic_name)
2963	return ERR_PTR(error: -ENOMEM);
2964
2965	strcpy(p: synthetic_name, q: "synthetic_");
2966	strcat(p: synthetic_name, q: field_name);
2967
2968	event_var = find_event_var(hist_data: target_hist_data, system, event_name, var_name: synthetic_name);
2969
2970	kfree(objp: synthetic_name);
2971
2972	return event_var;
2973	}
2974
2975	/**
2976	* create_field_var_hist - Automatically create a histogram and var for a field
2977	* @target_hist_data: The target hist trigger
2978	* @subsys_name: Optional subsystem name
2979	* @event_name: Optional event name
2980	* @field_name: The name of the field (and the resulting variable)
2981	*
2982	* Hist trigger actions fetch data from variables, not directly from
2983	* events. However, for convenience, users are allowed to directly
2984	* specify an event field in an action, which will be automatically
2985	* converted into a variable on their behalf.
2986	*
2987	* If a user specifies a field on an event that isn't the event the
2988	* histogram currently being defined (the target event histogram), the
2989	* only way that can be accomplished is if a new hist trigger is
2990	* created and the field variable defined on that.
2991	*
2992	* This function creates a new histogram compatible with the target
2993	* event (meaning a histogram with the same key as the target
2994	* histogram), and creates a variable for the specified field, but
2995	* with 'synthetic_' prepended to the variable name in order to avoid
2996	* collision with normal field variables.
2997	*
2998	* Return: The variable created for the field.
2999	*/
3000	static struct hist_field *
3001	create_field_var_hist(struct hist_trigger_data *target_hist_data,
3002	char *subsys_name, char *event_name, char *field_name)
3003	{
3004	struct trace_array *tr = target_hist_data->event_file->tr;
3005	struct hist_trigger_data *hist_data;
3006	unsigned int i, n, first = true;
3007	struct field_var_hist *var_hist;
3008	struct trace_event_file *file;
3009	struct hist_field *key_field;
3010	struct hist_field *event_var;
3011	char *saved_filter;
3012	char *cmd;
3013	int ret;
3014
3015	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3016	hist_err(tr, err_type: HIST_ERR_TOO_MANY_FIELD_VARS, err_pos: errpos(str: field_name));
3017	return ERR_PTR(error: -EINVAL);
3018	}
3019
3020	file = event_file(tr, system: subsys_name, event_name);
3021
3022	if (IS_ERR(ptr: file)) {
3023	hist_err(tr, err_type: HIST_ERR_EVENT_FILE_NOT_FOUND, err_pos: errpos(str: field_name));
3024	ret = PTR_ERR(ptr: file);
3025	return ERR_PTR(error: ret);
3026	}
3027
3028	/*
3029	* Look for a histogram compatible with target. We'll use the
3030	* found histogram specification to create a new matching
3031	* histogram with our variable on it. target_hist_data is not
3032	* yet a registered histogram so we can't use that.
3033	*/
3034	hist_data = find_compatible_hist(target_hist_data, file);
3035	if (!hist_data) {
3036	hist_err(tr, err_type: HIST_ERR_HIST_NOT_FOUND, err_pos: errpos(str: field_name));
3037	return ERR_PTR(error: -EINVAL);
3038	}
3039
3040	/* See if a synthetic field variable has already been created */
3041	event_var = find_synthetic_field_var(target_hist_data, system: subsys_name,
3042	event_name, field_name);
3043	if (!IS_ERR_OR_NULL(ptr: event_var))
3044	return event_var;
3045
3046	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3047	if (!var_hist)
3048	return ERR_PTR(error: -ENOMEM);
3049
3050	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3051	if (!cmd) {
3052	kfree(objp: var_hist);
3053	return ERR_PTR(error: -ENOMEM);
3054	}
3055
3056	/* Use the same keys as the compatible histogram */
3057	strcat(p: cmd, q: "keys=");
3058
3059	for_each_hist_key_field(i, hist_data) {
3060	key_field = hist_data->fields[i];
3061	if (!first)
3062	strcat(p: cmd, q: ",");
3063	strcat(p: cmd, q: key_field->field->name);
3064	first = false;
3065	}
3066
3067	/* Create the synthetic field variable specification */
3068	strcat(p: cmd, q: ":synthetic_");
3069	strcat(p: cmd, q: field_name);
3070	strcat(p: cmd, q: "=");
3071	strcat(p: cmd, q: field_name);
3072
3073	/* Use the same filter as the compatible histogram */
3074	saved_filter = find_trigger_filter(hist_data, file);
3075	if (saved_filter) {
3076	strcat(p: cmd, q: " if ");
3077	strcat(p: cmd, q: saved_filter);
3078	}
3079
3080	var_hist->cmd = kstrdup(s: cmd, GFP_KERNEL);
3081	if (!var_hist->cmd) {
3082	kfree(objp: cmd);
3083	kfree(objp: var_hist);
3084	return ERR_PTR(error: -ENOMEM);
3085	}
3086
3087	/* Save the compatible histogram information */
3088	var_hist->hist_data = hist_data;
3089
3090	/* Create the new histogram with our variable */
3091	ret = event_hist_trigger_parse(cmd_ops: &trigger_hist_cmd, file,
3092	glob: "", cmd: "hist", param_and_filter: cmd);
3093	if (ret) {
3094	kfree(objp: cmd);
3095	kfree(objp: var_hist->cmd);
3096	kfree(objp: var_hist);
3097	hist_err(tr, err_type: HIST_ERR_HIST_CREATE_FAIL, err_pos: errpos(str: field_name));
3098	return ERR_PTR(error: ret);
3099	}
3100
3101	kfree(objp: cmd);
3102
3103	/* If we can't find the variable, something went wrong */
3104	event_var = find_synthetic_field_var(target_hist_data, system: subsys_name,
3105	event_name, field_name);
3106	if (IS_ERR_OR_NULL(ptr: event_var)) {
3107	kfree(objp: var_hist->cmd);
3108	kfree(objp: var_hist);
3109	hist_err(tr, err_type: HIST_ERR_SYNTH_VAR_NOT_FOUND, err_pos: errpos(str: field_name));
3110	return ERR_PTR(error: -EINVAL);
3111	}
3112
3113	n = target_hist_data->n_field_var_hists;
3114	target_hist_data->field_var_hists[n] = var_hist;
3115	target_hist_data->n_field_var_hists++;
3116
3117	return event_var;
3118	}
3119
3120	static struct hist_field *
3121	find_target_event_var(struct hist_trigger_data *hist_data,
3122	char *subsys_name, char *event_name, char *var_name)
3123	{
3124	struct trace_event_file *file = hist_data->event_file;
3125	struct hist_field *hist_field = NULL;
3126
3127	if (subsys_name) {
3128	struct trace_event_call *call;
3129
3130	if (!event_name)
3131	return NULL;
3132
3133	call = file->event_call;
3134
3135	if (strcmp(subsys_name, call->class->system) != 0)
3136	return NULL;
3137
3138	if (strcmp(event_name, trace_event_name(call)) != 0)
3139	return NULL;
3140	}
3141
3142	hist_field = find_var_field(hist_data, var_name);
3143
3144	return hist_field;
3145	}
3146
3147	static inline void __update_field_vars(struct tracing_map_elt *elt,
3148	struct trace_buffer *buffer,
3149	struct ring_buffer_event *rbe,
3150	void *rec,
3151	struct field_var **field_vars,
3152	unsigned int n_field_vars,
3153	unsigned int field_var_str_start)
3154	{
3155	struct hist_elt_data *elt_data = elt->private_data;
3156	unsigned int i, j, var_idx;
3157	u64 var_val;
3158
3159	/* Make sure stacktrace can fit in the string variable length */
3160	BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX);
3161
3162	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3163	struct field_var *field_var = field_vars[i];
3164	struct hist_field *var = field_var->var;
3165	struct hist_field *val = field_var->val;
3166
3167	var_val = hist_fn_call(hist_field: val, elt, buffer, rbe, event: rec);
3168	var_idx = var->var.idx;
3169
3170	if (val->flags & (HIST_FIELD_FL_STRING |
3171	HIST_FIELD_FL_STACKTRACE)) {
3172	char *str = elt_data->field_var_str[j++];
3173	char *val_str = (char *)(uintptr_t)var_val;
3174	unsigned int size;
3175
3176	if (val->flags & HIST_FIELD_FL_STRING) {
3177	size = min(val->size, STR_VAR_LEN_MAX);
3178	strscpy(str, val_str, size);
3179	} else {
3180	char *stack_start = str + sizeof(unsigned long);
3181	int e;
3182
3183	e = stack_trace_save(store: (void *)stack_start,
3184	HIST_STACKTRACE_DEPTH,
3185	HIST_STACKTRACE_SKIP);
3186	if (e < HIST_STACKTRACE_DEPTH - 1)
3187	((unsigned long *)stack_start)[e] = 0;
3188	*((unsigned long *)str) = e;
3189	}
3190	var_val = (u64)(uintptr_t)str;
3191	}
3192	tracing_map_set_var(elt, i: var_idx, n: var_val);
3193	}
3194	}
3195
3196	static void update_field_vars(struct hist_trigger_data *hist_data,
3197	struct tracing_map_elt *elt,
3198	struct trace_buffer *buffer,
3199	struct ring_buffer_event *rbe,
3200	void *rec)
3201	{
3202	__update_field_vars(elt, buffer, rbe, rec, field_vars: hist_data->field_vars,
3203	n_field_vars: hist_data->n_field_vars, field_var_str_start: 0);
3204	}
3205
3206	static void save_track_data_vars(struct hist_trigger_data *hist_data,
3207	struct tracing_map_elt *elt,
3208	struct trace_buffer *buffer, void *rec,
3209	struct ring_buffer_event *rbe, void *key,
3210	struct action_data *data, u64 *var_ref_vals)
3211	{
3212	__update_field_vars(elt, buffer, rbe, rec, field_vars: hist_data->save_vars,
3213	n_field_vars: hist_data->n_save_vars, field_var_str_start: hist_data->n_field_var_str);
3214	}
3215
3216	static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3217	struct trace_event_file *file,
3218	char *name, int size, const char *type)
3219	{
3220	struct hist_field *var;
3221	int idx;
3222
3223	if (find_var(hist_data, file, var_name: name) && !hist_data->remove) {
3224	var = ERR_PTR(error: -EINVAL);
3225	goto out;
3226	}
3227
3228	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3229	if (!var) {
3230	var = ERR_PTR(error: -ENOMEM);
3231	goto out;
3232	}
3233
3234	idx = tracing_map_add_var(map: hist_data->map);
3235	if (idx < 0) {
3236	kfree(objp: var);
3237	var = ERR_PTR(error: -EINVAL);
3238	goto out;
3239	}
3240
3241	var->ref = 1;
3242	var->flags = HIST_FIELD_FL_VAR;
3243	var->var.idx = idx;
3244	var->var.hist_data = var->hist_data = hist_data;
3245	var->size = size;
3246	var->var.name = kstrdup(s: name, GFP_KERNEL);
3247	var->type = kstrdup_const(s: type, GFP_KERNEL);
3248	if (!var->var.name || !var->type) {
3249	kfree_const(x: var->type);
3250	kfree(objp: var->var.name);
3251	kfree(objp: var);
3252	var = ERR_PTR(error: -ENOMEM);
3253	}
3254	out:
3255	return var;
3256	}
3257
3258	static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3259	struct trace_event_file *file,
3260	char *field_name)
3261	{
3262	struct hist_field *val = NULL, *var = NULL;
3263	unsigned long flags = HIST_FIELD_FL_VAR;
3264	struct trace_array *tr = file->tr;
3265	struct field_var *field_var;
3266	int ret = 0;
3267
3268	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3269	hist_err(tr, err_type: HIST_ERR_TOO_MANY_FIELD_VARS, err_pos: errpos(str: field_name));
3270	ret = -EINVAL;
3271	goto err;
3272	}
3273
3274	val = parse_atom(hist_data, file, str: field_name, flags: &flags, NULL);
3275	if (IS_ERR(ptr: val)) {
3276	hist_err(tr, err_type: HIST_ERR_FIELD_VAR_PARSE_FAIL, err_pos: errpos(str: field_name));
3277	ret = PTR_ERR(ptr: val);
3278	goto err;
3279	}
3280
3281	var = create_var(hist_data, file, name: field_name, size: val->size, type: val->type);
3282	if (IS_ERR(ptr: var)) {
3283	hist_err(tr, err_type: HIST_ERR_VAR_CREATE_FIND_FAIL, err_pos: errpos(str: field_name));
3284	destroy_hist_field(hist_field: val, level: 0);
3285	ret = PTR_ERR(ptr: var);
3286	goto err;
3287	}
3288
3289	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3290	if (!field_var) {
3291	destroy_hist_field(hist_field: val, level: 0);
3292	kfree_const(x: var->type);
3293	kfree(objp: var->var.name);
3294	kfree(objp: var);
3295	ret = -ENOMEM;
3296	goto err;
3297	}
3298
3299	field_var->var = var;
3300	field_var->val = val;
3301	out:
3302	return field_var;
3303	err:
3304	field_var = ERR_PTR(error: ret);
3305	goto out;
3306	}
3307
3308	/**
3309	* create_target_field_var - Automatically create a variable for a field
3310	* @target_hist_data: The target hist trigger
3311	* @subsys_name: Optional subsystem name
3312	* @event_name: Optional event name
3313	* @var_name: The name of the field (and the resulting variable)
3314	*
3315	* Hist trigger actions fetch data from variables, not directly from
3316	* events. However, for convenience, users are allowed to directly
3317	* specify an event field in an action, which will be automatically
3318	* converted into a variable on their behalf.
3319	*
3320	* This function creates a field variable with the name var_name on
3321	* the hist trigger currently being defined on the target event. If
3322	* subsys_name and event_name are specified, this function simply
3323	* verifies that they do in fact match the target event subsystem and
3324	* event name.
3325	*
3326	* Return: The variable created for the field.
3327	*/
3328	static struct field_var *
3329	create_target_field_var(struct hist_trigger_data *target_hist_data,
3330	char *subsys_name, char *event_name, char *var_name)
3331	{
3332	struct trace_event_file *file = target_hist_data->event_file;
3333
3334	if (subsys_name) {
3335	struct trace_event_call *call;
3336
3337	if (!event_name)
3338	return NULL;
3339
3340	call = file->event_call;
3341
3342	if (strcmp(subsys_name, call->class->system) != 0)
3343	return NULL;
3344
3345	if (strcmp(event_name, trace_event_name(call)) != 0)
3346	return NULL;
3347	}
3348
3349	return create_field_var(hist_data: target_hist_data, file, field_name: var_name);
3350	}
3351
3352	static bool check_track_val_max(u64 track_val, u64 var_val)
3353	{
3354	if (var_val <= track_val)
3355	return false;
3356
3357	return true;
3358	}
3359
3360	static bool check_track_val_changed(u64 track_val, u64 var_val)
3361	{
3362	if (var_val == track_val)
3363	return false;
3364
3365	return true;
3366	}
3367
3368	static u64 get_track_val(struct hist_trigger_data *hist_data,
3369	struct tracing_map_elt *elt,
3370	struct action_data *data)
3371	{
3372	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3373	u64 track_val;
3374
3375	track_val = tracing_map_read_var(elt, i: track_var_idx);
3376
3377	return track_val;
3378	}
3379
3380	static void save_track_val(struct hist_trigger_data *hist_data,
3381	struct tracing_map_elt *elt,
3382	struct action_data *data, u64 var_val)
3383	{
3384	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3385
3386	tracing_map_set_var(elt, i: track_var_idx, n: var_val);
3387	}
3388
3389	static void save_track_data(struct hist_trigger_data *hist_data,
3390	struct tracing_map_elt *elt,
3391	struct trace_buffer *buffer, void *rec,
3392	struct ring_buffer_event *rbe, void *key,
3393	struct action_data *data, u64 *var_ref_vals)
3394	{
3395	if (data->track_data.save_data)
3396	data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3397	key, data, var_ref_vals);
3398	}
3399
3400	static bool check_track_val(struct tracing_map_elt *elt,
3401	struct action_data *data,
3402	u64 var_val)
3403	{
3404	struct hist_trigger_data *hist_data;
3405	u64 track_val;
3406
3407	hist_data = data->track_data.track_var->hist_data;
3408	track_val = get_track_val(hist_data, elt, data);
3409
3410	return data->track_data.check_val(track_val, var_val);
3411	}
3412
3413	#ifdef CONFIG_TRACER_SNAPSHOT
3414	static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3415	{
3416	/* called with tr->max_lock held */
3417	struct track_data *track_data = tr->cond_snapshot->cond_data;
3418	struct hist_elt_data *elt_data, *track_elt_data;
3419	struct snapshot_context *context = cond_data;
3420	struct action_data *action;
3421	u64 track_val;
3422
3423	if (!track_data)
3424	return false;
3425
3426	action = track_data->action_data;
3427
3428	track_val = get_track_val(hist_data: track_data->hist_data, elt: context->elt,
3429	data: track_data->action_data);
3430
3431	if (!action->track_data.check_val(track_data->track_val, track_val))
3432	return false;
3433
3434	track_data->track_val = track_val;
3435	memcpy(track_data->key, context->key, track_data->key_len);
3436
3437	elt_data = context->elt->private_data;
3438	track_elt_data = track_data->elt.private_data;
3439	if (elt_data->comm)
3440	strscpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3441
3442	track_data->updated = true;
3443
3444	return true;
3445	}
3446
3447	static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3448	struct tracing_map_elt *elt,
3449	struct trace_buffer *buffer, void *rec,
3450	struct ring_buffer_event *rbe, void *key,
3451	struct action_data *data,
3452	u64 *var_ref_vals)
3453	{
3454	struct trace_event_file *file = hist_data->event_file;
3455	struct snapshot_context context;
3456
3457	context.elt = elt;
3458	context.key = key;
3459
3460	tracing_snapshot_cond(tr: file->tr, cond_data: &context);
3461	}
3462
3463	static void hist_trigger_print_key(struct seq_file *m,
3464	struct hist_trigger_data *hist_data,
3465	void *key,
3466	struct tracing_map_elt *elt);
3467
3468	static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3469	{
3470	unsigned int i;
3471
3472	if (!hist_data->n_actions)
3473	return NULL;
3474
3475	for (i = 0; i < hist_data->n_actions; i++) {
3476	struct action_data *data = hist_data->actions[i];
3477
3478	if (data->action == ACTION_SNAPSHOT)
3479	return data;
3480	}
3481
3482	return NULL;
3483	}
3484
3485	static void track_data_snapshot_print(struct seq_file *m,
3486	struct hist_trigger_data *hist_data)
3487	{
3488	struct trace_event_file *file = hist_data->event_file;
3489	struct track_data *track_data;
3490	struct action_data *action;
3491
3492	track_data = tracing_cond_snapshot_data(tr: file->tr);
3493	if (!track_data)
3494	return;
3495
3496	if (!track_data->updated)
3497	return;
3498
3499	action = snapshot_action(hist_data);
3500	if (!action)
3501	return;
3502
3503	seq_puts(m, s: "\nSnapshot taken (see tracing/snapshot). Details:\n");
3504	seq_printf(m, fmt: "\ttriggering value { %s(%s) }: %10llu",
3505	action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3506	action->track_data.var_str, track_data->track_val);
3507
3508	seq_puts(m, s: "\ttriggered by event with key: ");
3509	hist_trigger_print_key(m, hist_data, key: track_data->key, elt: &track_data->elt);
3510	seq_putc(m, c: '\n');
3511	}
3512	#else
3513	static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3514	{
3515	return false;
3516	}
3517	static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3518	struct tracing_map_elt *elt,
3519	struct trace_buffer *buffer, void *rec,
3520	struct ring_buffer_event *rbe, void *key,
3521	struct action_data *data,
3522	u64 *var_ref_vals) {}
3523	static void track_data_snapshot_print(struct seq_file *m,
3524	struct hist_trigger_data *hist_data) {}
3525	#endif /* CONFIG_TRACER_SNAPSHOT */
3526
3527	static void track_data_print(struct seq_file *m,
3528	struct hist_trigger_data *hist_data,
3529	struct tracing_map_elt *elt,
3530	struct action_data *data)
3531	{
3532	u64 track_val = get_track_val(hist_data, elt, data);
3533	unsigned int i, save_var_idx;
3534
3535	if (data->handler == HANDLER_ONMAX)
3536	seq_printf(m, fmt: "\n\tmax: %10llu", track_val);
3537	else if (data->handler == HANDLER_ONCHANGE)
3538	seq_printf(m, fmt: "\n\tchanged: %10llu", track_val);
3539
3540	if (data->action == ACTION_SNAPSHOT)
3541	return;
3542
3543	for (i = 0; i < hist_data->n_save_vars; i++) {
3544	struct hist_field *save_val = hist_data->save_vars[i]->val;
3545	struct hist_field *save_var = hist_data->save_vars[i]->var;
3546	u64 val;
3547
3548	save_var_idx = save_var->var.idx;
3549
3550	val = tracing_map_read_var(elt, i: save_var_idx);
3551
3552	if (save_val->flags & HIST_FIELD_FL_STRING) {
3553	seq_printf(m, fmt: " %s: %-32s", save_var->var.name,
3554	(char *)(uintptr_t)(val));
3555	} else
3556	seq_printf(m, fmt: " %s: %10llu", save_var->var.name, val);
3557	}
3558	}
3559
3560	static void ontrack_action(struct hist_trigger_data *hist_data,
3561	struct tracing_map_elt *elt,
3562	struct trace_buffer *buffer, void *rec,
3563	struct ring_buffer_event *rbe, void *key,
3564	struct action_data *data, u64 *var_ref_vals)
3565	{
3566	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3567
3568	if (check_track_val(elt, data, var_val)) {
3569	save_track_val(hist_data, elt, data, var_val);
3570	save_track_data(hist_data, elt, buffer, rec, rbe,
3571	key, data, var_ref_vals);
3572	}
3573	}
3574
3575	static void action_data_destroy(struct action_data *data)
3576	{
3577	unsigned int i;
3578
3579	lockdep_assert_held(&event_mutex);
3580
3581	kfree(objp: data->action_name);
3582
3583	for (i = 0; i < data->n_params; i++)
3584	kfree(objp: data->params[i]);
3585
3586	if (data->synth_event)
3587	data->synth_event->ref--;
3588
3589	kfree(objp: data->synth_event_name);
3590
3591	kfree(objp: data);
3592	}
3593
3594	static void track_data_destroy(struct hist_trigger_data *hist_data,
3595	struct action_data *data)
3596	{
3597	struct trace_event_file *file = hist_data->event_file;
3598
3599	destroy_hist_field(hist_field: data->track_data.track_var, level: 0);
3600
3601	if (data->action == ACTION_SNAPSHOT) {
3602	struct track_data *track_data;
3603
3604	track_data = tracing_cond_snapshot_data(tr: file->tr);
3605	if (track_data && track_data->hist_data == hist_data) {
3606	tracing_snapshot_cond_disable(tr: file->tr);
3607	track_data_free(track_data);
3608	}
3609	}
3610
3611	kfree(objp: data->track_data.var_str);
3612
3613	action_data_destroy(data);
3614	}
3615
3616	static int action_create(struct hist_trigger_data *hist_data,
3617	struct action_data *data);
3618
3619	static int track_data_create(struct hist_trigger_data *hist_data,
3620	struct action_data *data)
3621	{
3622	struct hist_field *var_field, *ref_field, *track_var = NULL;
3623	struct trace_event_file *file = hist_data->event_file;
3624	struct trace_array *tr = file->tr;
3625	char *track_data_var_str;
3626	int ret = 0;
3627
3628	track_data_var_str = data->track_data.var_str;
3629	if (track_data_var_str[0] != '$') {
3630	hist_err(tr, err_type: HIST_ERR_ONX_NOT_VAR, err_pos: errpos(str: track_data_var_str));
3631	return -EINVAL;
3632	}
3633	track_data_var_str++;
3634
3635	var_field = find_target_event_var(hist_data, NULL, NULL, var_name: track_data_var_str);
3636	if (!var_field) {
3637	hist_err(tr, err_type: HIST_ERR_ONX_VAR_NOT_FOUND, err_pos: errpos(str: track_data_var_str));
3638	return -EINVAL;
3639	}
3640
3641	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3642	if (!ref_field)
3643	return -ENOMEM;
3644
3645	data->track_data.var_ref = ref_field;
3646
3647	if (data->handler == HANDLER_ONMAX)
3648	track_var = create_var(hist_data, file, name: "__max", size: sizeof(u64), type: "u64");
3649	if (IS_ERR(ptr: track_var)) {
3650	hist_err(tr, err_type: HIST_ERR_ONX_VAR_CREATE_FAIL, err_pos: 0);
3651	ret = PTR_ERR(ptr: track_var);
3652	goto out;
3653	}
3654
3655	if (data->handler == HANDLER_ONCHANGE)
3656	track_var = create_var(hist_data, file, name: "__change", size: sizeof(u64), type: "u64");
3657	if (IS_ERR(ptr: track_var)) {
3658	hist_err(tr, err_type: HIST_ERR_ONX_VAR_CREATE_FAIL, err_pos: 0);
3659	ret = PTR_ERR(ptr: track_var);
3660	goto out;
3661	}
3662	data->track_data.track_var = track_var;
3663
3664	ret = action_create(hist_data, data);
3665	out:
3666	return ret;
3667	}
3668
3669	static int parse_action_params(struct trace_array *tr, char *params,
3670	struct action_data *data)
3671	{
3672	char *param, *saved_param;
3673	bool first_param = true;
3674	int ret = 0;
3675
3676	while (params) {
3677	if (data->n_params >= SYNTH_FIELDS_MAX) {
3678	hist_err(tr, err_type: HIST_ERR_TOO_MANY_PARAMS, err_pos: 0);
3679	ret = -EINVAL;
3680	goto out;
3681	}
3682
3683	param = strsep(&params, ",");
3684	if (!param) {
3685	hist_err(tr, err_type: HIST_ERR_PARAM_NOT_FOUND, err_pos: 0);
3686	ret = -EINVAL;
3687	goto out;
3688	}
3689
3690	param = strstrip(str: param);
3691	if (strlen(param) < 2) {
3692	hist_err(tr, err_type: HIST_ERR_INVALID_PARAM, err_pos: errpos(str: param));
3693	ret = -EINVAL;
3694	goto out;
3695	}
3696
3697	saved_param = kstrdup(s: param, GFP_KERNEL);
3698	if (!saved_param) {
3699	ret = -ENOMEM;
3700	goto out;
3701	}
3702
3703	if (first_param && data->use_trace_keyword) {
3704	data->synth_event_name = saved_param;
3705	first_param = false;
3706	continue;
3707	}
3708	first_param = false;
3709
3710	data->params[data->n_params++] = saved_param;
3711	}
3712	out:
3713	return ret;
3714	}
3715
3716	static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3717	enum handler_id handler)
3718	{
3719	char *action_name;
3720	int ret = 0;
3721
3722	strsep(&str, ".");
3723	if (!str) {
3724	hist_err(tr, err_type: HIST_ERR_ACTION_NOT_FOUND, err_pos: 0);
3725	ret = -EINVAL;
3726	goto out;
3727	}
3728
3729	action_name = strsep(&str, "(");
3730	if (!action_name || !str) {
3731	hist_err(tr, err_type: HIST_ERR_ACTION_NOT_FOUND, err_pos: 0);
3732	ret = -EINVAL;
3733	goto out;
3734	}
3735
3736	if (str_has_prefix(str: action_name, prefix: "save")) {
3737	char *params = strsep(&str, ")");
3738
3739	if (!params) {
3740	hist_err(tr, err_type: HIST_ERR_NO_SAVE_PARAMS, err_pos: 0);
3741	ret = -EINVAL;
3742	goto out;
3743	}
3744
3745	ret = parse_action_params(tr, params, data);
3746	if (ret)
3747	goto out;
3748
3749	if (handler == HANDLER_ONMAX)
3750	data->track_data.check_val = check_track_val_max;
3751	else if (handler == HANDLER_ONCHANGE)
3752	data->track_data.check_val = check_track_val_changed;
3753	else {
3754	hist_err(tr, err_type: HIST_ERR_ACTION_MISMATCH, err_pos: errpos(str: action_name));
3755	ret = -EINVAL;
3756	goto out;
3757	}
3758
3759	data->track_data.save_data = save_track_data_vars;
3760	data->fn = ontrack_action;
3761	data->action = ACTION_SAVE;
3762	} else if (str_has_prefix(str: action_name, prefix: "snapshot")) {
3763	char *params = strsep(&str, ")");
3764
3765	if (!str) {
3766	hist_err(tr, err_type: HIST_ERR_NO_CLOSING_PAREN, err_pos: errpos(str: params));
3767	ret = -EINVAL;
3768	goto out;
3769	}
3770
3771	if (handler == HANDLER_ONMAX)
3772	data->track_data.check_val = check_track_val_max;
3773	else if (handler == HANDLER_ONCHANGE)
3774	data->track_data.check_val = check_track_val_changed;
3775	else {
3776	hist_err(tr, err_type: HIST_ERR_ACTION_MISMATCH, err_pos: errpos(str: action_name));
3777	ret = -EINVAL;
3778	goto out;
3779	}
3780
3781	data->track_data.save_data = save_track_data_snapshot;
3782	data->fn = ontrack_action;
3783	data->action = ACTION_SNAPSHOT;
3784	} else {
3785	char *params = strsep(&str, ")");
3786
3787	if (str_has_prefix(str: action_name, prefix: "trace"))
3788	data->use_trace_keyword = true;
3789
3790	if (params) {
3791	ret = parse_action_params(tr, params, data);
3792	if (ret)
3793	goto out;
3794	}
3795
3796	if (handler == HANDLER_ONMAX)
3797	data->track_data.check_val = check_track_val_max;
3798	else if (handler == HANDLER_ONCHANGE)
3799	data->track_data.check_val = check_track_val_changed;
3800
3801	if (handler != HANDLER_ONMATCH) {
3802	data->track_data.save_data = action_trace;
3803	data->fn = ontrack_action;
3804	} else
3805	data->fn = action_trace;
3806
3807	data->action = ACTION_TRACE;
3808	}
3809
3810	data->action_name = kstrdup(s: action_name, GFP_KERNEL);
3811	if (!data->action_name) {
3812	ret = -ENOMEM;
3813	goto out;
3814	}
3815
3816	data->handler = handler;
3817	out:
3818	return ret;
3819	}
3820
3821	static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3822	char *str, enum handler_id handler)
3823	{
3824	struct action_data *data;
3825	int ret = -EINVAL;
3826	char *var_str;
3827
3828	data = kzalloc(sizeof(*data), GFP_KERNEL);
3829	if (!data)
3830	return ERR_PTR(error: -ENOMEM);
3831
3832	var_str = strsep(&str, ")");
3833	if (!var_str || !str) {
3834	ret = -EINVAL;
3835	goto free;
3836	}
3837
3838	data->track_data.var_str = kstrdup(s: var_str, GFP_KERNEL);
3839	if (!data->track_data.var_str) {
3840	ret = -ENOMEM;
3841	goto free;
3842	}
3843
3844	ret = action_parse(tr: hist_data->event_file->tr, str, data, handler);
3845	if (ret)
3846	goto free;
3847	out:
3848	return data;
3849	free:
3850	track_data_destroy(hist_data, data);
3851	data = ERR_PTR(error: ret);
3852	goto out;
3853	}
3854
3855	static void onmatch_destroy(struct action_data *data)
3856	{
3857	kfree(objp: data->match_data.event);
3858	kfree(objp: data->match_data.event_system);
3859
3860	action_data_destroy(data);
3861	}
3862
3863	static void destroy_field_var(struct field_var *field_var)
3864	{
3865	if (!field_var)
3866	return;
3867
3868	destroy_hist_field(hist_field: field_var->var, level: 0);
3869	destroy_hist_field(hist_field: field_var->val, level: 0);
3870
3871	kfree(objp: field_var);
3872	}
3873
3874	static void destroy_field_vars(struct hist_trigger_data *hist_data)
3875	{
3876	unsigned int i;
3877
3878	for (i = 0; i < hist_data->n_field_vars; i++)
3879	destroy_field_var(field_var: hist_data->field_vars[i]);
3880
3881	for (i = 0; i < hist_data->n_save_vars; i++)
3882	destroy_field_var(field_var: hist_data->save_vars[i]);
3883	}
3884
3885	static void save_field_var(struct hist_trigger_data *hist_data,
3886	struct field_var *field_var)
3887	{
3888	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3889
3890	/* Stack traces are saved in the string storage too */
3891	if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
3892	hist_data->n_field_var_str++;
3893	}
3894
3895
3896	static int check_synth_field(struct synth_event *event,
3897	struct hist_field *hist_field,
3898	unsigned int field_pos)
3899	{
3900	struct synth_field *field;
3901
3902	if (field_pos >= event->n_fields)
3903	return -EINVAL;
3904
3905	field = event->fields[field_pos];
3906
3907	/*
3908	* A dynamic string synth field can accept static or
3909	* dynamic. A static string synth field can only accept a
3910	* same-sized static string, which is checked for later.
3911	*/
3912	if (strstr(hist_field->type, "char[") && field->is_string
3913	&& field->is_dynamic)
3914	return 0;
3915
3916	if (strstr(hist_field->type, "long[") && field->is_stack)
3917	return 0;
3918
3919	if (strcmp(field->type, hist_field->type) != 0) {
3920	if (field->size != hist_field->size ||
3921	(!field->is_string && field->is_signed != hist_field->is_signed))
3922	return -EINVAL;
3923	}
3924
3925	return 0;
3926	}
3927
3928	static struct hist_field *
3929	trace_action_find_var(struct hist_trigger_data *hist_data,
3930	struct action_data *data,
3931	char *system, char *event, char *var)
3932	{
3933	struct trace_array *tr = hist_data->event_file->tr;
3934	struct hist_field *hist_field;
3935
3936	var++; /* skip '$' */
3937
3938	hist_field = find_target_event_var(hist_data, subsys_name: system, event_name: event, var_name: var);
3939	if (!hist_field) {
3940	if (!system && data->handler == HANDLER_ONMATCH) {
3941	system = data->match_data.event_system;
3942	event = data->match_data.event;
3943	}
3944
3945	hist_field = find_event_var(hist_data, system, event_name: event, var_name: var);
3946	}
3947
3948	if (!hist_field)
3949	hist_err(tr, err_type: HIST_ERR_PARAM_NOT_FOUND, err_pos: errpos(str: var));
3950
3951	return hist_field;
3952	}
3953
3954	static struct hist_field *
3955	trace_action_create_field_var(struct hist_trigger_data *hist_data,
3956	struct action_data *data, char *system,
3957	char *event, char *var)
3958	{
3959	struct hist_field *hist_field = NULL;
3960	struct field_var *field_var;
3961
3962	/*
3963	* First try to create a field var on the target event (the
3964	* currently being defined). This will create a variable for
3965	* unqualified fields on the target event, or if qualified,
3966	* target fields that have qualified names matching the target.
3967	*/
3968	field_var = create_target_field_var(target_hist_data: hist_data, subsys_name: system, event_name: event, var_name: var);
3969
3970	if (field_var && !IS_ERR(ptr: field_var)) {
3971	save_field_var(hist_data, field_var);
3972	hist_field = field_var->var;
3973	} else {
3974	field_var = NULL;
3975	/*
3976	* If no explicit system.event is specified, default to
3977	* looking for fields on the onmatch(system.event.xxx)
3978	* event.
3979	*/
3980	if (!system && data->handler == HANDLER_ONMATCH) {
3981	system = data->match_data.event_system;
3982	event = data->match_data.event;
3983	}
3984
3985	if (!event)
3986	goto free;
3987	/*
3988	* At this point, we're looking at a field on another
3989	* event. Because we can't modify a hist trigger on
3990	* another event to add a variable for a field, we need
3991	* to create a new trigger on that event and create the
3992	* variable at the same time.
3993	*/
3994	hist_field = create_field_var_hist(target_hist_data: hist_data, subsys_name: system, event_name: event, field_name: var);
3995	if (IS_ERR(ptr: hist_field))
3996	goto free;
3997	}
3998	out:
3999	return hist_field;
4000	free:
4001	destroy_field_var(field_var);
4002	hist_field = NULL;
4003	goto out;
4004	}
4005
4006	static int trace_action_create(struct hist_trigger_data *hist_data,
4007	struct action_data *data)
4008	{
4009	struct trace_array *tr = hist_data->event_file->tr;
4010	char *event_name, *param, *system = NULL;
4011	struct hist_field *hist_field, *var_ref;
4012	unsigned int i;
4013	unsigned int field_pos = 0;
4014	struct synth_event *event;
4015	char *synth_event_name;
4016	int var_ref_idx, ret = 0;
4017
4018	lockdep_assert_held(&event_mutex);
4019
4020	/* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
4021	if (data->n_params > SYNTH_FIELDS_MAX)
4022	return -EINVAL;
4023
4024	if (data->use_trace_keyword)
4025	synth_event_name = data->synth_event_name;
4026	else
4027	synth_event_name = data->action_name;
4028
4029	event = find_synth_event(name: synth_event_name);
4030	if (!event) {
4031	hist_err(tr, err_type: HIST_ERR_SYNTH_EVENT_NOT_FOUND, err_pos: errpos(str: synth_event_name));
4032	return -EINVAL;
4033	}
4034
4035	event->ref++;
4036
4037	for (i = 0; i < data->n_params; i++) {
4038	char *p;
4039
4040	p = param = kstrdup(s: data->params[i], GFP_KERNEL);
4041	if (!param) {
4042	ret = -ENOMEM;
4043	goto err;
4044	}
4045
4046	system = strsep(&param, ".");
4047	if (!param) {
4048	param = (char *)system;
4049	system = event_name = NULL;
4050	} else {
4051	event_name = strsep(&param, ".");
4052	if (!param) {
4053	kfree(objp: p);
4054	ret = -EINVAL;
4055	goto err;
4056	}
4057	}
4058
4059	if (param[0] == '$')
4060	hist_field = trace_action_find_var(hist_data, data,
4061	system, event: event_name,
4062	var: param);
4063	else
4064	hist_field = trace_action_create_field_var(hist_data,
4065	data,
4066	system,
4067	event: event_name,
4068	var: param);
4069
4070	if (!hist_field) {
4071	kfree(objp: p);
4072	ret = -EINVAL;
4073	goto err;
4074	}
4075
4076	if (check_synth_field(event, hist_field, field_pos) == 0) {
4077	var_ref = create_var_ref(hist_data, var_field: hist_field,
4078	system, event_name);
4079	if (!var_ref) {
4080	kfree(objp: p);
4081	ret = -ENOMEM;
4082	goto err;
4083	}
4084
4085	var_ref_idx = find_var_ref_idx(hist_data, var_field: var_ref);
4086	if (WARN_ON(var_ref_idx < 0)) {
4087	kfree(objp: p);
4088	ret = var_ref_idx;
4089	goto err;
4090	}
4091
4092	data->var_ref_idx[i] = var_ref_idx;
4093
4094	field_pos++;
4095	kfree(objp: p);
4096	continue;
4097	}
4098
4099	hist_err(tr, err_type: HIST_ERR_SYNTH_TYPE_MISMATCH, err_pos: errpos(str: param));
4100	kfree(objp: p);
4101	ret = -EINVAL;
4102	goto err;
4103	}
4104
4105	if (field_pos != event->n_fields) {
4106	hist_err(tr, err_type: HIST_ERR_SYNTH_COUNT_MISMATCH, err_pos: errpos(str: event->name));
4107	ret = -EINVAL;
4108	goto err;
4109	}
4110
4111	data->synth_event = event;
4112	out:
4113	return ret;
4114	err:
4115	event->ref--;
4116
4117	goto out;
4118	}
4119
4120	static int action_create(struct hist_trigger_data *hist_data,
4121	struct action_data *data)
4122	{
4123	struct trace_event_file *file = hist_data->event_file;
4124	struct trace_array *tr = file->tr;
4125	struct track_data *track_data;
4126	struct field_var *field_var;
4127	unsigned int i;
4128	char *param;
4129	int ret = 0;
4130
4131	if (data->action == ACTION_TRACE)
4132	return trace_action_create(hist_data, data);
4133
4134	if (data->action == ACTION_SNAPSHOT) {
4135	track_data = track_data_alloc(key_len: hist_data->key_size, action_data: data, hist_data);
4136	if (IS_ERR(ptr: track_data)) {
4137	ret = PTR_ERR(ptr: track_data);
4138	goto out;
4139	}
4140
4141	ret = tracing_snapshot_cond_enable(tr: file->tr, cond_data: track_data,
4142	update: cond_snapshot_update);
4143	if (ret)
4144	track_data_free(track_data);
4145
4146	goto out;
4147	}
4148
4149	if (data->action == ACTION_SAVE) {
4150	if (hist_data->n_save_vars) {
4151	ret = -EEXIST;
4152	hist_err(tr, err_type: HIST_ERR_TOO_MANY_SAVE_ACTIONS, err_pos: 0);
4153	goto out;
4154	}
4155
4156	for (i = 0; i < data->n_params; i++) {
4157	param = kstrdup(s: data->params[i], GFP_KERNEL);
4158	if (!param) {
4159	ret = -ENOMEM;
4160	goto out;
4161	}
4162
4163	field_var = create_target_field_var(target_hist_data: hist_data, NULL, NULL, var_name: param);
4164	if (IS_ERR(ptr: field_var)) {
4165	hist_err(tr, err_type: HIST_ERR_FIELD_VAR_CREATE_FAIL,
4166	err_pos: errpos(str: param));
4167	ret = PTR_ERR(ptr: field_var);
4168	kfree(objp: param);
4169	goto out;
4170	}
4171
4172	hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4173	if (field_var->val->flags &
4174	(HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4175	hist_data->n_save_var_str++;
4176	kfree(objp: param);
4177	}
4178	}
4179	out:
4180	return ret;
4181	}
4182
4183	static int onmatch_create(struct hist_trigger_data *hist_data,
4184	struct action_data *data)
4185	{
4186	return action_create(hist_data, data);
4187	}
4188
4189	static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4190	{
4191	char *match_event, *match_event_system;
4192	struct action_data *data;
4193	int ret = -EINVAL;
4194
4195	data = kzalloc(sizeof(*data), GFP_KERNEL);
4196	if (!data)
4197	return ERR_PTR(error: -ENOMEM);
4198
4199	match_event = strsep(&str, ")");
4200	if (!match_event || !str) {
4201	hist_err(tr, err_type: HIST_ERR_NO_CLOSING_PAREN, err_pos: errpos(str: match_event));
4202	goto free;
4203	}
4204
4205	match_event_system = strsep(&match_event, ".");
4206	if (!match_event) {
4207	hist_err(tr, err_type: HIST_ERR_SUBSYS_NOT_FOUND, err_pos: errpos(str: match_event_system));
4208	goto free;
4209	}
4210
4211	if (IS_ERR(ptr: event_file(tr, system: match_event_system, event_name: match_event))) {
4212	hist_err(tr, err_type: HIST_ERR_INVALID_SUBSYS_EVENT, err_pos: errpos(str: match_event));
4213	goto free;
4214	}
4215
4216	data->match_data.event = kstrdup(s: match_event, GFP_KERNEL);
4217	if (!data->match_data.event) {
4218	ret = -ENOMEM;
4219	goto free;
4220	}
4221
4222	data->match_data.event_system = kstrdup(s: match_event_system, GFP_KERNEL);
4223	if (!data->match_data.event_system) {
4224	ret = -ENOMEM;
4225	goto free;
4226	}
4227
4228	ret = action_parse(tr, str, data, handler: HANDLER_ONMATCH);
4229	if (ret)
4230	goto free;
4231	out:
4232	return data;
4233	free:
4234	onmatch_destroy(data);
4235	data = ERR_PTR(error: ret);
4236	goto out;
4237	}
4238
4239	static int create_hitcount_val(struct hist_trigger_data *hist_data)
4240	{
4241	hist_data->fields[HITCOUNT_IDX] =
4242	create_hist_field(hist_data, NULL, flags: HIST_FIELD_FL_HITCOUNT, NULL);
4243	if (!hist_data->fields[HITCOUNT_IDX])
4244	return -ENOMEM;
4245
4246	hist_data->n_vals++;
4247	hist_data->n_fields++;
4248
4249	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4250	return -EINVAL;
4251
4252	return 0;
4253	}
4254
4255	static int __create_val_field(struct hist_trigger_data *hist_data,
4256	unsigned int val_idx,
4257	struct trace_event_file *file,
4258	char *var_name, char *field_str,
4259	unsigned long flags)
4260	{
4261	struct hist_field *hist_field;
4262	int ret = 0, n_subexprs = 0;
4263
4264	hist_field = parse_expr(hist_data, file, str: field_str, flags, var_name, n_subexprs: &n_subexprs);
4265	if (IS_ERR(ptr: hist_field)) {
4266	ret = PTR_ERR(ptr: hist_field);
4267	goto out;
4268	}
4269
4270	/* values and variables should not have some modifiers */
4271	if (hist_field->flags & HIST_FIELD_FL_VAR) {
4272	/* Variable */
4273	if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4274	HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2))
4275	goto err;
4276	} else {
4277	/* Value */
4278	if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4279	HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4280	HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4281	HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE))
4282	goto err;
4283	}
4284
4285	hist_data->fields[val_idx] = hist_field;
4286
4287	++hist_data->n_vals;
4288	++hist_data->n_fields;
4289
4290	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4291	ret = -EINVAL;
4292	out:
4293	return ret;
4294	err:
4295	hist_err(tr: file->tr, err_type: HIST_ERR_BAD_FIELD_MODIFIER, err_pos: errpos(str: field_str));
4296	return -EINVAL;
4297	}
4298
4299	static int create_val_field(struct hist_trigger_data *hist_data,
4300	unsigned int val_idx,
4301	struct trace_event_file *file,
4302	char *field_str)
4303	{
4304	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4305	return -EINVAL;
4306
4307	return __create_val_field(hist_data, val_idx, file, NULL, field_str, flags: 0);
4308	}
4309
4310	static const char no_comm[] = "(no comm)";
4311
4312	static u64 hist_field_execname(struct hist_field *hist_field,
4313	struct tracing_map_elt *elt,
4314	struct trace_buffer *buffer,
4315	struct ring_buffer_event *rbe,
4316	void *event)
4317	{
4318	struct hist_elt_data *elt_data;
4319
4320	if (WARN_ON_ONCE(!elt))
4321	return (u64)(unsigned long)no_comm;
4322
4323	elt_data = elt->private_data;
4324
4325	if (WARN_ON_ONCE(!elt_data->comm))
4326	return (u64)(unsigned long)no_comm;
4327
4328	return (u64)(unsigned long)(elt_data->comm);
4329	}
4330
4331	static u64 hist_field_stack(struct hist_field *hist_field,
4332	struct tracing_map_elt *elt,
4333	struct trace_buffer *buffer,
4334	struct ring_buffer_event *rbe,
4335	void *event)
4336	{
4337	u32 str_item = *(u32 *)(event + hist_field->field->offset);
4338	int str_loc = str_item & 0xffff;
4339	char *addr = (char *)(event + str_loc);
4340
4341	return (u64)(unsigned long)addr;
4342	}
4343
4344	static u64 hist_fn_call(struct hist_field *hist_field,
4345	struct tracing_map_elt *elt,
4346	struct trace_buffer *buffer,
4347	struct ring_buffer_event *rbe,
4348	void *event)
4349	{
4350	switch (hist_field->fn_num) {
4351	case HIST_FIELD_FN_VAR_REF:
4352	return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4353	case HIST_FIELD_FN_COUNTER:
4354	return hist_field_counter(field: hist_field, elt, buffer, rbe, event);
4355	case HIST_FIELD_FN_CONST:
4356	return hist_field_const(field: hist_field, elt, buffer, rbe, event);
4357	case HIST_FIELD_FN_LOG2:
4358	return hist_field_log2(hist_field, elt, buffer, rbe, event);
4359	case HIST_FIELD_FN_BUCKET:
4360	return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4361	case HIST_FIELD_FN_TIMESTAMP:
4362	return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4363	case HIST_FIELD_FN_CPU:
4364	return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4365	case HIST_FIELD_FN_COMM:
4366	return hist_field_comm(hist_field, elt, buffer, rbe, event);
4367	case HIST_FIELD_FN_STRING:
4368	return hist_field_string(hist_field, elt, buffer, rbe, event);
4369	case HIST_FIELD_FN_DYNSTRING:
4370	return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4371	case HIST_FIELD_FN_RELDYNSTRING:
4372	return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4373	case HIST_FIELD_FN_PSTRING:
4374	return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4375	case HIST_FIELD_FN_S64:
4376	return hist_field_s64(hist_field, elt, buffer, rbe, event);
4377	case HIST_FIELD_FN_U64:
4378	return hist_field_u64(hist_field, elt, buffer, rbe, event);
4379	case HIST_FIELD_FN_S32:
4380	return hist_field_s32(hist_field, elt, buffer, rbe, event);
4381	case HIST_FIELD_FN_U32:
4382	return hist_field_u32(hist_field, elt, buffer, rbe, event);
4383	case HIST_FIELD_FN_S16:
4384	return hist_field_s16(hist_field, elt, buffer, rbe, event);
4385	case HIST_FIELD_FN_U16:
4386	return hist_field_u16(hist_field, elt, buffer, rbe, event);
4387	case HIST_FIELD_FN_S8:
4388	return hist_field_s8(hist_field, elt, buffer, rbe, event);
4389	case HIST_FIELD_FN_U8:
4390	return hist_field_u8(hist_field, elt, buffer, rbe, event);
4391	case HIST_FIELD_FN_UMINUS:
4392	return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4393	case HIST_FIELD_FN_MINUS:
4394	return hist_field_minus(hist_field, elt, buffer, rbe, event);
4395	case HIST_FIELD_FN_PLUS:
4396	return hist_field_plus(hist_field, elt, buffer, rbe, event);
4397	case HIST_FIELD_FN_DIV:
4398	return hist_field_div(hist_field, elt, buffer, rbe, event);
4399	case HIST_FIELD_FN_MULT:
4400	return hist_field_mult(hist_field, elt, buffer, rbe, event);
4401	case HIST_FIELD_FN_DIV_POWER2:
4402	return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4403	case HIST_FIELD_FN_DIV_NOT_POWER2:
4404	return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4405	case HIST_FIELD_FN_DIV_MULT_SHIFT:
4406	return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4407	case HIST_FIELD_FN_EXECNAME:
4408	return hist_field_execname(hist_field, elt, buffer, rbe, event);
4409	case HIST_FIELD_FN_STACK:
4410	return hist_field_stack(hist_field, elt, buffer, rbe, event);
4411	default:
4412	return 0;
4413	}
4414	}
4415
4416	/* Convert a var that points to common_pid.execname to a string */
4417	static void update_var_execname(struct hist_field *hist_field)
4418	{
4419	hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4420	HIST_FIELD_FL_EXECNAME;
4421	hist_field->size = MAX_FILTER_STR_VAL;
4422	hist_field->is_signed = 0;
4423
4424	kfree_const(x: hist_field->type);
4425	hist_field->type = "char[]";
4426
4427	hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4428	}
4429
4430	static int create_var_field(struct hist_trigger_data *hist_data,
4431	unsigned int val_idx,
4432	struct trace_event_file *file,
4433	char *var_name, char *expr_str)
4434	{
4435	struct trace_array *tr = hist_data->event_file->tr;
4436	unsigned long flags = 0;
4437	int ret;
4438
4439	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4440	return -EINVAL;
4441
4442	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4443	hist_err(tr, err_type: HIST_ERR_DUPLICATE_VAR, err_pos: errpos(str: var_name));
4444	return -EINVAL;
4445	}
4446
4447	flags |= HIST_FIELD_FL_VAR;
4448	hist_data->n_vars++;
4449	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4450	return -EINVAL;
4451
4452	ret = __create_val_field(hist_data, val_idx, file, var_name, field_str: expr_str, flags);
4453
4454	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4455	update_var_execname(hist_field: hist_data->fields[val_idx]);
4456
4457	if (!ret && hist_data->fields[val_idx]->flags &
4458	(HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4459	hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4460
4461	return ret;
4462	}
4463
4464	static int create_val_fields(struct hist_trigger_data *hist_data,
4465	struct trace_event_file *file)
4466	{
4467	unsigned int i, j = 1, n_hitcount = 0;
4468	char *fields_str, *field_str;
4469	int ret;
4470
4471	ret = create_hitcount_val(hist_data);
4472	if (ret)
4473	goto out;
4474
4475	fields_str = hist_data->attrs->vals_str;
4476	if (!fields_str)
4477	goto out;
4478
4479	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4480	j < TRACING_MAP_VALS_MAX; i++) {
4481	field_str = strsep(&fields_str, ",");
4482	if (!field_str)
4483	break;
4484
4485	if (strcmp(field_str, "hitcount") == 0) {
4486	if (!n_hitcount++)
4487	continue;
4488	}
4489
4490	ret = create_val_field(hist_data, val_idx: j++, file, field_str);
4491	if (ret)
4492	goto out;
4493	}
4494
4495	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4496	ret = -EINVAL;
4497	out:
4498	/* There is only raw hitcount but nohitcount suppresses it. */
4499	if (j == 1 && hist_data->attrs->no_hitcount) {
4500	hist_err(tr: hist_data->event_file->tr, err_type: HIST_ERR_NEED_NOHC_VAL, err_pos: 0);
4501	ret = -ENOENT;
4502	}
4503
4504	return ret;
4505	}
4506
4507	static int create_key_field(struct hist_trigger_data *hist_data,
4508	unsigned int key_idx,
4509	unsigned int key_offset,
4510	struct trace_event_file *file,
4511	char *field_str)
4512	{
4513	struct trace_array *tr = hist_data->event_file->tr;
4514	struct hist_field *hist_field = NULL;
4515	unsigned long flags = 0;
4516	unsigned int key_size;
4517	int ret = 0, n_subexprs = 0;
4518
4519	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4520	return -EINVAL;
4521
4522	flags |= HIST_FIELD_FL_KEY;
4523
4524	if (strcmp(field_str, "stacktrace") == 0) {
4525	flags |= HIST_FIELD_FL_STACKTRACE;
4526	key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4527	hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4528	} else {
4529	hist_field = parse_expr(hist_data, file, str: field_str, flags,
4530	NULL, n_subexprs: &n_subexprs);
4531	if (IS_ERR(ptr: hist_field)) {
4532	ret = PTR_ERR(ptr: hist_field);
4533	goto out;
4534	}
4535
4536	if (field_has_hist_vars(hist_field, level: 0)) {
4537	hist_err(tr, err_type: HIST_ERR_INVALID_REF_KEY, err_pos: errpos(str: field_str));
4538	destroy_hist_field(hist_field, level: 0);
4539	ret = -EINVAL;
4540	goto out;
4541	}
4542
4543	key_size = hist_field->size;
4544	}
4545
4546	hist_data->fields[key_idx] = hist_field;
4547
4548	key_size = ALIGN(key_size, sizeof(u64));
4549	hist_data->fields[key_idx]->size = key_size;
4550	hist_data->fields[key_idx]->offset = key_offset;
4551
4552	hist_data->key_size += key_size;
4553
4554	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4555	ret = -EINVAL;
4556	goto out;
4557	}
4558
4559	hist_data->n_keys++;
4560	hist_data->n_fields++;
4561
4562	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4563	return -EINVAL;
4564
4565	ret = key_size;
4566	out:
4567	return ret;
4568	}
4569
4570	static int create_key_fields(struct hist_trigger_data *hist_data,
4571	struct trace_event_file *file)
4572	{
4573	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4574	char *fields_str, *field_str;
4575	int ret = -EINVAL;
4576
4577	fields_str = hist_data->attrs->keys_str;
4578	if (!fields_str)
4579	goto out;
4580
4581	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4582	field_str = strsep(&fields_str, ",");
4583	if (!field_str)
4584	break;
4585	ret = create_key_field(hist_data, key_idx: i, key_offset,
4586	file, field_str);
4587	if (ret < 0)
4588	goto out;
4589	key_offset += ret;
4590	}
4591	if (fields_str) {
4592	ret = -EINVAL;
4593	goto out;
4594	}
4595	ret = 0;
4596	out:
4597	return ret;
4598	}
4599
4600	static int create_var_fields(struct hist_trigger_data *hist_data,
4601	struct trace_event_file *file)
4602	{
4603	unsigned int i, j = hist_data->n_vals;
4604	int ret = 0;
4605
4606	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4607
4608	for (i = 0; i < n_vars; i++) {
4609	char *var_name = hist_data->attrs->var_defs.name[i];
4610	char *expr = hist_data->attrs->var_defs.expr[i];
4611
4612	ret = create_var_field(hist_data, val_idx: j++, file, var_name, expr_str: expr);
4613	if (ret)
4614	goto out;
4615	}
4616	out:
4617	return ret;
4618	}
4619
4620	static void free_var_defs(struct hist_trigger_data *hist_data)
4621	{
4622	unsigned int i;
4623
4624	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4625	kfree(objp: hist_data->attrs->var_defs.name[i]);
4626	kfree(objp: hist_data->attrs->var_defs.expr[i]);
4627	}
4628
4629	hist_data->attrs->var_defs.n_vars = 0;
4630	}
4631
4632	static int parse_var_defs(struct hist_trigger_data *hist_data)
4633	{
4634	struct trace_array *tr = hist_data->event_file->tr;
4635	char *s, *str, *var_name, *field_str;
4636	unsigned int i, j, n_vars = 0;
4637	int ret = 0;
4638
4639	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4640	str = hist_data->attrs->assignment_str[i];
4641	for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4642	field_str = strsep(&str, ",");
4643	if (!field_str)
4644	break;
4645
4646	var_name = strsep(&field_str, "=");
4647	if (!var_name || !field_str) {
4648	hist_err(tr, err_type: HIST_ERR_MALFORMED_ASSIGNMENT,
4649	err_pos: errpos(str: var_name));
4650	ret = -EINVAL;
4651	goto free;
4652	}
4653
4654	if (n_vars == TRACING_MAP_VARS_MAX) {
4655	hist_err(tr, err_type: HIST_ERR_TOO_MANY_VARS, err_pos: errpos(str: var_name));
4656	ret = -EINVAL;
4657	goto free;
4658	}
4659
4660	s = kstrdup(s: var_name, GFP_KERNEL);
4661	if (!s) {
4662	ret = -ENOMEM;
4663	goto free;
4664	}
4665	hist_data->attrs->var_defs.name[n_vars] = s;
4666
4667	s = kstrdup(s: field_str, GFP_KERNEL);
4668	if (!s) {
4669	kfree(objp: hist_data->attrs->var_defs.name[n_vars]);
4670	hist_data->attrs->var_defs.name[n_vars] = NULL;
4671	ret = -ENOMEM;
4672	goto free;
4673	}
4674	hist_data->attrs->var_defs.expr[n_vars++] = s;
4675
4676	hist_data->attrs->var_defs.n_vars = n_vars;
4677	}
4678	}
4679
4680	return ret;
4681	free:
4682	free_var_defs(hist_data);
4683
4684	return ret;
4685	}
4686
4687	static int create_hist_fields(struct hist_trigger_data *hist_data,
4688	struct trace_event_file *file)
4689	{
4690	int ret;
4691
4692	ret = parse_var_defs(hist_data);
4693	if (ret)
4694	return ret;
4695
4696	ret = create_val_fields(hist_data, file);
4697	if (ret)
4698	goto out;
4699
4700	ret = create_var_fields(hist_data, file);
4701	if (ret)
4702	goto out;
4703
4704	ret = create_key_fields(hist_data, file);
4705
4706	out:
4707	free_var_defs(hist_data);
4708
4709	return ret;
4710	}
4711
4712	static int is_descending(struct trace_array *tr, const char *str)
4713	{
4714	if (!str)
4715	return 0;
4716
4717	if (strcmp(str, "descending") == 0)
4718	return 1;
4719
4720	if (strcmp(str, "ascending") == 0)
4721	return 0;
4722
4723	hist_err(tr, err_type: HIST_ERR_INVALID_SORT_MODIFIER, err_pos: errpos(str: (char *)str));
4724
4725	return -EINVAL;
4726	}
4727
4728	static int create_sort_keys(struct hist_trigger_data *hist_data)
4729	{
4730	struct trace_array *tr = hist_data->event_file->tr;
4731	char *fields_str = hist_data->attrs->sort_key_str;
4732	struct tracing_map_sort_key *sort_key;
4733	int descending, ret = 0;
4734	unsigned int i, j, k;
4735
4736	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4737
4738	if (!fields_str)
4739	goto out;
4740
4741	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4742	struct hist_field *hist_field;
4743	char *field_str, *field_name;
4744	const char *test_name;
4745
4746	sort_key = &hist_data->sort_keys[i];
4747
4748	field_str = strsep(&fields_str, ",");
4749	if (!field_str)
4750	break;
4751
4752	if (!*field_str) {
4753	ret = -EINVAL;
4754	hist_err(tr, err_type: HIST_ERR_EMPTY_SORT_FIELD, err_pos: errpos(str: "sort="));
4755	break;
4756	}
4757
4758	if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4759	hist_err(tr, err_type: HIST_ERR_TOO_MANY_SORT_FIELDS, err_pos: errpos(str: "sort="));
4760	ret = -EINVAL;
4761	break;
4762	}
4763
4764	field_name = strsep(&field_str, ".");
4765	if (!field_name || !*field_name) {
4766	ret = -EINVAL;
4767	hist_err(tr, err_type: HIST_ERR_EMPTY_SORT_FIELD, err_pos: errpos(str: "sort="));
4768	break;
4769	}
4770
4771	if (strcmp(field_name, "hitcount") == 0) {
4772	descending = is_descending(tr, str: field_str);
4773	if (descending < 0) {
4774	ret = descending;
4775	break;
4776	}
4777	sort_key->descending = descending;
4778	continue;
4779	}
4780
4781	for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4782	unsigned int idx;
4783
4784	hist_field = hist_data->fields[j];
4785	if (hist_field->flags & HIST_FIELD_FL_VAR)
4786	continue;
4787
4788	idx = k++;
4789
4790	test_name = hist_field_name(field: hist_field, level: 0);
4791
4792	if (strcmp(field_name, test_name) == 0) {
4793	sort_key->field_idx = idx;
4794	descending = is_descending(tr, str: field_str);
4795	if (descending < 0) {
4796	ret = descending;
4797	goto out;
4798	}
4799	sort_key->descending = descending;
4800	break;
4801	}
4802	}
4803	if (j == hist_data->n_fields) {
4804	ret = -EINVAL;
4805	hist_err(tr, err_type: HIST_ERR_INVALID_SORT_FIELD, err_pos: errpos(str: field_name));
4806	break;
4807	}
4808	}
4809
4810	hist_data->n_sort_keys = i;
4811	out:
4812	return ret;
4813	}
4814
4815	static void destroy_actions(struct hist_trigger_data *hist_data)
4816	{
4817	unsigned int i;
4818
4819	for (i = 0; i < hist_data->n_actions; i++) {
4820	struct action_data *data = hist_data->actions[i];
4821
4822	if (data->handler == HANDLER_ONMATCH)
4823	onmatch_destroy(data);
4824	else if (data->handler == HANDLER_ONMAX ||
4825	data->handler == HANDLER_ONCHANGE)
4826	track_data_destroy(hist_data, data);
4827	else
4828	kfree(objp: data);
4829	}
4830	}
4831
4832	static int parse_actions(struct hist_trigger_data *hist_data)
4833	{
4834	struct trace_array *tr = hist_data->event_file->tr;
4835	struct action_data *data;
4836	unsigned int i;
4837	int ret = 0;
4838	char *str;
4839	int len;
4840
4841	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4842	enum handler_id hid = 0;
4843	char *action_str;
4844
4845	str = hist_data->attrs->action_str[i];
4846
4847	if ((len = str_has_prefix(str, prefix: "onmatch(")))
4848	hid = HANDLER_ONMATCH;
4849	else if ((len = str_has_prefix(str, prefix: "onmax(")))
4850	hid = HANDLER_ONMAX;
4851	else if ((len = str_has_prefix(str, prefix: "onchange(")))
4852	hid = HANDLER_ONCHANGE;
4853
4854	action_str = str + len;
4855
4856	switch (hid) {
4857	case HANDLER_ONMATCH:
4858	data = onmatch_parse(tr, str: action_str);
4859	break;
4860	case HANDLER_ONMAX:
4861	case HANDLER_ONCHANGE:
4862	data = track_data_parse(hist_data, str: action_str, handler: hid);
4863	break;
4864	default:
4865	data = ERR_PTR(error: -EINVAL);
4866	break;
4867	}
4868
4869	if (IS_ERR(ptr: data)) {
4870	ret = PTR_ERR(ptr: data);
4871	break;
4872	}
4873
4874	hist_data->actions[hist_data->n_actions++] = data;
4875	}
4876
4877	return ret;
4878	}
4879
4880	static int create_actions(struct hist_trigger_data *hist_data)
4881	{
4882	struct action_data *data;
4883	unsigned int i;
4884	int ret = 0;
4885
4886	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4887	data = hist_data->actions[i];
4888
4889	if (data->handler == HANDLER_ONMATCH) {
4890	ret = onmatch_create(hist_data, data);
4891	if (ret)
4892	break;
4893	} else if (data->handler == HANDLER_ONMAX ||
4894	data->handler == HANDLER_ONCHANGE) {
4895	ret = track_data_create(hist_data, data);
4896	if (ret)
4897	break;
4898	} else {
4899	ret = -EINVAL;
4900	break;
4901	}
4902	}
4903
4904	return ret;
4905	}
4906
4907	static void print_actions(struct seq_file *m,
4908	struct hist_trigger_data *hist_data,
4909	struct tracing_map_elt *elt)
4910	{
4911	unsigned int i;
4912
4913	for (i = 0; i < hist_data->n_actions; i++) {
4914	struct action_data *data = hist_data->actions[i];
4915
4916	if (data->action == ACTION_SNAPSHOT)
4917	continue;
4918
4919	if (data->handler == HANDLER_ONMAX ||
4920	data->handler == HANDLER_ONCHANGE)
4921	track_data_print(m, hist_data, elt, data);
4922	}
4923	}
4924
4925	static void print_action_spec(struct seq_file *m,
4926	struct hist_trigger_data *hist_data,
4927	struct action_data *data)
4928	{
4929	unsigned int i;
4930
4931	if (data->action == ACTION_SAVE) {
4932	for (i = 0; i < hist_data->n_save_vars; i++) {
4933	seq_printf(m, fmt: "%s", hist_data->save_vars[i]->var->var.name);
4934	if (i < hist_data->n_save_vars - 1)
4935	seq_puts(m, s: ",");
4936	}
4937	} else if (data->action == ACTION_TRACE) {
4938	if (data->use_trace_keyword)
4939	seq_printf(m, fmt: "%s", data->synth_event_name);
4940	for (i = 0; i < data->n_params; i++) {
4941	if (i || data->use_trace_keyword)
4942	seq_puts(m, s: ",");
4943	seq_printf(m, fmt: "%s", data->params[i]);
4944	}
4945	}
4946	}
4947
4948	static void print_track_data_spec(struct seq_file *m,
4949	struct hist_trigger_data *hist_data,
4950	struct action_data *data)
4951	{
4952	if (data->handler == HANDLER_ONMAX)
4953	seq_puts(m, s: ":onmax(");
4954	else if (data->handler == HANDLER_ONCHANGE)
4955	seq_puts(m, s: ":onchange(");
4956	seq_printf(m, fmt: "%s", data->track_data.var_str);
4957	seq_printf(m, fmt: ").%s(", data->action_name);
4958
4959	print_action_spec(m, hist_data, data);
4960
4961	seq_puts(m, s: ")");
4962	}
4963
4964	static void print_onmatch_spec(struct seq_file *m,
4965	struct hist_trigger_data *hist_data,
4966	struct action_data *data)
4967	{
4968	seq_printf(m, fmt: ":onmatch(%s.%s).", data->match_data.event_system,
4969	data->match_data.event);
4970
4971	seq_printf(m, fmt: "%s(", data->action_name);
4972
4973	print_action_spec(m, hist_data, data);
4974
4975	seq_puts(m, s: ")");
4976	}
4977
4978	static bool actions_match(struct hist_trigger_data *hist_data,
4979	struct hist_trigger_data *hist_data_test)
4980	{
4981	unsigned int i, j;
4982
4983	if (hist_data->n_actions != hist_data_test->n_actions)
4984	return false;
4985
4986	for (i = 0; i < hist_data->n_actions; i++) {
4987	struct action_data *data = hist_data->actions[i];
4988	struct action_data *data_test = hist_data_test->actions[i];
4989	char *action_name, *action_name_test;
4990
4991	if (data->handler != data_test->handler)
4992	return false;
4993	if (data->action != data_test->action)
4994	return false;
4995
4996	if (data->n_params != data_test->n_params)
4997	return false;
4998
4999	for (j = 0; j < data->n_params; j++) {
5000	if (strcmp(data->params[j], data_test->params[j]) != 0)
5001	return false;
5002	}
5003
5004	if (data->use_trace_keyword)
5005	action_name = data->synth_event_name;
5006	else
5007	action_name = data->action_name;
5008
5009	if (data_test->use_trace_keyword)
5010	action_name_test = data_test->synth_event_name;
5011	else
5012	action_name_test = data_test->action_name;
5013
5014	if (strcmp(action_name, action_name_test) != 0)
5015	return false;
5016
5017	if (data->handler == HANDLER_ONMATCH) {
5018	if (strcmp(data->match_data.event_system,
5019	data_test->match_data.event_system) != 0)
5020	return false;
5021	if (strcmp(data->match_data.event,
5022	data_test->match_data.event) != 0)
5023	return false;
5024	} else if (data->handler == HANDLER_ONMAX ||
5025	data->handler == HANDLER_ONCHANGE) {
5026	if (strcmp(data->track_data.var_str,
5027	data_test->track_data.var_str) != 0)
5028	return false;
5029	}
5030	}
5031
5032	return true;
5033	}
5034
5035
5036	static void print_actions_spec(struct seq_file *m,
5037	struct hist_trigger_data *hist_data)
5038	{
5039	unsigned int i;
5040
5041	for (i = 0; i < hist_data->n_actions; i++) {
5042	struct action_data *data = hist_data->actions[i];
5043
5044	if (data->handler == HANDLER_ONMATCH)
5045	print_onmatch_spec(m, hist_data, data);
5046	else if (data->handler == HANDLER_ONMAX ||
5047	data->handler == HANDLER_ONCHANGE)
5048	print_track_data_spec(m, hist_data, data);
5049	}
5050	}
5051
5052	static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5053	{
5054	unsigned int i;
5055
5056	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5057	kfree(objp: hist_data->field_var_hists[i]->cmd);
5058	kfree(objp: hist_data->field_var_hists[i]);
5059	}
5060	}
5061
5062	static void destroy_hist_data(struct hist_trigger_data *hist_data)
5063	{
5064	if (!hist_data)
5065	return;
5066
5067	destroy_hist_trigger_attrs(attrs: hist_data->attrs);
5068	destroy_hist_fields(hist_data);
5069	tracing_map_destroy(map: hist_data->map);
5070
5071	destroy_actions(hist_data);
5072	destroy_field_vars(hist_data);
5073	destroy_field_var_hists(hist_data);
5074
5075	kfree(objp: hist_data);
5076	}
5077
5078	static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5079	{
5080	struct tracing_map *map = hist_data->map;
5081	struct ftrace_event_field *field;
5082	struct hist_field *hist_field;
5083	int i, idx = 0;
5084
5085	for_each_hist_field(i, hist_data) {
5086	hist_field = hist_data->fields[i];
5087	if (hist_field->flags & HIST_FIELD_FL_KEY) {
5088	tracing_map_cmp_fn_t cmp_fn;
5089
5090	field = hist_field->field;
5091
5092	if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5093	cmp_fn = tracing_map_cmp_none;
5094	else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
5095	cmp_fn = tracing_map_cmp_num(field_size: hist_field->size,
5096	field_is_signed: hist_field->is_signed);
5097	else if (is_string_field(field))
5098	cmp_fn = tracing_map_cmp_string;
5099	else
5100	cmp_fn = tracing_map_cmp_num(field_size: field->size,
5101	field_is_signed: field->is_signed);
5102	idx = tracing_map_add_key_field(map,
5103	offset: hist_field->offset,
5104	cmp_fn);
5105	} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5106	idx = tracing_map_add_sum_field(map);
5107
5108	if (idx < 0)
5109	return idx;
5110
5111	if (hist_field->flags & HIST_FIELD_FL_VAR) {
5112	idx = tracing_map_add_var(map);
5113	if (idx < 0)
5114	return idx;
5115	hist_field->var.idx = idx;
5116	hist_field->var.hist_data = hist_data;
5117	}
5118	}
5119
5120	return 0;
5121	}
5122
5123	static struct hist_trigger_data *
5124	create_hist_data(unsigned int map_bits,
5125	struct hist_trigger_attrs *attrs,
5126	struct trace_event_file *file,
5127	bool remove)
5128	{
5129	const struct tracing_map_ops *map_ops = NULL;
5130	struct hist_trigger_data *hist_data;
5131	int ret = 0;
5132
5133	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5134	if (!hist_data)
5135	return ERR_PTR(error: -ENOMEM);
5136
5137	hist_data->attrs = attrs;
5138	hist_data->remove = remove;
5139	hist_data->event_file = file;
5140
5141	ret = parse_actions(hist_data);
5142	if (ret)
5143	goto free;
5144
5145	ret = create_hist_fields(hist_data, file);
5146	if (ret)
5147	goto free;
5148
5149	ret = create_sort_keys(hist_data);
5150	if (ret)
5151	goto free;
5152
5153	map_ops = &hist_trigger_elt_data_ops;
5154
5155	hist_data->map = tracing_map_create(map_bits, key_size: hist_data->key_size,
5156	ops: map_ops, private_data: hist_data);
5157	if (IS_ERR(ptr: hist_data->map)) {
5158	ret = PTR_ERR(ptr: hist_data->map);
5159	hist_data->map = NULL;
5160	goto free;
5161	}
5162
5163	ret = create_tracing_map_fields(hist_data);
5164	if (ret)
5165	goto free;
5166	out:
5167	return hist_data;
5168	free:
5169	hist_data->attrs = NULL;
5170
5171	destroy_hist_data(hist_data);
5172
5173	hist_data = ERR_PTR(error: ret);
5174
5175	goto out;
5176	}
5177
5178	static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5179	struct tracing_map_elt *elt,
5180	struct trace_buffer *buffer, void *rec,
5181	struct ring_buffer_event *rbe,
5182	u64 *var_ref_vals)
5183	{
5184	struct hist_elt_data *elt_data;
5185	struct hist_field *hist_field;
5186	unsigned int i, var_idx;
5187	u64 hist_val;
5188
5189	elt_data = elt->private_data;
5190	elt_data->var_ref_vals = var_ref_vals;
5191
5192	for_each_hist_val_field(i, hist_data) {
5193	hist_field = hist_data->fields[i];
5194	hist_val = hist_fn_call(hist_field, elt, buffer, rbe, event: rec);
5195	if (hist_field->flags & HIST_FIELD_FL_VAR) {
5196	var_idx = hist_field->var.idx;
5197
5198	if (hist_field->flags &
5199	(HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) {
5200	unsigned int str_start, var_str_idx, idx;
5201	char *str, *val_str;
5202	unsigned int size;
5203
5204	str_start = hist_data->n_field_var_str +
5205	hist_data->n_save_var_str;
5206	var_str_idx = hist_field->var_str_idx;
5207	idx = str_start + var_str_idx;
5208
5209	str = elt_data->field_var_str[idx];
5210	val_str = (char *)(uintptr_t)hist_val;
5211
5212	if (hist_field->flags & HIST_FIELD_FL_STRING) {
5213	size = min(hist_field->size, STR_VAR_LEN_MAX);
5214	strscpy(str, val_str, size);
5215	} else {
5216	char *stack_start = str + sizeof(unsigned long);
5217	int e;
5218
5219	e = stack_trace_save(store: (void *)stack_start,
5220	HIST_STACKTRACE_DEPTH,
5221	HIST_STACKTRACE_SKIP);
5222	if (e < HIST_STACKTRACE_DEPTH - 1)
5223	((unsigned long *)stack_start)[e] = 0;
5224	*((unsigned long *)str) = e;
5225	}
5226	hist_val = (u64)(uintptr_t)str;
5227	}
5228	tracing_map_set_var(elt, i: var_idx, n: hist_val);
5229	continue;
5230	}
5231	tracing_map_update_sum(elt, i, n: hist_val);
5232	}
5233
5234	for_each_hist_key_field(i, hist_data) {
5235	hist_field = hist_data->fields[i];
5236	if (hist_field->flags & HIST_FIELD_FL_VAR) {
5237	hist_val = hist_fn_call(hist_field, elt, buffer, rbe, event: rec);
5238	var_idx = hist_field->var.idx;
5239	tracing_map_set_var(elt, i: var_idx, n: hist_val);
5240	}
5241	}
5242
5243	update_field_vars(hist_data, elt, buffer, rbe, rec);
5244	}
5245
5246	static inline void add_to_key(char *compound_key, void *key,
5247	struct hist_field *key_field, void *rec)
5248	{
5249	size_t size = key_field->size;
5250
5251	if (key_field->flags & HIST_FIELD_FL_STRING) {
5252
5253	if (key_field->flags & HIST_FIELD_FL_COMM) {
5254	size = strlen((char *)key);
5255	} else {
5256	struct ftrace_event_field *field;
5257
5258	field = key_field->field;
5259	if (field->filter_type == FILTER_DYN_STRING ||
5260	field->filter_type == FILTER_RDYN_STRING)
5261	size = *(u32 *)(rec + field->offset) >> 16;
5262	else if (field->filter_type == FILTER_STATIC_STRING)
5263	size = field->size;
5264	}
5265
5266	/* ensure NULL-termination */
5267	if (size > key_field->size - 1)
5268	size = key_field->size - 1;
5269	}
5270	memcpy(compound_key + key_field->offset, key, size);
5271	}
5272
5273	static void
5274	hist_trigger_actions(struct hist_trigger_data *hist_data,
5275	struct tracing_map_elt *elt,
5276	struct trace_buffer *buffer, void *rec,
5277	struct ring_buffer_event *rbe, void *key,
5278	u64 *var_ref_vals)
5279	{
5280	struct action_data *data;
5281	unsigned int i;
5282
5283	for (i = 0; i < hist_data->n_actions; i++) {
5284	data = hist_data->actions[i];
5285	data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5286	}
5287	}
5288
5289	/*
5290	* The hist_pad structure is used to save information to create
5291	* a histogram from the histogram trigger. It's too big to store
5292	* on the stack, so when the histogram trigger is initialized
5293	* a percpu array of 4 hist_pad structures is allocated.
5294	* This will cover every context from normal, softirq, irq and NMI
5295	* in the very unlikely event that a trigger happens at each of
5296	* these contexts and interrupts a currently active trigger.
5297	*/
5298	struct hist_pad {
5299	unsigned long entries[HIST_STACKTRACE_DEPTH];
5300	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5301	char compound_key[HIST_KEY_SIZE_MAX];
5302	};
5303
5304	static struct hist_pad __percpu *hist_pads;
5305	static DEFINE_PER_CPU(int, hist_pad_cnt);
5306	static refcount_t hist_pad_ref;
5307
5308	/* One hist_pad for every context (normal, softirq, irq, NMI) */
5309	#define MAX_HIST_CNT 4
5310
5311	static int alloc_hist_pad(void)
5312	{
5313	lockdep_assert_held(&event_mutex);
5314
5315	if (refcount_read(r: &hist_pad_ref)) {
5316	refcount_inc(r: &hist_pad_ref);
5317	return 0;
5318	}
5319
5320	hist_pads = __alloc_percpu(sizeof(struct hist_pad) * MAX_HIST_CNT,
5321	__alignof__(struct hist_pad));
5322	if (!hist_pads)
5323	return -ENOMEM;
5324
5325	refcount_set(r: &hist_pad_ref, n: 1);
5326	return 0;
5327	}
5328
5329	static void free_hist_pad(void)
5330	{
5331	lockdep_assert_held(&event_mutex);
5332
5333	if (!refcount_dec_and_test(r: &hist_pad_ref))
5334	return;
5335
5336	free_percpu(pdata: hist_pads);
5337	hist_pads = NULL;
5338	}
5339
5340	static struct hist_pad *get_hist_pad(void)
5341	{
5342	struct hist_pad *hist_pad;
5343	int cnt;
5344
5345	if (WARN_ON_ONCE(!hist_pads))
5346	return NULL;
5347
5348	preempt_disable();
5349
5350	hist_pad = per_cpu_ptr(hist_pads, smp_processor_id());
5351
5352	if (this_cpu_read(hist_pad_cnt) == MAX_HIST_CNT) {
5353	preempt_enable();
5354	return NULL;
5355	}
5356
5357	cnt = this_cpu_inc_return(hist_pad_cnt) - 1;
5358
5359	return &hist_pad[cnt];
5360	}
5361
5362	static void put_hist_pad(void)
5363	{
5364	this_cpu_dec(hist_pad_cnt);
5365	preempt_enable();
5366	}
5367
5368	static void event_hist_trigger(struct event_trigger_data *data,
5369	struct trace_buffer *buffer, void *rec,
5370	struct ring_buffer_event *rbe)
5371	{
5372	struct hist_trigger_data *hist_data = data->private_data;
5373	bool use_compound_key = (hist_data->n_keys > 1);
5374	struct tracing_map_elt *elt = NULL;
5375	struct hist_field *key_field;
5376	struct hist_pad *hist_pad;
5377	u64 field_contents;
5378	void *key = NULL;
5379	unsigned int i;
5380
5381	if (unlikely(!rbe))
5382	return;
5383
5384	hist_pad = get_hist_pad();
5385	if (!hist_pad)
5386	return;
5387
5388	memset(hist_pad->compound_key, 0, hist_data->key_size);
5389
5390	for_each_hist_key_field(i, hist_data) {
5391	key_field = hist_data->fields[i];
5392
5393	if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5394	unsigned long *entries = hist_pad->entries;
5395
5396	memset(entries, 0, HIST_STACKTRACE_SIZE);
5397	if (key_field->field) {
5398	unsigned long *stack, n_entries;
5399
5400	field_contents = hist_fn_call(hist_field: key_field, elt, buffer, rbe, event: rec);
5401	stack = (unsigned long *)(long)field_contents;
5402	n_entries = *stack;
5403	memcpy(entries, ++stack, n_entries * sizeof(unsigned long));
5404	} else {
5405	stack_trace_save(store: entries, HIST_STACKTRACE_DEPTH,
5406	HIST_STACKTRACE_SKIP);
5407	}
5408	key = entries;
5409	} else {
5410	field_contents = hist_fn_call(hist_field: key_field, elt, buffer, rbe, event: rec);
5411	if (key_field->flags & HIST_FIELD_FL_STRING) {
5412	key = (void *)(unsigned long)field_contents;
5413	use_compound_key = true;
5414	} else
5415	key = (void *)&field_contents;
5416	}
5417
5418	if (use_compound_key)
5419	add_to_key(compound_key: hist_pad->compound_key, key, key_field, rec);
5420	}
5421
5422	if (use_compound_key)
5423	key = hist_pad->compound_key;
5424
5425	if (hist_data->n_var_refs &&
5426	!resolve_var_refs(hist_data, key, var_ref_vals: hist_pad->var_ref_vals, self: false))
5427	goto out;
5428
5429	elt = tracing_map_insert(map: hist_data->map, key);
5430	if (!elt)
5431	goto out;
5432
5433	hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals: hist_pad->var_ref_vals);
5434
5435	if (resolve_var_refs(hist_data, key, var_ref_vals: hist_pad->var_ref_vals, self: true)) {
5436	hist_trigger_actions(hist_data, elt, buffer, rec, rbe,
5437	key, var_ref_vals: hist_pad->var_ref_vals);
5438	}
5439
5440	hist_poll_wakeup();
5441
5442	out:
5443	put_hist_pad();
5444	}
5445
5446	static void hist_trigger_stacktrace_print(struct seq_file *m,
5447	unsigned long *stacktrace_entries,
5448	unsigned int max_entries)
5449	{
5450	unsigned int spaces = 8;
5451	unsigned int i;
5452
5453	for (i = 0; i < max_entries; i++) {
5454	if (!stacktrace_entries[i])
5455	return;
5456
5457	seq_printf(m, fmt: "%*c", 1 + spaces, ' ');
5458	seq_printf(m, fmt: "%pS\n", (void*)stacktrace_entries[i]);
5459	}
5460	}
5461
5462	static void hist_trigger_print_key(struct seq_file *m,
5463	struct hist_trigger_data *hist_data,
5464	void *key,
5465	struct tracing_map_elt *elt)
5466	{
5467	struct hist_field *key_field;
5468	bool multiline = false;
5469	const char *field_name;
5470	unsigned int i;
5471	u64 uval;
5472
5473	seq_puts(m, s: "{ ");
5474
5475	for_each_hist_key_field(i, hist_data) {
5476	key_field = hist_data->fields[i];
5477
5478	if (i > hist_data->n_vals)
5479	seq_puts(m, s: ", ");
5480
5481	field_name = hist_field_name(field: key_field, level: 0);
5482
5483	if (key_field->flags & HIST_FIELD_FL_HEX) {
5484	uval = *(u64 *)(key + key_field->offset);
5485	seq_printf(m, fmt: "%s: %llx", field_name, uval);
5486	} else if (key_field->flags & HIST_FIELD_FL_SYM) {
5487	uval = *(u64 *)(key + key_field->offset);
5488	seq_printf(m, fmt: "%s: [%llx] %-45ps", field_name,
5489	uval, (void *)(uintptr_t)uval);
5490	} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5491	uval = *(u64 *)(key + key_field->offset);
5492	seq_printf(m, fmt: "%s: [%llx] %-55pS", field_name,
5493	uval, (void *)(uintptr_t)uval);
5494	} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5495	struct hist_elt_data *elt_data = elt->private_data;
5496	char *comm;
5497
5498	if (WARN_ON_ONCE(!elt_data))
5499	return;
5500
5501	comm = elt_data->comm;
5502
5503	uval = *(u64 *)(key + key_field->offset);
5504	seq_printf(m, fmt: "%s: %-16s[%10llu]", field_name,
5505	comm, uval);
5506	} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5507	const char *syscall_name;
5508
5509	uval = *(u64 *)(key + key_field->offset);
5510	syscall_name = get_syscall_name(syscall: uval);
5511	if (!syscall_name)
5512	syscall_name = "unknown_syscall";
5513
5514	seq_printf(m, fmt: "%s: %-30s[%3llu]", field_name,
5515	syscall_name, uval);
5516	} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5517	if (key_field->field)
5518	seq_printf(m, fmt: "%s.stacktrace", key_field->field->name);
5519	else
5520	seq_puts(m, s: "common_stacktrace:\n");
5521	hist_trigger_stacktrace_print(m,
5522	stacktrace_entries: key + key_field->offset,
5523	HIST_STACKTRACE_DEPTH);
5524	multiline = true;
5525	} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5526	seq_printf(m, fmt: "%s: ~ 2^%-2llu", field_name,
5527	*(u64 *)(key + key_field->offset));
5528	} else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5529	unsigned long buckets = key_field->buckets;
5530	uval = *(u64 *)(key + key_field->offset);
5531	seq_printf(m, fmt: "%s: ~ %llu-%llu", field_name,
5532	uval, uval + buckets -1);
5533	} else if (key_field->flags & HIST_FIELD_FL_STRING) {
5534	seq_printf(m, fmt: "%s: %-50s", field_name,
5535	(char *)(key + key_field->offset));
5536	} else {
5537	uval = *(u64 *)(key + key_field->offset);
5538	seq_printf(m, fmt: "%s: %10llu", field_name, uval);
5539	}
5540	}
5541
5542	if (!multiline)
5543	seq_puts(m, s: " ");
5544
5545	seq_puts(m, s: "}");
5546	}
5547
5548	/* Get the 100 times of the percentage of @val in @total */
5549	static inline unsigned int __get_percentage(u64 val, u64 total)
5550	{
5551	if (!total)
5552	goto div0;
5553
5554	if (val < (U64_MAX / 10000))
5555	return (unsigned int)div64_ul(val * 10000, total);
5556
5557	total = div64_u64(dividend: total, divisor: 10000);
5558	if (!total)
5559	goto div0;
5560
5561	return (unsigned int)div64_ul(val, total);
5562	div0:
5563	return val ? UINT_MAX : 0;
5564	}
5565
5566	#define BAR_CHAR '#'
5567
5568	static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5569	{
5570	unsigned int len = __get_percentage(val, total: max);
5571	int i;
5572
5573	if (len == UINT_MAX) {
5574	snprintf(buf, size, fmt: "[ERROR]");
5575	return buf;
5576	}
5577
5578	len = len * size / 10000;
5579	for (i = 0; i < len && i < size; i++)
5580	buf[i] = BAR_CHAR;
5581	while (i < size)
5582	buf[i++] = ' ';
5583	buf[size] = '\0';
5584
5585	return buf;
5586	}
5587
5588	struct hist_val_stat {
5589	u64 max;
5590	u64 total;
5591	};
5592
5593	static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5594	const char *field_name, unsigned long flags,
5595	struct hist_val_stat *stats,
5596	struct tracing_map_elt *elt)
5597	{
5598	u64 val = tracing_map_read_sum(elt, i: idx);
5599	unsigned int pc;
5600	char bar[21];
5601
5602	if (flags & HIST_FIELD_FL_PERCENT) {
5603	pc = __get_percentage(val, total: stats[idx].total);
5604	if (pc == UINT_MAX)
5605	seq_printf(m, fmt: " %s (%%):[ERROR]", field_name);
5606	else
5607	seq_printf(m, fmt: " %s (%%): %3u.%02u", field_name,
5608	pc / 100, pc % 100);
5609	} else if (flags & HIST_FIELD_FL_GRAPH) {
5610	seq_printf(m, fmt: " %s: %20s", field_name,
5611	__fill_bar_str(buf: bar, size: 20, val, max: stats[idx].max));
5612	} else if (flags & HIST_FIELD_FL_HEX) {
5613	seq_printf(m, fmt: " %s: %10llx", field_name, val);
5614	} else {
5615	seq_printf(m, fmt: " %s: %10llu", field_name, val);
5616	}
5617	}
5618
5619	static void hist_trigger_entry_print(struct seq_file *m,
5620	struct hist_trigger_data *hist_data,
5621	struct hist_val_stat *stats,
5622	void *key,
5623	struct tracing_map_elt *elt)
5624	{
5625	const char *field_name;
5626	unsigned int i = HITCOUNT_IDX;
5627	unsigned long flags;
5628
5629	hist_trigger_print_key(m, hist_data, key, elt);
5630
5631	/* At first, show the raw hitcount if !nohitcount */
5632	if (!hist_data->attrs->no_hitcount)
5633	hist_trigger_print_val(m, idx: i, field_name: "hitcount", flags: 0, stats, elt);
5634
5635	for (i = 1; i < hist_data->n_vals; i++) {
5636	field_name = hist_field_name(field: hist_data->fields[i], level: 0);
5637	flags = hist_data->fields[i]->flags;
5638	if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5639	continue;
5640
5641	seq_puts(m, s: " ");
5642	hist_trigger_print_val(m, idx: i, field_name, flags, stats, elt);
5643	}
5644
5645	print_actions(m, hist_data, elt);
5646
5647	seq_puts(m, s: "\n");
5648	}
5649
5650	static int print_entries(struct seq_file *m,
5651	struct hist_trigger_data *hist_data)
5652	{
5653	struct tracing_map_sort_entry **sort_entries = NULL;
5654	struct tracing_map *map = hist_data->map;
5655	int i, j, n_entries;
5656	struct hist_val_stat *stats = NULL;
5657	u64 val;
5658
5659	n_entries = tracing_map_sort_entries(map, sort_keys: hist_data->sort_keys,
5660	n_sort_keys: hist_data->n_sort_keys,
5661	sort_entries: &sort_entries);
5662	if (n_entries < 0)
5663	return n_entries;
5664
5665	/* Calculate the max and the total for each field if needed. */
5666	for (j = 0; j < hist_data->n_vals; j++) {
5667	if (!(hist_data->fields[j]->flags &
5668	(HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5669	continue;
5670	if (!stats) {
5671	stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5672	GFP_KERNEL);
5673	if (!stats) {
5674	n_entries = -ENOMEM;
5675	goto out;
5676	}
5677	}
5678	for (i = 0; i < n_entries; i++) {
5679	val = tracing_map_read_sum(elt: sort_entries[i]->elt, i: j);
5680	stats[j].total += val;
5681	if (stats[j].max < val)
5682	stats[j].max = val;
5683	}
5684	}
5685
5686	for (i = 0; i < n_entries; i++)
5687	hist_trigger_entry_print(m, hist_data, stats,
5688	key: sort_entries[i]->key,
5689	elt: sort_entries[i]->elt);
5690
5691	kfree(objp: stats);
5692	out:
5693	tracing_map_destroy_sort_entries(entries: sort_entries, n_entries);
5694
5695	return n_entries;
5696	}
5697
5698	static void hist_trigger_show(struct seq_file *m,
5699	struct event_trigger_data *data, int n)
5700	{
5701	struct hist_trigger_data *hist_data;
5702	int n_entries;
5703
5704	if (n > 0)
5705	seq_puts(m, s: "\n\n");
5706
5707	seq_puts(m, s: "# event histogram\n#\n# trigger info: ");
5708	data->cmd_ops->print(m, data);
5709	seq_puts(m, s: "#\n\n");
5710
5711	hist_data = data->private_data;
5712	n_entries = print_entries(m, hist_data);
5713	if (n_entries < 0)
5714	n_entries = 0;
5715
5716	track_data_snapshot_print(m, hist_data);
5717
5718	seq_printf(m, fmt: "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
5719	(u64)atomic64_read(v: &hist_data->map->hits),
5720	n_entries, (u64)atomic64_read(v: &hist_data->map->drops));
5721	}
5722
5723	struct hist_file_data {
5724	struct file *file;
5725	u64 last_read;
5726	u64 last_act;
5727	};
5728
5729	static u64 get_hist_hit_count(struct trace_event_file *event_file)
5730	{
5731	struct hist_trigger_data *hist_data;
5732	struct event_trigger_data *data;
5733	u64 ret = 0;
5734
5735	list_for_each_entry(data, &event_file->triggers, list) {
5736	if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5737	hist_data = data->private_data;
5738	ret += atomic64_read(v: &hist_data->map->hits);
5739	}
5740	}
5741	return ret;
5742	}
5743
5744	static int hist_show(struct seq_file *m, void *v)
5745	{
5746	struct hist_file_data *hist_file = m->private;
5747	struct event_trigger_data *data;
5748	struct trace_event_file *event_file;
5749	int n = 0;
5750
5751	guard(mutex)(T: &event_mutex);
5752
5753	event_file = event_file_file(filp: hist_file->file);
5754	if (unlikely(!event_file))
5755	return -ENODEV;
5756
5757	list_for_each_entry(data, &event_file->triggers, list) {
5758	if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5759	hist_trigger_show(m, data, n: n++);
5760	}
5761	hist_file->last_read = get_hist_hit_count(event_file);
5762	/*
5763	* Update last_act too so that poll()/POLLPRI can wait for the next
5764	* event after any syscall on hist file.
5765	*/
5766	hist_file->last_act = hist_file->last_read;
5767
5768	return 0;
5769	}
5770
5771	static __poll_t event_hist_poll(struct file *file, struct poll_table_struct *wait)
5772	{
5773	struct trace_event_file *event_file;
5774	struct seq_file *m = file->private_data;
5775	struct hist_file_data *hist_file = m->private;
5776	__poll_t ret = 0;
5777	u64 cnt;
5778
5779	guard(mutex)(T: &event_mutex);
5780
5781	event_file = event_file_data(filp: file);
5782	if (!event_file)
5783	return EPOLLERR;
5784
5785	hist_poll_wait(file, wait);
5786
5787	cnt = get_hist_hit_count(event_file);
5788	if (hist_file->last_read != cnt)
5789	ret |= EPOLLIN | EPOLLRDNORM;
5790	if (hist_file->last_act != cnt) {
5791	hist_file->last_act = cnt;
5792	ret |= EPOLLPRI;
5793	}
5794
5795	return ret;
5796	}
5797
5798	static int event_hist_release(struct inode *inode, struct file *file)
5799	{
5800	struct seq_file *m = file->private_data;
5801	struct hist_file_data *hist_file = m->private;
5802
5803	kfree(objp: hist_file);
5804	return tracing_single_release_file_tr(inode, filp: file);
5805	}
5806
5807	static int event_hist_open(struct inode *inode, struct file *file)
5808	{
5809	struct trace_event_file *event_file;
5810	struct hist_file_data *hist_file;
5811	int ret;
5812
5813	ret = tracing_open_file_tr(inode, filp: file);
5814	if (ret)
5815	return ret;
5816
5817	guard(mutex)(T: &event_mutex);
5818
5819	event_file = event_file_data(filp: file);
5820	if (!event_file) {
5821	ret = -ENODEV;
5822	goto err;
5823	}
5824
5825	hist_file = kzalloc(sizeof(*hist_file), GFP_KERNEL);
5826	if (!hist_file) {
5827	ret = -ENOMEM;
5828	goto err;
5829	}
5830
5831	hist_file->file = file;
5832	hist_file->last_act = get_hist_hit_count(event_file);
5833
5834	/* Clear private_data to avoid warning in single_open() */
5835	file->private_data = NULL;
5836	ret = single_open(file, hist_show, hist_file);
5837	if (ret) {
5838	kfree(objp: hist_file);
5839	goto err;
5840	}
5841
5842	return 0;
5843	err:
5844	tracing_release_file_tr(inode, filp: file);
5845	return ret;
5846	}
5847
5848	const struct file_operations event_hist_fops = {
5849	.open = event_hist_open,
5850	.read = seq_read,
5851	.llseek = seq_lseek,
5852	.release = event_hist_release,
5853	.poll = event_hist_poll,
5854	};
5855
5856	#ifdef CONFIG_HIST_TRIGGERS_DEBUG
5857	static void hist_field_debug_show_flags(struct seq_file *m,
5858	unsigned long flags)
5859	{
5860	seq_puts(m, s: " flags:\n");
5861
5862	if (flags & HIST_FIELD_FL_KEY)
5863	seq_puts(m, s: " HIST_FIELD_FL_KEY\n");
5864	else if (flags & HIST_FIELD_FL_HITCOUNT)
5865	seq_puts(m, s: " VAL: HIST_FIELD_FL_HITCOUNT\n");
5866	else if (flags & HIST_FIELD_FL_VAR)
5867	seq_puts(m, s: " HIST_FIELD_FL_VAR\n");
5868	else if (flags & HIST_FIELD_FL_VAR_REF)
5869	seq_puts(m, s: " HIST_FIELD_FL_VAR_REF\n");
5870	else
5871	seq_puts(m, s: " VAL: normal u64 value\n");
5872
5873	if (flags & HIST_FIELD_FL_ALIAS)
5874	seq_puts(m, s: " HIST_FIELD_FL_ALIAS\n");
5875	else if (flags & HIST_FIELD_FL_CONST)
5876	seq_puts(m, s: " HIST_FIELD_FL_CONST\n");
5877	}
5878
5879	static int hist_field_debug_show(struct seq_file *m,
5880	struct hist_field *field, unsigned long flags)
5881	{
5882	if ((field->flags & flags) != flags) {
5883	seq_printf(m, fmt: "ERROR: bad flags - %lx\n", flags);
5884	return -EINVAL;
5885	}
5886
5887	hist_field_debug_show_flags(m, flags: field->flags);
5888	if (field->field)
5889	seq_printf(m, fmt: " ftrace_event_field name: %s\n",
5890	field->field->name);
5891
5892	if (field->flags & HIST_FIELD_FL_VAR) {
5893	seq_printf(m, fmt: " var.name: %s\n", field->var.name);
5894	seq_printf(m, fmt: " var.idx (into tracing_map_elt.vars[]): %u\n",
5895	field->var.idx);
5896	}
5897
5898	if (field->flags & HIST_FIELD_FL_CONST)
5899	seq_printf(m, fmt: " constant: %llu\n", field->constant);
5900
5901	if (field->flags & HIST_FIELD_FL_ALIAS)
5902	seq_printf(m, fmt: " var_ref_idx (into hist_data->var_refs[]): %u\n",
5903	field->var_ref_idx);
5904
5905	if (field->flags & HIST_FIELD_FL_VAR_REF) {
5906	seq_printf(m, fmt: " name: %s\n", field->name);
5907	seq_printf(m, fmt: " var.idx (into tracing_map_elt.vars[]): %u\n",
5908	field->var.idx);
5909	seq_printf(m, fmt: " var.hist_data: %p\n", field->var.hist_data);
5910	seq_printf(m, fmt: " var_ref_idx (into hist_data->var_refs[]): %u\n",
5911	field->var_ref_idx);
5912	if (field->system)
5913	seq_printf(m, fmt: " system: %s\n", field->system);
5914	if (field->event_name)
5915	seq_printf(m, fmt: " event_name: %s\n", field->event_name);
5916	}
5917
5918	seq_printf(m, fmt: " type: %s\n", field->type);
5919	seq_printf(m, fmt: " size: %u\n", field->size);
5920	seq_printf(m, fmt: " is_signed: %u\n", field->is_signed);
5921
5922	return 0;
5923	}
5924
5925	static int field_var_debug_show(struct seq_file *m,
5926	struct field_var *field_var, unsigned int i,
5927	bool save_vars)
5928	{
5929	const char *vars_name = save_vars ? "save_vars" : "field_vars";
5930	struct hist_field *field;
5931	int ret = 0;
5932
5933	seq_printf(m, fmt: "\n hist_data->%s[%d]:\n", vars_name, i);
5934
5935	field = field_var->var;
5936
5937	seq_printf(m, fmt: "\n %s[%d].var:\n", vars_name, i);
5938
5939	hist_field_debug_show_flags(m, flags: field->flags);
5940	seq_printf(m, fmt: " var.name: %s\n", field->var.name);
5941	seq_printf(m, fmt: " var.idx (into tracing_map_elt.vars[]): %u\n",
5942	field->var.idx);
5943
5944	field = field_var->val;
5945
5946	seq_printf(m, fmt: "\n %s[%d].val:\n", vars_name, i);
5947	if (field->field)
5948	seq_printf(m, fmt: " ftrace_event_field name: %s\n",
5949	field->field->name);
5950	else {
5951	ret = -EINVAL;
5952	goto out;
5953	}
5954
5955	seq_printf(m, fmt: " type: %s\n", field->type);
5956	seq_printf(m, fmt: " size: %u\n", field->size);
5957	seq_printf(m, fmt: " is_signed: %u\n", field->is_signed);
5958	out:
5959	return ret;
5960	}
5961
5962	static int hist_action_debug_show(struct seq_file *m,
5963	struct action_data *data, int i)
5964	{
5965	int ret = 0;
5966
5967	if (data->handler == HANDLER_ONMAX ||
5968	data->handler == HANDLER_ONCHANGE) {
5969	seq_printf(m, fmt: "\n hist_data->actions[%d].track_data.var_ref:\n", i);
5970	ret = hist_field_debug_show(m, field: data->track_data.var_ref,
5971	flags: HIST_FIELD_FL_VAR_REF);
5972	if (ret)
5973	goto out;
5974
5975	seq_printf(m, fmt: "\n hist_data->actions[%d].track_data.track_var:\n", i);
5976	ret = hist_field_debug_show(m, field: data->track_data.track_var,
5977	flags: HIST_FIELD_FL_VAR);
5978	if (ret)
5979	goto out;
5980	}
5981
5982	if (data->handler == HANDLER_ONMATCH) {
5983	seq_printf(m, fmt: "\n hist_data->actions[%d].match_data.event_system: %s\n",
5984	i, data->match_data.event_system);
5985	seq_printf(m, fmt: " hist_data->actions[%d].match_data.event: %s\n",
5986	i, data->match_data.event);
5987	}
5988	out:
5989	return ret;
5990	}
5991
5992	static int hist_actions_debug_show(struct seq_file *m,
5993	struct hist_trigger_data *hist_data)
5994	{
5995	int i, ret = 0;
5996
5997	if (hist_data->n_actions)
5998	seq_puts(m, s: "\n action tracking variables (for onmax()/onchange()/onmatch()):\n");
5999
6000	for (i = 0; i < hist_data->n_actions; i++) {
6001	struct action_data *action = hist_data->actions[i];
6002
6003	ret = hist_action_debug_show(m, data: action, i);
6004	if (ret)
6005	goto out;
6006	}
6007
6008	if (hist_data->n_save_vars)
6009	seq_puts(m, s: "\n save action variables (save() params):\n");
6010
6011	for (i = 0; i < hist_data->n_save_vars; i++) {
6012	ret = field_var_debug_show(m, field_var: hist_data->save_vars[i], i, save_vars: true);
6013	if (ret)
6014	goto out;
6015	}
6016	out:
6017	return ret;
6018	}
6019
6020	static void hist_trigger_debug_show(struct seq_file *m,
6021	struct event_trigger_data *data, int n)
6022	{
6023	struct hist_trigger_data *hist_data;
6024	int i, ret;
6025
6026	if (n > 0)
6027	seq_puts(m, s: "\n\n");
6028
6029	seq_puts(m, s: "# event histogram\n#\n# trigger info: ");
6030	data->cmd_ops->print(m, data);
6031	seq_puts(m, s: "#\n\n");
6032
6033	hist_data = data->private_data;
6034
6035	seq_printf(m, fmt: "hist_data: %p\n\n", hist_data);
6036	seq_printf(m, fmt: " n_vals: %u\n", hist_data->n_vals);
6037	seq_printf(m, fmt: " n_keys: %u\n", hist_data->n_keys);
6038	seq_printf(m, fmt: " n_fields: %u\n", hist_data->n_fields);
6039
6040	seq_puts(m, s: "\n val fields:\n\n");
6041
6042	seq_puts(m, s: " hist_data->fields[0]:\n");
6043	ret = hist_field_debug_show(m, field: hist_data->fields[0],
6044	flags: HIST_FIELD_FL_HITCOUNT);
6045	if (ret)
6046	return;
6047
6048	for (i = 1; i < hist_data->n_vals; i++) {
6049	seq_printf(m, fmt: "\n hist_data->fields[%d]:\n", i);
6050	ret = hist_field_debug_show(m, field: hist_data->fields[i], flags: 0);
6051	if (ret)
6052	return;
6053	}
6054
6055	seq_puts(m, s: "\n key fields:\n");
6056
6057	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
6058	seq_printf(m, fmt: "\n hist_data->fields[%d]:\n", i);
6059	ret = hist_field_debug_show(m, field: hist_data->fields[i],
6060	flags: HIST_FIELD_FL_KEY);
6061	if (ret)
6062	return;
6063	}
6064
6065	if (hist_data->n_var_refs)
6066	seq_puts(m, s: "\n variable reference fields:\n");
6067
6068	for (i = 0; i < hist_data->n_var_refs; i++) {
6069	seq_printf(m, fmt: "\n hist_data->var_refs[%d]:\n", i);
6070	ret = hist_field_debug_show(m, field: hist_data->var_refs[i],
6071	flags: HIST_FIELD_FL_VAR_REF);
6072	if (ret)
6073	return;
6074	}
6075
6076	if (hist_data->n_field_vars)
6077	seq_puts(m, s: "\n field variables:\n");
6078
6079	for (i = 0; i < hist_data->n_field_vars; i++) {
6080	ret = field_var_debug_show(m, field_var: hist_data->field_vars[i], i, save_vars: false);
6081	if (ret)
6082	return;
6083	}
6084
6085	ret = hist_actions_debug_show(m, hist_data);
6086	if (ret)
6087	return;
6088	}
6089
6090	static int hist_debug_show(struct seq_file *m, void *v)
6091	{
6092	struct event_trigger_data *data;
6093	struct trace_event_file *event_file;
6094	int n = 0;
6095
6096	guard(mutex)(T: &event_mutex);
6097
6098	event_file = event_file_file(filp: m->private);
6099	if (unlikely(!event_file))
6100	return -ENODEV;
6101
6102	list_for_each_entry(data, &event_file->triggers, list) {
6103	if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
6104	hist_trigger_debug_show(m, data, n: n++);
6105	}
6106	return 0;
6107	}
6108
6109	static int event_hist_debug_open(struct inode *inode, struct file *file)
6110	{
6111	int ret;
6112
6113	ret = tracing_open_file_tr(inode, filp: file);
6114	if (ret)
6115	return ret;
6116
6117	/* Clear private_data to avoid warning in single_open() */
6118	file->private_data = NULL;
6119	ret = single_open(file, hist_debug_show, file);
6120	if (ret)
6121	tracing_release_file_tr(inode, filp: file);
6122	return ret;
6123	}
6124
6125	const struct file_operations event_hist_debug_fops = {
6126	.open = event_hist_debug_open,
6127	.read = seq_read,
6128	.llseek = seq_lseek,
6129	.release = tracing_single_release_file_tr,
6130	};
6131	#endif
6132
6133	static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
6134	{
6135	const char *field_name = hist_field_name(field: hist_field, level: 0);
6136
6137	if (hist_field->var.name)
6138	seq_printf(m, fmt: "%s=", hist_field->var.name);
6139
6140	if (hist_field->flags & HIST_FIELD_FL_CPU)
6141	seq_puts(m, s: "common_cpu");
6142	if (hist_field->flags & HIST_FIELD_FL_COMM)
6143	seq_puts(m, s: "common_comm");
6144	else if (hist_field->flags & HIST_FIELD_FL_CONST)
6145	seq_printf(m, fmt: "%llu", hist_field->constant);
6146	else if (field_name) {
6147	if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
6148	hist_field->flags & HIST_FIELD_FL_ALIAS)
6149	seq_putc(m, c: '$');
6150	seq_printf(m, fmt: "%s", field_name);
6151	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
6152	seq_puts(m, s: "common_timestamp");
6153
6154	if (hist_field->flags) {
6155	if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
6156	!(hist_field->flags & HIST_FIELD_FL_EXPR) &&
6157	!(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) {
6158	const char *flags = get_hist_field_flags(hist_field);
6159
6160	if (flags)
6161	seq_printf(m, fmt: ".%s", flags);
6162	}
6163	}
6164	if (hist_field->buckets)
6165	seq_printf(m, fmt: "=%ld", hist_field->buckets);
6166	}
6167
6168	static int event_hist_trigger_print(struct seq_file *m,
6169	struct event_trigger_data *data)
6170	{
6171	struct hist_trigger_data *hist_data = data->private_data;
6172	struct hist_field *field;
6173	bool have_var = false;
6174	bool show_val = false;
6175	unsigned int i;
6176
6177	seq_puts(m, HIST_PREFIX);
6178
6179	if (data->name)
6180	seq_printf(m, fmt: "%s:", data->name);
6181
6182	seq_puts(m, s: "keys=");
6183
6184	for_each_hist_key_field(i, hist_data) {
6185	field = hist_data->fields[i];
6186
6187	if (i > hist_data->n_vals)
6188	seq_puts(m, s: ",");
6189
6190	if (field->flags & HIST_FIELD_FL_STACKTRACE) {
6191	if (field->field)
6192	seq_printf(m, fmt: "%s.stacktrace", field->field->name);
6193	else
6194	seq_puts(m, s: "common_stacktrace");
6195	} else
6196	hist_field_print(m, hist_field: field);
6197	}
6198
6199	seq_puts(m, s: ":vals=");
6200
6201	for_each_hist_val_field(i, hist_data) {
6202	field = hist_data->fields[i];
6203	if (field->flags & HIST_FIELD_FL_VAR) {
6204	have_var = true;
6205	continue;
6206	}
6207
6208	if (i == HITCOUNT_IDX) {
6209	if (hist_data->attrs->no_hitcount)
6210	continue;
6211	seq_puts(m, s: "hitcount");
6212	} else {
6213	if (show_val)
6214	seq_puts(m, s: ",");
6215	hist_field_print(m, hist_field: field);
6216	}
6217	show_val = true;
6218	}
6219
6220	if (have_var) {
6221	unsigned int n = 0;
6222
6223	seq_puts(m, s: ":");
6224
6225	for_each_hist_val_field(i, hist_data) {
6226	field = hist_data->fields[i];
6227
6228	if (field->flags & HIST_FIELD_FL_VAR) {
6229	if (n++)
6230	seq_puts(m, s: ",");
6231	hist_field_print(m, hist_field: field);
6232	}
6233	}
6234	}
6235
6236	seq_puts(m, s: ":sort=");
6237
6238	for (i = 0; i < hist_data->n_sort_keys; i++) {
6239	struct tracing_map_sort_key *sort_key;
6240	unsigned int idx, first_key_idx;
6241
6242	/* skip VAR vals */
6243	first_key_idx = hist_data->n_vals - hist_data->n_vars;
6244
6245	sort_key = &hist_data->sort_keys[i];
6246	idx = sort_key->field_idx;
6247
6248	if (WARN_ON(idx >= HIST_FIELDS_MAX))
6249	return -EINVAL;
6250
6251	if (i > 0)
6252	seq_puts(m, s: ",");
6253
6254	if (idx == HITCOUNT_IDX)
6255	seq_puts(m, s: "hitcount");
6256	else {
6257	if (idx >= first_key_idx)
6258	idx += hist_data->n_vars;
6259	hist_field_print(m, hist_field: hist_data->fields[idx]);
6260	}
6261
6262	if (sort_key->descending)
6263	seq_puts(m, s: ".descending");
6264	}
6265	seq_printf(m, fmt: ":size=%u", (1 << hist_data->map->map_bits));
6266	if (hist_data->enable_timestamps)
6267	seq_printf(m, fmt: ":clock=%s", hist_data->attrs->clock);
6268	if (hist_data->attrs->no_hitcount)
6269	seq_puts(m, s: ":nohitcount");
6270
6271	print_actions_spec(m, hist_data);
6272
6273	if (data->filter_str)
6274	seq_printf(m, fmt: " if %s", data->filter_str);
6275
6276	if (data->paused)
6277	seq_puts(m, s: " [paused]");
6278	else
6279	seq_puts(m, s: " [active]");
6280
6281	seq_putc(m, c: '\n');
6282
6283	return 0;
6284	}
6285
6286	static int event_hist_trigger_init(struct event_trigger_data *data)
6287	{
6288	struct hist_trigger_data *hist_data = data->private_data;
6289
6290	if (alloc_hist_pad() < 0)
6291	return -ENOMEM;
6292
6293	if (!data->ref && hist_data->attrs->name)
6294	save_named_trigger(name: hist_data->attrs->name, data);
6295
6296	data->ref++;
6297
6298	return 0;
6299	}
6300
6301	static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6302	{
6303	struct trace_event_file *file;
6304	unsigned int i;
6305	char *cmd;
6306	int ret;
6307
6308	for (i = 0; i < hist_data->n_field_var_hists; i++) {
6309	file = hist_data->field_var_hists[i]->hist_data->event_file;
6310	cmd = hist_data->field_var_hists[i]->cmd;
6311	ret = event_hist_trigger_parse(cmd_ops: &trigger_hist_cmd, file,
6312	glob: "!hist", cmd: "hist", param_and_filter: cmd);
6313	WARN_ON_ONCE(ret < 0);
6314	}
6315	}
6316
6317	static void event_hist_trigger_free(struct event_trigger_data *data)
6318	{
6319	struct hist_trigger_data *hist_data = data->private_data;
6320
6321	if (WARN_ON_ONCE(data->ref <= 0))
6322	return;
6323
6324	data->ref--;
6325	if (!data->ref) {
6326	if (data->name)
6327	del_named_trigger(data);
6328
6329	trigger_data_free(data);
6330
6331	remove_hist_vars(hist_data);
6332
6333	unregister_field_var_hists(hist_data);
6334
6335	destroy_hist_data(hist_data);
6336	}
6337	free_hist_pad();
6338	}
6339
6340	static int event_hist_trigger_named_init(struct event_trigger_data *data)
6341	{
6342	int ret;
6343
6344	data->ref++;
6345
6346	save_named_trigger(name: data->named_data->name, data);
6347
6348	ret = event_hist_trigger_init(data: data->named_data);
6349	if (ret < 0) {
6350	kfree(objp: data->cmd_ops);
6351	data->cmd_ops = &trigger_hist_cmd;
6352	}
6353
6354	return ret;
6355	}
6356
6357	static void event_hist_trigger_named_free(struct event_trigger_data *data)
6358	{
6359	if (WARN_ON_ONCE(data->ref <= 0))
6360	return;
6361
6362	event_hist_trigger_free(data: data->named_data);
6363
6364	data->ref--;
6365	if (!data->ref) {
6366	struct event_command *cmd_ops = data->cmd_ops;
6367
6368	del_named_trigger(data);
6369	trigger_data_free(data);
6370	kfree(objp: cmd_ops);
6371	}
6372	}
6373
6374	static void hist_clear(struct event_trigger_data *data)
6375	{
6376	struct hist_trigger_data *hist_data = data->private_data;
6377
6378	if (data->name)
6379	pause_named_trigger(data);
6380
6381	tracepoint_synchronize_unregister();
6382
6383	tracing_map_clear(map: hist_data->map);
6384
6385	if (data->name)
6386	unpause_named_trigger(data);
6387	}
6388
6389	static bool compatible_field(struct ftrace_event_field *field,
6390	struct ftrace_event_field *test_field)
6391	{
6392	if (field == test_field)
6393	return true;
6394	if (field == NULL || test_field == NULL)
6395	return false;
6396	if (strcmp(field->name, test_field->name) != 0)
6397	return false;
6398	if (strcmp(field->type, test_field->type) != 0)
6399	return false;
6400	if (field->size != test_field->size)
6401	return false;
6402	if (field->is_signed != test_field->is_signed)
6403	return false;
6404
6405	return true;
6406	}
6407
6408	static bool hist_trigger_match(struct event_trigger_data *data,
6409	struct event_trigger_data *data_test,
6410	struct event_trigger_data *named_data,
6411	bool ignore_filter)
6412	{
6413	struct tracing_map_sort_key *sort_key, *sort_key_test;
6414	struct hist_trigger_data *hist_data, *hist_data_test;
6415	struct hist_field *key_field, *key_field_test;
6416	unsigned int i;
6417
6418	if (named_data && (named_data != data_test) &&
6419	(named_data != data_test->named_data))
6420	return false;
6421
6422	if (!named_data && is_named_trigger(test: data_test))
6423	return false;
6424
6425	hist_data = data->private_data;
6426	hist_data_test = data_test->private_data;
6427
6428	if (hist_data->n_vals != hist_data_test->n_vals ||
6429	hist_data->n_fields != hist_data_test->n_fields ||
6430	hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6431	return false;
6432
6433	if (!ignore_filter) {
6434	if ((data->filter_str && !data_test->filter_str) ||
6435	(!data->filter_str && data_test->filter_str))
6436	return false;
6437	}
6438
6439	for_each_hist_field(i, hist_data) {
6440	key_field = hist_data->fields[i];
6441	key_field_test = hist_data_test->fields[i];
6442
6443	if (key_field->flags != key_field_test->flags)
6444	return false;
6445	if (!compatible_field(field: key_field->field, test_field: key_field_test->field))
6446	return false;
6447	if (key_field->offset != key_field_test->offset)
6448	return false;
6449	if (key_field->size != key_field_test->size)
6450	return false;
6451	if (key_field->is_signed != key_field_test->is_signed)
6452	return false;
6453	if (!!key_field->var.name != !!key_field_test->var.name)
6454	return false;
6455	if (key_field->var.name &&
6456	strcmp(key_field->var.name, key_field_test->var.name) != 0)
6457	return false;
6458	}
6459
6460	for (i = 0; i < hist_data->n_sort_keys; i++) {
6461	sort_key = &hist_data->sort_keys[i];
6462	sort_key_test = &hist_data_test->sort_keys[i];
6463
6464	if (sort_key->field_idx != sort_key_test->field_idx ||
6465	sort_key->descending != sort_key_test->descending)
6466	return false;
6467	}
6468
6469	if (!ignore_filter && data->filter_str &&
6470	(strcmp(data->filter_str, data_test->filter_str) != 0))
6471	return false;
6472
6473	if (!actions_match(hist_data, hist_data_test))
6474	return false;
6475
6476	return true;
6477	}
6478
6479	static bool existing_hist_update_only(char *glob,
6480	struct event_trigger_data *data,
6481	struct trace_event_file *file)
6482	{
6483	struct hist_trigger_data *hist_data = data->private_data;
6484	struct event_trigger_data *test, *named_data = NULL;
6485	bool updated = false;
6486
6487	if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6488	!hist_data->attrs->clear)
6489	goto out;
6490
6491	if (hist_data->attrs->name) {
6492	named_data = find_named_trigger(name: hist_data->attrs->name);
6493	if (named_data) {
6494	if (!hist_trigger_match(data, data_test: named_data, named_data,
6495	ignore_filter: true))
6496	goto out;
6497	}
6498	}
6499
6500	if (hist_data->attrs->name && !named_data)
6501	goto out;
6502
6503	list_for_each_entry(test, &file->triggers, list) {
6504	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6505	if (!hist_trigger_match(data, data_test: test, named_data, ignore_filter: false))
6506	continue;
6507	if (hist_data->attrs->pause)
6508	test->paused = true;
6509	else if (hist_data->attrs->cont)
6510	test->paused = false;
6511	else if (hist_data->attrs->clear)
6512	hist_clear(data: test);
6513	updated = true;
6514	goto out;
6515	}
6516	}
6517	out:
6518	return updated;
6519	}
6520
6521	static int hist_register_trigger(char *glob,
6522	struct event_trigger_data *data,
6523	struct trace_event_file *file)
6524	{
6525	struct hist_trigger_data *hist_data = data->private_data;
6526	struct event_trigger_data *test, *named_data = NULL;
6527	struct trace_array *tr = file->tr;
6528	int ret = 0;
6529
6530	if (hist_data->attrs->name) {
6531	named_data = find_named_trigger(name: hist_data->attrs->name);
6532	if (named_data) {
6533	if (!hist_trigger_match(data, data_test: named_data, named_data,
6534	ignore_filter: true)) {
6535	hist_err(tr, err_type: HIST_ERR_NAMED_MISMATCH, err_pos: errpos(str: hist_data->attrs->name));
6536	ret = -EINVAL;
6537	goto out;
6538	}
6539	}
6540	}
6541
6542	if (hist_data->attrs->name && !named_data)
6543	goto new;
6544
6545	lockdep_assert_held(&event_mutex);
6546
6547	list_for_each_entry(test, &file->triggers, list) {
6548	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6549	if (hist_trigger_match(data, data_test: test, named_data, ignore_filter: false)) {
6550	hist_err(tr, err_type: HIST_ERR_TRIGGER_EEXIST, err_pos: 0);
6551	ret = -EEXIST;
6552	goto out;
6553	}
6554	}
6555	}
6556	new:
6557	if (hist_data->attrs->cont || hist_data->attrs->clear) {
6558	hist_err(tr, err_type: HIST_ERR_TRIGGER_ENOENT_CLEAR, err_pos: 0);
6559	ret = -ENOENT;
6560	goto out;
6561	}
6562
6563	if (hist_data->attrs->pause)
6564	data->paused = true;
6565
6566	if (named_data) {
6567	struct event_command *cmd_ops;
6568
6569	data->private_data = named_data->private_data;
6570	set_named_trigger_data(data, named_data);
6571	/* Copy the command ops and update some of the functions */
6572	cmd_ops = kmalloc(sizeof(*cmd_ops), GFP_KERNEL);
6573	if (!cmd_ops) {
6574	ret = -ENOMEM;
6575	goto out;
6576	}
6577	*cmd_ops = *data->cmd_ops;
6578	cmd_ops->init = event_hist_trigger_named_init;
6579	cmd_ops->free = event_hist_trigger_named_free;
6580	data->cmd_ops = cmd_ops;
6581	}
6582
6583	if (data->cmd_ops->init) {
6584	ret = data->cmd_ops->init(data);
6585	if (ret < 0)
6586	goto out;
6587	}
6588
6589	if (hist_data->enable_timestamps) {
6590	char *clock = hist_data->attrs->clock;
6591
6592	ret = tracing_set_clock(tr: file->tr, clockstr: hist_data->attrs->clock);
6593	if (ret) {
6594	hist_err(tr, err_type: HIST_ERR_SET_CLOCK_FAIL, err_pos: errpos(str: clock));
6595	goto out;
6596	}
6597
6598	tracing_set_filter_buffering(tr: file->tr, set: true);
6599	}
6600
6601	if (named_data)
6602	destroy_hist_data(hist_data);
6603	out:
6604	return ret;
6605	}
6606
6607	static int hist_trigger_enable(struct event_trigger_data *data,
6608	struct trace_event_file *file)
6609	{
6610	int ret = 0;
6611
6612	list_add_tail_rcu(new: &data->list, head: &file->triggers);
6613
6614	update_cond_flag(file);
6615
6616	if (trace_event_trigger_enable_disable(file, trigger_enable: 1) < 0) {
6617	list_del_rcu(entry: &data->list);
6618	update_cond_flag(file);
6619	ret--;
6620	}
6621
6622	return ret;
6623	}
6624
6625	static bool have_hist_trigger_match(struct event_trigger_data *data,
6626	struct trace_event_file *file)
6627	{
6628	struct hist_trigger_data *hist_data = data->private_data;
6629	struct event_trigger_data *test, *named_data = NULL;
6630	bool match = false;
6631
6632	lockdep_assert_held(&event_mutex);
6633
6634	if (hist_data->attrs->name)
6635	named_data = find_named_trigger(name: hist_data->attrs->name);
6636
6637	list_for_each_entry(test, &file->triggers, list) {
6638	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6639	if (hist_trigger_match(data, data_test: test, named_data, ignore_filter: false)) {
6640	match = true;
6641	break;
6642	}
6643	}
6644	}
6645
6646	return match;
6647	}
6648
6649	static bool hist_trigger_check_refs(struct event_trigger_data *data,
6650	struct trace_event_file *file)
6651	{
6652	struct hist_trigger_data *hist_data = data->private_data;
6653	struct event_trigger_data *test, *named_data = NULL;
6654
6655	lockdep_assert_held(&event_mutex);
6656
6657	if (hist_data->attrs->name)
6658	named_data = find_named_trigger(name: hist_data->attrs->name);
6659
6660	list_for_each_entry(test, &file->triggers, list) {
6661	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6662	if (!hist_trigger_match(data, data_test: test, named_data, ignore_filter: false))
6663	continue;
6664	hist_data = test->private_data;
6665	if (check_var_refs(hist_data))
6666	return true;
6667	break;
6668	}
6669	}
6670
6671	return false;
6672	}
6673
6674	static void hist_unregister_trigger(char *glob,
6675	struct event_trigger_data *data,
6676	struct trace_event_file *file)
6677	{
6678	struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6679	struct hist_trigger_data *hist_data = data->private_data;
6680
6681	lockdep_assert_held(&event_mutex);
6682
6683	if (hist_data->attrs->name)
6684	named_data = find_named_trigger(name: hist_data->attrs->name);
6685
6686	list_for_each_entry(iter, &file->triggers, list) {
6687	if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6688	if (!hist_trigger_match(data, data_test: iter, named_data, ignore_filter: false))
6689	continue;
6690	test = iter;
6691	list_del_rcu(entry: &test->list);
6692	trace_event_trigger_enable_disable(file, trigger_enable: 0);
6693	update_cond_flag(file);
6694	break;
6695	}
6696	}
6697
6698	if (test && test->cmd_ops->free)
6699	test->cmd_ops->free(test);
6700
6701	if (hist_data->enable_timestamps) {
6702	if (!hist_data->remove || test)
6703	tracing_set_filter_buffering(tr: file->tr, set: false);
6704	}
6705	}
6706
6707	static bool hist_file_check_refs(struct trace_event_file *file)
6708	{
6709	struct hist_trigger_data *hist_data;
6710	struct event_trigger_data *test;
6711
6712	lockdep_assert_held(&event_mutex);
6713
6714	list_for_each_entry(test, &file->triggers, list) {
6715	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6716	hist_data = test->private_data;
6717	if (check_var_refs(hist_data))
6718	return true;
6719	}
6720	}
6721
6722	return false;
6723	}
6724
6725	static void hist_unreg_all(struct trace_event_file *file)
6726	{
6727	struct event_trigger_data *test, *n;
6728	struct hist_trigger_data *hist_data;
6729	struct synth_event *se;
6730	const char *se_name;
6731
6732	lockdep_assert_held(&event_mutex);
6733
6734	if (hist_file_check_refs(file))
6735	return;
6736
6737	list_for_each_entry_safe(test, n, &file->triggers, list) {
6738	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6739	hist_data = test->private_data;
6740	list_del_rcu(entry: &test->list);
6741	trace_event_trigger_enable_disable(file, trigger_enable: 0);
6742
6743	se_name = trace_event_name(call: file->event_call);
6744	se = find_synth_event(name: se_name);
6745	if (se)
6746	se->ref--;
6747
6748	update_cond_flag(file);
6749	if (hist_data->enable_timestamps)
6750	tracing_set_filter_buffering(tr: file->tr, set: false);
6751	if (test->cmd_ops->free)
6752	test->cmd_ops->free(test);
6753	}
6754	}
6755	}
6756
6757	static int event_hist_trigger_parse(struct event_command *cmd_ops,
6758	struct trace_event_file *file,
6759	char *glob, char *cmd,
6760	char *param_and_filter)
6761	{
6762	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6763	struct event_trigger_data *trigger_data;
6764	struct hist_trigger_attrs *attrs;
6765	struct hist_trigger_data *hist_data;
6766	char *param, *filter, *p, *start;
6767	struct synth_event *se;
6768	const char *se_name;
6769	bool remove;
6770	int ret = 0;
6771
6772	lockdep_assert_held(&event_mutex);
6773
6774	if (WARN_ON(!glob))
6775	return -EINVAL;
6776
6777	if (glob[0]) {
6778	hist_err_clear();
6779	last_cmd_set(file, str: param_and_filter);
6780	}
6781
6782	remove = event_trigger_check_remove(glob);
6783
6784	if (event_trigger_empty_param(param: param_and_filter))
6785	return -EINVAL;
6786
6787	/*
6788	* separate the trigger from the filter (k:v [if filter])
6789	* allowing for whitespace in the trigger
6790	*/
6791	p = param = param_and_filter;
6792	do {
6793	p = strstr(p, "if");
6794	if (!p)
6795	break;
6796	if (p == param_and_filter)
6797	return -EINVAL;
6798	if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6799	p++;
6800	continue;
6801	}
6802	if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6803	return -EINVAL;
6804	if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6805	p++;
6806	continue;
6807	}
6808	break;
6809	} while (1);
6810
6811	if (!p)
6812	filter = NULL;
6813	else {
6814	*(p - 1) = '\0';
6815	filter = strstrip(str: p);
6816	param = strstrip(str: param);
6817	}
6818
6819	/*
6820	* To simplify arithmetic expression parsing, replace occurrences of
6821	* '.sym-offset' modifier with '.symXoffset'
6822	*/
6823	start = strstr(param, ".sym-offset");
6824	while (start) {
6825	*(start + 4) = 'X';
6826	start = strstr(start + 11, ".sym-offset");
6827	}
6828
6829	attrs = parse_hist_trigger_attrs(tr: file->tr, trigger_str: param);
6830	if (IS_ERR(ptr: attrs))
6831	return PTR_ERR(ptr: attrs);
6832
6833	if (attrs->map_bits)
6834	hist_trigger_bits = attrs->map_bits;
6835
6836	hist_data = create_hist_data(map_bits: hist_trigger_bits, attrs, file, remove);
6837	if (IS_ERR(ptr: hist_data)) {
6838	destroy_hist_trigger_attrs(attrs);
6839	return PTR_ERR(ptr: hist_data);
6840	}
6841
6842	trigger_data = trigger_data_alloc(cmd_ops, cmd, param, private_data: hist_data);
6843	if (!trigger_data) {
6844	ret = -ENOMEM;
6845	goto out_free;
6846	}
6847
6848	ret = event_trigger_set_filter(cmd_ops, file, param: filter, trigger_data);
6849	if (ret < 0)
6850	goto out_free;
6851
6852	if (remove) {
6853	if (!have_hist_trigger_match(data: trigger_data, file))
6854	goto out_free;
6855
6856	if (hist_trigger_check_refs(data: trigger_data, file)) {
6857	ret = -EBUSY;
6858	goto out_free;
6859	}
6860
6861	event_trigger_unregister(cmd_ops, file, glob: glob+1, trigger_data);
6862	se_name = trace_event_name(call: file->event_call);
6863	se = find_synth_event(name: se_name);
6864	if (se)
6865	se->ref--;
6866	ret = 0;
6867	goto out_free;
6868	}
6869
6870	if (existing_hist_update_only(glob, data: trigger_data, file))
6871	goto out_free;
6872
6873	if (!get_named_trigger_data(data: trigger_data)) {
6874
6875	ret = create_actions(hist_data);
6876	if (ret)
6877	goto out_free;
6878
6879	if (has_hist_vars(hist_data) || hist_data->n_var_refs) {
6880	ret = save_hist_vars(hist_data);
6881	if (ret)
6882	goto out_free;
6883	}
6884
6885	ret = tracing_map_init(map: hist_data->map);
6886	if (ret)
6887	goto out_free;
6888	}
6889
6890	ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6891	if (ret < 0)
6892	goto out_free;
6893
6894	ret = hist_trigger_enable(data: trigger_data, file);
6895	if (ret)
6896	goto out_unreg;
6897
6898	se_name = trace_event_name(call: file->event_call);
6899	se = find_synth_event(name: se_name);
6900	if (se)
6901	se->ref++;
6902	out:
6903	if (ret == 0 && glob[0])
6904	hist_err_clear();
6905
6906	return ret;
6907	out_unreg:
6908	event_trigger_unregister(cmd_ops, file, glob: glob+1, trigger_data);
6909	out_free:
6910	event_trigger_reset_filter(cmd_ops, trigger_data);
6911
6912	remove_hist_vars(hist_data);
6913
6914	kfree(objp: trigger_data);
6915
6916	destroy_hist_data(hist_data);
6917	goto out;
6918	}
6919
6920	static struct event_command trigger_hist_cmd = {
6921	.name = "hist",
6922	.trigger_type = ETT_EVENT_HIST,
6923	.flags = EVENT_CMD_FL_NEEDS_REC,
6924	.parse = event_hist_trigger_parse,
6925	.reg = hist_register_trigger,
6926	.unreg = hist_unregister_trigger,
6927	.unreg_all = hist_unreg_all,
6928	.set_filter = set_trigger_filter,
6929	.trigger = event_hist_trigger,
6930	.print = event_hist_trigger_print,
6931	.init = event_hist_trigger_init,
6932	.free = event_hist_trigger_free,
6933	};
6934
6935	__init int register_trigger_hist_cmd(void)
6936	{
6937	int ret;
6938
6939	ret = register_event_command(cmd: &trigger_hist_cmd);
6940	WARN_ON(ret < 0);
6941
6942	return ret;
6943	}
6944
6945	static void
6946	hist_enable_trigger(struct event_trigger_data *data,
6947	struct trace_buffer *buffer, void *rec,
6948	struct ring_buffer_event *event)
6949	{
6950	struct enable_trigger_data *enable_data = data->private_data;
6951	struct event_trigger_data *test;
6952
6953	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6954	lockdep_is_held(&event_mutex)) {
6955	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6956	if (enable_data->enable)
6957	test->paused = false;
6958	else
6959	test->paused = true;
6960	}
6961	}
6962	}
6963
6964	static void hist_enable_unreg_all(struct trace_event_file *file)
6965	{
6966	struct event_trigger_data *test, *n;
6967
6968	list_for_each_entry_safe(test, n, &file->triggers, list) {
6969	if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6970	list_del_rcu(entry: &test->list);
6971	update_cond_flag(file);
6972	trace_event_trigger_enable_disable(file, trigger_enable: 0);
6973	if (test->cmd_ops->free)
6974	test->cmd_ops->free(test);
6975	}
6976	}
6977	}
6978
6979	static struct event_command trigger_hist_enable_cmd = {
6980	.name = ENABLE_HIST_STR,
6981	.trigger_type = ETT_HIST_ENABLE,
6982	.parse = event_enable_trigger_parse,
6983	.reg = event_enable_register_trigger,
6984	.unreg = event_enable_unregister_trigger,
6985	.unreg_all = hist_enable_unreg_all,
6986	.set_filter = set_trigger_filter,
6987	.trigger = hist_enable_trigger,
6988	.count_func = event_trigger_count,
6989	.print = event_enable_trigger_print,
6990	.init = event_trigger_init,
6991	.free = event_enable_trigger_free,
6992	};
6993
6994	static struct event_command trigger_hist_disable_cmd = {
6995	.name = DISABLE_HIST_STR,
6996	.trigger_type = ETT_HIST_ENABLE,
6997	.parse = event_enable_trigger_parse,
6998	.reg = event_enable_register_trigger,
6999	.unreg = event_enable_unregister_trigger,
7000	.unreg_all = hist_enable_unreg_all,
7001	.set_filter = set_trigger_filter,
7002	.trigger = hist_enable_trigger,
7003	.count_func = event_trigger_count,
7004	.print = event_enable_trigger_print,
7005	.init = event_trigger_init,
7006	.free = event_enable_trigger_free,
7007	};
7008
7009	static __init void unregister_trigger_hist_enable_disable_cmds(void)
7010	{
7011	unregister_event_command(cmd: &trigger_hist_enable_cmd);
7012	unregister_event_command(cmd: &trigger_hist_disable_cmd);
7013	}
7014
7015	__init int register_trigger_hist_enable_disable_cmds(void)
7016	{
7017	int ret;
7018
7019	ret = register_event_command(cmd: &trigger_hist_enable_cmd);
7020	if (WARN_ON(ret < 0))
7021	return ret;
7022	ret = register_event_command(cmd: &trigger_hist_disable_cmd);
7023	if (WARN_ON(ret < 0))
7024	unregister_trigger_hist_enable_disable_cmds();
7025
7026	return ret;
7027	}
7028