1	/*
2	* lib/dynamic_debug.c
3	*
4	* make pr_debug()/dev_dbg() calls runtime configurable based upon their
5	* source module.
6	*
7	* Copyright (C) 2008 Jason Baron <jbaron@redhat.com>
8	* By Greg Banks <gnb@melbourne.sgi.com>
9	* Copyright (c) 2008 Silicon Graphics Inc. All Rights Reserved.
10	* Copyright (C) 2011 Bart Van Assche. All Rights Reserved.
11	* Copyright (C) 2013 Du, Changbin <changbin.du@gmail.com>
12	*/
13
14	#define pr_fmt(fmt) "dyndbg: " fmt
15
16	#include <linux/kernel.h>
17	#include <linux/module.h>
18	#include <linux/moduleparam.h>
19	#include <linux/kallsyms.h>
20	#include <linux/types.h>
21	#include <linux/mutex.h>
22	#include <linux/proc_fs.h>
23	#include <linux/seq_file.h>
24	#include <linux/list.h>
25	#include <linux/sysctl.h>
26	#include <linux/ctype.h>
27	#include <linux/string.h>
28	#include <linux/parser.h>
29	#include <linux/string_helpers.h>
30	#include <linux/uaccess.h>
31	#include <linux/dynamic_debug.h>
32	#include <linux/debugfs.h>
33	#include <linux/slab.h>
34	#include <linux/jump_label.h>
35	#include <linux/hardirq.h>
36	#include <linux/sched.h>
37	#include <linux/device.h>
38	#include <linux/netdevice.h>
39
40	#include <rdma/ib_verbs.h>
41
42	extern struct _ddebug __start___dyndbg[];
43	extern struct _ddebug __stop___dyndbg[];
44	extern struct ddebug_class_map __start___dyndbg_classes[];
45	extern struct ddebug_class_map __stop___dyndbg_classes[];
46
47	struct ddebug_table {
48	struct list_head link, maps;
49	const char *mod_name;
50	unsigned int num_ddebugs;
51	struct _ddebug *ddebugs;
52	};
53
54	struct ddebug_query {
55	const char *filename;
56	const char *module;
57	const char *function;
58	const char *format;
59	const char *class_string;
60	unsigned int first_lineno, last_lineno;
61	};
62
63	struct ddebug_iter {
64	struct ddebug_table *table;
65	int idx;
66	};
67
68	struct flag_settings {
69	unsigned int flags;
70	unsigned int mask;
71	};
72
73	static DEFINE_MUTEX(ddebug_lock);
74	static LIST_HEAD(ddebug_tables);
75	static int verbose;
76	module_param(verbose, int, 0644);
77	MODULE_PARM_DESC(verbose, " dynamic_debug/control processing "
78	"( 0 = off (default), 1 = module add/rm, 2 = >control summary, 3 = parsing, 4 = per-site changes)");
79
80	/* Return the path relative to source root */
81	static inline const char *trim_prefix(const char *path)
82	{
83	int skip = strlen(__FILE__) - strlen("lib/dynamic_debug.c");
84
85	if (strncmp(path, __FILE__, skip))
86	skip = 0; /* prefix mismatch, don't skip */
87
88	return path + skip;
89	}
90
91	static const struct { unsigned flag:8; char opt_char; } opt_array[] = {
92	{ _DPRINTK_FLAGS_PRINT, 'p' },
93	{ _DPRINTK_FLAGS_INCL_MODNAME, 'm' },
94	{ _DPRINTK_FLAGS_INCL_FUNCNAME, 'f' },
95	{ _DPRINTK_FLAGS_INCL_SOURCENAME, 's' },
96	{ _DPRINTK_FLAGS_INCL_LINENO, 'l' },
97	{ _DPRINTK_FLAGS_INCL_TID, 't' },
98	{ _DPRINTK_FLAGS_INCL_STACK, 'd' },
99	{ _DPRINTK_FLAGS_NONE, '_' },
100	};
101
102	struct flagsbuf { char buf[ARRAY_SIZE(opt_array)+1]; };
103
104	/* format a string into buf[] which describes the _ddebug's flags */
105	static char *ddebug_describe_flags(unsigned int flags, struct flagsbuf *fb)
106	{
107	char *p = fb->buf;
108	int i;
109
110	for (i = 0; i < ARRAY_SIZE(opt_array); ++i)
111	if (flags & opt_array[i].flag)
112	*p++ = opt_array[i].opt_char;
113	if (p == fb->buf)
114	*p++ = '_';
115	*p = '\0';
116
117	return fb->buf;
118	}
119
120	#define vnpr_info(lvl, fmt, ...) \
121	do { \
122	if (verbose >= lvl) \
123	pr_info(fmt, ##__VA_ARGS__); \
124	} while (0)
125
126	#define vpr_info(fmt, ...) vnpr_info(1, fmt, ##__VA_ARGS__)
127	#define v2pr_info(fmt, ...) vnpr_info(2, fmt, ##__VA_ARGS__)
128	#define v3pr_info(fmt, ...) vnpr_info(3, fmt, ##__VA_ARGS__)
129	#define v4pr_info(fmt, ...) vnpr_info(4, fmt, ##__VA_ARGS__)
130
131	static void vpr_info_dq(const struct ddebug_query *query, const char *msg)
132	{
133	/* trim any trailing newlines */
134	int fmtlen = 0;
135
136	if (query->format) {
137	fmtlen = strlen(query->format);
138	while (fmtlen && query->format[fmtlen - 1] == '\n')
139	fmtlen--;
140	}
141
142	v3pr_info("%s: func=\"%s\" file=\"%s\" module=\"%s\" format=\"%.*s\" lineno=%u-%u class=%s\n",
143	msg,
144	query->function ?: "",
145	query->filename ?: "",
146	query->module ?: "",
147	fmtlen, query->format ?: "",
148	query->first_lineno, query->last_lineno, query->class_string);
149	}
150
151	static struct ddebug_class_map *ddebug_find_valid_class(struct ddebug_table const *dt,
152	const char *class_string, int *class_id)
153	{
154	struct ddebug_class_map *map;
155	int idx;
156
157	list_for_each_entry(map, &dt->maps, link) {
158	idx = match_string(array: map->class_names, n: map->length, string: class_string);
159	if (idx >= 0) {
160	*class_id = idx + map->base;
161	return map;
162	}
163	}
164	*class_id = -ENOENT;
165	return NULL;
166	}
167
168	#define __outvar /* filled by callee */
169	/*
170	* Search the tables for _ddebug's which match the given `query' and
171	* apply the `flags' and `mask' to them. Returns number of matching
172	* callsites, normally the same as number of changes. If verbose,
173	* logs the changes. Takes ddebug_lock.
174	*/
175	static int ddebug_change(const struct ddebug_query *query,
176	struct flag_settings *modifiers)
177	{
178	int i;
179	struct ddebug_table *dt;
180	unsigned int newflags;
181	unsigned int nfound = 0;
182	struct flagsbuf fbuf, nbuf;
183	struct ddebug_class_map *map = NULL;
184	int __outvar valid_class;
185
186	/* search for matching ddebugs */
187	mutex_lock(&ddebug_lock);
188	list_for_each_entry(dt, &ddebug_tables, link) {
189
190	/* match against the module name */
191	if (query->module &&
192	!match_wildcard(pattern: query->module, str: dt->mod_name))
193	continue;
194
195	if (query->class_string) {
196	map = ddebug_find_valid_class(dt, class_string: query->class_string, class_id: &valid_class);
197	if (!map)
198	continue;
199	} else {
200	/* constrain query, do not touch class'd callsites */
201	valid_class = _DPRINTK_CLASS_DFLT;
202	}
203
204	for (i = 0; i < dt->num_ddebugs; i++) {
205	struct _ddebug *dp = &dt->ddebugs[i];
206
207	/* match site against query-class */
208	if (dp->class_id != valid_class)
209	continue;
210
211	/* match against the source filename */
212	if (query->filename &&
213	!match_wildcard(pattern: query->filename, str: dp->filename) &&
214	!match_wildcard(pattern: query->filename,
215	str: kbasename(path: dp->filename)) &&
216	!match_wildcard(pattern: query->filename,
217	str: trim_prefix(path: dp->filename)))
218	continue;
219
220	/* match against the function */
221	if (query->function &&
222	!match_wildcard(pattern: query->function, str: dp->function))
223	continue;
224
225	/* match against the format */
226	if (query->format) {
227	if (*query->format == '^') {
228	char *p;
229	/* anchored search. match must be at beginning */
230	p = strstr(dp->format, query->format+1);
231	if (p != dp->format)
232	continue;
233	} else if (!strstr(dp->format, query->format))
234	continue;
235	}
236
237	/* match against the line number range */
238	if (query->first_lineno &&
239	dp->lineno < query->first_lineno)
240	continue;
241	if (query->last_lineno &&
242	dp->lineno > query->last_lineno)
243	continue;
244
245	nfound++;
246
247	newflags = (dp->flags & modifiers->mask) | modifiers->flags;
248	if (newflags == dp->flags)
249	continue;
250	#ifdef CONFIG_JUMP_LABEL
251	if (dp->flags & _DPRINTK_FLAGS_PRINT) {
252	if (!(newflags & _DPRINTK_FLAGS_PRINT))
253	static_branch_disable(&dp->key.dd_key_true);
254	} else if (newflags & _DPRINTK_FLAGS_PRINT) {
255	static_branch_enable(&dp->key.dd_key_true);
256	}
257	#endif
258	v4pr_info("changed %s:%d [%s]%s %s => %s\n",
259	trim_prefix(dp->filename), dp->lineno,
260	dt->mod_name, dp->function,
261	ddebug_describe_flags(dp->flags, &fbuf),
262	ddebug_describe_flags(newflags, &nbuf));
263	dp->flags = newflags;
264	}
265	}
266	mutex_unlock(lock: &ddebug_lock);
267
268	if (!nfound && verbose)
269	pr_info("no matches for query\n");
270
271	return nfound;
272	}
273
274	/*
275	* Split the buffer `buf' into space-separated words.
276	* Handles simple " and ' quoting, i.e. without nested,
277	* embedded or escaped \". Return the number of words
278	* or <0 on error.
279	*/
280	static int ddebug_tokenize(char *buf, char *words[], int maxwords)
281	{
282	int nwords = 0;
283
284	while (*buf) {
285	char *end;
286
287	/* Skip leading whitespace */
288	buf = skip_spaces(buf);
289	if (!*buf)
290	break; /* oh, it was trailing whitespace */
291	if (*buf == '#')
292	break; /* token starts comment, skip rest of line */
293
294	/* find `end' of word, whitespace separated or quoted */
295	if (*buf == '"' || *buf == '\'') {
296	int quote = *buf++;
297	for (end = buf; *end && *end != quote; end++)
298	;
299	if (!*end) {
300	pr_err("unclosed quote: %s\n", buf);
301	return -EINVAL; /* unclosed quote */
302	}
303	} else {
304	for (end = buf; *end && !isspace(*end); end++)
305	;
306	if (end == buf) {
307	pr_err("parse err after word:%d=%s\n", nwords,
308	nwords ? words[nwords - 1] : "<none>");
309	return -EINVAL;
310	}
311	}
312
313	/* `buf' is start of word, `end' is one past its end */
314	if (nwords == maxwords) {
315	pr_err("too many words, legal max <=%d\n", maxwords);
316	return -EINVAL; /* ran out of words[] before bytes */
317	}
318	if (*end)
319	*end++ = '\0'; /* terminate the word */
320	words[nwords++] = buf;
321	buf = end;
322	}
323
324	if (verbose >= 3) {
325	int i;
326	pr_info("split into words:");
327	for (i = 0; i < nwords; i++)
328	pr_cont(" \"%s\"", words[i]);
329	pr_cont("\n");
330	}
331
332	return nwords;
333	}
334
335	/*
336	* Parse a single line number. Note that the empty string ""
337	* is treated as a special case and converted to zero, which
338	* is later treated as a "don't care" value.
339	*/
340	static inline int parse_lineno(const char *str, unsigned int *val)
341	{
342	BUG_ON(str == NULL);
343	if (*str == '\0') {
344	*val = 0;
345	return 0;
346	}
347	if (kstrtouint(s: str, base: 10, res: val) < 0) {
348	pr_err("bad line-number: %s\n", str);
349	return -EINVAL;
350	}
351	return 0;
352	}
353
354	static int parse_linerange(struct ddebug_query *query, const char *first)
355	{
356	char *last = strchr(first, '-');
357
358	if (query->first_lineno || query->last_lineno) {
359	pr_err("match-spec: line used 2x\n");
360	return -EINVAL;
361	}
362	if (last)
363	*last++ = '\0';
364	if (parse_lineno(str: first, val: &query->first_lineno) < 0)
365	return -EINVAL;
366	if (last) {
367	/* range <first>-<last> */
368	if (parse_lineno(str: last, val: &query->last_lineno) < 0)
369	return -EINVAL;
370
371	/* special case for last lineno not specified */
372	if (query->last_lineno == 0)
373	query->last_lineno = UINT_MAX;
374
375	if (query->last_lineno < query->first_lineno) {
376	pr_err("last-line:%d < 1st-line:%d\n",
377	query->last_lineno,
378	query->first_lineno);
379	return -EINVAL;
380	}
381	} else {
382	query->last_lineno = query->first_lineno;
383	}
384	v3pr_info("parsed line %d-%d\n", query->first_lineno,
385	query->last_lineno);
386	return 0;
387	}
388
389	static int check_set(const char **dest, char *src, char *name)
390	{
391	int rc = 0;
392
393	if (*dest) {
394	rc = -EINVAL;
395	pr_err("match-spec:%s val:%s overridden by %s\n",
396	name, *dest, src);
397	}
398	*dest = src;
399	return rc;
400	}
401
402	/*
403	* Parse words[] as a ddebug query specification, which is a series
404	* of (keyword, value) pairs chosen from these possibilities:
405	*
406	* func <function-name>
407	* file <full-pathname>
408	* file <base-filename>
409	* module <module-name>
410	* format <escaped-string-to-find-in-format>
411	* line <lineno>
412	* line <first-lineno>-<last-lineno> // where either may be empty
413	*
414	* Only 1 of each type is allowed.
415	* Returns 0 on success, <0 on error.
416	*/
417	static int ddebug_parse_query(char *words[], int nwords,
418	struct ddebug_query *query, const char *modname)
419	{
420	unsigned int i;
421	int rc = 0;
422	char *fline;
423
424	/* check we have an even number of words */
425	if (nwords % 2 != 0) {
426	pr_err("expecting pairs of match-spec <value>\n");
427	return -EINVAL;
428	}
429
430	for (i = 0; i < nwords; i += 2) {
431	char *keyword = words[i];
432	char *arg = words[i+1];
433
434	if (!strcmp(keyword, "func")) {
435	rc = check_set(dest: &query->function, src: arg, name: "func");
436	} else if (!strcmp(keyword, "file")) {
437	if (check_set(dest: &query->filename, src: arg, name: "file"))
438	return -EINVAL;
439
440	/* tail :$info is function or line-range */
441	fline = strchr(query->filename, ':');
442	if (!fline)
443	continue;
444	*fline++ = '\0';
445	if (isalpha(*fline) || *fline == '*' || *fline == '?') {
446	/* take as function name */
447	if (check_set(dest: &query->function, src: fline, name: "func"))
448	return -EINVAL;
449	} else {
450	if (parse_linerange(query, first: fline))
451	return -EINVAL;
452	}
453	} else if (!strcmp(keyword, "module")) {
454	rc = check_set(dest: &query->module, src: arg, name: "module");
455	} else if (!strcmp(keyword, "format")) {
456	string_unescape_inplace(buf: arg, UNESCAPE_SPACE |
457	UNESCAPE_OCTAL |
458	UNESCAPE_SPECIAL);
459	rc = check_set(dest: &query->format, src: arg, name: "format");
460	} else if (!strcmp(keyword, "line")) {
461	if (parse_linerange(query, first: arg))
462	return -EINVAL;
463	} else if (!strcmp(keyword, "class")) {
464	rc = check_set(dest: &query->class_string, src: arg, name: "class");
465	} else {
466	pr_err("unknown keyword \"%s\"\n", keyword);
467	return -EINVAL;
468	}
469	if (rc)
470	return rc;
471	}
472	if (!query->module && modname)
473	/*
474	* support $modname.dyndbg=<multiple queries>, when
475	* not given in the query itself
476	*/
477	query->module = modname;
478
479	vpr_info_dq(query, msg: "parsed");
480	return 0;
481	}
482
483	/*
484	* Parse `str' as a flags specification, format [-+=][p]+.
485	* Sets up *maskp and *flagsp to be used when changing the
486	* flags fields of matched _ddebug's. Returns 0 on success
487	* or <0 on error.
488	*/
489	static int ddebug_parse_flags(const char *str, struct flag_settings *modifiers)
490	{
491	int op, i;
492
493	switch (*str) {
494	case '+':
495	case '-':
496	case '=':
497	op = *str++;
498	break;
499	default:
500	pr_err("bad flag-op %c, at start of %s\n", *str, str);
501	return -EINVAL;
502	}
503	v3pr_info("op='%c'\n", op);
504
505	for (; *str ; ++str) {
506	for (i = ARRAY_SIZE(opt_array) - 1; i >= 0; i--) {
507	if (*str == opt_array[i].opt_char) {
508	modifiers->flags |= opt_array[i].flag;
509	break;
510	}
511	}
512	if (i < 0) {
513	pr_err("unknown flag '%c'\n", *str);
514	return -EINVAL;
515	}
516	}
517	v3pr_info("flags=0x%x\n", modifiers->flags);
518
519	/* calculate final flags, mask based upon op */
520	switch (op) {
521	case '=':
522	/* modifiers->flags already set */
523	modifiers->mask = 0;
524	break;
525	case '+':
526	modifiers->mask = ~0U;
527	break;
528	case '-':
529	modifiers->mask = ~modifiers->flags;
530	modifiers->flags = 0;
531	break;
532	}
533	v3pr_info("*flagsp=0x%x *maskp=0x%x\n", modifiers->flags, modifiers->mask);
534
535	return 0;
536	}
537
538	static int ddebug_exec_query(char *query_string, const char *modname)
539	{
540	struct flag_settings modifiers = {};
541	struct ddebug_query query = {};
542	#define MAXWORDS 9
543	int nwords, nfound;
544	char *words[MAXWORDS];
545
546	nwords = ddebug_tokenize(buf: query_string, words, MAXWORDS);
547	if (nwords <= 0) {
548	pr_err("tokenize failed\n");
549	return -EINVAL;
550	}
551	/* check flags 1st (last arg) so query is pairs of spec,val */
552	if (ddebug_parse_flags(str: words[nwords-1], modifiers: &modifiers)) {
553	pr_err("flags parse failed\n");
554	return -EINVAL;
555	}
556	if (ddebug_parse_query(words, nwords: nwords-1, query: &query, modname)) {
557	pr_err("query parse failed\n");
558	return -EINVAL;
559	}
560	/* actually go and implement the change */
561	nfound = ddebug_change(query: &query, modifiers: &modifiers);
562	vpr_info_dq(query: &query, msg: nfound ? "applied" : "no-match");
563
564	return nfound;
565	}
566
567	/* handle multiple queries in query string, continue on error, return
568	last error or number of matching callsites. Module name is either
569	in param (for boot arg) or perhaps in query string.
570	*/
571	static int ddebug_exec_queries(char *query, const char *modname)
572	{
573	char *split;
574	int i, errs = 0, exitcode = 0, rc, nfound = 0;
575
576	for (i = 0; query; query = split) {
577	split = strpbrk(query, ";\n");
578	if (split)
579	*split++ = '\0';
580
581	query = skip_spaces(query);
582	if (!query || !*query || *query == '#')
583	continue;
584
585	vpr_info("query %d: \"%s\" mod:%s\n", i, query, modname ?: "*");
586
587	rc = ddebug_exec_query(query_string: query, modname);
588	if (rc < 0) {
589	errs++;
590	exitcode = rc;
591	} else {
592	nfound += rc;
593	}
594	i++;
595	}
596	if (i)
597	v2pr_info("processed %d queries, with %d matches, %d errs\n",
598	i, nfound, errs);
599
600	if (exitcode)
601	return exitcode;
602	return nfound;
603	}
604
605	/* apply a new bitmap to the sys-knob's current bit-state */
606	static int ddebug_apply_class_bitmap(const struct ddebug_class_param *dcp,
607	unsigned long *new_bits, unsigned long *old_bits)
608	{
609	#define QUERY_SIZE 128
610	char query[QUERY_SIZE];
611	const struct ddebug_class_map *map = dcp->map;
612	int matches = 0;
613	int bi, ct;
614
615	v2pr_info("apply: 0x%lx to: 0x%lx\n", *new_bits, *old_bits);
616
617	for (bi = 0; bi < map->length; bi++) {
618	if (test_bit(bi, new_bits) == test_bit(bi, old_bits))
619	continue;
620
621	snprintf(buf: query, QUERY_SIZE, fmt: "class %s %c%s", map->class_names[bi],
622	test_bit(bi, new_bits) ? '+' : '-', dcp->flags);
623
624	ct = ddebug_exec_queries(query, NULL);
625	matches += ct;
626
627	v2pr_info("bit_%d: %d matches on class: %s -> 0x%lx\n", bi,
628	ct, map->class_names[bi], *new_bits);
629	}
630	return matches;
631	}
632
633	/* stub to later conditionally add "$module." prefix where not already done */
634	#define KP_NAME(kp) kp->name
635
636	#define CLASSMAP_BITMASK(width) ((1UL << (width)) - 1)
637
638	/* accept comma-separated-list of [+-] classnames */
639	static int param_set_dyndbg_classnames(const char *instr, const struct kernel_param *kp)
640	{
641	const struct ddebug_class_param *dcp = kp->arg;
642	const struct ddebug_class_map *map = dcp->map;
643	unsigned long curr_bits, old_bits;
644	char *cl_str, *p, *tmp;
645	int cls_id, totct = 0;
646	bool wanted;
647
648	cl_str = tmp = kstrdup_and_replace(src: instr, old: '\n', new: '\0', GFP_KERNEL);
649	if (!tmp)
650	return -ENOMEM;
651
652	/* start with previously set state-bits, then modify */
653	curr_bits = old_bits = *dcp->bits;
654	vpr_info("\"%s\" > %s:0x%lx\n", cl_str, KP_NAME(kp), curr_bits);
655
656	for (; cl_str; cl_str = p) {
657	p = strchr(cl_str, ',');
658	if (p)
659	*p++ = '\0';
660
661	if (*cl_str == '-') {
662	wanted = false;
663	cl_str++;
664	} else {
665	wanted = true;
666	if (*cl_str == '+')
667	cl_str++;
668	}
669	cls_id = match_string(array: map->class_names, n: map->length, string: cl_str);
670	if (cls_id < 0) {
671	pr_err("%s unknown to %s\n", cl_str, KP_NAME(kp));
672	continue;
673	}
674
675	/* have one or more valid class_ids of one *_NAMES type */
676	switch (map->map_type) {
677	case DD_CLASS_TYPE_DISJOINT_NAMES:
678	/* the +/- pertains to a single bit */
679	if (test_bit(cls_id, &curr_bits) == wanted) {
680	v3pr_info("no change on %s\n", cl_str);
681	continue;
682	}
683	curr_bits ^= BIT(cls_id);
684	totct += ddebug_apply_class_bitmap(dcp, new_bits: &curr_bits, old_bits: dcp->bits);
685	*dcp->bits = curr_bits;
686	v2pr_info("%s: changed bit %d:%s\n", KP_NAME(kp), cls_id,
687	map->class_names[cls_id]);
688	break;
689	case DD_CLASS_TYPE_LEVEL_NAMES:
690	/* cls_id = N in 0..max. wanted +/- determines N or N-1 */
691	old_bits = CLASSMAP_BITMASK(*dcp->lvl);
692	curr_bits = CLASSMAP_BITMASK(cls_id + (wanted ? 1 : 0 ));
693
694	totct += ddebug_apply_class_bitmap(dcp, new_bits: &curr_bits, old_bits: &old_bits);
695	*dcp->lvl = (cls_id + (wanted ? 1 : 0));
696	v2pr_info("%s: changed bit-%d: \"%s\" %lx->%lx\n", KP_NAME(kp), cls_id,
697	map->class_names[cls_id], old_bits, curr_bits);
698	break;
699	default:
700	pr_err("illegal map-type value %d\n", map->map_type);
701	}
702	}
703	kfree(objp: tmp);
704	vpr_info("total matches: %d\n", totct);
705	return 0;
706	}
707
708	/**
709	* param_set_dyndbg_classes - class FOO >control
710	* @instr: string echo>d to sysfs, input depends on map_type
711	* @kp: kp->arg has state: bits/lvl, map, map_type
712	*
713	* Enable/disable prdbgs by their class, as given in the arguments to
714	* DECLARE_DYNDBG_CLASSMAP. For LEVEL map-types, enforce relative
715	* levels by bitpos.
716	*
717	* Returns: 0 or <0 if error.
718	*/
719	int param_set_dyndbg_classes(const char *instr, const struct kernel_param *kp)
720	{
721	const struct ddebug_class_param *dcp = kp->arg;
722	const struct ddebug_class_map *map = dcp->map;
723	unsigned long inrep, new_bits, old_bits;
724	int rc, totct = 0;
725
726	switch (map->map_type) {
727
728	case DD_CLASS_TYPE_DISJOINT_NAMES:
729	case DD_CLASS_TYPE_LEVEL_NAMES:
730	/* handle [+-]classnames list separately, we are done here */
731	return param_set_dyndbg_classnames(instr, kp);
732
733	case DD_CLASS_TYPE_DISJOINT_BITS:
734	case DD_CLASS_TYPE_LEVEL_NUM:
735	/* numeric input, accept and fall-thru */
736	rc = kstrtoul(s: instr, base: 0, res: &inrep);
737	if (rc) {
738	pr_err("expecting numeric input: %s > %s\n", instr, KP_NAME(kp));
739	return -EINVAL;
740	}
741	break;
742	default:
743	pr_err("%s: bad map type: %d\n", KP_NAME(kp), map->map_type);
744	return -EINVAL;
745	}
746
747	/* only _BITS,_NUM (numeric) map-types get here */
748	switch (map->map_type) {
749	case DD_CLASS_TYPE_DISJOINT_BITS:
750	/* expect bits. mask and warn if too many */
751	if (inrep & ~CLASSMAP_BITMASK(map->length)) {
752	pr_warn("%s: input: 0x%lx exceeds mask: 0x%lx, masking\n",
753	KP_NAME(kp), inrep, CLASSMAP_BITMASK(map->length));
754	inrep &= CLASSMAP_BITMASK(map->length);
755	}
756	v2pr_info("bits:%lx > %s\n", inrep, KP_NAME(kp));
757	totct += ddebug_apply_class_bitmap(dcp, new_bits: &inrep, old_bits: dcp->bits);
758	*dcp->bits = inrep;
759	break;
760	case DD_CLASS_TYPE_LEVEL_NUM:
761	/* input is bitpos, of highest verbosity to be enabled */
762	if (inrep > map->length) {
763	pr_warn("%s: level:%ld exceeds max:%d, clamping\n",
764	KP_NAME(kp), inrep, map->length);
765	inrep = map->length;
766	}
767	old_bits = CLASSMAP_BITMASK(*dcp->lvl);
768	new_bits = CLASSMAP_BITMASK(inrep);
769	v2pr_info("lvl:%ld bits:0x%lx > %s\n", inrep, new_bits, KP_NAME(kp));
770	totct += ddebug_apply_class_bitmap(dcp, new_bits: &new_bits, old_bits: &old_bits);
771	*dcp->lvl = inrep;
772	break;
773	default:
774	pr_warn("%s: bad map type: %d\n", KP_NAME(kp), map->map_type);
775	}
776	vpr_info("%s: total matches: %d\n", KP_NAME(kp), totct);
777	return 0;
778	}
779	EXPORT_SYMBOL(param_set_dyndbg_classes);
780
781	/**
782	* param_get_dyndbg_classes - classes reader
783	* @buffer: string description of controlled bits -> classes
784	* @kp: kp->arg has state: bits, map
785	*
786	* Reads last written state, underlying prdbg state may have been
787	* altered by direct >control. Displays 0x for DISJOINT, 0-N for
788	* LEVEL Returns: #chars written or <0 on error
789	*/
790	int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp)
791	{
792	const struct ddebug_class_param *dcp = kp->arg;
793	const struct ddebug_class_map *map = dcp->map;
794
795	switch (map->map_type) {
796
797	case DD_CLASS_TYPE_DISJOINT_NAMES:
798	case DD_CLASS_TYPE_DISJOINT_BITS:
799	return scnprintf(buf: buffer, PAGE_SIZE, fmt: "0x%lx\n", *dcp->bits);
800
801	case DD_CLASS_TYPE_LEVEL_NAMES:
802	case DD_CLASS_TYPE_LEVEL_NUM:
803	return scnprintf(buf: buffer, PAGE_SIZE, fmt: "%d\n", *dcp->lvl);
804	default:
805	return -1;
806	}
807	}
808	EXPORT_SYMBOL(param_get_dyndbg_classes);
809
810	const struct kernel_param_ops param_ops_dyndbg_classes = {
811	.set = param_set_dyndbg_classes,
812	.get = param_get_dyndbg_classes,
813	};
814	EXPORT_SYMBOL(param_ops_dyndbg_classes);
815
816	#define PREFIX_SIZE 128
817
818	static int remaining(int wrote)
819	{
820	if (PREFIX_SIZE - wrote > 0)
821	return PREFIX_SIZE - wrote;
822	return 0;
823	}
824
825	static char *__dynamic_emit_prefix(const struct _ddebug *desc, char *buf)
826	{
827	int pos_after_tid;
828	int pos = 0;
829
830	if (desc->flags & _DPRINTK_FLAGS_INCL_TID) {
831	if (in_interrupt())
832	pos += snprintf(buf: buf + pos, size: remaining(wrote: pos), fmt: "<intr> ");
833	else
834	pos += snprintf(buf: buf + pos, size: remaining(wrote: pos), fmt: "[%d] ",
835	task_pid_vnr(current));
836	}
837	pos_after_tid = pos;
838	if (desc->flags & _DPRINTK_FLAGS_INCL_MODNAME)
839	pos += snprintf(buf: buf + pos, size: remaining(wrote: pos), fmt: "%s:",
840	desc->modname);
841	if (desc->flags & _DPRINTK_FLAGS_INCL_FUNCNAME)
842	pos += snprintf(buf: buf + pos, size: remaining(wrote: pos), fmt: "%s:",
843	desc->function);
844	if (desc->flags & _DPRINTK_FLAGS_INCL_SOURCENAME)
845	pos += snprintf(buf: buf + pos, size: remaining(wrote: pos), fmt: "%s:",
846	trim_prefix(path: desc->filename));
847	if (desc->flags & _DPRINTK_FLAGS_INCL_LINENO)
848	pos += snprintf(buf: buf + pos, size: remaining(wrote: pos), fmt: "%d:",
849	desc->lineno);
850	if (pos - pos_after_tid)
851	pos += snprintf(buf: buf + pos, size: remaining(wrote: pos), fmt: " ");
852	if (pos >= PREFIX_SIZE)
853	buf[PREFIX_SIZE - 1] = '\0';
854
855	return buf;
856	}
857
858	static inline char *dynamic_emit_prefix(struct _ddebug *desc, char *buf)
859	{
860	if (unlikely(desc->flags & _DPRINTK_FLAGS_INCL_ANY))
861	return __dynamic_emit_prefix(desc, buf);
862	return buf;
863	}
864
865	void __dynamic_pr_debug(struct _ddebug *descriptor, const char *fmt, ...)
866	{
867	va_list args;
868	struct va_format vaf;
869	char buf[PREFIX_SIZE] = "";
870
871	BUG_ON(!descriptor);
872	BUG_ON(!fmt);
873
874	va_start(args, fmt);
875
876	vaf.fmt = fmt;
877	vaf.va = &args;
878
879	printk(KERN_DEBUG "%s%pV", dynamic_emit_prefix(descriptor, buf), &vaf);
880
881	va_end(args);
882	}
883	EXPORT_SYMBOL(__dynamic_pr_debug);
884
885	void __dynamic_dev_dbg(struct _ddebug *descriptor,
886	const struct device *dev, const char *fmt, ...)
887	{
888	struct va_format vaf;
889	va_list args;
890
891	BUG_ON(!descriptor);
892	BUG_ON(!fmt);
893
894	va_start(args, fmt);
895
896	vaf.fmt = fmt;
897	vaf.va = &args;
898
899	if (!dev) {
900	printk(KERN_DEBUG "(NULL device *): %pV", &vaf);
901	} else {
902	char buf[PREFIX_SIZE] = "";
903
904	dev_printk_emit(LOGLEVEL_DEBUG, dev, fmt: "%s%s %s: %pV",
905	dynamic_emit_prefix(desc: descriptor, buf),
906	dev_driver_string(dev), dev_name(dev),
907	&vaf);
908	}
909
910	va_end(args);
911	}
912	EXPORT_SYMBOL(__dynamic_dev_dbg);
913
914	#ifdef CONFIG_NET
915
916	void __dynamic_netdev_dbg(struct _ddebug *descriptor,
917	const struct net_device *dev, const char *fmt, ...)
918	{
919	struct va_format vaf;
920	va_list args;
921
922	BUG_ON(!descriptor);
923	BUG_ON(!fmt);
924
925	va_start(args, fmt);
926
927	vaf.fmt = fmt;
928	vaf.va = &args;
929
930	if (dev && dev->dev.parent) {
931	char buf[PREFIX_SIZE] = "";
932
933	dev_printk_emit(LOGLEVEL_DEBUG, dev: dev->dev.parent,
934	fmt: "%s%s %s %s%s: %pV",
935	dynamic_emit_prefix(desc: descriptor, buf),
936	dev_driver_string(dev: dev->dev.parent),
937	dev_name(dev: dev->dev.parent),
938	netdev_name(dev), netdev_reg_state(dev),
939	&vaf);
940	} else if (dev) {
941	printk(KERN_DEBUG "%s%s: %pV", netdev_name(dev),
942	netdev_reg_state(dev), &vaf);
943	} else {
944	printk(KERN_DEBUG "(NULL net_device): %pV", &vaf);
945	}
946
947	va_end(args);
948	}
949	EXPORT_SYMBOL(__dynamic_netdev_dbg);
950
951	#endif
952
953	#if IS_ENABLED(CONFIG_INFINIBAND)
954
955	void __dynamic_ibdev_dbg(struct _ddebug *descriptor,
956	const struct ib_device *ibdev, const char *fmt, ...)
957	{
958	struct va_format vaf;
959	va_list args;
960
961	va_start(args, fmt);
962
963	vaf.fmt = fmt;
964	vaf.va = &args;
965
966	if (ibdev && ibdev->dev.parent) {
967	char buf[PREFIX_SIZE] = "";
968
969	dev_printk_emit(LOGLEVEL_DEBUG, dev: ibdev->dev.parent,
970	fmt: "%s%s %s %s: %pV",
971	dynamic_emit_prefix(desc: descriptor, buf),
972	dev_driver_string(dev: ibdev->dev.parent),
973	dev_name(dev: ibdev->dev.parent),
974	dev_name(dev: &ibdev->dev),
975	&vaf);
976	} else if (ibdev) {
977	printk(KERN_DEBUG "%s: %pV", dev_name(&ibdev->dev), &vaf);
978	} else {
979	printk(KERN_DEBUG "(NULL ib_device): %pV", &vaf);
980	}
981
982	va_end(args);
983	}
984	EXPORT_SYMBOL(__dynamic_ibdev_dbg);
985
986	#endif
987
988	/*
989	* Install a noop handler to make dyndbg look like a normal kernel cli param.
990	* This avoids warnings about dyndbg being an unknown cli param when supplied
991	* by a user.
992	*/
993	static __init int dyndbg_setup(char *str)
994	{
995	return 1;
996	}
997
998	__setup("dyndbg=", dyndbg_setup);
999
1000	/*
1001	* File_ops->write method for <debugfs>/dynamic_debug/control. Gathers the
1002	* command text from userspace, parses and executes it.
1003	*/
1004	#define USER_BUF_PAGE 4096
1005	static ssize_t ddebug_proc_write(struct file *file, const char __user *ubuf,
1006	size_t len, loff_t *offp)
1007	{
1008	char *tmpbuf;
1009	int ret;
1010
1011	if (len == 0)
1012	return 0;
1013	if (len > USER_BUF_PAGE - 1) {
1014	pr_warn("expected <%d bytes into control\n", USER_BUF_PAGE);
1015	return -E2BIG;
1016	}
1017	tmpbuf = memdup_user_nul(ubuf, len);
1018	if (IS_ERR(ptr: tmpbuf))
1019	return PTR_ERR(ptr: tmpbuf);
1020	v2pr_info("read %zu bytes from userspace\n", len);
1021
1022	ret = ddebug_exec_queries(query: tmpbuf, NULL);
1023	kfree(objp: tmpbuf);
1024	if (ret < 0)
1025	return ret;
1026
1027	*offp += len;
1028	return len;
1029	}
1030
1031	/*
1032	* Set the iterator to point to the first _ddebug object
1033	* and return a pointer to that first object. Returns
1034	* NULL if there are no _ddebugs at all.
1035	*/
1036	static struct _ddebug *ddebug_iter_first(struct ddebug_iter *iter)
1037	{
1038	if (list_empty(head: &ddebug_tables)) {
1039	iter->table = NULL;
1040	return NULL;
1041	}
1042	iter->table = list_entry(ddebug_tables.next,
1043	struct ddebug_table, link);
1044	iter->idx = iter->table->num_ddebugs;
1045	return &iter->table->ddebugs[--iter->idx];
1046	}
1047
1048	/*
1049	* Advance the iterator to point to the next _ddebug
1050	* object from the one the iterator currently points at,
1051	* and returns a pointer to the new _ddebug. Returns
1052	* NULL if the iterator has seen all the _ddebugs.
1053	*/
1054	static struct _ddebug *ddebug_iter_next(struct ddebug_iter *iter)
1055	{
1056	if (iter->table == NULL)
1057	return NULL;
1058	if (--iter->idx < 0) {
1059	/* iterate to next table */
1060	if (list_is_last(list: &iter->table->link, head: &ddebug_tables)) {
1061	iter->table = NULL;
1062	return NULL;
1063	}
1064	iter->table = list_entry(iter->table->link.next,
1065	struct ddebug_table, link);
1066	iter->idx = iter->table->num_ddebugs;
1067	--iter->idx;
1068	}
1069	return &iter->table->ddebugs[iter->idx];
1070	}
1071
1072	/*
1073	* Seq_ops start method. Called at the start of every
1074	* read() call from userspace. Takes the ddebug_lock and
1075	* seeks the seq_file's iterator to the given position.
1076	*/
1077	static void *ddebug_proc_start(struct seq_file *m, loff_t *pos)
1078	{
1079	struct ddebug_iter *iter = m->private;
1080	struct _ddebug *dp;
1081	int n = *pos;
1082
1083	mutex_lock(&ddebug_lock);
1084
1085	if (!n)
1086	return SEQ_START_TOKEN;
1087	if (n < 0)
1088	return NULL;
1089	dp = ddebug_iter_first(iter);
1090	while (dp != NULL && --n > 0)
1091	dp = ddebug_iter_next(iter);
1092	return dp;
1093	}
1094
1095	/*
1096	* Seq_ops next method. Called several times within a read()
1097	* call from userspace, with ddebug_lock held. Walks to the
1098	* next _ddebug object with a special case for the header line.
1099	*/
1100	static void *ddebug_proc_next(struct seq_file *m, void *p, loff_t *pos)
1101	{
1102	struct ddebug_iter *iter = m->private;
1103	struct _ddebug *dp;
1104
1105	if (p == SEQ_START_TOKEN)
1106	dp = ddebug_iter_first(iter);
1107	else
1108	dp = ddebug_iter_next(iter);
1109	++*pos;
1110	return dp;
1111	}
1112
1113	#define class_in_range(class_id, map) \
1114	(class_id >= map->base && class_id < map->base + map->length)
1115
1116	static const char *ddebug_class_name(struct ddebug_iter *iter, struct _ddebug *dp)
1117	{
1118	struct ddebug_class_map *map;
1119
1120	list_for_each_entry(map, &iter->table->maps, link)
1121	if (class_in_range(dp->class_id, map))
1122	return map->class_names[dp->class_id - map->base];
1123
1124	return NULL;
1125	}
1126
1127	/*
1128	* Seq_ops show method. Called several times within a read()
1129	* call from userspace, with ddebug_lock held. Formats the
1130	* current _ddebug as a single human-readable line, with a
1131	* special case for the header line.
1132	*/
1133	static int ddebug_proc_show(struct seq_file *m, void *p)
1134	{
1135	struct ddebug_iter *iter = m->private;
1136	struct _ddebug *dp = p;
1137	struct flagsbuf flags;
1138	char const *class;
1139
1140	if (p == SEQ_START_TOKEN) {
1141	seq_puts(m,
1142	s: "# filename:lineno [module]function flags format\n");
1143	return 0;
1144	}
1145
1146	seq_printf(m, fmt: "%s:%u [%s]%s =%s \"",
1147	trim_prefix(path: dp->filename), dp->lineno,
1148	iter->table->mod_name, dp->function,
1149	ddebug_describe_flags(flags: dp->flags, fb: &flags));
1150	seq_escape_str(m, src: dp->format, ESCAPE_SPACE, esc: "\t\r\n\"");
1151	seq_putc(m, c: '"');
1152
1153	if (dp->class_id != _DPRINTK_CLASS_DFLT) {
1154	class = ddebug_class_name(iter, dp);
1155	if (class)
1156	seq_printf(m, fmt: " class:%s", class);
1157	else
1158	seq_printf(m, fmt: " class unknown, _id:%d", dp->class_id);
1159	}
1160	seq_putc(m, c: '\n');
1161
1162	return 0;
1163	}
1164
1165	/*
1166	* Seq_ops stop method. Called at the end of each read()
1167	* call from userspace. Drops ddebug_lock.
1168	*/
1169	static void ddebug_proc_stop(struct seq_file *m, void *p)
1170	{
1171	mutex_unlock(lock: &ddebug_lock);
1172	}
1173
1174	static const struct seq_operations ddebug_proc_seqops = {
1175	.start = ddebug_proc_start,
1176	.next = ddebug_proc_next,
1177	.show = ddebug_proc_show,
1178	.stop = ddebug_proc_stop
1179	};
1180
1181	static int ddebug_proc_open(struct inode *inode, struct file *file)
1182	{
1183	return seq_open_private(file, &ddebug_proc_seqops,
1184	sizeof(struct ddebug_iter));
1185	}
1186
1187	static const struct file_operations ddebug_proc_fops = {
1188	.owner = THIS_MODULE,
1189	.open = ddebug_proc_open,
1190	.read = seq_read,
1191	.llseek = seq_lseek,
1192	.release = seq_release_private,
1193	.write = ddebug_proc_write
1194	};
1195
1196	static const struct proc_ops proc_fops = {
1197	.proc_open = ddebug_proc_open,
1198	.proc_read = seq_read,
1199	.proc_lseek = seq_lseek,
1200	.proc_release = seq_release_private,
1201	.proc_write = ddebug_proc_write
1202	};
1203
1204	static void ddebug_attach_module_classes(struct ddebug_table *dt,
1205	struct ddebug_class_map *classes,
1206	int num_classes)
1207	{
1208	struct ddebug_class_map *cm;
1209	int i, j, ct = 0;
1210
1211	for (cm = classes, i = 0; i < num_classes; i++, cm++) {
1212
1213	if (!strcmp(cm->mod_name, dt->mod_name)) {
1214
1215	v2pr_info("class[%d]: module:%s base:%d len:%d ty:%d\n", i,
1216	cm->mod_name, cm->base, cm->length, cm->map_type);
1217
1218	for (j = 0; j < cm->length; j++)
1219	v3pr_info(" %d: %d %s\n", j + cm->base, j,
1220	cm->class_names[j]);
1221
1222	list_add(new: &cm->link, head: &dt->maps);
1223	ct++;
1224	}
1225	}
1226	if (ct)
1227	vpr_info("module:%s attached %d classes\n", dt->mod_name, ct);
1228	}
1229
1230	/*
1231	* Allocate a new ddebug_table for the given module
1232	* and add it to the global list.
1233	*/
1234	static int ddebug_add_module(struct _ddebug_info *di, const char *modname)
1235	{
1236	struct ddebug_table *dt;
1237
1238	v3pr_info("add-module: %s.%d sites\n", modname, di->num_descs);
1239	if (!di->num_descs) {
1240	v3pr_info(" skip %s\n", modname);
1241	return 0;
1242	}
1243
1244	dt = kzalloc(sizeof(*dt), GFP_KERNEL);
1245	if (dt == NULL) {
1246	pr_err("error adding module: %s\n", modname);
1247	return -ENOMEM;
1248	}
1249	/*
1250	* For built-in modules, name lives in .rodata and is
1251	* immortal. For loaded modules, name points at the name[]
1252	* member of struct module, which lives at least as long as
1253	* this struct ddebug_table.
1254	*/
1255	dt->mod_name = modname;
1256	dt->ddebugs = di->descs;
1257	dt->num_ddebugs = di->num_descs;
1258
1259	INIT_LIST_HEAD(list: &dt->link);
1260	INIT_LIST_HEAD(list: &dt->maps);
1261
1262	if (di->classes && di->num_classes)
1263	ddebug_attach_module_classes(dt, classes: di->classes, num_classes: di->num_classes);
1264
1265	mutex_lock(&ddebug_lock);
1266	list_add_tail(new: &dt->link, head: &ddebug_tables);
1267	mutex_unlock(lock: &ddebug_lock);
1268
1269	vpr_info("%3u debug prints in module %s\n", di->num_descs, modname);
1270	return 0;
1271	}
1272
1273	/* helper for ddebug_dyndbg_(boot|module)_param_cb */
1274	static int ddebug_dyndbg_param_cb(char *param, char *val,
1275	const char *modname, int on_err)
1276	{
1277	char *sep;
1278
1279	sep = strchr(param, '.');
1280	if (sep) {
1281	/* needed only for ddebug_dyndbg_boot_param_cb */
1282	*sep = '\0';
1283	modname = param;
1284	param = sep + 1;
1285	}
1286	if (strcmp(param, "dyndbg"))
1287	return on_err; /* determined by caller */
1288
1289	ddebug_exec_queries(query: (val ? val : "+p"), modname);
1290
1291	return 0; /* query failure shouldn't stop module load */
1292	}
1293
1294	/* handle both dyndbg and $module.dyndbg params at boot */
1295	static int ddebug_dyndbg_boot_param_cb(char *param, char *val,
1296	const char *unused, void *arg)
1297	{
1298	vpr_info("%s=\"%s\"\n", param, val);
1299	return ddebug_dyndbg_param_cb(param, val, NULL, on_err: 0);
1300	}
1301
1302	/*
1303	* modprobe foo finds foo.params in boot-args, strips "foo.", and
1304	* passes them to load_module(). This callback gets unknown params,
1305	* processes dyndbg params, rejects others.
1306	*/
1307	int ddebug_dyndbg_module_param_cb(char *param, char *val, const char *module)
1308	{
1309	vpr_info("module: %s %s=\"%s\"\n", module, param, val);
1310	return ddebug_dyndbg_param_cb(param, val, modname: module, on_err: -ENOENT);
1311	}
1312
1313	static void ddebug_table_free(struct ddebug_table *dt)
1314	{
1315	list_del_init(entry: &dt->link);
1316	kfree(objp: dt);
1317	}
1318
1319	#ifdef CONFIG_MODULES
1320
1321	/*
1322	* Called in response to a module being unloaded. Removes
1323	* any ddebug_table's which point at the module.
1324	*/
1325	static int ddebug_remove_module(const char *mod_name)
1326	{
1327	struct ddebug_table *dt, *nextdt;
1328	int ret = -ENOENT;
1329
1330	mutex_lock(&ddebug_lock);
1331	list_for_each_entry_safe(dt, nextdt, &ddebug_tables, link) {
1332	if (dt->mod_name == mod_name) {
1333	ddebug_table_free(dt);
1334	ret = 0;
1335	break;
1336	}
1337	}
1338	mutex_unlock(lock: &ddebug_lock);
1339	if (!ret)
1340	v2pr_info("removed module \"%s\"\n", mod_name);
1341	return ret;
1342	}
1343
1344	static int ddebug_module_notify(struct notifier_block *self, unsigned long val,
1345	void *data)
1346	{
1347	struct module *mod = data;
1348	int ret = 0;
1349
1350	switch (val) {
1351	case MODULE_STATE_COMING:
1352	ret = ddebug_add_module(di: &mod->dyndbg_info, modname: mod->name);
1353	if (ret)
1354	WARN(1, "Failed to allocate memory: dyndbg may not work properly.\n");
1355	break;
1356	case MODULE_STATE_GOING:
1357	ddebug_remove_module(mod_name: mod->name);
1358	break;
1359	}
1360
1361	return notifier_from_errno(err: ret);
1362	}
1363
1364	static struct notifier_block ddebug_module_nb = {
1365	.notifier_call = ddebug_module_notify,
1366	.priority = 0, /* dynamic debug depends on jump label */
1367	};
1368
1369	#endif /* CONFIG_MODULES */
1370
1371	static void ddebug_remove_all_tables(void)
1372	{
1373	mutex_lock(&ddebug_lock);
1374	while (!list_empty(head: &ddebug_tables)) {
1375	struct ddebug_table *dt = list_entry(ddebug_tables.next,
1376	struct ddebug_table,
1377	link);
1378	ddebug_table_free(dt);
1379	}
1380	mutex_unlock(lock: &ddebug_lock);
1381	}
1382
1383	static __initdata int ddebug_init_success;
1384
1385	static int __init dynamic_debug_init_control(void)
1386	{
1387	struct proc_dir_entry *procfs_dir;
1388	struct dentry *debugfs_dir;
1389
1390	if (!ddebug_init_success)
1391	return -ENODEV;
1392
1393	/* Create the control file in debugfs if it is enabled */
1394	if (debugfs_initialized()) {
1395	debugfs_dir = debugfs_create_dir(name: "dynamic_debug", NULL);
1396	debugfs_create_file("control", 0644, debugfs_dir, NULL,
1397	&ddebug_proc_fops);
1398	}
1399
1400	/* Also create the control file in procfs */
1401	procfs_dir = proc_mkdir("dynamic_debug", NULL);
1402	if (procfs_dir)
1403	proc_create(name: "control", mode: 0644, parent: procfs_dir, proc_ops: &proc_fops);
1404
1405	return 0;
1406	}
1407
1408	static int __init dynamic_debug_init(void)
1409	{
1410	struct _ddebug *iter, *iter_mod_start;
1411	int ret, i, mod_sites, mod_ct;
1412	const char *modname;
1413	char *cmdline;
1414
1415	struct _ddebug_info di = {
1416	.descs = __start___dyndbg,
1417	.classes = __start___dyndbg_classes,
1418	.num_descs = __stop___dyndbg - __start___dyndbg,
1419	.num_classes = __stop___dyndbg_classes - __start___dyndbg_classes,
1420	};
1421
1422	#ifdef CONFIG_MODULES
1423	ret = register_module_notifier(nb: &ddebug_module_nb);
1424	if (ret) {
1425	pr_warn("Failed to register dynamic debug module notifier\n");
1426	return ret;
1427	}
1428	#endif /* CONFIG_MODULES */
1429
1430	if (&__start___dyndbg == &__stop___dyndbg) {
1431	if (IS_ENABLED(CONFIG_DYNAMIC_DEBUG)) {
1432	pr_warn("_ddebug table is empty in a CONFIG_DYNAMIC_DEBUG build\n");
1433	return 1;
1434	}
1435	pr_info("Ignore empty _ddebug table in a CONFIG_DYNAMIC_DEBUG_CORE build\n");
1436	ddebug_init_success = 1;
1437	return 0;
1438	}
1439
1440	iter = iter_mod_start = __start___dyndbg;
1441	modname = iter->modname;
1442	i = mod_sites = mod_ct = 0;
1443
1444	for (; iter < __stop___dyndbg; iter++, i++, mod_sites++) {
1445
1446	if (strcmp(modname, iter->modname)) {
1447	mod_ct++;
1448	di.num_descs = mod_sites;
1449	di.descs = iter_mod_start;
1450	ret = ddebug_add_module(di: &di, modname);
1451	if (ret)
1452	goto out_err;
1453
1454	mod_sites = 0;
1455	modname = iter->modname;
1456	iter_mod_start = iter;
1457	}
1458	}
1459	di.num_descs = mod_sites;
1460	di.descs = iter_mod_start;
1461	ret = ddebug_add_module(di: &di, modname);
1462	if (ret)
1463	goto out_err;
1464
1465	ddebug_init_success = 1;
1466	vpr_info("%d prdebugs in %d modules, %d KiB in ddebug tables, %d kiB in __dyndbg section\n",
1467	i, mod_ct, (int)((mod_ct * sizeof(struct ddebug_table)) >> 10),
1468	(int)((i * sizeof(struct _ddebug)) >> 10));
1469
1470	if (di.num_classes)
1471	v2pr_info(" %d builtin ddebug class-maps\n", di.num_classes);
1472
1473	/* now that ddebug tables are loaded, process all boot args
1474	* again to find and activate queries given in dyndbg params.
1475	* While this has already been done for known boot params, it
1476	* ignored the unknown ones (dyndbg in particular). Reusing
1477	* parse_args avoids ad-hoc parsing. This will also attempt
1478	* to activate queries for not-yet-loaded modules, which is
1479	* slightly noisy if verbose, but harmless.
1480	*/
1481	cmdline = kstrdup(s: saved_command_line, GFP_KERNEL);
1482	parse_args(name: "dyndbg params", args: cmdline, NULL,
1483	num: 0, level_min: 0, level_max: 0, NULL, unknown: &ddebug_dyndbg_boot_param_cb);
1484	kfree(objp: cmdline);
1485	return 0;
1486
1487	out_err:
1488	ddebug_remove_all_tables();
1489	return 0;
1490	}
1491	/* Allow early initialization for boot messages via boot param */
1492	early_initcall(dynamic_debug_init);
1493
1494	/* Debugfs setup must be done later */
1495	fs_initcall(dynamic_debug_init_control);
1496