// SPDX-License-Identifier: GPL-2.0-only

/*

* kernel/lockdep.c

*

* Runtime locking correctness validator

*

* Started by Ingo Molnar:

*

* Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>

* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra

*

* this code maps all the lock dependencies as they occur in a live kernel

* and will warn about the following classes of locking bugs:

*

* - lock inversion scenarios

* - circular lock dependencies

* - hardirq/softirq safe/unsafe locking bugs

*

* Bugs are reported even if the current locking scenario does not cause

* any deadlock at this point.

*

* I.e. if anytime in the past two locks were taken in a different order,

* even if it happened for another task, even if those were different

* locks (but of the same class as this lock), this code will detect it.

*

* Thanks to Arjan van de Ven for coming up with the initial idea of

* mapping lock dependencies runtime.

*/

#define DISABLE_BRANCH_PROFILING

#include <linux/mutex.h>

#include <linux/sched.h>

#include <linux/sched/clock.h>

#include <linux/sched/task.h>

#include <linux/sched/mm.h>

#include <linux/delay.h>

#include <linux/module.h>

#include <linux/proc_fs.h>

#include <linux/seq_file.h>

#include <linux/spinlock.h>

#include <linux/kallsyms.h>

#include <linux/interrupt.h>

#include <linux/stacktrace.h>

#include <linux/debug_locks.h>

#include <linux/irqflags.h>

#include <linux/utsname.h>

#include <linux/hash.h>

#include <linux/ftrace.h>

#include <linux/stringify.h>

#include <linux/bitmap.h>

#include <linux/bitops.h>

#include <linux/gfp.h>

#include <linux/random.h>

#include <linux/jhash.h>

#include <linux/nmi.h>

#include <linux/rcupdate.h>

#include <linux/kprobes.h>

#include <linux/lockdep.h>

#include <linux/context_tracking.h>

#include <asm/sections.h>

#include "lockdep_internals.h"

#include <trace/events/lock.h>

#ifdef CONFIG_PROVE_LOCKING

static int prove_locking = 1;

module_param(prove_locking, int, 0644);

#else

#define prove_locking 0

#endif

#ifdef CONFIG_LOCK_STAT

static int lock_stat = 1;

module_param(lock_stat, int, 0644);

#else

#define lock_stat 0

#endif

#ifdef CONFIG_SYSCTL

static struct ctl_table kern_lockdep_table[] = {

#ifdef CONFIG_PROVE_LOCKING

{

.procname = "prove_locking",

.data = &prove_locking,

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = proc_dointvec,

},

#endif /* CONFIG_PROVE_LOCKING */

#ifdef CONFIG_LOCK_STAT

{

.procname = "lock_stat",

.data = &lock_stat,

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = proc_dointvec,

},

#endif /* CONFIG_LOCK_STAT */

};

static __init int kernel_lockdep_sysctls_init(void)

{

register_sysctl_init("kernel", kern_lockdep_table);

return 0;

}

late_initcall(kernel_lockdep_sysctls_init);

#endif /* CONFIG_SYSCTL */

DEFINE_PER_CPU(unsigned int, lockdep_recursion);

EXPORT_PER_CPU_SYMBOL_GPL(lockdep_recursion);

static __always_inline bool lockdep_enabled(void)

{

if (!debug_locks)

return false;

if (this_cpu_read(lockdep_recursion))

return false;

if (current->lockdep_recursion)

return false;

return true;

}

/*

* lockdep_lock: protects the lockdep graph, the hashes and the

* class/list/hash allocators.

*

* This is one of the rare exceptions where it's justified

* to use a raw spinlock - we really dont want the spinlock

* code to recurse back into the lockdep code...

*/

static arch_spinlock_t __lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;

static struct task_struct *__owner;

static inline void lockdep_lock(void)

{

DEBUG_LOCKS_WARN_ON(!irqs_disabled());

__this_cpu_inc(lockdep_recursion);

arch_spin_lock(&__lock);

__owner = current;

}

static inline void lockdep_unlock(void)

{

DEBUG_LOCKS_WARN_ON(!irqs_disabled());

if (debug_locks && DEBUG_LOCKS_WARN_ON(__owner != current))

return;

__owner = NULL;

arch_spin_unlock(&__lock);

__this_cpu_dec(lockdep_recursion);

}

static inline bool lockdep_assert_locked(void)

{

return DEBUG_LOCKS_WARN_ON(__owner != current);

}

static struct task_struct *lockdep_selftest_task_struct;

static int graph_lock(void)

{

lockdep_lock();

/*

* Make sure that if another CPU detected a bug while

* walking the graph we dont change it (while the other

* CPU is busy printing out stuff with the graph lock

* dropped already)

*/

if (!debug_locks) {

lockdep_unlock();

return 0;

}

return 1;

}

static inline void graph_unlock(void)

{

lockdep_unlock();

}

/*

* Turn lock debugging off and return with 0 if it was off already,

* and also release the graph lock:

*/

static inline int debug_locks_off_graph_unlock(void)

{

int ret = debug_locks_off();

lockdep_unlock();

return ret;

}

unsigned long nr_list_entries;

static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES];

static DECLARE_BITMAP(list_entries_in_use, MAX_LOCKDEP_ENTRIES);

/*

* All data structures here are protected by the global debug_lock.

*

* nr_lock_classes is the number of elements of lock_classes[] that is

* in use.

*/

#define KEYHASH_BITS (MAX_LOCKDEP_KEYS_BITS - 1)

#define KEYHASH_SIZE (1UL << KEYHASH_BITS)

static struct hlist_head lock_keys_hash[KEYHASH_SIZE];

unsigned long nr_lock_classes;

unsigned long nr_zapped_classes;

unsigned long max_lock_class_idx;

struct lock_class lock_classes[MAX_LOCKDEP_KEYS];

DECLARE_BITMAP(lock_classes_in_use, MAX_LOCKDEP_KEYS);

static inline struct lock_class *hlock_class(struct held_lock *hlock)

{

unsigned int class_idx = hlock->class_idx;

/* Don't re-read hlock->class_idx, can't use READ_ONCE() on bitfield */

barrier();

if (!test_bit(class_idx, lock_classes_in_use)) {

/*

* Someone passed in garbage, we give up.

*/

DEBUG_LOCKS_WARN_ON(1);

return NULL;

}

/*

* At this point, if the passed hlock->class_idx is still garbage,

* we just have to live with it

*/

return lock_classes + class_idx;

}

#ifdef CONFIG_LOCK_STAT

static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], cpu_lock_stats);

static inline u64 lockstat_clock(void)

{

return local_clock();

}

static int lock_point(unsigned long points[], unsigned long ip)

{

int i;

for (i = 0; i < LOCKSTAT_POINTS; i++) {

if (points[i] == 0) {

points[i] = ip;

break;

}

if (points[i] == ip)

break;

}

return i;

}

static void lock_time_inc(struct lock_time *lt, u64 time)

{

if (time > lt->max)

lt->max = time;

if (time < lt->min || !lt->nr)

lt->min = time;

lt->total += time;

lt->nr++;

}

static inline void lock_time_add(struct lock_time *src, struct lock_time *dst)

{

if (!src->nr)

return;

if (src->max > dst->max)

dst->max = src->max;

if (src->min < dst->min || !dst->nr)

dst->min = src->min;

dst->total += src->total;

dst->nr += src->nr;

}

struct lock_class_stats lock_stats(struct lock_class *class)

{

struct lock_class_stats stats;

int cpu, i;

memset(&stats, 0, sizeof(struct lock_class_stats));

for_each_possible_cpu(cpu) {

struct lock_class_stats *pcs =

&per_cpu(cpu_lock_stats, cpu)[class - lock_classes];

for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)

stats.contention_point[i] += pcs->contention_point[i];

for (i = 0; i < ARRAY_SIZE(stats.contending_point); i++)

stats.contending_point[i] += pcs->contending_point[i];

lock_time_add(&pcs->read_waittime, &stats.read_waittime);

lock_time_add(&pcs->write_waittime, &stats.write_waittime);

lock_time_add(&pcs->read_holdtime, &stats.read_holdtime);

lock_time_add(&pcs->write_holdtime, &stats.write_holdtime);

for (i = 0; i < ARRAY_SIZE(stats.bounces); i++)

stats.bounces[i] += pcs->bounces[i];

}

return stats;

}

void clear_lock_stats(struct lock_class *class)

{

int cpu;

for_each_possible_cpu(cpu) {

struct lock_class_stats *cpu_stats =

&per_cpu(cpu_lock_stats, cpu)[class - lock_classes];

memset(cpu_stats, 0, sizeof(struct lock_class_stats));

}

memset(class->contention_point, 0, sizeof(class->contention_point));

memset(class->contending_point, 0, sizeof(class->contending_point));

}

static struct lock_class_stats *get_lock_stats(struct lock_class *class)

{

return &this_cpu_ptr(cpu_lock_stats)[class - lock_classes];

}

static void lock_release_holdtime(struct held_lock *hlock)

{

struct lock_class_stats *stats;

u64 holdtime;

if (!lock_stat)

return;

holdtime = lockstat_clock() - hlock->holdtime_stamp;

stats = get_lock_stats(hlock_class(hlock));

if (hlock->read)

lock_time_inc(&stats->read_holdtime, holdtime);

else

lock_time_inc(&stats->write_holdtime, holdtime);

}

#else

static inline void lock_release_holdtime(struct held_lock *hlock)

{

}

#endif

/*

* We keep a global list of all lock classes. The list is only accessed with

* the lockdep spinlock lock held. free_lock_classes is a list with free

* elements. These elements are linked together by the lock_entry member in

* struct lock_class.

*/

static LIST_HEAD(all_lock_classes);

static LIST_HEAD(free_lock_classes);

/**

* struct pending_free - information about data structures about to be freed

* @zapped: Head of a list with struct lock_class elements.

* @lock_chains_being_freed: Bitmap that indicates which lock_chains[] elements

* are about to be freed.

*/

struct pending_free {

struct list_head zapped;

DECLARE_BITMAP(lock_chains_being_freed, MAX_LOCKDEP_CHAINS);

};

/**

* struct delayed_free - data structures used for delayed freeing

*

* A data structure for delayed freeing of data structures that may be

* accessed by RCU readers at the time these were freed.

*

* @rcu_head: Used to schedule an RCU callback for freeing data structures.

* @index: Index of @pf to which freed data structures are added.

* @scheduled: Whether or not an RCU callback has been scheduled.

* @pf: Array with information about data structures about to be freed.

*/

static struct delayed_free {

struct rcu_head rcu_head;

int index;

int scheduled;

struct pending_free pf[2];

} delayed_free;

/*

* The lockdep classes are in a hash-table as well, for fast lookup:

*/

#define CLASSHASH_BITS (MAX_LOCKDEP_KEYS_BITS - 1)

#define CLASSHASH_SIZE (1UL << CLASSHASH_BITS)

#define __classhashfn(key) hash_long((unsigned long)key, CLASSHASH_BITS)

#define classhashentry(key) (classhash_table + __classhashfn((key)))

static struct hlist_head classhash_table[CLASSHASH_SIZE];

/*

* We put the lock dependency chains into a hash-table as well, to cache

* their existence:

*/

#define CHAINHASH_BITS (MAX_LOCKDEP_CHAINS_BITS-1)

#define CHAINHASH_SIZE (1UL << CHAINHASH_BITS)

#define __chainhashfn(chain) hash_long(chain, CHAINHASH_BITS)

#define chainhashentry(chain) (chainhash_table + __chainhashfn((chain)))

static struct hlist_head chainhash_table[CHAINHASH_SIZE];

/*

* the id of held_lock

*/

static inline u16 hlock_id(struct held_lock *hlock)

{

BUILD_BUG_ON(MAX_LOCKDEP_KEYS_BITS + 2 > 16);

return (hlock->class_idx | (hlock->read << MAX_LOCKDEP_KEYS_BITS));

}

static inline unsigned int chain_hlock_class_idx(u16 hlock_id)

{

return hlock_id & (MAX_LOCKDEP_KEYS - 1);

}

/*

* The hash key of the lock dependency chains is a hash itself too:

* it's a hash of all locks taken up to that lock, including that lock.

* It's a 64-bit hash, because it's important for the keys to be

* unique.

*/

static inline u64 iterate_chain_key(u64 key, u32 idx)

{

u32 k0 = key, k1 = key >> 32;

__jhash_mix(idx, k0, k1); /* Macro that modifies arguments! */

return k0 | (u64)k1 << 32;

}

void lockdep_init_task(struct task_struct *task)

{

task->lockdep_depth = 0; /* no locks held yet */

task->curr_chain_key = INITIAL_CHAIN_KEY;

task->lockdep_recursion = 0;

}

static __always_inline void lockdep_recursion_inc(void)

{

__this_cpu_inc(lockdep_recursion);

}

static __always_inline void lockdep_recursion_finish(void)

{

if (WARN_ON_ONCE(__this_cpu_dec_return(lockdep_recursion)))

__this_cpu_write(lockdep_recursion, 0);

}

void lockdep_set_selftest_task(struct task_struct *task)

{

lockdep_selftest_task_struct = task;

}

/*

* Debugging switches:

*/

#define VERBOSE 0

#define VERY_VERBOSE 0

#if VERBOSE

# define HARDIRQ_VERBOSE 1

# define SOFTIRQ_VERBOSE 1

#else

# define HARDIRQ_VERBOSE 0

# define SOFTIRQ_VERBOSE 0

#endif

#if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE

/*

* Quick filtering for interesting events:

*/

static int class_filter(struct lock_class *class)

{

#if 0

/* Example */

if (class->name_version == 1 &&

!strcmp(class->name, "lockname"))

return 1;

if (class->name_version == 1 &&

!strcmp(class->name, "&struct->lockfield"))

return 1;

#endif

/* Filter everything else. 1 would be to allow everything else */

return 0;

}

#endif

static int verbose(struct lock_class *class)

{

#if VERBOSE

return class_filter(class);

#endif

return 0;

}

static void print_lockdep_off(const char *bug_msg)

{

printk(KERN_DEBUG "%s\n", bug_msg);

printk(KERN_DEBUG "turning off the locking correctness validator.\n");

#ifdef CONFIG_LOCK_STAT

printk(KERN_DEBUG "Please attach the output of /proc/lock_stat to the bug report\n");

#endif

}

unsigned long nr_stack_trace_entries;

#ifdef CONFIG_PROVE_LOCKING

/**

* struct lock_trace - single stack backtrace

* @hash_entry: Entry in a stack_trace_hash[] list.

* @hash: jhash() of @entries.

* @nr_entries: Number of entries in @entries.

* @entries: Actual stack backtrace.

*/

struct lock_trace {

struct hlist_node hash_entry;

u32 hash;

u32 nr_entries;

unsigned long entries[] __aligned(sizeof(unsigned long));

};

#define LOCK_TRACE_SIZE_IN_LONGS \

(sizeof(struct lock_trace) / sizeof(unsigned long))

/*

* Stack-trace: sequence of lock_trace structures. Protected by the graph_lock.

*/

static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES];

static struct hlist_head stack_trace_hash[STACK_TRACE_HASH_SIZE];

static bool traces_identical(struct lock_trace *t1, struct lock_trace *t2)

{

return t1->hash == t2->hash && t1->nr_entries == t2->nr_entries &&

memcmp(t1->entries, t2->entries,

t1->nr_entries * sizeof(t1->entries[0])) == 0;

}

static struct lock_trace *save_trace(void)

{

struct lock_trace *trace, *t2;

struct hlist_head *hash_head;

u32 hash;

int max_entries;

BUILD_BUG_ON_NOT_POWER_OF_2(STACK_TRACE_HASH_SIZE);

BUILD_BUG_ON(LOCK_TRACE_SIZE_IN_LONGS >= MAX_STACK_TRACE_ENTRIES);

trace = (struct lock_trace *)(stack_trace + nr_stack_trace_entries);

max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries -

LOCK_TRACE_SIZE_IN_LONGS;

if (max_entries <= 0) {

if (!debug_locks_off_graph_unlock())

return NULL;

print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");

dump_stack();

return NULL;

}

trace->nr_entries = stack_trace_save(trace->entries, max_entries, 3);

hash = jhash(trace->entries, trace->nr_entries *

sizeof(trace->entries[0]), 0);

trace->hash = hash;

hash_head = stack_trace_hash + (hash & (STACK_TRACE_HASH_SIZE - 1));

hlist_for_each_entry(t2, hash_head, hash_entry) {

if (traces_identical(trace, t2))

return t2;

}

nr_stack_trace_entries += LOCK_TRACE_SIZE_IN_LONGS + trace->nr_entries;

hlist_add_head(&trace->hash_entry, hash_head);

return trace;

}

/* Return the number of stack traces in the stack_trace[] array. */

u64 lockdep_stack_trace_count(void)

{

struct lock_trace *trace;

u64 c = 0;

int i;

for (i = 0; i < ARRAY_SIZE(stack_trace_hash); i++) {

hlist_for_each_entry(trace, &stack_trace_hash[i], hash_entry) {

c++;

}

}

return c;

}

/* Return the number of stack hash chains that have at least one stack trace. */

u64 lockdep_stack_hash_count(void)

{

u64 c = 0;

int i;

for (i = 0; i < ARRAY_SIZE(stack_trace_hash); i++)

if (!hlist_empty(&stack_trace_hash[i]))

c++;

return c;

}

#endif

unsigned int nr_hardirq_chains;

unsigned int nr_softirq_chains;

unsigned int nr_process_chains;

unsigned int max_lockdep_depth;

#ifdef CONFIG_DEBUG_LOCKDEP

/*

* Various lockdep statistics:

*/

DEFINE_PER_CPU(struct lockdep_stats, lockdep_stats);

#endif

#ifdef CONFIG_PROVE_LOCKING

/*

* Locking printouts:

*/

#define __USAGE(__STATE) \

[LOCK_USED_IN_##__STATE] = "IN-"__stringify(__STATE)"-W", \

[LOCK_ENABLED_##__STATE] = __stringify(__STATE)"-ON-W", \

[LOCK_USED_IN_##__STATE##_READ] = "IN-"__stringify(__STATE)"-R",\

[LOCK_ENABLED_##__STATE##_READ] = __stringify(__STATE)"-ON-R",

static const char *usage_str[] =

{

#define LOCKDEP_STATE(__STATE) __USAGE(__STATE)

#include "lockdep_states.h"

#undef LOCKDEP_STATE

[LOCK_USED] = "INITIAL USE",

[LOCK_USED_READ] = "INITIAL READ USE",

/* abused as string storage for verify_lock_unused() */

[LOCK_USAGE_STATES] = "IN-NMI",

};

#endif

const char *__get_key_name(const struct lockdep_subclass_key *key, char *str)

{

return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str);

}

static inline unsigned long lock_flag(enum lock_usage_bit bit)

{

return 1UL << bit;

}

static char get_usage_char(struct lock_class *class, enum lock_usage_bit bit)

{

/*

* The usage character defaults to '.' (i.e., irqs disabled and not in

* irq context), which is the safest usage category.

*/

char c = '.';

/*

* The order of the following usage checks matters, which will

* result in the outcome character as follows:

*

* - '+': irq is enabled and not in irq context

* - '-': in irq context and irq is disabled

* - '?': in irq context and irq is enabled

*/

if (class->usage_mask & lock_flag(bit + LOCK_USAGE_DIR_MASK)) {

c = '+';

if (class->usage_mask & lock_flag(bit))

c = '?';

} else if (class->usage_mask & lock_flag(bit))

c = '-';

return c;

}

void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS])

{

int i = 0;

#define LOCKDEP_STATE(__STATE) \

usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE); \

usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE##_READ);

#include "lockdep_states.h"

#undef LOCKDEP_STATE

usage[i] = '\0';

}

static void __print_lock_name(struct held_lock *hlock, struct lock_class *class)

{

char str[KSYM_NAME_LEN];

const char *name;

name = class->name;

if (!name) {

name = __get_key_name(class->key, str);

printk(KERN_CONT "%s", name);

} else {

printk(KERN_CONT "%s", name);

if (class->name_version > 1)

printk(KERN_CONT "#%d", class->name_version);

if (class->subclass)

printk(KERN_CONT "/%d", class->subclass);

if (hlock && class->print_fn)

class->print_fn(hlock->instance);

}

}

static void print_lock_name(struct held_lock *hlock, struct lock_class *class)

{

char usage[LOCK_USAGE_CHARS];

get_usage_chars(class, usage);

printk(KERN_CONT " (");

__print_lock_name(hlock, class);

printk(KERN_CONT "){%s}-{%d:%d}", usage,

class->wait_type_outer ?: class->wait_type_inner,

class->wait_type_inner);

}

static void print_lockdep_cache(struct lockdep_map *lock)

{

const char *name;

char str[KSYM_NAME_LEN];

name = lock->name;

if (!name)

name = __get_key_name(lock->key->subkeys, str);

printk(KERN_CONT "%s", name);

}

static void print_lock(struct held_lock *hlock)

{

/*

* We can be called locklessly through debug_show_all_locks() so be

* extra careful, the hlock might have been released and cleared.

*

* If this indeed happens, lets pretend it does not hurt to continue

* to print the lock unless the hlock class_idx does not point to a

* registered class. The rationale here is: since we don't attempt

* to distinguish whether we are in this situation, if it just

* happened we can't count on class_idx to tell either.

*/

struct lock_class *lock = hlock_class(hlock);

if (!lock) {

printk(KERN_CONT "<RELEASED>\n");

return;

}

printk(KERN_CONT "%px", hlock->instance);

print_lock_name(hlock, lock);

printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip);

}

static void lockdep_print_held_locks(struct task_struct *p)

{

int i, depth = READ_ONCE(p->lockdep_depth);

if (!depth)

printk("no locks held by %s/%d.\n", p->comm, task_pid_nr(p));

else

printk("%d lock%s held by %s/%d:\n", depth,

depth > 1 ? "s" : "", p->comm, task_pid_nr(p));

/*

* It's not reliable to print a task's held locks if it's not sleeping

* and it's not the current task.

*/

if (p != current && task_is_running(p))

return;

for (i = 0; i < depth; i++) {

printk(" #%d: ", i);

print_lock(p->held_locks + i);

}

}

static void print_kernel_ident(void)

{

printk("%s %.*s %s\n", init_utsname()->release,

(int)strcspn(init_utsname()->version, " "),

init_utsname()->version,

print_tainted());

}

static int very_verbose(struct lock_class *class)

{

#if VERY_VERBOSE

return class_filter(class);

#endif

return 0;

}

/*

* Is this the address of a static object:

*/

#ifdef __KERNEL__

static int static_obj(const void *obj)

{

unsigned long addr = (unsigned long) obj;

if (is_kernel_core_data(addr))

return 1;

/*

* keys are allowed in the __ro_after_init section.

*/

if (is_kernel_rodata(addr))

return 1;

/*

* in initdata section and used during bootup only?

* NOTE: On some platforms the initdata section is

* outside of the _stext ... _end range.

*/

if (system_state < SYSTEM_FREEING_INITMEM &&

init_section_contains((void *)addr, 1))

return 1;

/*

* in-kernel percpu var?

*/

if (is_kernel_percpu_address(addr))

return 1;

/*

* module static or percpu var?

*/

return is_module_address(addr) || is_module_percpu_address(addr);

}

#endif

/*

* To make lock name printouts unique, we calculate a unique

* class->name_version generation counter. The caller must hold the graph

* lock.

*/

static int count_matching_names(struct lock_class *new_class)

{

struct lock_class *class;

int count = 0;

if (!new_class->name)

return 0;

list_for_each_entry(class, &all_lock_classes, lock_entry) {

if (new_class->key - new_class->subclass == class->key)

return class->name_version;

if (class->name && !strcmp(class->name, new_class->name))

count = max(count, class->name_version);

}

return count + 1;

}

/* used from NMI context -- must be lockless */

static noinstr struct lock_class *

look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)

{

struct lockdep_subclass_key *key;

struct hlist_head *hash_head;

struct lock_class *class;

if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {

instrumentation_begin();

debug_locks_off();

printk(KERN_ERR

"BUG: looking up invalid subclass: %u\n", subclass);

printk(KERN_ERR

"turning off the locking correctness validator.\n");

dump_stack();

instrumentation_end();

return NULL;

}

/*

* If it is not initialised then it has never been locked,

* so it won't be present in the hash table.

*/

if (unlikely(!lock->key))

return NULL;

/*

* NOTE: the class-key must be unique. For dynamic locks, a static

* lock_class_key variable is passed in through the mutex_init()

* (or spin_lock_init()) call - which acts as the key. For static

* locks we use the lock object itself as the key.

*/

BUILD_BUG_ON(sizeof(struct lock_class_key) >

sizeof(struct lockdep_map));

key = lock->key->subkeys + subclass;

hash_head = classhashentry(key);

/*

* We do an RCU walk of the hash, see lockdep_free_key_range().

*/

if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))

return NULL;

hlist_for_each_entry_rcu_notrace(class, hash_head, hash_entry) {

if (class->key == key) {

/*

* Huh! same key, different name? Did someone trample

* on some memory? We're most confused.

*/

WARN_ONCE(class->name != lock->name &&

lock->key != &__lockdep_no_validate__,

"Looking for class \"%s\" with key %ps, but found a different class \"%s\" with the same key\n",

lock->name, lock->key, class->name);

return class;

}

}

return NULL;

}

/*

* Static locks do not have their class-keys yet - for them the key is

* the lock object itself. If the lock is in the per cpu area, the

* canonical address of the lock (per cpu offset removed) is used.

*/

static bool assign_lock_key(struct lockdep_map *lock)

{

unsigned long can_addr, addr = (unsigned long)lock;

#ifdef __KERNEL__

/*

* lockdep_free_key_range() assumes that struct lock_class_key

* objects do not overlap. Since we use the address of lock

* objects as class key for static objects, check whether the

* size of lock_class_key objects does not exceed the size of

* the smallest lock object.

*/

BUILD_BUG_ON(sizeof(struct lock_class_key) > sizeof(raw_spinlock_t));

#endif

if (__is_kernel_percpu_address(addr, &can_addr))

lock->key = (void *)can_addr;

else if (__is_module_percpu_address(addr, &can_addr))

lock->key = (void *)can_addr;

else if (static_obj(lock))

lock->key = (void *)lock;

else {

/* Debug-check: all keys must be persistent! */

debug_locks_off();

pr_err("INFO: trying to register non-static key.\n");

pr_err("The code is fine but needs lockdep annotation, or maybe\n");

pr_err("you didn't initialize this object before use?\n");

pr_err("turning off the locking correctness validator.\n");

dump_stack();

return false;

}

return true;

}

#ifdef CONFIG_DEBUG_LOCKDEP

/* Check whether element @e occurs in list @h */

static bool in_list(struct list_head *e, struct list_head *h)

{

struct list_head *f;

list_for_each(f, h) {

if (e == f)

return true;

}

return false;

}

/*

* Check whether entry @e occurs in any of the locks_after or locks_before

* lists.

*/