// SPDX-License-Identifier: GPL-2.0-only

/*

* fs/dcache.c

*

* Complete reimplementation

* (C) 1997 Thomas Schoebel-Theuer,

* with heavy changes by Linus Torvalds

*/

/*

* Notes on the allocation strategy:

*

* The dcache is a master of the icache - whenever a dcache entry

* exists, the inode will always exist. "iput()" is done either when

* the dcache entry is deleted or garbage collected.

*/

#include <linux/ratelimit.h>

#include <linux/string.h>

#include <linux/mm.h>

#include <linux/fs.h>

#include <linux/fscrypt.h>

#include <linux/fsnotify.h>

#include <linux/slab.h>

#include <linux/init.h>

#include <linux/hash.h>

#include <linux/cache.h>

#include <linux/export.h>

#include <linux/security.h>

#include <linux/seqlock.h>

#include <linux/memblock.h>

#include <linux/bit_spinlock.h>

#include <linux/rculist_bl.h>

#include <linux/list_lru.h>

#include "internal.h"

#include "mount.h"

#include <asm/runtime-const.h>

/*

* Usage:

* dcache->d_inode->i_lock protects:

* - i_dentry, d_alias, d_inode of aliases

* dcache_hash_bucket lock protects:

* - the dcache hash table

* s_roots bl list spinlock protects:

* - the s_roots list (see __d_drop)

* dentry->d_sb->s_dentry_lru_lock protects:

* - the dcache lru lists and counters

* d_lock protects:

* - d_flags

* - d_name

* - d_lru

* - d_count

* - d_unhashed()

* - d_parent and d_chilren

* - childrens' d_sib and d_parent

* - d_alias, d_inode

*

* Ordering:

* dentry->d_inode->i_lock

* dentry->d_lock

* dentry->d_sb->s_dentry_lru_lock

* dcache_hash_bucket lock

* s_roots lock

*

* If there is an ancestor relationship:

* dentry->d_parent->...->d_parent->d_lock

* ...

* dentry->d_parent->d_lock

* dentry->d_lock

*

* If no ancestor relationship:

* arbitrary, since it's serialized on rename_lock

*/

static int sysctl_vfs_cache_pressure __read_mostly = 100;

static int sysctl_vfs_cache_pressure_denom __read_mostly = 100;

unsigned long vfs_pressure_ratio(unsigned long val)

{

return mult_frac(val, sysctl_vfs_cache_pressure, sysctl_vfs_cache_pressure_denom);

}

EXPORT_SYMBOL_GPL(vfs_pressure_ratio);

__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);

EXPORT_SYMBOL(rename_lock);

static struct kmem_cache *__dentry_cache __ro_after_init;

#define dentry_cache runtime_const_ptr(__dentry_cache)

const struct qstr empty_name = QSTR_INIT("", 0);

EXPORT_SYMBOL(empty_name);

const struct qstr slash_name = QSTR_INIT("/", 1);

EXPORT_SYMBOL(slash_name);

const struct qstr dotdot_name = QSTR_INIT("..", 2);

EXPORT_SYMBOL(dotdot_name);

/*

* This is the single most critical data structure when it comes

* to the dcache: the hashtable for lookups. Somebody should try

* to make this good - I've just made it work.

*

* This hash-function tries to avoid losing too many bits of hash

* information, yet avoid using a prime hash-size or similar.

*

* Marking the variables "used" ensures that the compiler doesn't

* optimize them away completely on architectures with runtime

* constant infrastructure, this allows debuggers to see their

* values. But updating these values has no effect on those arches.

*/

static unsigned int d_hash_shift __ro_after_init __used;

static struct hlist_bl_head *dentry_hashtable __ro_after_init __used;

static inline struct hlist_bl_head *d_hash(unsigned long hashlen)

{

return runtime_const_ptr(dentry_hashtable) +

runtime_const_shift_right_32(hashlen, d_hash_shift);

}

#define IN_LOOKUP_SHIFT 10

static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];

static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,

unsigned int hash)

{

hash += (unsigned long) parent / L1_CACHE_BYTES;

return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);

}

struct dentry_stat_t {

long nr_dentry;

long nr_unused;

long age_limit; /* age in seconds */

long want_pages; /* pages requested by system */

long nr_negative; /* # of unused negative dentries */

long dummy; /* Reserved for future use */

};

static DEFINE_PER_CPU(long, nr_dentry);

static DEFINE_PER_CPU(long, nr_dentry_unused);

static DEFINE_PER_CPU(long, nr_dentry_negative);

static int dentry_negative_policy;

#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)

/* Statistics gathering. */

static struct dentry_stat_t dentry_stat = {

.age_limit = 45,

};

/*

* Here we resort to our own counters instead of using generic per-cpu counters

* for consistency with what the vfs inode code does. We are expected to harvest

* better code and performance by having our own specialized counters.

*

* Please note that the loop is done over all possible CPUs, not over all online

* CPUs. The reason for this is that we don't want to play games with CPUs going

* on and off. If one of them goes off, we will just keep their counters.

*

* glommer: See cffbc8a for details, and if you ever intend to change this,

* please update all vfs counters to match.

*/

static long get_nr_dentry(void)

{

int i;

long sum = 0;

for_each_possible_cpu(i)

sum += per_cpu(nr_dentry, i);

return sum < 0 ? 0 : sum;

}

static long get_nr_dentry_unused(void)

{

int i;

long sum = 0;

for_each_possible_cpu(i)

sum += per_cpu(nr_dentry_unused, i);

return sum < 0 ? 0 : sum;

}

static long get_nr_dentry_negative(void)

{

int i;

long sum = 0;

for_each_possible_cpu(i)

sum += per_cpu(nr_dentry_negative, i);

return sum < 0 ? 0 : sum;

}

static int proc_nr_dentry(const struct ctl_table *table, int write, void *buffer,

size_t *lenp, loff_t *ppos)

{

dentry_stat.nr_dentry = get_nr_dentry();

dentry_stat.nr_unused = get_nr_dentry_unused();

dentry_stat.nr_negative = get_nr_dentry_negative();

return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);

}

static const struct ctl_table fs_dcache_sysctls[] = {

{

.procname = "dentry-state",

.data = &dentry_stat,

.maxlen = 6*sizeof(long),

.mode = 0444,

.proc_handler = proc_nr_dentry,

},

{

.procname = "dentry-negative",

.data = &dentry_negative_policy,

.maxlen = sizeof(dentry_negative_policy),

.mode = 0644,

.proc_handler = proc_dointvec_minmax,

.extra1 = SYSCTL_ZERO,

.extra2 = SYSCTL_ONE,

},

};

static const struct ctl_table vm_dcache_sysctls[] = {

{

.procname = "vfs_cache_pressure",

.data = &sysctl_vfs_cache_pressure,

.maxlen = sizeof(sysctl_vfs_cache_pressure),

.mode = 0644,

.proc_handler = proc_dointvec_minmax,

.extra1 = SYSCTL_ZERO,

},

{

.procname = "vfs_cache_pressure_denom",

.data = &sysctl_vfs_cache_pressure_denom,

.maxlen = sizeof(sysctl_vfs_cache_pressure_denom),

.mode = 0644,

.proc_handler = proc_dointvec_minmax,

.extra1 = SYSCTL_ONE_HUNDRED,

},

};

static int __init init_fs_dcache_sysctls(void)

{

register_sysctl_init("vm", vm_dcache_sysctls);

register_sysctl_init("fs", fs_dcache_sysctls);

return 0;

}

fs_initcall(init_fs_dcache_sysctls);

#endif

/*

* Compare 2 name strings, return 0 if they match, otherwise non-zero.

* The strings are both count bytes long, and count is non-zero.

*/

#ifdef CONFIG_DCACHE_WORD_ACCESS

#include <asm/word-at-a-time.h>

/*

* NOTE! 'cs' and 'scount' come from a dentry, so it has a

* aligned allocation for this particular component. We don't

* strictly need the load_unaligned_zeropad() safety, but it

* doesn't hurt either.

*

* In contrast, 'ct' and 'tcount' can be from a pathname, and do

* need the careful unaligned handling.

*/

static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)

{

unsigned long a,b,mask;

for (;;) {

a = read_word_at_a_time(cs);

b = load_unaligned_zeropad(ct);

if (tcount < sizeof(unsigned long))

break;

if (unlikely(a != b))

return 1;

cs += sizeof(unsigned long);

ct += sizeof(unsigned long);

tcount -= sizeof(unsigned long);

if (!tcount)

return 0;

}

mask = bytemask_from_count(tcount);

return unlikely(!!((a ^ b) & mask));

}

#else

static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)

{

do {

if (*cs != *ct)

return 1;

cs++;

ct++;

tcount--;

} while (tcount);

return 0;

}

#endif

static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)

{

/*

* Be careful about RCU walk racing with rename:

* use 'READ_ONCE' to fetch the name pointer.

*

* NOTE! Even if a rename will mean that the length

* was not loaded atomically, we don't care. The

* RCU walk will check the sequence count eventually,

* and catch it. And we won't overrun the buffer,

* because we're reading the name pointer atomically,

* and a dentry name is guaranteed to be properly

* terminated with a NUL byte.

*

* End result: even if 'len' is wrong, we'll exit

* early because the data cannot match (there can

* be no NUL in the ct/tcount data)

*/

const unsigned char *cs = READ_ONCE(dentry->d_name.name);

return dentry_string_cmp(cs, ct, tcount);

}

/*

* long names are allocated separately from dentry and never modified.

* Refcounted, freeing is RCU-delayed. See take_dentry_name_snapshot()

* for the reason why ->count and ->head can't be combined into a union.

* dentry_string_cmp() relies upon ->name[] being word-aligned.

*/

struct external_name {

atomic_t count;

struct rcu_head head;

unsigned char name[] __aligned(sizeof(unsigned long));

};

static inline struct external_name *external_name(struct dentry *dentry)

{

return container_of(dentry->d_name.name, struct external_name, name[0]);

}

static void __d_free(struct rcu_head *head)

{

struct dentry *dentry = container_of(head, struct dentry, d_rcu);

kmem_cache_free(dentry_cache, dentry);

}

static void __d_free_external(struct rcu_head *head)

{

struct dentry *dentry = container_of(head, struct dentry, d_rcu);

kfree(external_name(dentry));

kmem_cache_free(dentry_cache, dentry);

}

static inline int dname_external(const struct dentry *dentry)

{

return dentry->d_name.name != dentry->d_shortname.string;

}

void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)

{

unsigned seq;

const unsigned char *s;

rcu_read_lock();

retry:

seq = read_seqcount_begin(&dentry->d_seq);

s = READ_ONCE(dentry->d_name.name);

name->name.hash_len = dentry->d_name.hash_len;

name->name.name = name->inline_name.string;

if (likely(s == dentry->d_shortname.string)) {

name->inline_name = dentry->d_shortname;

} else {

struct external_name *p;

p = container_of(s, struct external_name, name[0]);

// get a valid reference

if (unlikely(!atomic_inc_not_zero(&p->count)))

goto retry;

name->name.name = s;

}

if (read_seqcount_retry(&dentry->d_seq, seq)) {

release_dentry_name_snapshot(name);

goto retry;

}

rcu_read_unlock();

}

EXPORT_SYMBOL(take_dentry_name_snapshot);

void release_dentry_name_snapshot(struct name_snapshot *name)

{

if (unlikely(name->name.name != name->inline_name.string)) {

struct external_name *p;

p = container_of(name->name.name, struct external_name, name[0]);

if (unlikely(atomic_dec_and_test(&p->count)))

kfree_rcu(p, head);

}

}

EXPORT_SYMBOL(release_dentry_name_snapshot);

static inline void __d_set_inode_and_type(struct dentry *dentry,

struct inode *inode,

unsigned type_flags)

{

unsigned flags;

dentry->d_inode = inode;

flags = READ_ONCE(dentry->d_flags);

flags &= ~DCACHE_ENTRY_TYPE;

flags |= type_flags;

smp_store_release(&dentry->d_flags, flags);

}

static inline void __d_clear_type_and_inode(struct dentry *dentry)

{

unsigned flags = READ_ONCE(dentry->d_flags);

flags &= ~DCACHE_ENTRY_TYPE;

WRITE_ONCE(dentry->d_flags, flags);

dentry->d_inode = NULL;

/*

* The negative counter only tracks dentries on the LRU. Don't inc if

* d_lru is on another list.

*/

if ((flags & (DCACHE_LRU_LIST|DCACHE_SHRINK_LIST)) == DCACHE_LRU_LIST)

this_cpu_inc(nr_dentry_negative);

}

static void dentry_free(struct dentry *dentry)

{

WARN_ON(d_really_is_positive(dentry));

if (unlikely(dname_external(dentry))) {

struct external_name *p = external_name(dentry);

if (likely(atomic_dec_and_test(&p->count))) {

call_rcu(&dentry->d_rcu, __d_free_external);

return;

}

}

/* if dentry was never visible to RCU, immediate free is OK */

if (dentry->d_flags & DCACHE_NORCU)

__d_free(&dentry->d_rcu);

else

call_rcu(&dentry->d_rcu, __d_free);

}

/*

* Release the dentry's inode, using the filesystem

* d_iput() operation if defined.

*/

static void dentry_unlink_inode(struct dentry * dentry)

__releases(dentry->d_lock)

__releases(dentry->d_inode->i_lock)

{

struct inode *inode = dentry->d_inode;

raw_write_seqcount_begin(&dentry->d_seq);

__d_clear_type_and_inode(dentry);

hlist_del_init(&dentry->d_alias);

/*

* dentry becomes negative, so the space occupied by ->d_alias

* belongs to ->waiters now; we could use __hlist_del() instead

* of hlist_del_init(), if not for the stunt pulled by nfs

* dummy root dentries - positive dentry *not* included into

* the alias list of its inode. Open-coding hlist_del_init()

* and removing zeroing would be too clumsy...

*/

dentry->waiters = NULL;

raw_write_seqcount_end(&dentry->d_seq);

spin_unlock(&dentry->d_lock);

spin_unlock(&inode->i_lock);

if (!inode->i_nlink)

fsnotify_inoderemove(inode);

if (dentry->d_op && dentry->d_op->d_iput)

dentry->d_op->d_iput(dentry, inode);

else

iput(inode);

}

/*

* The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry

* is in use - which includes both the "real" per-superblock

* LRU list _and_ the DCACHE_SHRINK_LIST use.

*

* The DCACHE_SHRINK_LIST bit is set whenever the dentry is

* on the shrink list (ie not on the superblock LRU list).

*

* The per-cpu "nr_dentry_unused" counters are updated with

* the DCACHE_LRU_LIST bit.

*

* The per-cpu "nr_dentry_negative" counters are only updated

* when deleted from or added to the per-superblock LRU list, not

* from/to the shrink list. That is to avoid an unneeded dec/inc

* pair when moving from LRU to shrink list in select_collect().

*

* These helper functions make sure we always follow the

* rules. d_lock must be held by the caller.

*/

#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))

static void d_lru_add(struct dentry *dentry)

{

D_FLAG_VERIFY(dentry, 0);

dentry->d_flags |= DCACHE_LRU_LIST;

this_cpu_inc(nr_dentry_unused);

if (d_is_negative(dentry))

this_cpu_inc(nr_dentry_negative);

WARN_ON_ONCE(!list_lru_add_obj(

&dentry->d_sb->s_dentry_lru, &dentry->d_lru));

}

static void d_lru_del(struct dentry *dentry)

{

D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);

dentry->d_flags &= ~DCACHE_LRU_LIST;

this_cpu_dec(nr_dentry_unused);

if (d_is_negative(dentry))

this_cpu_dec(nr_dentry_negative);

WARN_ON_ONCE(!list_lru_del_obj(

&dentry->d_sb->s_dentry_lru, &dentry->d_lru));

}

static void d_shrink_del(struct dentry *dentry)

{

D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);

list_del_init(&dentry->d_lru);

dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);

this_cpu_dec(nr_dentry_unused);

}

static void d_shrink_add(struct dentry *dentry, struct list_head *list)

{

D_FLAG_VERIFY(dentry, 0);

list_add(&dentry->d_lru, list);

dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;

this_cpu_inc(nr_dentry_unused);

}

/*

* These can only be called under the global LRU lock, ie during the

* callback for freeing the LRU list. "isolate" removes it from the

* LRU lists entirely, while shrink_move moves it to the indicated

* private list.

*/

static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)

{

D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);

dentry->d_flags &= ~DCACHE_LRU_LIST;

this_cpu_dec(nr_dentry_unused);

if (d_is_negative(dentry))

this_cpu_dec(nr_dentry_negative);

list_lru_isolate(lru, &dentry->d_lru);

}

static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,

struct list_head *list)

{

D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);

dentry->d_flags |= DCACHE_SHRINK_LIST;

if (d_is_negative(dentry))

this_cpu_dec(nr_dentry_negative);

list_lru_isolate_move(lru, &dentry->d_lru, list);

}

static void ___d_drop(struct dentry *dentry)

{

struct hlist_bl_head *b;

/*

* Hashed dentries are normally on the dentry hashtable,

* with the exception of those newly allocated by

* d_obtain_root, which are always IS_ROOT:

*/

if (unlikely(IS_ROOT(dentry)))

b = &dentry->d_sb->s_roots;

else

b = d_hash(dentry->d_name.hash);

hlist_bl_lock(b);

__hlist_bl_del(&dentry->d_hash);

hlist_bl_unlock(b);

}

void __d_drop(struct dentry *dentry)

{

if (!d_unhashed(dentry)) {

___d_drop(dentry);

dentry->d_hash.pprev = NULL;

write_seqcount_invalidate(&dentry->d_seq);

}

}

EXPORT_SYMBOL(__d_drop);

/**

* d_drop - drop a dentry

* @dentry: dentry to drop

*

* d_drop() unhashes the entry from the parent dentry hashes, so that it won't

* be found through a VFS lookup any more. Note that this is different from

* deleting the dentry - d_delete will try to mark the dentry negative if

* possible, giving a successful _negative_ lookup, while d_drop will

* just make the cache lookup fail.

*

* d_drop() is used mainly for stuff that wants to invalidate a dentry for some

* reason (NFS timeouts or autofs deletes).

*

* __d_drop requires dentry->d_lock

*

* ___d_drop doesn't mark dentry as "unhashed"

* (dentry->d_hash.pprev will be LIST_POISON2, not NULL).

*/

void d_drop(struct dentry *dentry)

{

spin_lock(&dentry->d_lock);

__d_drop(dentry);

spin_unlock(&dentry->d_lock);

}

EXPORT_SYMBOL(d_drop);

struct completion_list {

struct completion_list *next;

struct completion completion;

};

/*

* shrink_dcache_tree() needs to be notified when dentry in process of

* being evicted finally gets unlisted. Such dentries are

* already with negative ->d_count

* already negative

* already not in in-lookup hash

* reachable only via ->d_sib.

*

* Use ->waiters for a single-linked list of struct completion_list of

* waiters.

*/

static inline void d_add_waiter(struct dentry *dentry, struct completion_list *p)

{

struct completion_list *v = dentry->waiters;

init_completion(&p->completion);

p->next = v;

dentry->waiters = p;

}

static inline void d_complete_waiters(struct dentry *dentry)

{

struct completion_list *v = dentry->waiters;

if (unlikely(v)) {

/* some shrink_dcache_tree() instances are waiting */

dentry->waiters = NULL;

while (v) {

struct completion *r = &v->completion;

v = v->next;

complete(r);

}

}

}

static inline void dentry_unlist(struct dentry *dentry)

{

struct dentry *next;

/*

* Inform d_walk() and shrink_dentry_list() that we are no longer

* attached to the dentry tree

*/

dentry->d_flags |= DCACHE_DENTRY_KILLED;

d_complete_waiters(dentry);

if (unlikely(hlist_unhashed(&dentry->d_sib)))

return;

__hlist_del(&dentry->d_sib);

/*

* Cursors can move around the list of children. While we'd been

* a normal list member, it didn't matter - ->d_sib.next would've

* been updated. However, from now on it won't be and for the

* things like d_walk() it might end up with a nasty surprise.

* Normally d_walk() doesn't care about cursors moving around -

* ->d_lock on parent prevents that and since a cursor has no children

* of its own, we get through it without ever unlocking the parent.

* There is one exception, though - if we ascend from a child that

* gets killed as soon as we unlock it, the next sibling is found

* using the value left in its ->d_sib.next. And if _that_

* pointed to a cursor, and cursor got moved (e.g. by lseek())

* before d_walk() regains parent->d_lock, we'll end up skipping

* everything the cursor had been moved past.

*

* Solution: make sure that the pointer left behind in ->d_sib.next

* points to something that won't be moving around. I.e. skip the

* cursors.

*/

while (dentry->d_sib.next) {

next = hlist_entry(dentry->d_sib.next, struct dentry, d_sib);

if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))

break;

dentry->d_sib.next = next->d_sib.next;

}

}

static struct dentry *__dentry_kill(struct dentry *dentry)

{

struct dentry *parent = NULL;

bool can_free = true;

/*

* The dentry is now unrecoverably dead to the world.

*/

lockref_mark_dead(&dentry->d_lockref);

/*

* inform the fs via d_prune that this dentry is about to be

* unhashed and destroyed.

*/

if (dentry->d_flags & DCACHE_OP_PRUNE)

dentry->d_op->d_prune(dentry);

if (dentry->d_flags & DCACHE_LRU_LIST) {

if (!(dentry->d_flags & DCACHE_SHRINK_LIST))

d_lru_del(dentry);

}

/* if it was on the hash then remove it */

__d_drop(dentry);

if (dentry->d_inode)

dentry_unlink_inode(dentry);

else

spin_unlock(&dentry->d_lock);

this_cpu_dec(nr_dentry);

if (dentry->d_op && dentry->d_op->d_release)

dentry->d_op->d_release(dentry);

cond_resched();

/* now that it's negative, ->d_parent is stable */

if (!IS_ROOT(dentry)) {

parent = dentry->d_parent;

spin_lock(&parent->d_lock);

}

spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);

dentry_unlist(dentry);

if (dentry->d_flags & DCACHE_SHRINK_LIST)

can_free = false;

spin_unlock(&dentry->d_lock);

if (likely(can_free))

dentry_free(dentry);

if (parent && --parent->d_lockref.count) {

spin_unlock(&parent->d_lock);

return NULL;

}

return parent;

}

/*

* Lock a dentry for feeding it to __dentry_kill().

* Called under rcu_read_lock() and dentry->d_lock; the former

* guarantees that nothing we access will be freed under us.

* Note that dentry is *not* protected from concurrent dentry_kill(),

* d_delete(), etc.

*

* Return false if dentry is busy. Otherwise, return true and have

* that dentry's inode locked.

*/

static bool lock_for_kill(struct dentry *dentry)

{

struct inode *inode = dentry->d_inode;

if (unlikely(dentry->d_lockref.count))

return false;

if (!inode || likely(spin_trylock(&inode->i_lock)))

return true;

do {

spin_unlock(&dentry->d_lock);

spin_lock(&inode->i_lock);

spin_lock(&dentry->d_lock);

if (likely(inode == dentry->d_inode))

break;

spin_unlock(&inode->i_lock);

inode = dentry->d_inode;

} while (inode);

if (likely(!dentry->d_lockref.count))

return true;

if (inode)

spin_unlock(&inode->i_lock);

return false;

}

/*

* Decide if dentry is worth retaining. Usually this is called with dentry

* locked; if not locked, we are more limited and might not be able to tell

* without a lock. False in this case means "punt to locked path and recheck".

*

* In case we aren't locked, these predicates are not "stable". However, it is

* sufficient that at some point after we dropped the reference the dentry was

* hashed and the flags had the proper value. Other dentry users may have

* re-gotten a reference to the dentry and change that, but our work is done -

* we can leave the dentry around with a zero refcount.

*/

static inline bool retain_dentry(struct dentry *dentry, bool locked)

{

unsigned int d_flags;

smp_rmb();

d_flags = READ_ONCE(dentry->d_flags);

// Unreachable? Nobody would be able to look it up, no point retaining

if (unlikely(d_unhashed(dentry)))

return false;

// Same if it's disconnected

if (unlikely(d_flags & DCACHE_DISCONNECTED))

return false;

// ->d_delete() might tell us not to bother, but that requires

// ->d_lock; can't decide without it

if (unlikely(d_flags & DCACHE_OP_DELETE)) {

if (!locked || dentry->d_op->d_delete(dentry))

return false;

}

// Explicitly told not to bother

if (unlikely(d_flags & DCACHE_DONTCACHE))

return false;

// At this point it looks like we ought to keep it. We also might

// need to do something - put it on LRU if it wasn't there already

// and mark it referenced if it was on LRU, but not marked yet.

// Unfortunately, both actions require ->d_lock, so in lockless

// case we'd have to punt rather than doing those.

if (unlikely(!(d_flags & DCACHE_LRU_LIST))) {

if (!locked)

return false;

d_lru_add(dentry);

} else if (unlikely(!(d_flags & DCACHE_REFERENCED))) {

if (!locked)

return false;

dentry->d_flags |= DCACHE_REFERENCED;

}

return true;

}

void d_mark_dontcache(struct inode *inode)

{

struct dentry *de;

spin_lock(&inode->i_lock);

for_each_alias(de, inode) {

spin_lock(&de->d_lock);

de->d_flags |= DCACHE_DONTCACHE;

spin_unlock(&de->d_lock);

}

inode_state_set(inode, I_DONTCACHE);

spin_unlock(&inode->i_lock);

}

EXPORT_SYMBOL(d_mark_dontcache);

/*

* Try to do a lockless dput(), and return whether that was successful.

*

* If unsuccessful, we return false, having already taken the dentry lock.

* In that case refcount is guaranteed to be zero and we have already

* decided that it's not worth keeping around.

*

* The caller needs to hold the RCU read lock, so that the dentry is

* guaranteed to stay around even if the refcount goes down to zero!

*/

static inline bool fast_dput(struct dentry *dentry)

{

int ret;

/*

* try to decrement the lockref optimistically.

*/

ret = lockref_put_return(&dentry->d_lockref);

/*

* If the lockref_put_return() failed due to the lock being held

* by somebody else, the fast path has failed. We will need to

* get the lock, and then check the count again.

*/

if (unlikely(ret < 0)) {

spin_lock(&dentry->d_lock);

if (WARN_ON_ONCE(dentry->d_lockref.count <= 0)) {

spin_unlock(&dentry->d_lock);

return true;

}

dentry->d_lockref.count--;

goto locked;

}

/*

* If we weren't the last ref, we're done.

*/

if (ret)

return true;

/*

* Can we decide that decrement of refcount is all we needed without

* taking the lock? There's a very common case when it's all we need -

* dentry looks like it ought to be retained and there's nothing else

* to do.

*/

if (retain_dentry(dentry, false))

return true;

/*

* Either not worth retaining or we can't tell without the lock.

* Get the lock, then. We've already decremented the refcount to 0,

* but we'll need to re-check the situation after getting the lock.

*/

spin_lock(&dentry->d_lock);

/*

* Did somebody else grab a reference to it in the meantime, and

* we're no longer the last user after all? Alternatively, somebody

* else could have killed it and marked it dead. Either way, we

* don't need to do anything else.

*/

locked:

if (dentry->d_lockref.count || retain_dentry(dentry, true)) {

spin_unlock(&dentry->d_lock);

return true;

}

return false;

}

static void finish_dput(struct dentry *dentry)

__releases(dentry->d_lock)

__releases(RCU)

{

while (lock_for_kill(dentry)) {

rcu_read_unlock();

dentry = __dentry_kill(dentry);

if (!dentry)

return;

if (retain_dentry(dentry, true)) {

spin_unlock(&dentry->d_lock);

return;

}

rcu_read_lock();

}

rcu_read_unlock();

spin_unlock(&dentry->d_lock);

}

/*

* This is dput

*

* This is complicated by the fact that we do not want to put

* dentries that are no longer on any hash chain on the unused

* list: we'd much rather just get rid of them immediately.

*

* However, that implies that we have to traverse the dentry

* tree upwards to the parents which might _also_ now be

* scheduled for deletion (it may have been only waiting for

* its last child to go away).

*

* This tail recursion is done by hand as we don't want to depend

* on the compiler to always get this right (gcc generally doesn't).

* Real recursion would eat up our stack space.

*/

/*

* dput - release a dentry

* @dentry: dentry to release

*

* Release a dentry. This will drop the usage count and if appropriate

* call the dentry unlink method as well as removing it from the queues and

* releasing its resources. If the parent dentries were scheduled for release

* they too may now get deleted.

*/

void dput(struct dentry *dentry)

{

if (!dentry)

return;

might_sleep();

rcu_read_lock();

if (likely(fast_dput(dentry))) {

rcu_read_unlock();

return;

}

finish_dput(dentry);

}

EXPORT_SYMBOL(dput);

void d_make_discardable(struct dentry *dentry)

{

spin_lock(&dentry->d_lock);

WARN_ON(!(dentry->d_flags & DCACHE_PERSISTENT));

dentry->d_flags &= ~DCACHE_PERSISTENT;

dentry->d_lockref.count--;

rcu_read_lock();

finish_dput(dentry);

}

EXPORT_SYMBOL(d_make_discardable);

static void to_shrink_list(struct dentry *dentry, struct list_head *list)

__must_hold(&dentry->d_lock)

{

if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {

if (dentry->d_flags & DCACHE_LRU_LIST)

d_lru_del(dentry);

d_shrink_add(dentry, list);

}

}