// SPDX-License-Identifier: GPL-2.0-or-later

/*

* Generic address resolution entity

*

* Authors:

* Pedro Roque <roque@di.fc.ul.pt>

* Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>

*

* Fixes:

* Vitaly E. Lavrov releasing NULL neighbor in neigh_add.

* Harald Welte Add neighbour cache statistics like rtstat

*/

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/slab.h>

#include <linux/types.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/socket.h>

#include <linux/netdevice.h>

#include <linux/proc_fs.h>

#ifdef CONFIG_SYSCTL

#include <linux/sysctl.h>

#endif

#include <linux/times.h>

#include <net/net_namespace.h>

#include <net/neighbour.h>

#include <net/arp.h>

#include <net/dst.h>

#include <net/ip.h>

#include <net/sock.h>

#include <net/netevent.h>

#include <net/netlink.h>

#include <linux/rtnetlink.h>

#include <linux/random.h>

#include <linux/string.h>

#include <linux/log2.h>

#include <linux/inetdevice.h>

#include <net/addrconf.h>

#include <trace/events/neigh.h>

#define NEIGH_DEBUG 1

#define neigh_dbg(level, fmt, ...) \

do { \

if (level <= NEIGH_DEBUG) \

pr_debug(fmt, ##__VA_ARGS__); \

} while (0)

#define PNEIGH_HASHMASK 0xF

static void neigh_timer_handler(struct timer_list *t);

static void neigh_notify(struct neighbour *n, int type, int flags, u32 pid);

static void __neigh_notify(struct neighbour *n, int type, int flags, u32 pid);

static void pneigh_ifdown(struct neigh_table *tbl, struct net_device *dev,

bool skip_perm);

#ifdef CONFIG_PROC_FS

static const struct seq_operations neigh_stat_seq_ops;

#endif

static struct hlist_head *neigh_get_dev_table(struct net_device *dev, int family)

{

int i;

switch (family) {

default:

DEBUG_NET_WARN_ON_ONCE(1);

fallthrough; /* to avoid panic by null-ptr-deref */

case AF_INET:

i = NEIGH_ARP_TABLE;

break;

case AF_INET6:

i = NEIGH_ND_TABLE;

break;

}

return &dev->neighbours[i];

}

/*

Neighbour hash table buckets are protected with tbl->lock.

- All the scans/updates to hash buckets MUST be made under this lock.

- NOTHING clever should be made under this lock: no callbacks

to protocol backends, no attempts to send something to network.

It will result in deadlocks, if backend/driver wants to use neighbour

cache.

- If the entry requires some non-trivial actions, increase

its reference count and release table lock.

Neighbour entries are protected:

- with reference count.

- with rwlock neigh->lock

Reference count prevents destruction.

neigh->lock mainly serializes ll address data and its validity state.

However, the same lock is used to protect another entry fields:

- timer

- resolution queue

Again, nothing clever shall be made under neigh->lock,

the most complicated procedure, which we allow is dev->hard_header.

It is supposed, that dev->hard_header is simplistic and does

not make callbacks to neighbour tables.

*/

static int neigh_blackhole(struct neighbour *neigh, struct sk_buff *skb)

{

kfree_skb(skb);

return -ENETDOWN;

}

static void neigh_cleanup_and_release(struct neighbour *neigh)

{

trace_neigh_cleanup_and_release(neigh, 0);

neigh_notify(neigh, RTM_DELNEIGH, 0, 0);

call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, neigh);

neigh_release(neigh);

}

/*

* It is random distribution in the interval (1/2)*base...(3/2)*base.

* It corresponds to default IPv6 settings and is not overridable,

* because it is really reasonable choice.

*/

unsigned long neigh_rand_reach_time(unsigned long base)

{

return base ? get_random_u32_below(base) + (base >> 1) : 0;

}

EXPORT_SYMBOL(neigh_rand_reach_time);

static void neigh_mark_dead(struct neighbour *n)

{

n->dead = 1;

if (!list_empty(&n->gc_list)) {

list_del_init(&n->gc_list);

atomic_dec(&n->tbl->gc_entries);

}

if (!list_empty(&n->managed_list))

list_del_init(&n->managed_list);

}

static void neigh_update_gc_list(struct neighbour *n)

{

bool on_gc_list, exempt_from_gc;

spin_lock_bh(&n->tbl->lock);

write_lock(&n->lock);

if (n->dead)

goto out;

/* remove from the gc list if new state is permanent or if neighbor is

* externally learned / validated; otherwise entry should be on the gc

* list

*/

exempt_from_gc = n->nud_state & NUD_PERMANENT ||

n->flags & (NTF_EXT_LEARNED | NTF_EXT_VALIDATED);

on_gc_list = !list_empty(&n->gc_list);

if (exempt_from_gc && on_gc_list) {

list_del_init(&n->gc_list);

atomic_dec(&n->tbl->gc_entries);

} else if (!exempt_from_gc && !on_gc_list) {

/* add entries to the tail; cleaning removes from the front */

list_add_tail(&n->gc_list, &n->tbl->gc_list);

atomic_inc(&n->tbl->gc_entries);

}

out:

write_unlock(&n->lock);

spin_unlock_bh(&n->tbl->lock);

}

static void neigh_update_managed_list(struct neighbour *n)

{

bool on_managed_list, add_to_managed;

spin_lock_bh(&n->tbl->lock);

write_lock(&n->lock);

if (n->dead)

goto out;

add_to_managed = n->flags & NTF_MANAGED;

on_managed_list = !list_empty(&n->managed_list);

if (!add_to_managed && on_managed_list)

list_del_init(&n->managed_list);

else if (add_to_managed && !on_managed_list)

list_add_tail(&n->managed_list, &n->tbl->managed_list);

out:

write_unlock(&n->lock);

spin_unlock_bh(&n->tbl->lock);

}

static void neigh_update_flags(struct neighbour *neigh, u32 flags, int *notify,

bool *gc_update, bool *managed_update)

{

u32 ndm_flags, old_flags = neigh->flags;

if (!(flags & NEIGH_UPDATE_F_ADMIN))

return;

ndm_flags = (flags & NEIGH_UPDATE_F_EXT_LEARNED) ? NTF_EXT_LEARNED : 0;

ndm_flags |= (flags & NEIGH_UPDATE_F_MANAGED) ? NTF_MANAGED : 0;

ndm_flags |= (flags & NEIGH_UPDATE_F_EXT_VALIDATED) ? NTF_EXT_VALIDATED : 0;

if ((old_flags ^ ndm_flags) & NTF_EXT_LEARNED) {

if (ndm_flags & NTF_EXT_LEARNED)

neigh->flags |= NTF_EXT_LEARNED;

else

neigh->flags &= ~NTF_EXT_LEARNED;

*notify = 1;

*gc_update = true;

}

if ((old_flags ^ ndm_flags) & NTF_MANAGED) {

if (ndm_flags & NTF_MANAGED)

neigh->flags |= NTF_MANAGED;

else

neigh->flags &= ~NTF_MANAGED;

*notify = 1;

*managed_update = true;

}

if ((old_flags ^ ndm_flags) & NTF_EXT_VALIDATED) {

if (ndm_flags & NTF_EXT_VALIDATED)

neigh->flags |= NTF_EXT_VALIDATED;

else

neigh->flags &= ~NTF_EXT_VALIDATED;

*notify = 1;

*gc_update = true;

}

}

bool neigh_remove_one(struct neighbour *n)

{

bool retval = false;

write_lock(&n->lock);

if (refcount_read(&n->refcnt) == 1) {

hlist_del_rcu(&n->hash);

hlist_del_rcu(&n->dev_list);

neigh_mark_dead(n);

retval = true;

}

write_unlock(&n->lock);

if (retval)

neigh_cleanup_and_release(n);

return retval;

}

static int neigh_forced_gc(struct neigh_table *tbl)

{

int max_clean = atomic_read(&tbl->gc_entries) -

READ_ONCE(tbl->gc_thresh2);

u64 tmax = ktime_get_ns() + NSEC_PER_MSEC;

unsigned long tref = jiffies - 5 * HZ;

struct neighbour *n, *tmp;

int shrunk = 0;

int loop = 0;

NEIGH_CACHE_STAT_INC(tbl, forced_gc_runs);

spin_lock_bh(&tbl->lock);

list_for_each_entry_safe(n, tmp, &tbl->gc_list, gc_list) {

if (refcount_read(&n->refcnt) == 1) {

bool remove = false;

write_lock(&n->lock);

if ((n->nud_state == NUD_FAILED) ||

(n->nud_state == NUD_NOARP) ||

(tbl->is_multicast &&

tbl->is_multicast(n->primary_key)) ||

!time_in_range(n->updated, tref, jiffies))

remove = true;

write_unlock(&n->lock);

if (remove && neigh_remove_one(n))

shrunk++;

if (shrunk >= max_clean)

break;

if (++loop == 16) {

if (ktime_get_ns() > tmax)

goto unlock;

loop = 0;

}

}

}

WRITE_ONCE(tbl->last_flush, jiffies);

unlock:

spin_unlock_bh(&tbl->lock);

return shrunk;

}

static void neigh_add_timer(struct neighbour *n, unsigned long when)

{

/* Use safe distance from the jiffies - LONG_MAX point while timer

* is running in DELAY/PROBE state but still show to user space

* large times in the past.

*/

unsigned long mint = jiffies - (LONG_MAX - 86400 * HZ);

neigh_hold(n);

if (!time_in_range(n->confirmed, mint, jiffies))

n->confirmed = mint;

if (time_before(n->used, n->confirmed))

n->used = n->confirmed;

if (unlikely(mod_timer(&n->timer, when))) {

printk("NEIGH: BUG, double timer add, state is %x\n",

n->nud_state);

dump_stack();

}

}

static int neigh_del_timer(struct neighbour *n)

{

if ((n->nud_state & NUD_IN_TIMER) &&

timer_delete(&n->timer)) {

neigh_release(n);

return 1;

}

return 0;

}

static struct neigh_parms *neigh_get_dev_parms_rcu(struct net_device *dev,

int family)

{

switch (family) {

case AF_INET:

return __in_dev_arp_parms_get_rcu(dev);

case AF_INET6:

return __in6_dev_nd_parms_get_rcu(dev);

}

return NULL;

}

static void neigh_parms_qlen_dec(struct net_device *dev, int family)

{

struct neigh_parms *p;

rcu_read_lock();

p = neigh_get_dev_parms_rcu(dev, family);

if (p)

p->qlen--;

rcu_read_unlock();

}

static void pneigh_queue_purge(struct sk_buff_head *list, struct net *net,

int family)

{

struct sk_buff_head tmp;

unsigned long flags;

struct sk_buff *skb;

skb_queue_head_init(&tmp);

spin_lock_irqsave(&list->lock, flags);

skb = skb_peek(list);

while (skb != NULL) {

struct sk_buff *skb_next = skb_peek_next(skb, list);

struct net_device *dev = skb->dev;

if (net == NULL || net_eq(dev_net(dev), net)) {

neigh_parms_qlen_dec(dev, family);

__skb_unlink(skb, list);

__skb_queue_tail(&tmp, skb);

}

skb = skb_next;

}

spin_unlock_irqrestore(&list->lock, flags);

while ((skb = __skb_dequeue(&tmp))) {

dev_put(skb->dev);

kfree_skb(skb);

}

}

static void neigh_flush_one(struct neighbour *n)

{

hlist_del_rcu(&n->hash);

hlist_del_rcu(&n->dev_list);

write_lock(&n->lock);

neigh_del_timer(n);

neigh_mark_dead(n);

if (refcount_read(&n->refcnt) != 1) {

/* The most unpleasant situation.

* We must destroy neighbour entry,

* but someone still uses it.

*

* The destroy will be delayed until

* the last user releases us, but

* we must kill timers etc. and move

* it to safe state.

*/

__skb_queue_purge(&n->arp_queue);

n->arp_queue_len_bytes = 0;

WRITE_ONCE(n->output, neigh_blackhole);

if (n->nud_state & NUD_VALID)

n->nud_state = NUD_NOARP;

else

n->nud_state = NUD_NONE;

neigh_dbg(2, "neigh %p is stray\n", n);

}

write_unlock(&n->lock);

neigh_cleanup_and_release(n);

}

static void neigh_flush_dev(struct neigh_table *tbl, struct net_device *dev,

bool skip_perm)

{

struct hlist_head *dev_head;

struct hlist_node *tmp;

struct neighbour *n;

dev_head = neigh_get_dev_table(dev, tbl->family);

hlist_for_each_entry_safe(n, tmp, dev_head, dev_list) {

if (skip_perm &&

(n->nud_state & NUD_PERMANENT ||

n->flags & NTF_EXT_VALIDATED))

continue;

neigh_flush_one(n);

}

}

static void neigh_flush_table(struct neigh_table *tbl)

{

struct neigh_hash_table *nht;

int i;

nht = rcu_dereference_protected(tbl->nht,

lockdep_is_held(&tbl->lock));

for (i = 0; i < (1 << nht->hash_shift); i++) {

struct hlist_node *tmp;

struct neighbour *n;

neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[i])

neigh_flush_one(n);

}

}

void neigh_changeaddr(struct neigh_table *tbl, struct net_device *dev)

{

spin_lock_bh(&tbl->lock);

neigh_flush_dev(tbl, dev, false);

spin_unlock_bh(&tbl->lock);

}

EXPORT_SYMBOL(neigh_changeaddr);

static int __neigh_ifdown(struct neigh_table *tbl, struct net_device *dev,

bool skip_perm)

{

spin_lock_bh(&tbl->lock);

if (likely(dev)) {

neigh_flush_dev(tbl, dev, skip_perm);

} else {

DEBUG_NET_WARN_ON_ONCE(skip_perm);

neigh_flush_table(tbl);

}

spin_unlock_bh(&tbl->lock);

pneigh_ifdown(tbl, dev, skip_perm);

pneigh_queue_purge(&tbl->proxy_queue, dev ? dev_net(dev) : NULL,

tbl->family);

if (skb_queue_empty_lockless(&tbl->proxy_queue))

timer_delete_sync(&tbl->proxy_timer);

return 0;

}

int neigh_carrier_down(struct neigh_table *tbl, struct net_device *dev)

{

__neigh_ifdown(tbl, dev, true);

return 0;

}

EXPORT_SYMBOL(neigh_carrier_down);

int neigh_ifdown(struct neigh_table *tbl, struct net_device *dev)

{

__neigh_ifdown(tbl, dev, false);

return 0;

}

EXPORT_SYMBOL(neigh_ifdown);

static struct neighbour *neigh_alloc(struct neigh_table *tbl,

struct net_device *dev,

u32 flags, bool exempt_from_gc)

{

struct neighbour *n = NULL;

unsigned long now = jiffies;

int entries, gc_thresh3;

if (exempt_from_gc)

goto do_alloc;

entries = atomic_inc_return(&tbl->gc_entries) - 1;

gc_thresh3 = READ_ONCE(tbl->gc_thresh3);

if (entries >= gc_thresh3 ||

(entries >= READ_ONCE(tbl->gc_thresh2) &&

time_after(now, READ_ONCE(tbl->last_flush) + 5 * HZ))) {

if (!neigh_forced_gc(tbl) && entries >= gc_thresh3) {

net_info_ratelimited("%s: neighbor table overflow!\n",

tbl->id);

NEIGH_CACHE_STAT_INC(tbl, table_fulls);

goto out_entries;

}

}

do_alloc:

n = kzalloc(tbl->entry_size + dev->neigh_priv_len, GFP_ATOMIC);

if (!n)

goto out_entries;

__skb_queue_head_init(&n->arp_queue);

rwlock_init(&n->lock);

seqlock_init(&n->ha_lock);

n->updated = n->used = now;

n->nud_state = NUD_NONE;

n->output = neigh_blackhole;

n->flags = flags;

seqlock_init(&n->hh.hh_lock);

n->parms = neigh_parms_clone(&tbl->parms);

timer_setup(&n->timer, neigh_timer_handler, 0);

NEIGH_CACHE_STAT_INC(tbl, allocs);

n->tbl = tbl;

refcount_set(&n->refcnt, 1);

n->dead = 1;

INIT_LIST_HEAD(&n->gc_list);

INIT_LIST_HEAD(&n->managed_list);

atomic_inc(&tbl->entries);

out:

return n;

out_entries:

if (!exempt_from_gc)

atomic_dec(&tbl->gc_entries);

goto out;

}

static void neigh_get_hash_rnd(u32 *x)

{

*x = get_random_u32() | 1;

}

static struct neigh_hash_table *neigh_hash_alloc(unsigned int shift)

{

size_t size = (1 << shift) * sizeof(struct hlist_head);

struct hlist_head *hash_heads;

struct neigh_hash_table *ret;

int i;

ret = kmalloc_obj(*ret, GFP_ATOMIC);

if (!ret)

return NULL;

hash_heads = kzalloc(size, GFP_ATOMIC);

if (!hash_heads) {

kfree(ret);

return NULL;

}

ret->hash_heads = hash_heads;

ret->hash_shift = shift;

for (i = 0; i < NEIGH_NUM_HASH_RND; i++)

neigh_get_hash_rnd(&ret->hash_rnd[i]);

return ret;

}

static void neigh_hash_free_rcu(struct rcu_head *head)

{

struct neigh_hash_table *nht = container_of(head,

struct neigh_hash_table,

rcu);

kfree(nht->hash_heads);

kfree(nht);

}

static struct neigh_hash_table *neigh_hash_grow(struct neigh_table *tbl,

unsigned long new_shift)

{

unsigned int i, hash;

struct neigh_hash_table *new_nht, *old_nht;

NEIGH_CACHE_STAT_INC(tbl, hash_grows);

old_nht = rcu_dereference_protected(tbl->nht,

lockdep_is_held(&tbl->lock));

new_nht = neigh_hash_alloc(new_shift);

if (!new_nht)

return old_nht;

for (i = 0; i < (1 << old_nht->hash_shift); i++) {

struct hlist_node *tmp;

struct neighbour *n;

neigh_for_each_in_bucket_safe(n, tmp, &old_nht->hash_heads[i]) {

hash = tbl->hash(n->primary_key, n->dev,

new_nht->hash_rnd);

hash >>= (32 - new_nht->hash_shift);

hlist_del_rcu(&n->hash);

hlist_add_head_rcu(&n->hash, &new_nht->hash_heads[hash]);

}

}

rcu_assign_pointer(tbl->nht, new_nht);

call_rcu(&old_nht->rcu, neigh_hash_free_rcu);

return new_nht;

}

struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey,

struct net_device *dev)

{

struct neighbour *n;

NEIGH_CACHE_STAT_INC(tbl, lookups);

rcu_read_lock();

n = __neigh_lookup_noref(tbl, pkey, dev);

if (n) {

if (!refcount_inc_not_zero(&n->refcnt))

n = NULL;

NEIGH_CACHE_STAT_INC(tbl, hits);

}

rcu_read_unlock();

return n;

}

EXPORT_SYMBOL(neigh_lookup);

static struct neighbour *

___neigh_create(struct neigh_table *tbl, const void *pkey,

struct net_device *dev, u32 flags,

bool exempt_from_gc, bool want_ref)

{

u32 hash_val, key_len = tbl->key_len;

struct neighbour *n1, *rc, *n;

struct neigh_hash_table *nht;

int error;

n = neigh_alloc(tbl, dev, flags, exempt_from_gc);

trace_neigh_create(tbl, dev, pkey, n, exempt_from_gc);

if (!n) {

rc = ERR_PTR(-ENOBUFS);

goto out;

}

memcpy(n->primary_key, pkey, key_len);

n->dev = dev;

netdev_hold(dev, &n->dev_tracker, GFP_ATOMIC);

/* Protocol specific setup. */

if (tbl->constructor && (error = tbl->constructor(n)) < 0) {

rc = ERR_PTR(error);

goto out_neigh_release;

}

if (dev->netdev_ops->ndo_neigh_construct) {

error = dev->netdev_ops->ndo_neigh_construct(dev, n);

if (error < 0) {

rc = ERR_PTR(error);

goto out_neigh_release;

}

}

/* Device specific setup. */

if (n->parms->neigh_setup &&

(error = n->parms->neigh_setup(n)) < 0) {

rc = ERR_PTR(error);

goto out_neigh_release;

}

n->confirmed = jiffies - (NEIGH_VAR(n->parms, BASE_REACHABLE_TIME) << 1);

spin_lock_bh(&tbl->lock);

nht = rcu_dereference_protected(tbl->nht,

lockdep_is_held(&tbl->lock));

if (atomic_read(&tbl->entries) > (1 << nht->hash_shift))

nht = neigh_hash_grow(tbl, nht->hash_shift + 1);

hash_val = tbl->hash(n->primary_key, dev, nht->hash_rnd) >> (32 - nht->hash_shift);

if (n->parms->dead) {

rc = ERR_PTR(-EINVAL);

goto out_tbl_unlock;

}

neigh_for_each_in_bucket(n1, &nht->hash_heads[hash_val]) {

if (dev == n1->dev && !memcmp(n1->primary_key, n->primary_key, key_len)) {

if (want_ref)

neigh_hold(n1);

rc = n1;

goto out_tbl_unlock;

}

}

n->dead = 0;

if (!exempt_from_gc)

list_add_tail(&n->gc_list, &n->tbl->gc_list);

if (n->flags & NTF_MANAGED)

list_add_tail(&n->managed_list, &n->tbl->managed_list);

if (want_ref)

neigh_hold(n);

hlist_add_head_rcu(&n->hash, &nht->hash_heads[hash_val]);

hlist_add_head_rcu(&n->dev_list,

neigh_get_dev_table(dev, tbl->family));

spin_unlock_bh(&tbl->lock);

neigh_dbg(2, "neigh %p is created\n", n);

rc = n;

out:

return rc;

out_tbl_unlock:

spin_unlock_bh(&tbl->lock);

out_neigh_release:

if (!exempt_from_gc)

atomic_dec(&tbl->gc_entries);

neigh_release(n);

goto out;

}

struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,

struct net_device *dev, bool want_ref)

{

bool exempt_from_gc = !!(dev->flags & IFF_LOOPBACK);

return ___neigh_create(tbl, pkey, dev, 0, exempt_from_gc, want_ref);

}

EXPORT_SYMBOL(__neigh_create);

static u32 pneigh_hash(const void *pkey, unsigned int key_len)

{

u32 hash_val = *(u32 *)(pkey + key_len - 4);

hash_val ^= (hash_val >> 16);

hash_val ^= hash_val >> 8;

hash_val ^= hash_val >> 4;

hash_val &= PNEIGH_HASHMASK;

return hash_val;

}

struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl,

struct net *net, const void *pkey,

struct net_device *dev)

{

struct pneigh_entry *n;

unsigned int key_len;

u32 hash_val;

key_len = tbl->key_len;

hash_val = pneigh_hash(pkey, key_len);

n = rcu_dereference_check(tbl->phash_buckets[hash_val],

lockdep_is_held(&tbl->phash_lock));

while (n) {

if (!memcmp(n->key, pkey, key_len) &&

net_eq(pneigh_net(n), net) &&

(n->dev == dev || !n->dev))

return n;

n = rcu_dereference_check(n->next, lockdep_is_held(&tbl->phash_lock));

}

return NULL;

}

int pneigh_create(struct neigh_table *tbl, struct net *net,

const void *pkey, struct net_device *dev,

u32 flags, u8 protocol, bool permanent)

{

struct pneigh_entry *n;

unsigned int key_len;

u32 hash_val;

int err = 0;

mutex_lock(&tbl->phash_lock);

n = pneigh_lookup(tbl, net, pkey, dev);

if (n)

goto update;

key_len = tbl->key_len;

n = kzalloc(sizeof(*n) + key_len, GFP_KERNEL);

if (!n) {

err = -ENOBUFS;

goto out;

}

write_pnet(&n->net, net);

memcpy(n->key, pkey, key_len);

n->dev = dev;

netdev_hold(dev, &n->dev_tracker, GFP_KERNEL);

if (tbl->pconstructor && tbl->pconstructor(n)) {

netdev_put(dev, &n->dev_tracker);

kfree(n);

err = -ENOBUFS;

goto out;

}

hash_val = pneigh_hash(pkey, key_len);

n->next = tbl->phash_buckets[hash_val];

rcu_assign_pointer(tbl->phash_buckets[hash_val], n);

update:

WRITE_ONCE(n->flags, flags);

n->permanent = permanent;

if (protocol)

WRITE_ONCE(n->protocol, protocol);

out:

mutex_unlock(&tbl->phash_lock);

return err;

}

static void pneigh_destroy(struct rcu_head *rcu)

{

struct pneigh_entry *n = container_of(rcu, struct pneigh_entry, rcu);

netdev_put(n->dev, &n->dev_tracker);

kfree(n);

}

int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *pkey,

struct net_device *dev)

{

struct pneigh_entry *n, __rcu **np;

unsigned int key_len;

u32 hash_val;

key_len = tbl->key_len;

hash_val = pneigh_hash(pkey, key_len);

mutex_lock(&tbl->phash_lock);

for (np = &tbl->phash_buckets[hash_val];

(n = rcu_dereference_protected(*np, 1)) != NULL;

np = &n->next) {

if (!memcmp(n->key, pkey, key_len) && n->dev == dev &&

net_eq(pneigh_net(n), net)) {

rcu_assign_pointer(*np, n->next);

mutex_unlock(&tbl->phash_lock);

if (tbl->pdestructor)

tbl->pdestructor(n);

call_rcu(&n->rcu, pneigh_destroy);

return 0;

}

}

mutex_unlock(&tbl->phash_lock);

return -ENOENT;

}

static void pneigh_ifdown(struct neigh_table *tbl, struct net_device *dev,

bool skip_perm)

{

struct pneigh_entry *n, __rcu **np;

LIST_HEAD(head);

u32 h;

mutex_lock(&tbl->phash_lock);

for (h = 0; h <= PNEIGH_HASHMASK; h++) {

np = &tbl->phash_buckets[h];

while ((n = rcu_dereference_protected(*np, 1)) != NULL) {

if (skip_perm && n->permanent)

goto skip;

if (!dev || n->dev == dev) {

rcu_assign_pointer(*np, n->next);

list_add(&n->free_node, &head);

continue;

}

skip:

np = &n->next;

}

}

mutex_unlock(&tbl->phash_lock);

while (!list_empty(&head)) {

n = list_first_entry(&head, typeof(*n), free_node);

list_del(&n->free_node);

if (tbl->pdestructor)

tbl->pdestructor(n);

call_rcu(&n->rcu, pneigh_destroy);

}

}

static inline void neigh_parms_put(struct neigh_parms *parms)

{

if (refcount_dec_and_test(&parms->refcnt))

kfree(parms);

}

/*

* neighbour must already be out of the table;

*

*/

void neigh_destroy(struct neighbour *neigh)

{

struct net_device *dev = neigh->dev;

NEIGH_CACHE_STAT_INC(neigh->tbl, destroys);

if (!neigh->dead) {

pr_warn("Destroying alive neighbour %p\n", neigh);

dump_stack();

return;

}

if (neigh_del_timer(neigh))

pr_warn("Impossible event\n");

write_lock_bh(&neigh->lock);

__skb_queue_purge(&neigh->arp_queue);

write_unlock_bh(&neigh->lock);

neigh->arp_queue_len_bytes = 0;

if (dev->netdev_ops->ndo_neigh_destroy)

dev->netdev_ops->ndo_neigh_destroy(dev, neigh);

netdev_put(dev, &neigh->dev_tracker);

neigh_parms_put(neigh->parms);

neigh_dbg(2, "neigh %p is destroyed\n", neigh);

atomic_dec(&neigh->tbl->entries);

kfree_rcu(neigh, rcu);

}

EXPORT_SYMBOL(neigh_destroy);

/* Neighbour state is suspicious;

disable fast path.

Called with write_locked neigh.

*/

static void neigh_suspect(struct neighbour *neigh)

{

neigh_dbg(2, "neigh %p is suspected\n", neigh);

WRITE_ONCE(neigh->output, neigh->ops->output);

}

/* Neighbour state is OK;

enable fast path.

Called with write_locked neigh.

*/

static void neigh_connect(struct neighbour *neigh)

{

neigh_dbg(2, "neigh %p is connected\n", neigh);

WRITE_ONCE(neigh->output, neigh->ops->connected_output);

}

static void neigh_periodic_work(struct work_struct *work)

{

struct neigh_table *tbl = container_of(work, struct neigh_table, gc_work.work);

struct neigh_hash_table *nht;

struct hlist_node *tmp;

struct neighbour *n;

unsigned int i;

NEIGH_CACHE_STAT_INC(tbl, periodic_gc_runs);

spin_lock_bh(&tbl->lock);

nht = rcu_dereference_protected(tbl->nht,

lockdep_is_held(&tbl->lock));

/*

* periodically recompute ReachableTime from random function

*/

if (time_after(jiffies, tbl->last_rand + 300 * HZ)) {

struct neigh_parms *p;

WRITE_ONCE(tbl->last_rand, jiffies);

list_for_each_entry(p, &tbl->parms_list, list)

neigh_set_reach_time(p);

}

if (atomic_read(&tbl->entries) < READ_ONCE(tbl->gc_thresh1))