Skip to content

Commit b5cdae3

Browse files
davem330davem330
committed
net: Generic XDP
This provides a generic SKB based non-optimized XDP path which is used if either the driver lacks a specific XDP implementation, or the user requests it via a new IFLA_XDP_FLAGS value named XDP_FLAGS_SKB_MODE. It is arguable that perhaps I should have required something like this as part of the initial XDP feature merge. I believe this is critical for two reasons: 1) Accessibility. More people can play with XDP with less dependencies. Yes I know we have XDP support in virtio_net, but that just creates another depedency for learning how to use this facility. I wrote this to make life easier for the XDP newbies. 2) As a model for what the expected semantics are. If there is a pure generic core implementation, it serves as a semantic example for driver folks adding XDP support. One thing I have not tried to address here is the issue of XDP_PACKET_HEADROOM, thanks to Daniel for spotting that. It seems incredibly expensive to do a skb_cow(skb, XDP_PACKET_HEADROOM) or whatever even if the XDP program doesn't try to push headers at all. I think we really need the verifier to somehow propagate whether certain XDP helpers are used or not. v5: - Handle both negative and positive offset after running prog - Fix mac length in XDP_TX case (Alexei) - Use rcu_dereference_protected() in free_netdev (kbuild test robot) v4: - Fix MAC header adjustmnet before calling prog (David Ahern) - Disable LRO when generic XDP is installed (Michael Chan) - Bypass qdisc et al. on XDP_TX and record the event (Alexei) - Do not perform generic XDP on reinjected packets (DaveM) v3: - Make sure XDP program sees packet at MAC header, push back MAC header if we do XDP_TX. (Alexei) - Elide GRO when generic XDP is in use. (Alexei) - Add XDP_FLAG_SKB_MODE flag which the user can use to request generic XDP even if the driver has an XDP implementation. (Alexei) - Report whether SKB mode is in use in rtnl_xdp_fill() via XDP_FLAGS attribute. (Daniel) v2: - Add some "fall through" comments in switch statements based upon feedback from Andrew Lunn - Use RCU for generic xdp_prog, thanks to Johannes Berg. Tested-by: Andy Gospodarek <andy@greyhouse.net> Tested-by: Jesper Dangaard Brouer <brouer@redhat.com> Tested-by: David Ahern <dsa@cumulusnetworks.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 2f7878c commit b5cdae3

5 files changed

Lines changed: 187 additions & 22 deletions

File tree

‎include/linux/netdevice.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1905,9 +1905,17 @@ struct net_device {
19051905
struct lock_class_key *qdisc_tx_busylock;
19061906
struct lock_class_key *qdisc_running_key;
19071907
bool proto_down;
1908+
struct bpf_prog __rcu *xdp_prog;
19081909
};
19091910
#define to_net_dev(d) container_of(d, struct net_device, dev)
19101911

1912+
static inline bool netif_elide_gro(const struct net_device *dev)
1913+
{
1914+
if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
1915+
return true;
1916+
return false;
1917+
}
1918+
19111919
#define NETDEV_ALIGN 32
19121920

19131921
static inline

‎include/uapi/linux/if_link.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -887,7 +887,9 @@ enum {
887887
/* XDP section */
888888

889889
#define XDP_FLAGS_UPDATE_IF_NOEXIST (1U << 0)
890-
#define XDP_FLAGS_MASK (XDP_FLAGS_UPDATE_IF_NOEXIST)
890+
#define XDP_FLAGS_SKB_MODE (2U << 0)
891+
#define XDP_FLAGS_MASK (XDP_FLAGS_UPDATE_IF_NOEXIST | \
892+
XDP_FLAGS_SKB_MODE)
891893

892894
enum {
893895
IFLA_XDP_UNSPEC,

‎net/core/dev.c‎

Lines changed: 150 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@
9595
#include <linux/notifier.h>
9696
#include <linux/skbuff.h>
9797
#include <linux/bpf.h>
98+
#include <linux/bpf_trace.h>
9899
#include <net/net_namespace.h>
99100
#include <net/sock.h>
100101
#include <net/busy_poll.h>
@@ -4251,6 +4252,125 @@ static int __netif_receive_skb(struct sk_buff *skb)
42514252
return ret;
42524253
}
42534254

4255+
static struct static_key generic_xdp_needed __read_mostly;
4256+
4257+
static int generic_xdp_install(struct net_device *dev, struct netdev_xdp *xdp)
4258+
{
4259+
struct bpf_prog *new = xdp->prog;
4260+
int ret = 0;
4261+
4262+
switch (xdp->command) {
4263+
case XDP_SETUP_PROG: {
4264+
struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
4265+
4266+
rcu_assign_pointer(dev->xdp_prog, new);
4267+
if (old)
4268+
bpf_prog_put(old);
4269+
4270+
if (old && !new) {
4271+
static_key_slow_dec(&generic_xdp_needed);
4272+
} else if (new && !old) {
4273+
static_key_slow_inc(&generic_xdp_needed);
4274+
dev_disable_lro(dev);
4275+
}
4276+
break;
4277+
}
4278+
4279+
case XDP_QUERY_PROG:
4280+
xdp->prog_attached = !!rcu_access_pointer(dev->xdp_prog);
4281+
break;
4282+
4283+
default:
4284+
ret = -EINVAL;
4285+
break;
4286+
}
4287+
4288+
return ret;
4289+
}
4290+
4291+
static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4292+
struct bpf_prog *xdp_prog)
4293+
{
4294+
struct xdp_buff xdp;
4295+
u32 act = XDP_DROP;
4296+
void *orig_data;
4297+
int hlen, off;
4298+
u32 mac_len;
4299+
4300+
/* Reinjected packets coming from act_mirred or similar should
4301+
* not get XDP generic processing.
4302+
*/
4303+
if (skb_cloned(skb))
4304+
return XDP_PASS;
4305+
4306+
if (skb_linearize(skb))
4307+
goto do_drop;
4308+
4309+
/* The XDP program wants to see the packet starting at the MAC
4310+
* header.
4311+
*/
4312+
mac_len = skb->data - skb_mac_header(skb);
4313+
hlen = skb_headlen(skb) + mac_len;
4314+
xdp.data = skb->data - mac_len;
4315+
xdp.data_end = xdp.data + hlen;
4316+
xdp.data_hard_start = skb->data - skb_headroom(skb);
4317+
orig_data = xdp.data;
4318+
4319+
act = bpf_prog_run_xdp(xdp_prog, &xdp);
4320+
4321+
off = xdp.data - orig_data;
4322+
if (off > 0)
4323+
__skb_pull(skb, off);
4324+
else if (off < 0)
4325+
__skb_push(skb, -off);
4326+
4327+
switch (act) {
4328+
case XDP_TX:
4329+
__skb_push(skb, mac_len);
4330+
/* fall through */
4331+
case XDP_PASS:
4332+
break;
4333+
4334+
default:
4335+
bpf_warn_invalid_xdp_action(act);
4336+
/* fall through */
4337+
case XDP_ABORTED:
4338+
trace_xdp_exception(skb->dev, xdp_prog, act);
4339+
/* fall through */
4340+
case XDP_DROP:
4341+
do_drop:
4342+
kfree_skb(skb);
4343+
break;
4344+
}
4345+
4346+
return act;
4347+
}
4348+
4349+
/* When doing generic XDP we have to bypass the qdisc layer and the
4350+
* network taps in order to match in-driver-XDP behavior.
4351+
*/
4352+
static void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4353+
{
4354+
struct net_device *dev = skb->dev;
4355+
struct netdev_queue *txq;
4356+
bool free_skb = true;
4357+
int cpu, rc;
4358+
4359+
txq = netdev_pick_tx(dev, skb, NULL);
4360+
cpu = smp_processor_id();
4361+
HARD_TX_LOCK(dev, txq, cpu);
4362+
if (!netif_xmit_stopped(txq)) {
4363+
rc = netdev_start_xmit(skb, dev, txq, 0);
4364+
if (dev_xmit_complete(rc))
4365+
free_skb = false;
4366+
}
4367+
HARD_TX_UNLOCK(dev, txq);
4368+
if (free_skb) {
4369+
trace_xdp_exception(dev, xdp_prog, XDP_TX);
4370+
kfree_skb(skb);
4371+
}
4372+
}
4373+
42544374
static int netif_receive_skb_internal(struct sk_buff *skb)
42554375
{
42564376
int ret;
@@ -4262,6 +4382,21 @@ static int netif_receive_skb_internal(struct sk_buff *skb)
42624382

42634383
rcu_read_lock();
42644384

4385+
if (static_key_false(&generic_xdp_needed)) {
4386+
struct bpf_prog *xdp_prog = rcu_dereference(skb->dev->xdp_prog);
4387+
4388+
if (xdp_prog) {
4389+
u32 act = netif_receive_generic_xdp(skb, xdp_prog);
4390+
4391+
if (act != XDP_PASS) {
4392+
rcu_read_unlock();
4393+
if (act == XDP_TX)
4394+
generic_xdp_tx(skb, xdp_prog);
4395+
return NET_RX_DROP;
4396+
}
4397+
}
4398+
}
4399+
42654400
#ifdef CONFIG_RPS
42664401
if (static_key_false(&rps_needed)) {
42674402
struct rps_dev_flow voidflow, *rflow = &voidflow;
@@ -4494,7 +4629,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
44944629
enum gro_result ret;
44954630
int grow;
44964631

4497-
if (!(skb->dev->features & NETIF_F_GRO))
4632+
if (netif_elide_gro(skb->dev))
44984633
goto normal;
44994634

45004635
if (skb->csum_bad)
@@ -6723,21 +6858,24 @@ EXPORT_SYMBOL(dev_change_proto_down);
67236858
*/
67246859
int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags)
67256860
{
6861+
int (*xdp_op)(struct net_device *dev, struct netdev_xdp *xdp);
67266862
const struct net_device_ops *ops = dev->netdev_ops;
67276863
struct bpf_prog *prog = NULL;
67286864
struct netdev_xdp xdp;
67296865
int err;
67306866

67316867
ASSERT_RTNL();
67326868

6733-
if (!ops->ndo_xdp)
6734-
return -EOPNOTSUPP;
6869+
xdp_op = ops->ndo_xdp;
6870+
if (!xdp_op || (flags & XDP_FLAGS_SKB_MODE))
6871+
xdp_op = generic_xdp_install;
6872+
67356873
if (fd >= 0) {
67366874
if (flags & XDP_FLAGS_UPDATE_IF_NOEXIST) {
67376875
memset(&xdp, 0, sizeof(xdp));
67386876
xdp.command = XDP_QUERY_PROG;
67396877

6740-
err = ops->ndo_xdp(dev, &xdp);
6878+
err = xdp_op(dev, &xdp);
67416879
if (err < 0)
67426880
return err;
67436881
if (xdp.prog_attached)
@@ -6753,7 +6891,7 @@ int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags)
67536891
xdp.command = XDP_SETUP_PROG;
67546892
xdp.prog = prog;
67556893

6756-
err = ops->ndo_xdp(dev, &xdp);
6894+
err = xdp_op(dev, &xdp);
67576895
if (err < 0 && prog)
67586896
bpf_prog_put(prog);
67596897

@@ -7793,6 +7931,7 @@ EXPORT_SYMBOL(alloc_netdev_mqs);
77937931
void free_netdev(struct net_device *dev)
77947932
{
77957933
struct napi_struct *p, *n;
7934+
struct bpf_prog *prog;
77967935

77977936
might_sleep();
77987937
netif_free_tx_queues(dev);
@@ -7811,6 +7950,12 @@ void free_netdev(struct net_device *dev)
78117950
free_percpu(dev->pcpu_refcnt);
78127951
dev->pcpu_refcnt = NULL;
78137952

7953+
prog = rcu_dereference_protected(dev->xdp_prog, 1);
7954+
if (prog) {
7955+
bpf_prog_put(prog);
7956+
static_key_slow_dec(&generic_xdp_needed);
7957+
}
7958+
78147959
/* Compatibility with error handling in drivers */
78157960
if (dev->reg_state == NETREG_UNINITIALIZED) {
78167961
netdev_freemem(dev);

‎net/core/gro_cells.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb)
1313
struct net_device *dev = skb->dev;
1414
struct gro_cell *cell;
1515

16-
if (!gcells->cells || skb_cloned(skb) || !(dev->features & NETIF_F_GRO))
16+
if (!gcells->cells || skb_cloned(skb) || netif_elide_gro(dev))
1717
return netif_rx(skb);
1818

1919
cell = this_cpu_ptr(gcells->cells);

‎net/core/rtnetlink.c‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -896,15 +896,13 @@ static size_t rtnl_port_size(const struct net_device *dev,
896896
return port_self_size;
897897
}
898898

899-
static size_t rtnl_xdp_size(const struct net_device *dev)
899+
static size_t rtnl_xdp_size(void)
900900
{
901901
size_t xdp_size = nla_total_size(0) + /* nest IFLA_XDP */
902-
nla_total_size(1); /* XDP_ATTACHED */
902+
nla_total_size(1) + /* XDP_ATTACHED */
903+
nla_total_size(4); /* XDP_FLAGS */
903904

904-
if (!dev->netdev_ops->ndo_xdp)
905-
return 0;
906-
else
907-
return xdp_size;
905+
return xdp_size;
908906
}
909907

910908
static noinline size_t if_nlmsg_size(const struct net_device *dev,
@@ -943,7 +941,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev,
943941
+ nla_total_size(MAX_PHYS_ITEM_ID_LEN) /* IFLA_PHYS_PORT_ID */
944942
+ nla_total_size(MAX_PHYS_ITEM_ID_LEN) /* IFLA_PHYS_SWITCH_ID */
945943
+ nla_total_size(IFNAMSIZ) /* IFLA_PHYS_PORT_NAME */
946-
+ rtnl_xdp_size(dev) /* IFLA_XDP */
944+
+ rtnl_xdp_size() /* IFLA_XDP */
947945
+ nla_total_size(1); /* IFLA_PROTO_DOWN */
948946

949947
}
@@ -1251,23 +1249,35 @@ static int rtnl_fill_link_ifmap(struct sk_buff *skb, struct net_device *dev)
12511249

12521250
static int rtnl_xdp_fill(struct sk_buff *skb, struct net_device *dev)
12531251
{
1254-
struct netdev_xdp xdp_op = {};
12551252
struct nlattr *xdp;
1253+
u32 xdp_flags = 0;
1254+
u8 val = 0;
12561255
int err;
12571256

1258-
if (!dev->netdev_ops->ndo_xdp)
1259-
return 0;
12601257
xdp = nla_nest_start(skb, IFLA_XDP);
12611258
if (!xdp)
12621259
return -EMSGSIZE;
1263-
xdp_op.command = XDP_QUERY_PROG;
1264-
err = dev->netdev_ops->ndo_xdp(dev, &xdp_op);
1265-
if (err)
1266-
goto err_cancel;
1267-
err = nla_put_u8(skb, IFLA_XDP_ATTACHED, xdp_op.prog_attached);
1260+
if (rcu_access_pointer(dev->xdp_prog)) {
1261+
xdp_flags = XDP_FLAGS_SKB_MODE;
1262+
val = 1;
1263+
} else if (dev->netdev_ops->ndo_xdp) {
1264+
struct netdev_xdp xdp_op = {};
1265+
1266+
xdp_op.command = XDP_QUERY_PROG;
1267+
err = dev->netdev_ops->ndo_xdp(dev, &xdp_op);
1268+
if (err)
1269+
goto err_cancel;
1270+
val = xdp_op.prog_attached;
1271+
}
1272+
err = nla_put_u8(skb, IFLA_XDP_ATTACHED, val);
12681273
if (err)
12691274
goto err_cancel;
12701275

1276+
if (xdp_flags) {
1277+
err = nla_put_u32(skb, IFLA_XDP_FLAGS, xdp_flags);
1278+
if (err)
1279+
goto err_cancel;
1280+
}
12711281
nla_nest_end(skb, xdp);
12721282
return 0;
12731283

0 commit comments

Comments
 (0)