1/* SPDX-License-Identifier: GPL-2.0 */
2/* AF_XDP internal functions
3 * Copyright(c) 2018 Intel Corporation.
4 */
5
6#ifndef _LINUX_XDP_SOCK_H
7#define _LINUX_XDP_SOCK_H
8
9#include <linux/bpf.h>
10#include <linux/workqueue.h>
11#include <linux/if_xdp.h>
12#include <linux/mutex.h>
13#include <linux/spinlock.h>
14#include <linux/mm.h>
15#include <net/sock.h>
16
17#define XDP_UMEM_SG_FLAG (1 << 1)
18
19struct net_device;
20struct xsk_queue;
21struct xdp_buff;
22
23struct xdp_umem {
24 void *addrs;
25 u64 size;
26 u32 headroom;
27 u32 chunk_size;
28 u32 chunks;
29 u32 npgs;
30 struct user_struct *user;
31 refcount_t users;
32 u8 flags;
33 u8 tx_metadata_len;
34 bool zc;
35 struct page **pgs;
36 int id;
37 struct list_head xsk_dma_list;
38 struct work_struct work;
39};
40
41struct xsk_map {
42 struct bpf_map map;
43 spinlock_t lock; /* Synchronize map updates */
44 atomic_t count;
45 struct xdp_sock __rcu *xsk_map[];
46};
47
48struct xdp_sock {
49 /* struct sock must be the first member of struct xdp_sock */
50 struct sock sk;
51 struct xsk_queue *rx ____cacheline_aligned_in_smp;
52 struct net_device *dev;
53 struct xdp_umem *umem;
54 struct list_head flush_node;
55 struct xsk_buff_pool *pool;
56 u16 queue_id;
57 bool zc;
58 bool sg;
59 enum {
60 XSK_READY = 0,
61 XSK_BOUND,
62 XSK_UNBOUND,
63 } state;
64
65 struct xsk_queue *tx ____cacheline_aligned_in_smp;
66 struct list_head tx_list;
67 /* record the number of tx descriptors sent by this xsk and
68 * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs
69 * to be given to other xsks for sending tx descriptors, thereby
70 * preventing other XSKs from being starved.
71 */
72 u32 tx_budget_spent;
73
74 /* Statistics */
75 u64 rx_dropped;
76 u64 rx_queue_full;
77
78 /* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
79 * packet, the partially built skb is saved here so that packet building can resume in next
80 * call of __xsk_generic_xmit().
81 */
82 struct sk_buff *skb;
83
84 struct list_head map_list;
85 /* Protects map_list */
86 spinlock_t map_list_lock;
87 u32 max_tx_budget;
88 /* Protects multiple processes in the control path */
89 struct mutex mutex;
90 struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
91 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
92};
93
94/*
95 * AF_XDP TX metadata hooks for network devices.
96 * The following hooks can be defined; unless noted otherwise, they are
97 * optional and can be filled with a null pointer.
98 *
99 * void (*tmo_request_timestamp)(void *priv)
100 * Called when AF_XDP frame requested egress timestamp.
101 *
102 * u64 (*tmo_fill_timestamp)(void *priv)
103 * Called when AF_XDP frame, that had requested egress timestamp,
104 * received a completion. The hook needs to return the actual HW timestamp.
105 *
106 * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
107 * Called when AF_XDP frame requested HW checksum offload. csum_start
108 * indicates position where checksumming should start.
109 * csum_offset indicates position where checksum should be stored.
110 *
111 * void (*tmo_request_launch_time)(u64 launch_time, void *priv)
112 * Called when AF_XDP frame requested launch time HW offload support.
113 * launch_time indicates the PTP time at which the device can schedule the
114 * packet for transmission.
115 */
116struct xsk_tx_metadata_ops {
117 void (*tmo_request_timestamp)(void *priv);
118 u64 (*tmo_fill_timestamp)(void *priv);
119 void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
120 void (*tmo_request_launch_time)(u64 launch_time, void *priv);
121};
122
123#ifdef CONFIG_XDP_SOCKETS
124
125int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
126int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
127void __xsk_map_flush(struct list_head *flush_list);
128INDIRECT_CALLABLE_DECLARE(void xsk_destruct_skb(struct sk_buff *));
129
130/**
131 * xsk_tx_metadata_to_compl - Save enough relevant metadata information
132 * to perform tx completion in the future.
133 * @meta: pointer to AF_XDP metadata area
134 * @compl: pointer to output struct xsk_tx_metadata_to_compl
135 *
136 * This function should be called by the networking device when
137 * it prepares AF_XDP egress packet. The value of @compl should be stored
138 * and passed to xsk_tx_metadata_complete upon TX completion.
139 */
140static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
141 struct xsk_tx_metadata_compl *compl)
142{
143 if (!meta)
144 return;
145
146 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
147 compl->tx_timestamp = &meta->completion.tx_timestamp;
148 else
149 compl->tx_timestamp = NULL;
150}
151
152/**
153 * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
154 * and call appropriate xsk_tx_metadata_ops operation.
155 * @meta: pointer to AF_XDP metadata area
156 * @ops: pointer to struct xsk_tx_metadata_ops
157 * @priv: pointer to driver-private aread
158 *
159 * This function should be called by the networking device when
160 * it prepares AF_XDP egress packet.
161 */
162static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
163 const struct xsk_tx_metadata_ops *ops,
164 void *priv)
165{
166 if (!meta)
167 return;
168
169 if (ops->tmo_request_launch_time)
170 if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
171 ops->tmo_request_launch_time(meta->request.launch_time,
172 priv);
173
174 if (ops->tmo_request_timestamp)
175 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
176 ops->tmo_request_timestamp(priv);
177
178 if (ops->tmo_request_checksum)
179 if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
180 ops->tmo_request_checksum(meta->request.csum_start,
181 meta->request.csum_offset, priv);
182}
183
184/**
185 * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
186 * and call appropriate xsk_tx_metadata_ops operation.
187 * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
188 * @ops: pointer to struct xsk_tx_metadata_ops
189 * @priv: pointer to driver-private aread
190 *
191 * This function should be called by the networking device upon
192 * AF_XDP egress completion.
193 */
194static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
195 const struct xsk_tx_metadata_ops *ops,
196 void *priv)
197{
198 if (!compl)
199 return;
200 if (!compl->tx_timestamp)
201 return;
202
203 *compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
204}
205
206#else
207
208static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
209{
210 return -ENOTSUPP;
211}
212
213static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
214{
215 return -EOPNOTSUPP;
216}
217
218static inline void __xsk_map_flush(struct list_head *flush_list)
219{
220}
221
222#ifdef CONFIG_MITIGATION_RETPOLINE
223static inline void xsk_destruct_skb(struct sk_buff *skb)
224{
225}
226#endif
227
228static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
229 struct xsk_tx_metadata_compl *compl)
230{
231}
232
233static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
234 const struct xsk_tx_metadata_ops *ops,
235 void *priv)
236{
237}
238
239static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
240 const struct xsk_tx_metadata_ops *ops,
241 void *priv)
242{
243}
244
245#endif /* CONFIG_XDP_SOCKETS */
246#endif /* _LINUX_XDP_SOCK_H */
247

source code of linux/include/net/xdp_sock.h