// SPDX-License-Identifier: GPL-2.0

#include <linux/kernel.h>

#include <linux/errno.h>

#include <linux/file.h>

#include <linux/slab.h>

#include <linux/net.h>

#include <linux/compat.h>

#include <net/compat.h>

#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "filetable.h"

#include "io_uring.h"

#include "kbuf.h"

#include "alloc_cache.h"

#include "net.h"

#include "notif.h"

#include "rsrc.h"

#include "zcrx.h"

struct io_shutdown {

struct file *file;

int how;

};

struct io_accept {

struct file *file;

struct sockaddr __user *addr;

int __user *addr_len;

int flags;

int iou_flags;

u32 file_slot;

unsigned long nofile;

};

struct io_socket {

struct file *file;

int domain;

int type;

int protocol;

int flags;

u32 file_slot;

unsigned long nofile;

};

struct io_connect {

struct file *file;

struct sockaddr __user *addr;

int addr_len;

bool in_progress;

bool seen_econnaborted;

};

struct io_bind {

struct file *file;

int addr_len;

};

struct io_listen {

struct file *file;

int backlog;

};

struct io_sr_msg {

struct file *file;

union {

struct compat_msghdr __user *umsg_compat;

struct user_msghdr __user *umsg;

void __user *buf;

};

int len;

unsigned done_io;

unsigned msg_flags;

unsigned nr_multishot_loops;

u16 flags;

/* initialised and used only by !msg send variants */

u16 buf_group;

/* per-invocation mshot limit */

unsigned mshot_len;

/* overall mshot byte limit */

unsigned mshot_total_len;

void __user *msg_control;

/* used only for send zerocopy */

struct io_kiocb *notif;

};

/*

* The UAPI flags are the lower 8 bits, as that's all sqe->ioprio will hold

* anyway. Use the upper 8 bits for internal uses.

*/

enum sr_retry_flags {

IORING_RECV_RETRY = (1U << 15),

IORING_RECV_PARTIAL_MAP = (1U << 14),

IORING_RECV_MSHOT_CAP = (1U << 13),

IORING_RECV_MSHOT_LIM = (1U << 12),

IORING_RECV_MSHOT_DONE = (1U << 11),

IORING_RECV_RETRY_CLEAR = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP,

IORING_RECV_NO_RETRY = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP |

IORING_RECV_MSHOT_CAP | IORING_RECV_MSHOT_DONE,

};

/*

* Number of times we'll try and do receives if there's more data. If we

* exceed this limit, then add us to the back of the queue and retry from

* there. This helps fairness between flooding clients.

*/

#define MULTISHOT_MAX_RETRY 32

struct io_recvzc {

struct file *file;

u16 flags;

u32 len;

struct io_zcrx_ifq *ifq;

};

static int io_sg_from_iter_iovec(struct sk_buff *skb,

struct iov_iter *from, size_t length);

static int io_sg_from_iter(struct sk_buff *skb,

struct iov_iter *from, size_t length);

int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);

if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||

sqe->buf_index || sqe->splice_fd_in))

return -EINVAL;

shutdown->how = READ_ONCE(sqe->len);

req->flags |= REQ_F_FORCE_ASYNC;

return 0;

}

int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)

{

struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);

struct socket *sock;

int ret;

WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);

sock = sock_from_file(req->file);

if (unlikely(!sock))

return -ENOTSOCK;

ret = __sys_shutdown_sock(sock, shutdown->how);

io_req_set_res(req, ret, 0);

return IOU_COMPLETE;

}

static bool io_net_retry(struct socket *sock, int flags)

{

if (!(flags & MSG_WAITALL))

return false;

return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;

}

static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)

{

if (kmsg->vec.iovec)

io_vec_free(&kmsg->vec);

}

static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)

{

struct io_async_msghdr *hdr = req->async_data;

/* can't recycle, ensure we free the iovec if we have one */

if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {

io_netmsg_iovec_free(hdr);

return;

}

/* Let normal cleanup path reap it if we fail adding to the cache */

io_alloc_cache_vec_kasan(&hdr->vec);

if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP)

io_vec_free(&hdr->vec);

if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr))

io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);

}

static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)

{

struct io_ring_ctx *ctx = req->ctx;

struct io_async_msghdr *hdr;

hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req);

if (!hdr)

return NULL;

/* If the async data was cached, we might have an iov cached inside. */

if (hdr->vec.iovec)

req->flags |= REQ_F_NEED_CLEANUP;

return hdr;

}

static inline void io_mshot_prep_retry(struct io_kiocb *req,

struct io_async_msghdr *kmsg)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

req->flags &= ~REQ_F_BL_EMPTY;

sr->done_io = 0;

sr->flags &= ~IORING_RECV_RETRY_CLEAR;

sr->len = sr->mshot_len;

}

static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg,

const struct iovec __user *uiov, unsigned uvec_seg,

int ddir)

{

struct iovec *iov;

int ret, nr_segs;

if (iomsg->vec.iovec) {

nr_segs = iomsg->vec.nr;

iov = iomsg->vec.iovec;

} else {

nr_segs = 1;

iov = &iomsg->fast_iov;

}

ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov,

&iomsg->msg.msg_iter, io_is_compat(req->ctx));

if (unlikely(ret < 0))

return ret;

if (iov) {

req->flags |= REQ_F_NEED_CLEANUP;

io_vec_reset_iovec(&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs);

}

return 0;

}

static int io_compat_msg_copy_hdr(struct io_kiocb *req,

struct io_async_msghdr *iomsg,

struct compat_msghdr *msg, int ddir,

struct sockaddr __user **save_addr)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

struct compat_iovec __user *uiov;

int ret;

if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg)))

return -EFAULT;

ret = __get_compat_msghdr(&iomsg->msg, msg, save_addr);

if (ret)

return ret;

uiov = compat_ptr(msg->msg_iov);

if (req->flags & REQ_F_BUFFER_SELECT) {

if (msg->msg_iovlen == 0) {

sr->len = 0;

} else if (msg->msg_iovlen > 1) {

return -EINVAL;

} else {

struct compat_iovec tmp_iov;

if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov)))

return -EFAULT;

sr->len = tmp_iov.iov_len;

}

}

return 0;

}

static int io_copy_msghdr_from_user(struct user_msghdr *msg,

struct user_msghdr __user *umsg)

{

if (!user_access_begin(umsg, sizeof(*umsg)))

return -EFAULT;

unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end);

unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end);

unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end);

unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end);

unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end);

unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end);

user_access_end();

return 0;

ua_end:

user_access_end();

return -EFAULT;

}

static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,

struct user_msghdr *msg, int ddir,

struct sockaddr __user **save_addr)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

struct user_msghdr __user *umsg = sr->umsg;

int ret;

iomsg->msg.msg_name = &iomsg->addr;

iomsg->msg.msg_iter.nr_segs = 0;

if (io_is_compat(req->ctx)) {

struct compat_msghdr cmsg;

ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ddir, save_addr);

if (ret)

return ret;

memset(msg, 0, sizeof(*msg));

msg->msg_namelen = cmsg.msg_namelen;

msg->msg_controllen = cmsg.msg_controllen;

msg->msg_iov = compat_ptr(cmsg.msg_iov);

msg->msg_iovlen = cmsg.msg_iovlen;

return 0;

}

ret = io_copy_msghdr_from_user(msg, umsg);

if (unlikely(ret))

return ret;

msg->msg_flags = 0;

ret = __copy_msghdr(&iomsg->msg, msg, save_addr);

if (ret)

return ret;

if (req->flags & REQ_F_BUFFER_SELECT) {

if (msg->msg_iovlen == 0) {

sr->len = 0;

} else if (msg->msg_iovlen > 1) {

return -EINVAL;

} else {

struct iovec __user *uiov = msg->msg_iov;

struct iovec tmp_iov;

if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov)))

return -EFAULT;

sr->len = tmp_iov.iov_len;

}

}

return 0;

}

void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)

{

struct io_async_msghdr *io = req->async_data;

io_netmsg_iovec_free(io);

}

static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

struct io_async_msghdr *kmsg = req->async_data;

void __user *addr;

u16 addr_len;

int ret;

sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));

if (READ_ONCE(sqe->__pad3[0]))

return -EINVAL;

kmsg->msg.msg_name = NULL;

kmsg->msg.msg_namelen = 0;

kmsg->msg.msg_control = NULL;

kmsg->msg.msg_controllen = 0;

kmsg->msg.msg_ubuf = NULL;

addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));

addr_len = READ_ONCE(sqe->addr_len);

if (addr) {

ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr);

if (unlikely(ret < 0))

return ret;

kmsg->msg.msg_name = &kmsg->addr;

kmsg->msg.msg_namelen = addr_len;

}

if (sr->flags & IORING_RECVSEND_FIXED_BUF) {

if (!(sr->flags & IORING_SEND_VECTORIZED)) {

req->flags |= REQ_F_IMPORT_BUFFER;

return 0;

}

kmsg->msg.msg_iter.nr_segs = sr->len;

return io_prep_reg_iovec(req, &kmsg->vec, sr->buf, sr->len);

}

if (req->flags & REQ_F_BUFFER_SELECT)

return 0;

if (sr->flags & IORING_SEND_VECTORIZED)

return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE);

return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter);

}

static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

struct io_async_msghdr *kmsg = req->async_data;

struct user_msghdr msg;

int ret;

sr->flags |= IORING_SEND_VECTORIZED;

sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));

ret = io_msg_copy_hdr(req, kmsg, &msg, ITER_SOURCE, NULL);

if (unlikely(ret))

return ret;

/* save msg_control as sys_sendmsg() overwrites it */

sr->msg_control = kmsg->msg.msg_control_user;

if (sr->flags & IORING_RECVSEND_FIXED_BUF) {

kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen;

return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov,

msg.msg_iovlen);

}

if (req->flags & REQ_F_BUFFER_SELECT)

return 0;

return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE);

}

#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE | IORING_SEND_VECTORIZED)

int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

sr->done_io = 0;

sr->len = READ_ONCE(sqe->len);

if (unlikely(sr->len < 0))

return -EINVAL;

sr->flags = READ_ONCE(sqe->ioprio);

if (sr->flags & ~SENDMSG_FLAGS)

return -EINVAL;

sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;

if (sr->msg_flags & MSG_DONTWAIT)

req->flags |= REQ_F_NOWAIT;

if (req->flags & REQ_F_BUFFER_SELECT)

sr->buf_group = req->buf_index;

if (sr->flags & IORING_RECVSEND_BUNDLE) {

if (req->opcode == IORING_OP_SENDMSG)

return -EINVAL;

sr->msg_flags |= MSG_WAITALL;

req->flags |= REQ_F_MULTISHOT;

}

if (io_is_compat(req->ctx))

sr->msg_flags |= MSG_CMSG_COMPAT;

if (unlikely(!io_msg_alloc_async(req)))

return -ENOMEM;

if (req->opcode != IORING_OP_SENDMSG)

return io_send_setup(req, sqe);

if (unlikely(sqe->addr2 || sqe->file_index))

return -EINVAL;

return io_sendmsg_setup(req, sqe);

}

static void io_req_msg_cleanup(struct io_kiocb *req,

unsigned int issue_flags)

{

io_netmsg_recycle(req, issue_flags);

}

/*

* For bundle completions, we need to figure out how many segments we consumed.

* A bundle could be using a single ITER_UBUF if that's all we mapped, or it

* could be using an ITER_IOVEC. If the latter, then if we consumed all of

* the segments, then it's a trivial questiont o answer. If we have residual

* data in the iter, then loop the segments to figure out how much we

* transferred.

*/

static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)

{

struct iovec *iov;

int nbufs;

/* no data is always zero segments, and a ubuf is always 1 segment */

if (ret <= 0)

return 0;

if (iter_is_ubuf(&kmsg->msg.msg_iter))

return 1;

iov = kmsg->vec.iovec;

if (!iov)

iov = &kmsg->fast_iov;

/* if all data was transferred, it's basic pointer math */

if (!iov_iter_count(&kmsg->msg.msg_iter))

return iter_iov(&kmsg->msg.msg_iter) - iov;

/* short transfer, count segments */

nbufs = 0;

do {

int this_len = min_t(int, iov[nbufs].iov_len, ret);

nbufs++;

ret -= this_len;

} while (ret);

return nbufs;

}

static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl,

struct io_async_msghdr *kmsg, int len)

{

req->flags |= REQ_F_BL_NO_RECYCLE;

if (req->flags & REQ_F_BUFFERS_COMMIT)

io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len));

return IOU_RETRY;

}

static inline bool io_send_finish(struct io_kiocb *req,

struct io_async_msghdr *kmsg,

struct io_br_sel *sel)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

bool bundle_finished = sel->val <= 0;

unsigned int cflags;

if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {

cflags = io_put_kbuf(req, sel->val, sel->buf_list);

goto finish;

}

cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val));

/*

* Don't start new bundles if the buffer list is empty, or if the

* current operation needed to go through polling to complete.

*/

if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED))

goto finish;

/*

* Fill CQE for this receive and see if we should keep trying to

* receive from this socket.

*/

if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {

io_mshot_prep_retry(req, kmsg);

return false;

}

/* Otherwise stop bundle and use the current result. */

finish:

io_req_set_res(req, sel->val, cflags);

sel->val = IOU_COMPLETE;

return true;

}

int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

struct io_async_msghdr *kmsg = req->async_data;

struct socket *sock;

unsigned flags;

int min_ret = 0;

int ret;

sock = sock_from_file(req->file);

if (unlikely(!sock))

return -ENOTSOCK;

if (!(req->flags & REQ_F_POLLED) &&

(sr->flags & IORING_RECVSEND_POLL_FIRST))

return -EAGAIN;

flags = sr->msg_flags;

if (issue_flags & IO_URING_F_NONBLOCK)

flags |= MSG_DONTWAIT;

if (flags & MSG_WAITALL)

min_ret = iov_iter_count(&kmsg->msg.msg_iter);

kmsg->msg.msg_control_user = sr->msg_control;

ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);

if (ret < min_ret) {

if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))

return -EAGAIN;

if (ret > 0 && io_net_retry(sock, flags)) {

kmsg->msg.msg_controllen = 0;

kmsg->msg.msg_control = NULL;

sr->done_io += ret;

return -EAGAIN;

}

if (ret == -ERESTARTSYS)

ret = -EINTR;

req_set_fail(req);

}

io_req_msg_cleanup(req, issue_flags);

if (ret >= 0)

ret += sr->done_io;

else if (sr->done_io)

ret = sr->done_io;

io_req_set_res(req, ret, 0);

return IOU_COMPLETE;

}

static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,

struct io_br_sel *sel, struct io_async_msghdr *kmsg)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

struct buf_sel_arg arg = {

.iovs = &kmsg->fast_iov,

.max_len = min_not_zero(sr->len, INT_MAX),

.nr_iovs = 1,

.buf_group = sr->buf_group,

};

int ret;

if (kmsg->vec.iovec) {

arg.nr_iovs = kmsg->vec.nr;

arg.iovs = kmsg->vec.iovec;

arg.mode = KBUF_MODE_FREE;

}

if (!(sr->flags & IORING_RECVSEND_BUNDLE))

arg.nr_iovs = 1;

else

arg.mode |= KBUF_MODE_EXPAND;

ret = io_buffers_select(req, &arg, sel, issue_flags);

if (unlikely(ret < 0))

return ret;

if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {

kmsg->vec.nr = ret;

kmsg->vec.iovec = arg.iovs;

req->flags |= REQ_F_NEED_CLEANUP;

}

sr->len = arg.out_len;

if (ret == 1) {

sr->buf = arg.iovs[0].iov_base;

ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,

&kmsg->msg.msg_iter);

if (unlikely(ret))

return ret;

} else {

iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE,

arg.iovs, ret, arg.out_len);

}

return 0;

}

int io_send(struct io_kiocb *req, unsigned int issue_flags)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

struct io_async_msghdr *kmsg = req->async_data;

struct io_br_sel sel = { };

struct socket *sock;

unsigned flags;

int min_ret = 0;

int ret;

sock = sock_from_file(req->file);

if (unlikely(!sock))

return -ENOTSOCK;

if (!(req->flags & REQ_F_POLLED) &&

(sr->flags & IORING_RECVSEND_POLL_FIRST))

return -EAGAIN;

flags = sr->msg_flags;

if (issue_flags & IO_URING_F_NONBLOCK)

flags |= MSG_DONTWAIT;

retry_bundle:

sel.buf_list = NULL;

if (io_do_buffer_select(req)) {

ret = io_send_select_buffer(req, issue_flags, &sel, kmsg);

if (ret)

return ret;

}

/*

* If MSG_WAITALL is set, or this is a bundle send, then we need

* the full amount. If just bundle is set, if we do a short send

* then we complete the bundle sequence rather than continue on.

*/

if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE)

min_ret = iov_iter_count(&kmsg->msg.msg_iter);

flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;

kmsg->msg.msg_flags = flags;

ret = sock_sendmsg(sock, &kmsg->msg);

if (ret < min_ret) {

if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))

return -EAGAIN;

if (ret > 0 && io_net_retry(sock, flags)) {

sr->len -= ret;

sr->buf += ret;

sr->done_io += ret;

return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);

}

if (ret == -ERESTARTSYS)

ret = -EINTR;

req_set_fail(req);

}

if (ret >= 0)

ret += sr->done_io;

else if (sr->done_io)

ret = sr->done_io;

sel.val = ret;

if (!io_send_finish(req, kmsg, &sel))

goto retry_bundle;

io_req_msg_cleanup(req, issue_flags);

return sel.val;

}

static int io_recvmsg_mshot_prep(struct io_kiocb *req,

struct io_async_msghdr *iomsg,

int namelen, size_t controllen)

{

if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) ==

(REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) {

int hdr;

if (unlikely(namelen < 0))

return -EOVERFLOW;

if (check_add_overflow(sizeof(struct io_uring_recvmsg_out),

namelen, &hdr))

return -EOVERFLOW;

if (check_add_overflow(hdr, controllen, &hdr))

return -EOVERFLOW;

iomsg->namelen = namelen;

iomsg->controllen = controllen;

return 0;

}

return 0;

}

static int io_recvmsg_copy_hdr(struct io_kiocb *req,

struct io_async_msghdr *iomsg)

{

struct user_msghdr msg;

int ret;

ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_DEST, &iomsg->uaddr);

if (unlikely(ret))

return ret;

if (!(req->flags & REQ_F_BUFFER_SELECT)) {

ret = io_net_import_vec(req, iomsg, msg.msg_iov, msg.msg_iovlen,

ITER_DEST);

if (unlikely(ret))

return ret;

}

return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen,

msg.msg_controllen);

}

static int io_recvmsg_prep_setup(struct io_kiocb *req)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

struct io_async_msghdr *kmsg;

kmsg = io_msg_alloc_async(req);

if (unlikely(!kmsg))

return -ENOMEM;

if (req->opcode == IORING_OP_RECV) {

kmsg->msg.msg_name = NULL;

kmsg->msg.msg_namelen = 0;

kmsg->msg.msg_inq = 0;

kmsg->msg.msg_control = NULL;

kmsg->msg.msg_get_inq = 1;

kmsg->msg.msg_controllen = 0;

kmsg->msg.msg_iocb = NULL;

kmsg->msg.msg_ubuf = NULL;

if (req->flags & REQ_F_BUFFER_SELECT)

return 0;

return import_ubuf(ITER_DEST, sr->buf, sr->len,

&kmsg->msg.msg_iter);

}

return io_recvmsg_copy_hdr(req, kmsg);

}

#define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \

IORING_RECVSEND_BUNDLE)

int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

sr->done_io = 0;

if (unlikely(sqe->addr2))

return -EINVAL;

sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));

sr->len = READ_ONCE(sqe->len);

if (unlikely(sr->len < 0))

return -EINVAL;

sr->flags = READ_ONCE(sqe->ioprio);

if (sr->flags & ~RECVMSG_FLAGS)

return -EINVAL;

sr->msg_flags = READ_ONCE(sqe->msg_flags);

if (sr->msg_flags & MSG_DONTWAIT)

req->flags |= REQ_F_NOWAIT;

if (sr->msg_flags & MSG_ERRQUEUE)

req->flags |= REQ_F_CLEAR_POLLIN;

if (req->flags & REQ_F_BUFFER_SELECT)

sr->buf_group = req->buf_index;

sr->mshot_total_len = sr->mshot_len = 0;

if (sr->flags & IORING_RECV_MULTISHOT) {

if (!(req->flags & REQ_F_BUFFER_SELECT))

return -EINVAL;

if (sr->msg_flags & MSG_WAITALL)

return -EINVAL;

if (req->opcode == IORING_OP_RECV) {

sr->mshot_len = sr->len;

sr->mshot_total_len = READ_ONCE(sqe->optlen);

if (sr->mshot_total_len)

sr->flags |= IORING_RECV_MSHOT_LIM;

} else if (sqe->optlen) {

return -EINVAL;

}

req->flags |= REQ_F_APOLL_MULTISHOT;

} else if (sqe->optlen) {

return -EINVAL;

}

if (sr->flags & IORING_RECVSEND_BUNDLE) {

if (req->opcode == IORING_OP_RECVMSG)

return -EINVAL;

}

if (io_is_compat(req->ctx))

sr->msg_flags |= MSG_CMSG_COMPAT;

sr->nr_multishot_loops = 0;

return io_recvmsg_prep_setup(req);

}

/* bits to clear in old and inherit in new cflags on bundle retry */

#define CQE_F_MASK (IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE)

/*

* Finishes io_recv and io_recvmsg.

*

* Returns true if it is actually finished, or false if it should run

* again (for multishot).

*/

static inline bool io_recv_finish(struct io_kiocb *req,

struct io_async_msghdr *kmsg,

struct io_br_sel *sel, bool mshot_finished,

unsigned issue_flags)

{

struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

unsigned int cflags = 0;

if (kmsg->msg.msg_inq > 0)

cflags |= IORING_CQE_F_SOCK_NONEMPTY;

if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) {

/*

* If sr->len hits zero, the limit has been reached. Mark

* mshot as finished, and flag MSHOT_DONE as well to prevent

* a potential bundle from being retried.

*/

sr->mshot_total_len -= min_t(int, sel->val, sr->mshot_total_len);

if (!sr->mshot_total_len) {

sr->flags |= IORING_RECV_MSHOT_DONE;

mshot_finished = true;

}

}

if (sr->flags & IORING_RECVSEND_BUNDLE) {

size_t this_ret = sel->val - sr->done_io;

cflags |= io_put_kbufs(req, this_ret, sel->buf_list, io_bundle_nbufs(kmsg, this_ret));

if (sr->flags & IORING_RECV_RETRY)

cflags = req->cqe.flags | (cflags & CQE_F_MASK);

if (sr->mshot_len && sel->val >= sr->mshot_len)

sr->flags |= IORING_RECV_MSHOT_CAP;

/* bundle with no more immediate buffers, we're done */

if (req->flags & REQ_F_BL_EMPTY)

goto finish;

/*

* If more is available AND it was a full transfer, retry and

* append to this one

*/

if (!(sr->flags & IORING_RECV_NO_RETRY) &&

kmsg->msg.msg_inq > 1 && this_ret > 0 &&

!iov_iter_count(&kmsg->msg.msg_iter)) {

req->cqe.flags = cflags & ~CQE_F_MASK;

sr->len = kmsg->msg.msg_inq;

sr->done_io += this_ret;

sr->flags |= IORING_RECV_RETRY;

return false;

}

} else {

cflags |= io_put_kbuf(req, sel->val, sel->buf_list);

}

/*

* Fill CQE for this receive and see if we should keep trying to

* receive from this socket.

*/

if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished &&

io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {

sel->val = IOU_RETRY;

io_mshot_prep_retry(req, kmsg);

/* Known not-empty or unknown state, retry */

if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) {

if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY &&

!(sr->flags & IORING_RECV_MSHOT_CAP)) {

return false;

}

/* mshot retries exceeded, force a requeue */

sr->nr_multishot_loops = 0;

sr->flags &= ~IORING_RECV_MSHOT_CAP;

if (issue_flags & IO_URING_F_MULTISHOT)

sel->val = IOU_REQUEUE;

}

return true;

}

/* Finish the request / stop multishot. */

finish:

io_req_set_res(req, sel->val, cflags);

sel->val = IOU_COMPLETE;

io_req_msg_cleanup(req, issue_flags);

return true;

}

static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg,

struct io_sr_msg *sr, void __user **buf,

size_t *len)

{

unsigned long ubuf = (unsigned long) *buf;

unsigned long hdr;

hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +

kmsg->controllen;

if (*len < hdr)

return -EFAULT;

if (kmsg->controllen) {

unsigned long control = ubuf + hdr - kmsg->controllen;

kmsg->msg.msg_control_user = (void __user *) control;

kmsg->msg.msg_controllen = kmsg->controllen;

}

sr->buf = *buf; /* stash for later copy */

*buf = (void __user *) (ubuf + hdr);

kmsg->payloadlen = *len = *len - hdr;

return 0;

}

struct io_recvmsg_multishot_hdr {

struct io_uring_recvmsg_out msg;

struct sockaddr_storage addr;

};

static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io,

struct io_async_msghdr *kmsg,

unsigned int flags, bool *finished)

{

int err;

int copy_len;

struct io_recvmsg_multishot_hdr hdr;

if (kmsg->namelen)

kmsg->msg.msg_name = &hdr.addr;

kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT);

kmsg->msg.msg_namelen = 0;

if (sock->file->f_flags & O_NONBLOCK)

flags |= MSG_DONTWAIT;

err = sock_recvmsg(sock, &kmsg->msg, flags);

*finished = err <= 0;

if (err < 0)

return err;

hdr.msg = (struct io_uring_recvmsg_out) {

.controllen = kmsg->controllen - kmsg->msg.msg_controllen,

.flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT

};

hdr.msg.payloadlen = err;

if (err > kmsg->payloadlen)

err = kmsg->payloadlen;

copy_len = sizeof(struct io_uring_recvmsg_out);

if (kmsg->msg.msg_namelen > kmsg->namelen)

copy_len += kmsg->namelen;

else

copy_len += kmsg->msg.msg_namelen;