// SPDX-License-Identifier: BSD-2-Clause or GPL-2.0+

/* This file consists mostly of "glue" that hooks Homa into the rest of

* the Linux kernel. The guts of the protocol are in other files.

*/

#include "homa_impl.h"

#include "homa_peer.h"

#include "homa_pool.h"

#ifndef __STRIP__ /* See strip.py */

#include "homa_grant.h"

#include "homa_hijack.h"

#include "homa_offload.h"

#include "homa_pacer.h"

#include "homa_qdisc.h"

#endif /* See strip.py */

/* Identifier for retrieving Homa-specific data for a struct net. */

unsigned int homa_net_id;

/* This structure defines functions that allow Homa to be used as a

* pernet subsystem.

*/

static struct pernet_operations homa_net_ops = {

.init = homa_net_start,

.exit = homa_net_exit,

.id = &homa_net_id,

.size = sizeof(struct homa_net)

};

/* Global data for Homa. Avoid referencing directly except when there is

* no alternative (instead, use a homa pointer stored in a struct or

* passed via a parameter). This allows overriding during unit tests.

*/

static struct homa homa_data;

/* This structure defines functions that handle various operations on

* Homa sockets. These functions are relatively generic: they are called

* to implement top-level system calls. Many of these operations can

* be implemented by PF_INET6 functions that are independent of the

* Homa protocol.

*/

static const struct proto_ops homa_proto_ops = {

.family = PF_INET,

.owner = THIS_MODULE,

.release = inet_release,

.bind = homa_bind,

.connect = inet_dgram_connect,

.socketpair = sock_no_socketpair,

.accept = sock_no_accept,

.getname = inet_getname,

.poll = homa_poll,

.ioctl = homa_ioctl,

.listen = sock_no_listen,

.shutdown = homa_shutdown,

.setsockopt = sock_common_setsockopt,

.getsockopt = sock_common_getsockopt,

.sendmsg = inet_sendmsg,

.recvmsg = inet_recvmsg,

.mmap = sock_no_mmap,

.set_peek_off = sk_set_peek_off,

};

static const struct proto_ops homav6_proto_ops = {

.family = PF_INET6,

.owner = THIS_MODULE,

.release = inet6_release,

.bind = homa_bind,

.connect = inet_dgram_connect,

.socketpair = sock_no_socketpair,

.accept = sock_no_accept,

.getname = inet6_getname,

.poll = homa_poll,

.ioctl = homa_ioctl,

.listen = sock_no_listen,

.shutdown = homa_shutdown,

.setsockopt = sock_common_setsockopt,

.getsockopt = sock_common_getsockopt,

.sendmsg = inet_sendmsg,

.recvmsg = inet_recvmsg,

.mmap = sock_no_mmap,

.set_peek_off = sk_set_peek_off,

};

/* This structure also defines functions that handle various operations

* on Homa sockets. However, these functions are lower-level than those

* in homa_proto_ops: they are specific to the PF_INET or PF_INET6

* protocol family, and in many cases they are invoked by functions in

* homa_proto_ops. Most of these functions have Homa-specific implementations.

*/

static struct proto homa_prot = {

.name = "HOMA",

.owner = THIS_MODULE,

.close = homa_close,

.connect = ip4_datagram_connect,

.init = homa_socket,

.destroy = homa_sock_destroy,

.setsockopt = homa_setsockopt,

.getsockopt = homa_getsockopt,

.sendmsg = homa_sendmsg,

.recvmsg = homa_recvmsg,

.hash = homa_hash,

.unhash = homa_unhash,

.obj_size = sizeof(struct homa_sock),

.no_autobind = 1,

};

static struct proto homav6_prot = {

.name = "HOMAv6",

.owner = THIS_MODULE,

.close = homa_close,

.connect = ip6_datagram_connect,

.init = homa_socket,

.destroy = homa_sock_destroy,

.setsockopt = homa_setsockopt,

.getsockopt = homa_getsockopt,

.sendmsg = homa_sendmsg,

.recvmsg = homa_recvmsg,

.hash = homa_hash,

.unhash = homa_unhash,

.obj_size = sizeof(struct homa_v6_sock),

.ipv6_pinfo_offset = offsetof(struct homa_v6_sock, inet6),

.no_autobind = 1,

};

/* Top-level structure describing the Homa protocol. */

static struct inet_protosw homa_protosw = {

.type = SOCK_DGRAM,

.protocol = IPPROTO_HOMA,

.prot = &homa_prot,

.ops = &homa_proto_ops,

.flags = INET_PROTOSW_REUSE,

};

static struct inet_protosw homav6_protosw = {

.type = SOCK_DGRAM,

.protocol = IPPROTO_HOMA,

.prot = &homav6_prot,

.ops = &homav6_proto_ops,

.flags = INET_PROTOSW_REUSE,

};

/* This structure is used by IP to deliver incoming Homa packets to us. */

static struct net_protocol homa_protocol = {

.handler = homa_softirq,

.err_handler = homa_err_handler_v4,

.no_policy = 1,

};

static struct inet6_protocol homav6_protocol = {

.handler = homa_softirq,

.err_handler = homa_err_handler_v6,

.flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,

};

#ifndef __STRIP__ /* See strip.py */

/* Used to configure sysctl access to Homa configuration parameters. The

* @data fields are actually offsets within a struct homa; these are converted

* to pointers into a net-specific struct homa later.

*/

#define OFFSET(field) ((void *)offsetof(struct homa, field))

static struct ctl_table homa_ctl_table[] = {

{

.procname = "accept_bits",

.data = OFFSET(accept_bits),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "action",

.data = OFFSET(sysctl_action),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "bpage_lease_usecs",

.data = OFFSET(bpage_lease_usecs),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "busy_usecs",

.data = OFFSET(busy_usecs),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "cutoff_version",

.data = OFFSET(cutoff_version),

.maxlen = sizeof(int),

.mode = 0444,

.proc_handler = homa_dointvec

},

{

.procname = "dead_buffs_limit",

.data = OFFSET(dead_buffs_limit),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "drop_bits",

.data = OFFSET(drop_bits),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "flags",

.data = OFFSET(flags),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "freeze_type",

.data = OFFSET(freeze_type),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "gen3_softirq_cores",

.data = NULL,

.maxlen = 0,

.mode = 0644,

.proc_handler = homa_sysctl_softirq_cores

},

{

.procname = "gro_busy_usecs",

.data = OFFSET(gro_busy_usecs),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "gro_policy",

.data = OFFSET(gro_policy),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "gso_force_software",

.data = OFFSET(gso_force_software),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "hijack_tcp",

.data = OFFSET(hijack_tcp),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "link_mbps",

.data = OFFSET(link_mbps),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "max_dead_buffs",

.data = OFFSET(max_dead_buffs),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "max_gro_skbs",

.data = OFFSET(max_gro_skbs),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "max_gso_size",

.data = OFFSET(max_gso_size),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "max_sched_prio",

.data = OFFSET(max_sched_prio),

.maxlen = sizeof(int),

.mode = 0444,

.proc_handler = homa_dointvec

},

{

.procname = "next_id",

.data = OFFSET(next_id),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "num_priorities",

.data = OFFSET(num_priorities),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "poll_usecs",

.data = OFFSET(poll_usecs),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "priority_map",

.data = OFFSET(priority_map),

.maxlen = HOMA_MAX_PRIORITIES * sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "reap_limit",

.data = OFFSET(reap_limit),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "request_ack_ticks",

.data = OFFSET(request_ack_ticks),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "resend_interval",

.data = OFFSET(resend_interval),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "resend_ticks",

.data = OFFSET(resend_ticks),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "skb_page_frees_per_sec",

.data = OFFSET(skb_page_frees_per_sec),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "skb_page_pool_min_kb",

.data = OFFSET(skb_page_pool_min_kb),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "temp",

.data = OFFSET(temp[0]),

.maxlen = sizeof(((struct homa *)0)->temp),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "timeout_resends",

.data = OFFSET(timeout_resends),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "timeout_ticks",

.data = OFFSET(timeout_ticks),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "unsched_bytes",

.data = OFFSET(unsched_bytes),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "unsched_cutoffs",

.data = OFFSET(unsched_cutoffs),

.maxlen = HOMA_MAX_PRIORITIES * sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "verbose",

.data = OFFSET(verbose),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

{

.procname = "wmem_max",

.data = OFFSET(wmem_max),

.maxlen = sizeof(int),

.mode = 0644,

.proc_handler = homa_dointvec

},

};

#endif /* See strip.py */

/* Sizes of the headers for each Homa packet type, in bytes. */

#ifndef __STRIP__ /* See strip.py */

static u16 header_lengths[] = {

sizeof(struct homa_data_hdr),

sizeof(struct homa_grant_hdr),

sizeof(struct homa_resend_hdr),

sizeof(struct homa_rpc_unknown_hdr),

sizeof(struct homa_busy_hdr),

sizeof(struct homa_cutoffs_hdr),

sizeof(struct homa_freeze_hdr),

sizeof(struct homa_need_ack_hdr),

sizeof(struct homa_ack_hdr)

};

#else /* See strip.py */

static u16 header_lengths[] = {

sizeof(struct homa_data_hdr),

0,

sizeof(struct homa_resend_hdr),

sizeof(struct homa_rpc_unknown_hdr),

sizeof(struct homa_busy_hdr),

0,

0,

sizeof(struct homa_need_ack_hdr),

sizeof(struct homa_ack_hdr)

};

#endif /* See strip.py */

#ifndef __STRIP__ /* See strip.py */

/* Used to remove sysctl values when the module is unloaded. */

static struct ctl_table_header *homa_ctl_header;

#endif /* See strip.py */

/* Thread that runs timer code to detect lost packets and crashed peers. */

static struct task_struct *timer_kthread;

static DECLARE_COMPLETION(timer_thread_done);

/* Used to wakeup timer_kthread at regular intervals. */

static struct hrtimer hrtimer;

/* Nonzero is an indication to the timer thread that it should exit. */

static int timer_thread_exit;

/**

* homa_load() - invoked when this module is loaded into the Linux kernel

* Return: 0 on success, otherwise a negative errno.

*/

int __init homa_load(void)

{

struct homa *homa = &homa_data;

bool init_protocol6 = false;

bool init_protosw6 = false;

bool init_protocol = false;

bool init_protosw = false;

bool init_net_ops = false;

bool init_proto6 = false;

bool init_proto = false;

bool init_homa = false;

int status;

IF_NO_STRIP(bool init_metrics = false);

IF_NO_STRIP(bool init_offload = false);

IF_NO_STRIP(bool init_sysctl = false);

IF_NO_STRIP(bool init_qdisc = false);

/* Compile-time validations that no packet header is longer

* than HOMA_MAX_HEADER.

*/

BUILD_BUG_ON(sizeof(struct homa_data_hdr) > HOMA_MAX_HEADER);

#ifndef __STRIP__ /* See strip.py */

BUILD_BUG_ON(sizeof(struct homa_grant_hdr) > HOMA_MAX_HEADER);

#endif /* See strip.py */

BUILD_BUG_ON(sizeof(struct homa_resend_hdr) > HOMA_MAX_HEADER);

BUILD_BUG_ON(sizeof(struct homa_rpc_unknown_hdr) > HOMA_MAX_HEADER);

BUILD_BUG_ON(sizeof(struct homa_busy_hdr) > HOMA_MAX_HEADER);

#ifndef __STRIP__ /* See strip.py */

BUILD_BUG_ON(sizeof(struct homa_cutoffs_hdr) > HOMA_MAX_HEADER);

#endif /* See strip.py */

#ifndef __UPSTREAM__ /* See strip.py */

BUILD_BUG_ON(sizeof(struct homa_freeze_hdr) > HOMA_MAX_HEADER);

#endif /* See strip.py */

BUILD_BUG_ON(sizeof(struct homa_need_ack_hdr) > HOMA_MAX_HEADER);

BUILD_BUG_ON(sizeof(struct homa_ack_hdr) > HOMA_MAX_HEADER);

/* Extra constraints on data packets:

* - Ensure minimum header length so Homa doesn't have to worry about

* padding data packets.

* - Make sure data packet headers are a multiple of 4 bytes (needed

* for TCP/TSO compatibility).

*/

BUILD_BUG_ON(sizeof(struct homa_data_hdr) < HOMA_MIN_PKT_LENGTH);

BUILD_BUG_ON((sizeof(struct homa_data_hdr) -

sizeof(struct homa_seg_hdr)) & 0x3);

#ifndef __STRIP__ /* See strip.py */

/* Homa requires at least 8 priority levels. */

BUILD_BUG_ON(HOMA_MAX_PRIORITIES < 8);

#endif /* See strip.py */

/* Detect size changes in uAPI structs. */

BUILD_BUG_ON(sizeof(struct homa_sendmsg_args) != 24);

BUILD_BUG_ON(sizeof(struct homa_recvmsg_args) != 88);

#ifndef __STRIP__ /* See strip.py */

BUILD_BUG_ON(sizeof(struct homa_abort_args) != 32);

#endif /* See strip.py */

#ifndef __UPSTREAM__ /* See strip.py */

pr_err("Homa module loading\n");

pr_notice("Homa structure sizes: homa_data_hdr %lu, homa_seg_hdr %lu, ack %lu, peer %lu, ip_hdr %lu flowi %lu ipv6_hdr %lu, flowi6 %lu tcp_sock %lu homa_rpc %lu sk_buff %lu skb_shared_info %lu rcvmsg_control %lu union sockaddr_in_union %lu HOMA_MAX_BPAGES %u NR_CPUS %u nr_cpu_ids %u, MAX_NUMNODES %d\n",

sizeof(struct homa_data_hdr),

sizeof(struct homa_seg_hdr),

sizeof(struct homa_ack),

sizeof(struct homa_peer),

sizeof(struct iphdr),

sizeof(struct flowi),

sizeof(struct ipv6hdr),

sizeof(struct flowi6),

sizeof(struct tcp_sock),

sizeof(struct homa_rpc),

sizeof(struct sk_buff),

sizeof(struct skb_shared_info),

sizeof(struct homa_recvmsg_args),

sizeof(union sockaddr_in_union),

HOMA_MAX_BPAGES,

NR_CPUS,

nr_cpu_ids,

MAX_NUMNODES);

#endif /* See strip.py */

#ifndef __UPSTREAM__ /* See strip.py */

tt_init("timetrace");

#endif /* See strip.py */

status = homa_init(homa);

if (status)

goto error;

init_homa = true;

status = proto_register(&homa_prot, 1);

if (status != 0) {

pr_err("proto_register failed for homa_prot: %d\n", status);

goto error;

}

init_proto = true;

status = proto_register(&homav6_prot, 1);

if (status != 0) {

pr_err("proto_register failed for homav6_prot: %d\n", status);

goto error;

}

init_proto6 = true;

inet_register_protosw(&homa_protosw);

init_protosw = true;

status = inet6_register_protosw(&homav6_protosw);

if (status != 0) {

pr_err("inet6_register_protosw failed in %s: %d\n", __func__,

status);

goto error;

}

init_protosw6 = true;

status = inet_add_protocol(&homa_protocol, IPPROTO_HOMA);

if (status != 0) {

pr_err("inet_add_protocol failed in %s: %d\n", __func__,

status);

goto error;

}

init_protocol = true;

status = inet6_add_protocol(&homav6_protocol, IPPROTO_HOMA);

if (status != 0) {

pr_err("inet6_add_protocol failed in %s: %d\n", __func__,

status);

goto error;

}

init_protocol6 = true;

#ifndef __STRIP__ /* See strip.py */

status = homa_metrics_init();

if (status != 0)

goto error;

init_metrics = true;

homa_ctl_header = register_net_sysctl(&init_net, "net/homa",

homa_ctl_table);

if (!homa_ctl_header) {

pr_err("couldn't register Homa sysctl parameters\n");

status = -ENOMEM;

goto error;

}

init_sysctl = true;

status = homa_offload_init();

if (status != 0) {

pr_err("Homa couldn't init offloads\n");

goto error;

}

init_offload = true;

status = homa_qdisc_register();

if (status != 0) {

pr_err("Homa couldn't load its qdisc: error %d\n", status);

goto error;

}

init_qdisc = true;

#endif /* See strip.py */

status = register_pernet_subsys(&homa_net_ops);

if (status != 0) {

pr_err("Homa got error from register_pernet_subsys: %d\n",

status);

goto error;

}

init_net_ops = true;

timer_kthread = kthread_run(homa_timer_main, homa, "homa_timer");

if (IS_ERR(timer_kthread)) {

status = PTR_ERR(timer_kthread);

pr_err("couldn't create Homa timer thread: error %d\n",

status);

timer_kthread = NULL;

goto error;

}

#ifndef __STRIP__ /* See strip.py */

homa_hijack_init();

#endif /* See strip.py */

#ifndef __UPSTREAM__ /* See strip.py */

tt_set_temp(homa->temp);

#endif /* See strip.py */

return 0;

error:

if (timer_kthread) {

timer_thread_exit = 1;

wake_up_process(timer_kthread);

wait_for_completion(&timer_thread_done);

}

#ifndef __STRIP__ /* See strip.py */

if (init_qdisc)

homa_qdisc_unregister();

if (init_offload)

homa_offload_end();

if (init_sysctl)

unregister_net_sysctl_table(homa_ctl_header);

if (init_metrics)

homa_metrics_end();

#endif /* See strip.py */

if (init_net_ops)

unregister_pernet_subsys(&homa_net_ops);

if (init_homa)

homa_destroy(homa);

if (init_protocol)

inet_del_protocol(&homa_protocol, IPPROTO_HOMA);

if (init_protocol6)

inet6_del_protocol(&homav6_protocol, IPPROTO_HOMA);

if (init_protosw)

inet_unregister_protosw(&homa_protosw);

if (init_protosw6)

inet6_unregister_protosw(&homav6_protosw);

if (init_proto)

proto_unregister(&homa_prot);

if (init_proto6)

proto_unregister(&homav6_prot);

return status;

}

/**

* homa_unload() - invoked when this module is unloaded from the Linux kernel.

*/

void __exit homa_unload(void)

{

struct homa *homa = &homa_data;

pr_notice("Homa module unloading\n");

#ifndef __STRIP__ /* See strip.py */

homa_hijack_end();

#endif /* See strip.py */

if (timer_kthread) {

timer_thread_exit = 1;

wake_up_process(timer_kthread);

wait_for_completion(&timer_thread_done);

}

#ifndef __STRIP__ /* See strip.py */

homa_qdisc_unregister();

if (homa_offload_end() != 0)

pr_err("Homa couldn't stop offloads\n");

unregister_net_sysctl_table(homa_ctl_header);

homa_metrics_end();

#endif /* See strip.py */

unregister_pernet_subsys(&homa_net_ops);

inet_del_protocol(&homa_protocol, IPPROTO_HOMA);

inet_unregister_protosw(&homa_protosw);

inet6_del_protocol(&homav6_protocol, IPPROTO_HOMA);

inet6_unregister_protosw(&homav6_protosw);

proto_unregister(&homa_prot);

proto_unregister(&homav6_prot);

homa_destroy(homa);

#ifndef __UPSTREAM__ /* See strip.py */

tt_destroy();

#endif /* See strip.py */

}

module_init(homa_load);

module_exit(homa_unload);

/**

* homa_net_start() - Initialize Homa for a new network namespace.

* @net: The net that Homa will be associated with.

* Return: 0 on success, otherwise a negative errno.

*/

int homa_net_start(struct net *net)

{

pr_notice("Homa attaching to net namespace\n");

return homa_net_init(homa_net(net), net, &homa_data);

}

/**

* homa_net_exit() - Perform Homa cleanup needed when a network namespace

* is destroyed.

* @net: The net from which Homa should be removed.

*/

void homa_net_exit(struct net *net)

{

pr_notice("Homa detaching from net namespace\n");

homa_net_destroy(homa_net(net));

}

/**

* homa_bind() - Implements the bind system call for Homa sockets: associates

* a well-known service port with a socket. Unlike other AF_INET6 protocols,

* there is no need to invoke this system call for sockets that are only

* used as clients.

* @sock: Socket on which the system call was invoked.

* @addr: Contains the desired port number.

* @addr_len: Number of bytes in uaddr.

* Return: 0 on success, otherwise a negative errno. Sets hsk->error_msg

* on errors.

*/

int homa_bind(struct socket *sock, struct sockaddr *addr, int addr_len)

{

union sockaddr_in_union *addr_in = (union sockaddr_in_union *)addr;

struct homa_sock *hsk = homa_sk(sock->sk);

int port = 0;

if (unlikely(addr->sa_family != sock->sk->sk_family)) {

hsk->error_msg = "address family in bind address didn't match socket";

return -EAFNOSUPPORT;

}

if (addr_in->in6.sin6_family == AF_INET6) {

if (addr_len < sizeof(struct sockaddr_in6)) {

hsk->error_msg = "ipv6 address too short";

return -EINVAL;

}

port = ntohs(addr_in->in4.sin_port);

} else if (addr_in->in4.sin_family == AF_INET) {

if (addr_len < sizeof(struct sockaddr_in)) {

hsk->error_msg = "ipv4 address too short";

return -EINVAL;

}

port = ntohs(addr_in->in6.sin6_port);

}

return homa_sock_bind(hsk->hnet, hsk, port);

}

/**

* homa_close() - Invoked when close system call is invoked on a Homa socket.

* @sk: Socket being closed

* @timeout: ??

*/

void homa_close(struct sock *sk, long timeout)

{

struct homa_sock *hsk = homa_sk(sk);

#ifndef __UPSTREAM__ /* See strip.py */

int port = hsk->port;

#endif /* See strip.py */

homa_sock_shutdown(hsk);

sk_common_release(sk);

tt_record1("closed socket, port %d", port);

}

/**

* homa_shutdown() - Implements the shutdown system call for Homa sockets.

* @sock: Socket to shut down.

* @how: Ignored: for other sockets, can independently shut down

* sending and receiving, but for Homa any shutdown will

* shut down everything.

*

* Return: 0 on success, otherwise a negative errno.

*/

int homa_shutdown(struct socket *sock, int how)

{

homa_sock_shutdown(homa_sk(sock->sk));

return 0;

}

#ifndef __STRIP__ /* See strip.py */

/**

* homa_ioc_abort() - The top-level function for the ioctl that implements

* the homa_abort user-level API.

* @sock: Socket for this request.

* @arg: User-space address of a homa_abort_args struct.

*

* Return: 0 on success, otherwise a negative errno. Sets hsk->error_msg

* on errors.

*/

int homa_ioc_abort(struct socket *sock, unsigned long arg)

{

struct homa_sock *hsk = homa_sk(sock->sk);

struct homa_abort_args args;

struct homa_rpc *rpc;

int ret = 0;

if (unlikely(copy_from_user(&args, (void __user *)arg, sizeof(args)))) {

hsk->error_msg = "invalid address for homa_abort_args";

return -EFAULT;

}

if (args._pad1 || args._pad2[0] || args._pad2[1]) {

hsk->error_msg = "reserved fields in homa_abort_args must be zero";

return -EINVAL;

}

if (args.id == 0) {

homa_abort_sock_rpcs(hsk, -args.error);

return 0;

}

rpc = homa_rpc_find_client(hsk, args.id);

if (!rpc) {

hsk->error_msg = "RPC identifier did not match any existing RPC";

return -EINVAL;

}

if (args.error == 0)

homa_rpc_end(rpc);

else

homa_rpc_abort(rpc, -args.error);

homa_rpc_unlock(rpc); /* Locked by homa_rpc_find_client. */

return ret;

}

#endif /* See strip.py */

/**

* homa_ioc_info() - The top-level function that implements the

* HOMAIOCINFO ioctl for Homa sockets.

* @sock: Socket for this request

* @arg: The address in user space of the argument to ioctl, which

* is a homa_info struct.

*

* Return: 0 on success, otherwise a negative errno. Sets hsk->error_msg

* on errors.

*/

int homa_ioc_info(struct socket *sock, unsigned long arg)

{

struct homa_sock *hsk = homa_sk(sock->sk);

struct homa_rpc_info rinfo;

struct homa_info hinfo;

struct homa_rpc *rpc;

int bytes_avl;

char *dst;

if (unlikely(copy_from_user(&hinfo, (void __user *)arg,

sizeof(hinfo)))) {

hsk->error_msg = "invalid address for homa_info";

return -EFAULT;

}

if (!homa_protect_rpcs(hsk)) {

hsk->error_msg = "socket has been shut down";

return -ESHUTDOWN;

}

rcu_read_lock();

hinfo.bpool_avail_bytes = homa_pool_avail_bytes(hsk->buffer_pool);

hinfo.port = hsk->port;

dst = (char *)hinfo.rpc_info;

bytes_avl = hinfo.rpc_info_length;

hinfo.num_rpcs = 0;

list_for_each_entry_rcu(rpc, &hsk->active_rpcs, active_links) {

homa_rpc_lock(rpc);

if (rpc->state == RPC_DEAD) {

homa_rpc_unlock(rpc);

continue;

}

homa_rpc_get_info(rpc, &rinfo);

homa_rpc_unlock(rpc);

if (dst && bytes_avl >= sizeof(rinfo)) {

if (copy_to_user((void __user *)dst, &rinfo,

sizeof(rinfo))) {

rcu_read_unlock();

homa_unprotect_rpcs(hsk);

hsk->error_msg = "couldn't copy homa_rpc_info to user space: invalid or read-only address?";

return -EFAULT;

}

dst += sizeof(rinfo);

bytes_avl -= sizeof(rinfo);

}

hinfo.num_rpcs++;

}

rcu_read_unlock();

homa_unprotect_rpcs(hsk);

if (hsk->error_msg)

snprintf(hinfo.error_msg, HOMA_ERROR_MSG_SIZE, "%s",

hsk->error_msg);

else

hinfo.error_msg[0] = 0;

if (copy_to_user((void __user *)arg, &hinfo, sizeof(hinfo))) {

hsk->error_msg = "couldn't copy homa_info to user space: read-only address?";

return -EFAULT;

}

return 0;

}

/**

* homa_ioctl() - Implements the ioctl system call for Homa sockets.

* @sock: Socket on which the system call was invoked.

* @cmd: Identifier for a particular ioctl operation.

* @arg: Operation-specific argument; typically the address of a block

* of data in user address space.

*

* Return: 0 on success, otherwise a negative errno. Sets hsk->error_msg

* on errors.

*/

int homa_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)

{

#ifndef __STRIP__ /* See strip.py */

if (cmd == HOMAIOCABORT) {

u64 start = homa_clock();

int result;

result = homa_ioc_abort(sock, arg);

INC_METRIC(abort_calls, 1);

INC_METRIC(abort_cycles, homa_clock() - start);

return result;

}

if (cmd == HOMAIOCFREEZE) {

tt_record1("Freezing timetrace because of HOMAIOCFREEZE ioctl, pid %d",

current->pid);

tt_freeze();

return 0;

}

#endif /* See strip.py */

if (cmd == HOMAIOCINFO)

return homa_ioc_info(sock, arg);

homa_sk(sock->sk)->error_msg = "ioctl opcode isn't supported by Homa";

return -EINVAL;

}

/**

* homa_socket() - Implements the socket(2) system call for sockets.

* @sk: Socket on which the system call was invoked. The non-Homa

* parts have already been initialized.

*

* Return: always 0 (success).

*/

int homa_socket(struct sock *sk)

{

return homa_sock_init(homa_sk(sk));

}

/**

* homa_setsockopt() - Implements the getsockopt system call for Homa sockets.

* @sk: Socket on which the system call was invoked.

* @level: Level at which the operation should be handled; will always

* be IPPROTO_HOMA.

* @optname: Identifies a particular setsockopt operation.

* @optval: Address in user space of information about the option.

* @optlen: Number of bytes of data at @optval.

* Return: 0 on success, otherwise a negative errno. Sets hsk->error_msg

* on errors.

*/

int homa_setsockopt(struct sock *sk, int level, int optname,

sockptr_t optval, unsigned int optlen)

{

struct homa_sock *hsk = homa_sk(sk);

int ret;

if (level != IPPROTO_HOMA) {