Page MenuHomeFreeBSD
Files F170980609

D59501.diff
No OneTemporary
Actions

D59501.diff
View Options

diff --git a/lib/libsys/getsockopt.2 b/lib/libsys/getsockopt.2
--- a/lib/libsys/getsockopt.2
+++ b/lib/libsys/getsockopt.2
@@ -25,7 +25,7 @@
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
-.Dd June 3, 2026
+.Dd August 20, 2026
.Dt GETSOCKOPT 2
.Os
.Sh NAME
@@ -150,6 +150,7 @@
.It Dv SO_REUSEADDR Ta "enables local address reuse"
.It Dv SO_REUSEPORT Ta "enables duplicate address and port bindings"
.It Dv SO_REUSEPORT_LB Ta "enables duplicate address and port bindings with load balancing"
+.It Dv SO_REUSEPORT_LB_CPU Ta "get or set receive-CPU affinity within a load-balancing group"
.It Dv SO_KEEPALIVE Ta "enables keep connections alive"
.It Dv SO_DONTROUTE Ta "enables routing bypass for outgoing messages"
.It Dv SO_LINGER Ta "linger on close if data present"
@@ -226,6 +227,31 @@
.Xr divert 4
manual page for details.
.Pp
+.Dv SO_REUSEPORT_LB_CPU
+sets a receive-CPU affinity on a socket that is a member of a
+.Dv SO_REUSEPORT_LB
+group.
+The argument is an integer CPU id.
+When an incoming TCP or UDP packet is processed on that CPU, the group
+delivers it to this socket in preference to selecting a member with the
+hash function.
+An application that pins each worker thread to a distinct CPU can thereby
+receive the flows that the network stack, following the receive-side
+scaling steering performed by the hardware, processes on the same CPU.
+The special value
+.Dv SO_REUSEPORT_LB_CPU_CURRENT
+affinitizes the socket to the CPU of the calling thread, and
+.Dv SO_REUSEPORT_LB_CPU_ANY
+clears the affinity so that the socket again participates in the hashed
+distribution.
+Packets whose processing CPU matches no affinitized member of the group
+are distributed by the hash function as usual.
+Reading the option with
+.Xr getsockopt 2
+returns the CPU id currently affinitized to the socket, or
+.Dv SO_REUSEPORT_LB_CPU_ANY
+if no affinity is set.
+.Pp
.Dv SO_KEEPALIVE
enables the
periodic transmission of messages on a connected socket.
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -3963,6 +3963,10 @@
error = so->so_proto->pr_ctloutput(so, sopt);
break;
+ case SO_REUSEPORT_LB_CPU:
+ error = so->so_proto->pr_ctloutput(so, sopt);
+ break;
+
case SO_USER_COOKIE:
error = sooptcopyin(sopt, &val32, sizeof val32,
sizeof val32);
@@ -4349,6 +4353,10 @@
optval = so->so_max_pacing_rate;
goto integer;
+ case SO_REUSEPORT_LB_CPU:
+ error = so->so_proto->pr_ctloutput(so, sopt);
+ break;
+
case SO_SPLICE: {
off_t n;
diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h
--- a/sys/netinet/in_pcb.h
+++ b/sys/netinet/in_pcb.h
@@ -341,6 +341,7 @@
int inp_flags; /* (i) generic IP/datagram flags */
int inp_flags2; /* (i) generic IP/datagram flags #2*/
uint8_t inp_numa_domain; /* numa domain */
+ int inp_lb_cpu; /* (i) SO_REUSEPORT_LB receive-CPU affinity */
struct socket *inp_socket; /* (i) back pointer to socket */
struct inpcbinfo *inp_pcbinfo; /* (c) PCB list info */
struct ucred *inp_cred; /* (c) cache of socket cred */
@@ -633,6 +634,7 @@
int in_pcbladdr(const struct inpcb *, struct in_addr *, struct in_addr *,
struct ucred *);
int in_pcblbgroup_numa(struct inpcb *, int arg);
+int in_pcblbgroup_cpu(struct inpcb *, int cpu);
void in_pcblisten(struct inpcb *);
struct inpcb *
in_pcblookup(struct inpcbinfo *, struct in_addr, u_int,
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -332,6 +332,9 @@
grp->il_pendcnt++;
} else {
grp->il_inp[grp->il_inpcnt] = inp;
+ if (inp->inp_lb_cpu != NOCPU)
+ atomic_store_rel_int(&grp->il_cpucnt,
+ grp->il_cpucnt + 1);
/*
* Synchronize with in_pcblookup_lbgroup(): make sure that we
@@ -365,6 +368,7 @@
for (i = 0; i < old_grp->il_inpcnt; ++i)
grp->il_inp[i] = old_grp->il_inp[i];
grp->il_inpcnt = old_grp->il_inpcnt;
+ grp->il_cpucnt = old_grp->il_cpucnt;
CK_LIST_INSERT_HEAD(&bucket->head, grp, il_list);
LIST_SWAP(&old_grp->il_pending, &grp->il_pending, inpcb,
inp_lbgroup_list);
@@ -475,6 +479,9 @@
if (grp->il_inp[i] != inp)
continue;
+ if (inp->inp_lb_cpu != NOCPU)
+ atomic_store_rel_int(&grp->il_cpucnt,
+ grp->il_cpucnt - 1);
if (grp->il_inpcnt == 1 &&
LIST_EMPTY(&grp->il_pending)) {
/* We are the last, free this local group. */
@@ -536,6 +543,52 @@
return (0);
}
+int
+in_pcblbgroup_cpu(struct inpcb *inp, int cpu)
+{
+ struct lbgroupbucket *bucket;
+ struct inpcblbgroup *grp;
+ int old;
+ u_int i;
+
+ INP_WLOCK_ASSERT(inp);
+
+ switch (cpu) {
+ case SO_REUSEPORT_LB_CPU_ANY:
+ cpu = NOCPU;
+ break;
+ case SO_REUSEPORT_LB_CPU_CURRENT:
+ cpu = curcpu;
+ break;
+ default:
+ if (cpu < 0 || cpu > mp_maxid || CPU_ABSENT(cpu))
+ return (EINVAL);
+ }
+
+ old = inp->inp_lb_cpu;
+ atomic_store_int(&inp->inp_lb_cpu, cpu);
+
+ if ((inp->inp_flags & INP_INLBGROUP) == 0)
+ return (0);
+
+ grp = in_pcblbgroup_find(inp, &bucket);
+ if (grp == NULL)
+ return (0);
+ for (i = 0; i < grp->il_inpcnt; i++) {
+ if (grp->il_inp[i] != inp)
+ continue;
+ if (old == NOCPU && cpu != NOCPU)
+ atomic_store_rel_int(&grp->il_cpucnt,
+ grp->il_cpucnt + 1);
+ else if (old != NOCPU && cpu == NOCPU)
+ atomic_store_rel_int(&grp->il_cpucnt,
+ grp->il_cpucnt - 1);
+ break;
+ }
+ INPBUCKET_UNLOCK(bucket);
+ return (0);
+}
+
/*
* Initialize an inpcbinfo - a per-VNET instance of connections db.
*/
@@ -655,6 +708,7 @@
#ifdef NUMA
inp->inp_numa_domain = M_NODOM;
#endif
+ inp->inp_lb_cpu = NOCPU;
inp->inp_pcbinfo = pcbinfo;
inp->inp_socket = so;
inp->inp_cred = crhold(so->so_cred);
@@ -2286,6 +2340,16 @@
count = atomic_load_acq_int(&grp->il_inpcnt);
if (count == 0)
return (NULL);
+ if (atomic_load_acq_int(&grp->il_cpucnt) > 0) {
+ int cpu = curcpu;
+
+ for (u_int i = 0; i < count; i++) {
+ inp = grp->il_inp[i];
+ if (inp != NULL &&
+ atomic_load_int(&inp->inp_lb_cpu) == cpu)
+ return (inp);
+ }
+ }
inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count];
KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
return (inp);
diff --git a/sys/netinet/in_pcb_var.h b/sys/netinet/in_pcb_var.h
--- a/sys/netinet/in_pcb_var.h
+++ b/sys/netinet/in_pcb_var.h
@@ -153,6 +153,7 @@
uint32_t il_inpsiz; /* max count in il_inp[] (h) */
uint32_t il_inpcnt; /* cur count in il_inp[] (h) */
uint32_t il_pendcnt; /* cur count in il_pending (h) */
+ uint32_t il_cpucnt; /* cur il_inp[] with CPU affinity (h) */
struct inpcb *il_inp[]; /* (h) */
};
diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c
--- a/sys/netinet/ip_output.c
+++ b/sys/netinet/ip_output.c
@@ -1124,6 +1124,16 @@
inp->inp_inc.inc_fibnum = optval;
INP_WUNLOCK(inp);
break;
+ case SO_REUSEPORT_LB_CPU:
+ error = sooptcopyin(sopt, &optval,
+ sizeof(optval), sizeof(optval));
+ if (error != 0)
+ break;
+
+ INP_WLOCK(inp);
+ error = in_pcblbgroup_cpu(inp, optval);
+ INP_WUNLOCK(inp);
+ break;
case SO_MAX_PACING_RATE:
#ifdef RATELIMIT
INP_WLOCK(inp);
@@ -1137,6 +1147,19 @@
default:
break;
}
+ } else if (sopt->sopt_level == SOL_SOCKET &&
+ sopt->sopt_dir == SOPT_GET) {
+ switch (sopt->sopt_name) {
+ case SO_REUSEPORT_LB_CPU:
+ INP_RLOCK(inp);
+ optval = inp->inp_lb_cpu;
+ INP_RUNLOCK(inp);
+ error = sooptcopyout(sopt, &optval,
+ sizeof(optval));
+ break;
+ default:
+ break;
+ }
}
return (error);
}
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -1003,6 +1003,16 @@
count = atomic_load_acq_int(&grp->il_inpcnt);
if (count == 0)
return (NULL);
+ if (atomic_load_acq_int(&grp->il_cpucnt) > 0) {
+ int cpu = curcpu;
+
+ for (u_int i = 0; i < count; i++) {
+ inp = grp->il_inp[i];
+ if (inp != NULL &&
+ atomic_load_int(&inp->inp_lb_cpu) == cpu)
+ return (inp);
+ }
+ }
inp = grp->il_inp[INP6_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count];
KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
return (inp);
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -1525,6 +1525,16 @@
inp->inp_inc.inc_fibnum = optval;
INP_WUNLOCK(inp);
break;
+ case SO_REUSEPORT_LB_CPU:
+ error = sooptcopyin(sopt, &optval,
+ sizeof(optval), sizeof(optval));
+ if (error != 0)
+ break;
+
+ INP_WLOCK(inp);
+ error = in_pcblbgroup_cpu(inp, optval);
+ INP_WUNLOCK(inp);
+ break;
case SO_MAX_PACING_RATE:
#ifdef RATELIMIT
INP_WLOCK(inp);
@@ -1538,6 +1548,19 @@
default:
break;
}
+ } else if (sopt->sopt_level == SOL_SOCKET &&
+ sopt->sopt_dir == SOPT_GET) {
+ switch (sopt->sopt_name) {
+ case SO_REUSEPORT_LB_CPU:
+ INP_RLOCK(inp);
+ optval = inp->inp_lb_cpu;
+ INP_RUNLOCK(inp);
+ error = sooptcopyout(sopt, &optval,
+ sizeof(optval));
+ break;
+ default:
+ break;
+ }
}
} else { /* level == IPPROTO_IPV6 */
switch (op) {
diff --git a/sys/sys/socket.h b/sys/sys/socket.h
--- a/sys/sys/socket.h
+++ b/sys/sys/socket.h
@@ -174,6 +174,7 @@
#define SO_TS_CLOCK 0x1017 /* int; clock type used for SO_TIMESTAMP */
#define SO_MAX_PACING_RATE 0x1018 /* uint32_t; socket's max TX pacing rate (Linux name) */
#define SO_DOMAIN 0x1019 /* int; get socket domain */
+#define SO_REUSEPORT_LB_CPU 0x101a /* int; SO_REUSEPORT_LB receive-CPU affinity */
#define SO_SPLICE 0x1023 /* struct splice; splice data to other socket */
#endif
@@ -184,6 +185,10 @@
#define SO_TS_MONOTONIC 3 /* nanosecond resolution, monotonic */
#define SO_TS_DEFAULT SO_TS_REALTIME_MICRO
#define SO_TS_CLOCK_MAX SO_TS_MONOTONIC
+
+/* Special values for SO_REUSEPORT_LB_CPU. */
+#define SO_REUSEPORT_LB_CPU_ANY (-1) /* clear receive-CPU affinity */
+#define SO_REUSEPORT_LB_CPU_CURRENT (-2) /* affinitize to calling CPU */
#endif
/*

File Metadata

Mime Type
text/plain
Expires
Wed, Sep 9, 12:04 AM (15 h, 57 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
38547892
Default Alt Text
D59501.diff (9 KB)

Event Timeline