Page MenuHomeFreeBSD
Files F163431135

D58290.id182099.diff
No OneTemporary
Actions

D58290.id182099.diff
View Options

diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h
--- a/sys/netinet/in_pcb.h
+++ b/sys/netinet/in_pcb.h
@@ -384,7 +384,7 @@
};
CK_LIST_ENTRY(inpcb) inp_portlist; /* (r:e/w:h) port list */
uint64_t inp_gencnt; /* (c) generation count */
- void *spare_ptr; /* Spare pointer. */
+ struct lport_cache *inp_lport_cache;
rt_gen_t inp_rt_cookie; /* generation for route entry */
union { /* cached L3 information */
struct route inp_route;
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -735,6 +735,110 @@
#endif
}
+static MALLOC_DEFINE(M_LPORTS, "inp_lports", "inpcb lport cache");
+#define PORTBITS (IPPORT_MAX + 1)
+BITSET_DEFINE(lportbits, PORTBITS);
+struct lport_cache {
+ struct lportbits lports;
+ u_int refcount;
+#ifdef INVARIANTS
+ union in_dependaddr faddr;
+ uint16_t fport;
+#endif
+};
+
+static inline void
+lport_acquire(struct lportbits *lports, uint16_t port)
+{
+ BIT_SET_ATOMIC(PORTBITS, port, lports);
+}
+
+static inline bool
+lport_check(struct lportbits *lports, uint16_t port)
+{
+ return (BIT_ISSET(PORTBITS, port, lports));
+}
+
+static inline void
+lport_merge(struct lportbits *dst, struct lportbits *src)
+{
+ BIT_OR_ATOMIC(PORTBITS, dst, src);
+}
+
+static void
+in_pcb_cache_lport(struct inpcb *new, struct inpcb *old)
+{
+ struct lport_cache *cache;
+
+ INP_WLOCK_ASSERT(new);
+ INP_WLOCK_ASSERT(old);
+
+ if (new->inp_lport_cache == NULL &&
+ old->inp_lport_cache == NULL) {
+ cache = malloc(sizeof(*cache), M_LPORTS, M_ZERO | M_NOWAIT);
+ if (__predict_false(cache == NULL))
+ goto out;
+ refcount_init(&cache->refcount, 2);
+#ifdef INVARIANTS
+ cache->faddr = old->inp_inc.inc_ie.ie_dependfaddr;
+ cache->fport = old->inp_fport;
+#endif
+ lport_acquire(&cache->lports, ntohs(old->inp_lport));
+ new->inp_lport_cache = old->inp_lport_cache = cache;
+ } else if (new->inp_lport_cache == NULL) {
+ cache = old->inp_lport_cache;
+ MPASS(memcmp(&old->inp_inc.inc_ie.ie_dependfaddr,
+ &cache->faddr, sizeof(union in_dependaddr)) == 0);
+ MPASS(old->inp_fport == cache->fport);
+ refcount_acquire(&cache->refcount);
+ new->inp_lport_cache = cache;
+ } else if (old->inp_lport_cache == NULL) {
+ cache = new->inp_lport_cache;
+ MPASS(memcmp(&old->inp_inc.inc_ie.ie_dependfaddr,
+ &cache->faddr, sizeof(union in_dependaddr)) == 0);
+ MPASS(old->inp_fport == cache->fport);
+ lport_acquire(&cache->lports, ntohs(old->inp_lport));
+ refcount_acquire(&cache->refcount);
+ old->inp_lport_cache = cache;
+ } else {
+ struct lport_cache *second;
+
+ if (refcount_load(&new->inp_lport_cache->refcount) >
+ refcount_load(&old->inp_lport_cache->refcount)) {
+ cache = new->inp_lport_cache;
+ second = old->inp_lport_cache;
+ old->inp_lport_cache = cache;
+ } else {
+ cache = old->inp_lport_cache;
+ second = new->inp_lport_cache;
+ new->inp_lport_cache = cache;
+ }
+ MPASS(memcmp(&old->inp_inc.inc_ie.ie_dependfaddr,
+ &cache->faddr, sizeof(union in_dependaddr)) == 0);
+ MPASS(old->inp_fport == cache->fport);
+ MPASS(memcmp(&second->faddr, &cache->faddr,
+ sizeof(union in_dependaddr)) == 0);
+ MPASS(second->fport == cache->fport);
+ lport_merge(&cache->lports, &second->lports);
+ refcount_acquire(&cache->refcount);
+ if (refcount_release(&second->refcount))
+ free(second, M_LPORTS);
+ }
+out:
+ INP_WUNLOCK(old);
+}
+
+static void
+in_pcb_lport_cache_free(struct inpcb *inp)
+{
+ INP_WLOCK_ASSERT(inp);
+
+ if (inp->inp_lport_cache != NULL &&
+ refcount_release(&inp->inp_lport_cache->refcount))
+ free(inp->inp_lport_cache, M_LPORTS);
+ inp->inp_lport_cache = NULL;
+}
+
/*
* Assign a local port like in_pcb_lport(), but also used with connect()
* and a foreign address and port. If fsa is non-NULL, choose a local port
@@ -757,8 +861,10 @@
#ifdef INET6
const struct in6_addr *laddr6, *faddr6;
#endif
+ const bool use_cache = INP_WLOCKED(inp) && fsa != NULL;
INP_LOCK_ASSERT(inp);
+ MPASS(inp->inp_lport_cache == NULL);
if (inp->inp_flags & INP_HIGHPORT) {
first = V_ipport_hifirstauto; /* sysctl */
@@ -825,6 +931,10 @@
};
struct inpcb *tmpinp = NULL;
+ if (use_cache && inp->inp_lport_cache != NULL &&
+ lport_check(&inp->inp_lport_cache->lports, port))
+ goto next;
+
cycles++;
lport = htons(port);
@@ -843,6 +953,9 @@
M_NODOM, RT_ALL_FIBS);
}
#endif
+ if (use_cache && tmpinp != NULL &&
+ inp_trylock(tmpinp, INPLOOKUP_WLOCKPCB))
+ in_pcb_cache_lport(inp, tmpinp);
} else {
#ifdef INET6
if ((inp->inp_vflag & INP_IPV6) != 0) {
@@ -875,13 +988,19 @@
break;
}
inpcbinfo_ctx_release(&tmpctx);
+next:
++port;
if (port < first || port > last)
port = first;
}
- if (count == 0) /* completely used? */
+ if (count == 0) { /* completely used? */
+ in_pcb_lport_cache_free(inp);
return (EADDRNOTAVAIL);
+ }
+
+ if (use_cache && inp->inp_lport_cache != NULL)
+ lport_acquire(&inp->inp_lport_cache->lports, ntohs(lport));
*lportp = lport;
@@ -1525,6 +1644,7 @@
if (inp->inp_flags & INP_UNCONNECTED)
return;
+ in_pcb_lport_cache_free(inp);
in_pcbremhash(inp);
IPI_LOCK(inp->inp_pcbinfo);
CK_LIST_INSERT_HEAD(&inp->inp_pcbinfo->ipi_list_unconn.head, inp,
@@ -1952,6 +2072,7 @@
inp->inp_socket->so_pcb = NULL;
inp->inp_socket = NULL;
+ in_pcb_lport_cache_free(inp);
RO_INVALIDATE_CACHE(&inp->inp_route);
#ifdef MAC
mac_inpcb_destroy(inp);

File Metadata

Mime Type
text/plain
Expires
Fri, Jul 24, 3:49 AM (14 h, 22 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35183723
Default Alt Text
D58290.id182099.diff (5 KB)

Event Timeline