Index: sys/netinet6/frag6.c =================================================================== --- sys/netinet6/frag6.c +++ sys/netinet6/frag6.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -75,20 +76,26 @@ #define IP6REASS_NHASH (1 << IP6REASS_NHASH_LOG2) #define IP6REASS_HMASK (IP6REASS_NHASH - 1) -static void frag6_enq(struct ip6asfrag *, struct ip6asfrag *, - uint32_t bucket __unused); -static void frag6_deq(struct ip6asfrag *, uint32_t bucket __unused); -static void frag6_insque_head(struct ip6q *, struct ip6q *, - uint32_t bucket); -static void frag6_remque(struct ip6q *, uint32_t bucket); -static void frag6_freef(struct ip6q *, uint32_t bucket); - +TAILQ_HEAD(ip6qhead, ip6q); struct ip6qbucket { - struct ip6q ip6q; + struct ip6qhead packets; struct mtx lock; int count; }; +struct ip6asfrag { + TAILQ_ENTRY(ip6asfrag) ip6af_tq; + struct mbuf *ip6af_m; + int ip6af_offset; /* offset in ip6af_m to next header */ + int ip6af_frglen; /* fragmentable part length */ + int ip6af_off; /* fragment offset */ + u_int16_t ip6af_mff; /* more fragment bit in frag off */ +}; + +#define IP6_REASS_MBUF(ip6af) (*(struct mbuf **)&((ip6af)->ip6af_m)) + +static void frag6_freef(struct ip6q *, uint32_t bucket); + VNET_DEFINE_STATIC(volatile u_int, frag6_nfragpackets); volatile u_int frag6_nfrags = 0; VNET_DEFINE_STATIC(struct ip6qbucket, ip6q[IP6REASS_NHASH]); @@ -102,7 +109,7 @@ #define IP6Q_TRYLOCK(i) mtx_trylock(&V_ip6q[(i)].lock) #define IP6Q_LOCK_ASSERT(i) mtx_assert(&V_ip6q[(i)].lock, MA_OWNED) #define IP6Q_UNLOCK(i) mtx_unlock(&V_ip6q[(i)].lock) -#define IP6Q_HEAD(i) (&V_ip6q[(i)].ip6q) +#define IP6Q_HEAD(i) (&V_ip6q[(i)].packets) static MALLOC_DEFINE(M_FTABLE, "fragment", "fragment reassembly header"); @@ -153,14 +160,12 @@ void frag6_init(void) { - struct ip6q *q6; int i; V_ip6_maxfragpackets = IP6_MAXFRAGPACKETS; frag6_set_bucketsize(); for (i = 0; i < IP6REASS_NHASH; i++) { - q6 = IP6Q_HEAD(i); - q6->ip6q_next = q6->ip6q_prev = q6; + TAILQ_INIT(IP6Q_HEAD(i)); mtx_init(&V_ip6q[i].lock, "ip6qlock", NULL, MTX_DEF); V_ip6q[i].count = 0; } @@ -212,8 +217,9 @@ struct mbuf *m = *mp, *t; struct ip6_hdr *ip6; struct ip6_frag *ip6f; - struct ip6q *head, *q6; - struct ip6asfrag *af6, *ip6af, *af6dwn; + struct ip6q *q6; + struct ip6qhead *head; + struct ip6asfrag *af6, *ip6af, *af6tmp; struct in6_ifaddr *ia; int offset = *offp, nxt, i, next; int first_frag = 0; @@ -318,7 +324,7 @@ else if (atomic_load_int(&frag6_nfrags) >= (u_int)ip6_maxfrags) goto dropfrag; - for (q6 = head->ip6q_next; q6 != head; q6 = q6->ip6q_next) + TAILQ_FOREACH(q6, head, ip6q_tq) if (ip6f->ip6f_ident == q6->ip6q_ident && IN6_ARE_ADDR_EQUAL(&ip6->ip6_src, &q6->ip6q_src) && IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst, &q6->ip6q_dst) @@ -328,7 +334,7 @@ ) break; - if (q6 == head) { + if (q6 == NULL) { /* * the first fragment to arrive, create a reassembly queue. */ @@ -360,10 +366,11 @@ } mac_ip6q_create(m, q6); #endif - frag6_insque_head(q6, head, hash); + TAILQ_INSERT_HEAD(head, q6, ip6q_tq); + V_ip6q[hash].count++; /* ip6q_nxt will be filled afterwards, from 1st fragment */ - q6->ip6q_down = q6->ip6q_up = (struct ip6asfrag *)q6; + TAILQ_INIT(&q6->ip6q_frags); #ifdef notyet q6->ip6q_nxtp = (u_char *)nxtp; #endif @@ -415,10 +422,7 @@ * fragment already stored in the reassembly queue. */ if (fragoff == 0) { - for (af6 = q6->ip6q_down; af6 != (struct ip6asfrag *)q6; - af6 = af6dwn) { - af6dwn = af6->ip6af_down; - + TAILQ_FOREACH_SAFE(af6, &q6->ip6q_frags, ip6af_tq, af6tmp) { if (q6->ip6q_unfrglen + af6->ip6af_off + af6->ip6af_frglen > IPV6_MAXPACKET) { struct mbuf *merr = IP6_REASS_MBUF(af6); @@ -426,7 +430,7 @@ int erroff = af6->ip6af_offset; /* dequeue the fragment. */ - frag6_deq(af6, hash); + TAILQ_REMOVE(&q6->ip6q_frags, af6, ip6af_tq); free(af6, M_FTABLE); /* adjust pointer. */ @@ -486,8 +490,7 @@ /* * Find a segment which begins after this one does. */ - for (af6 = q6->ip6q_down; af6 != (struct ip6asfrag *)q6; - af6 = af6->ip6af_down) + TAILQ_FOREACH(af6, &q6->ip6q_frags, ip6af_tq) if (af6->ip6af_off > ip6af->ip6af_off) break; @@ -497,8 +500,12 @@ * our data already. If so, drop the data from the incoming * segment. If it provides all of our data, drop us. */ - if (af6->ip6af_up != (struct ip6asfrag *)q6) { - i = af6->ip6af_up->ip6af_off + af6->ip6af_up->ip6af_frglen + if (af6 != NULL) + af6tmp = TAILQ_PREV(af6, ip6fraghead, ip6af_tq); + else + af6tmp = TAILQ_LAST(&q6->ip6q_frags, ip6fraghead); + if (af6tmp != NULL) { + i = af6tmp->ip6af_off + af6tmp->ip6af_frglen - ip6af->ip6af_off; if (i > 0) { if (i >= ip6af->ip6af_frglen) @@ -513,18 +520,20 @@ * While we overlap succeeding segments trim them or, * if they are completely covered, dequeue them. */ - while (af6 != (struct ip6asfrag *)q6 && - ip6af->ip6af_off + ip6af->ip6af_frglen > af6->ip6af_off) { + while (af6 != NULL) { i = (ip6af->ip6af_off + ip6af->ip6af_frglen) - af6->ip6af_off; + if (i <= 0) + break; if (i < af6->ip6af_frglen) { af6->ip6af_frglen -= i; af6->ip6af_off += i; m_adj(IP6_REASS_MBUF(af6), i); break; } - af6 = af6->ip6af_down; - m_freem(IP6_REASS_MBUF(af6->ip6af_up)); - frag6_deq(af6->ip6af_up, hash); + af6tmp = TAILQ_NEXT(af6, ip6af_tq); + m_freem(IP6_REASS_MBUF(af6)); + TAILQ_REMOVE(&q6->ip6q_frags, af6, ip6af_tq); + af6 = af6tmp; } #else /* @@ -537,8 +546,12 @@ * Note: due to changes after disabling this part, mbuf passed to * m_adj() below now does not meet the requirement. */ - if (af6->ip6af_up != (struct ip6asfrag *)q6) { - i = af6->ip6af_up->ip6af_off + af6->ip6af_up->ip6af_frglen + if (af6 != NULL) + af6tmp = TAILQ_PREV(af6, ip6fraghead, ip6af_tq); + else + af6tmp = TAILQ_LAST(&q6->ip6q_frags, ip6fraghead); + if (af6tmp != NULL) { + i = af6tmp->ip6af_off + af6tmp->ip6af_frglen - ip6af->ip6af_off; if (i > 0) { #if 0 /* suppress the noisy log */ @@ -550,7 +563,7 @@ goto dropfrag; } } - if (af6 != (struct ip6asfrag *)q6) { + if (af6 != NULL) { i = (ip6af->ip6af_off + ip6af->ip6af_frglen) - af6->ip6af_off; if (i > 0) { #if 0 /* suppress the noisy log */ @@ -577,18 +590,20 @@ * Move to front of packet queue, as we are * the most recently active fragmented packet. */ - frag6_enq(ip6af, af6->ip6af_up, hash); + if (af6 != NULL) + TAILQ_INSERT_BEFORE(af6, ip6af, ip6af_tq); + else + TAILQ_INSERT_TAIL(&q6->ip6q_frags, ip6af, ip6af_tq); atomic_add_int(&frag6_nfrags, 1); q6->ip6q_nfrag++; #if 0 /* xxx */ - if (q6 != head->ip6q_next) { - frag6_remque(q6, hash); - frag6_insque_head(q6, head, hash); + if (q6 != TAILQ_FIRST(head)) { + TAILQ_REMOVE(head, q6, ip6q_tq); + TAILQ_INSERT_HEAD(head, q6, ip6q_tq); } #endif next = 0; - for (af6 = q6->ip6q_down; af6 != (struct ip6asfrag *)q6; - af6 = af6->ip6af_down) { + TAILQ_FOREACH(af6, &q6->ip6q_frags, ip6af_tq) { if (af6->ip6af_off != next) { if (q6->ip6q_nfrag > V_ip6_maxfragsperpacket) { IP6STAT_INC(ip6s_fragdropped); @@ -599,7 +614,8 @@ } next += af6->ip6af_frglen; } - if (af6->ip6af_up->ip6af_mff) { + af6 = TAILQ_LAST(&q6->ip6q_frags, ip6fraghead); + if (af6->ip6af_mff) { if (q6->ip6q_nfrag > V_ip6_maxfragsperpacket) { IP6STAT_INC(ip6s_fragdropped); frag6_freef(q6, hash); @@ -611,25 +627,22 @@ /* * Reassembly is complete; concatenate fragments. */ - ip6af = q6->ip6q_down; + ip6af = TAILQ_FIRST(&q6->ip6q_frags); t = m = IP6_REASS_MBUF(ip6af); - af6 = ip6af->ip6af_down; - frag6_deq(ip6af, hash); - while (af6 != (struct ip6asfrag *)q6) { + TAILQ_REMOVE(&q6->ip6q_frags, ip6af, ip6af_tq); + while ((af6 = TAILQ_FIRST(&q6->ip6q_frags)) != NULL) { m->m_pkthdr.csum_flags &= IP6_REASS_MBUF(af6)->m_pkthdr.csum_flags; m->m_pkthdr.csum_data += IP6_REASS_MBUF(af6)->m_pkthdr.csum_data; - af6dwn = af6->ip6af_down; - frag6_deq(af6, hash); + TAILQ_REMOVE(&q6->ip6q_frags, af6, ip6af_tq); while (t->m_next) t = t->m_next; m_adj(IP6_REASS_MBUF(af6), af6->ip6af_offset); m_demote_pkthdr(IP6_REASS_MBUF(af6)); m_cat(t, IP6_REASS_MBUF(af6)); free(af6, M_FTABLE); - af6 = af6dwn; } while (m->m_pkthdr.csum_data & 0xffff0000) @@ -649,7 +662,8 @@ #endif if (ip6_deletefraghdr(m, offset, M_NOWAIT) != 0) { - frag6_remque(q6, hash); + TAILQ_REMOVE(head, q6, ip6q_tq); + V_ip6q[hash].count--; atomic_subtract_int(&frag6_nfrags, q6->ip6q_nfrag); #ifdef MAC mac_ip6q_destroy(q6); @@ -666,7 +680,8 @@ m_copyback(m, ip6_get_prevhdr(m, offset), sizeof(uint8_t), (caddr_t)&nxt); - frag6_remque(q6, hash); + TAILQ_REMOVE(head, q6, ip6q_tq); + V_ip6q[hash].count--; atomic_subtract_int(&frag6_nfrags, q6->ip6q_nfrag); #ifdef MAC mac_ip6q_reassemble(q6, m); @@ -731,16 +746,14 @@ static void frag6_freef(struct ip6q *q6, uint32_t bucket) { - struct ip6asfrag *af6, *down6; + struct ip6asfrag *af6; IP6Q_LOCK_ASSERT(bucket); - for (af6 = q6->ip6q_down; af6 != (struct ip6asfrag *)q6; - af6 = down6) { + while ((af6 = TAILQ_FIRST(&q6->ip6q_frags)) != NULL) { struct mbuf *m = IP6_REASS_MBUF(af6); - down6 = af6->ip6af_down; - frag6_deq(af6, bucket); + TAILQ_REMOVE(&q6->ip6q_frags, af6, ip6af_tq); /* * Return ICMP time exceeded error for the 1st fragment. @@ -762,7 +775,8 @@ m_freem(m); free(af6, M_FTABLE); } - frag6_remque(q6, bucket); + TAILQ_REMOVE(IP6Q_HEAD(bucket), q6, ip6q_tq); + V_ip6q[bucket].count--; atomic_subtract_int(&frag6_nfrags, q6->ip6q_nfrag); #ifdef MAC mac_ip6q_destroy(q6); @@ -771,63 +785,6 @@ atomic_subtract_int(&V_frag6_nfragpackets, 1); } -/* - * Put an ip fragment on a reassembly chain. - * Like insque, but pointers in middle of structure. - */ -static void -frag6_enq(struct ip6asfrag *af6, struct ip6asfrag *up6, - uint32_t bucket __unused) -{ - - IP6Q_LOCK_ASSERT(bucket); - - af6->ip6af_up = up6; - af6->ip6af_down = up6->ip6af_down; - up6->ip6af_down->ip6af_up = af6; - up6->ip6af_down = af6; -} - -/* - * To frag6_enq as remque is to insque. - */ -static void -frag6_deq(struct ip6asfrag *af6, uint32_t bucket __unused) -{ - - IP6Q_LOCK_ASSERT(bucket); - - af6->ip6af_up->ip6af_down = af6->ip6af_down; - af6->ip6af_down->ip6af_up = af6->ip6af_up; -} - -static void -frag6_insque_head(struct ip6q *new, struct ip6q *old, uint32_t bucket) -{ - - IP6Q_LOCK_ASSERT(bucket); - KASSERT(IP6Q_HEAD(bucket) == old, - ("%s: attempt to insert at head of wrong bucket" - " (bucket=%u, old=%p)", __func__, bucket, old)); - - new->ip6q_prev = old; - new->ip6q_next = old->ip6q_next; - old->ip6q_next->ip6q_prev= new; - old->ip6q_next = new; - V_ip6q[bucket].count++; -} - -static void -frag6_remque(struct ip6q *p6, uint32_t bucket) -{ - - IP6Q_LOCK_ASSERT(bucket); - - p6->ip6q_prev->ip6q_next = p6->ip6q_next; - p6->ip6q_next->ip6q_prev = p6->ip6q_prev; - V_ip6q[bucket].count--; -} - /* * IPv6 reassembling timer processing; * if a timer expires on a reassembly @@ -837,7 +794,8 @@ frag6_slowtimo(void) { VNET_ITERATOR_DECL(vnet_iter); - struct ip6q *head, *q6; + struct ip6q *q6, *q6tmp; + struct ip6qhead *head; int i; VNET_LIST_RLOCK_NOSLEEP(); @@ -846,24 +804,12 @@ for (i = 0; i < IP6REASS_NHASH; i++) { IP6Q_LOCK(i); head = IP6Q_HEAD(i); - q6 = head->ip6q_next; - if (q6 == NULL) { - /* - * XXXJTL: This should never happen. This - * should turn into an assertion. - */ - IP6Q_UNLOCK(i); - continue; - } - while (q6 != head) { - --q6->ip6q_ttl; - q6 = q6->ip6q_next; - if (q6->ip6q_prev->ip6q_ttl == 0) { + TAILQ_FOREACH_SAFE(q6, head, ip6q_tq, q6tmp) + if (--q6->ip6q_ttl == 0) { IP6STAT_INC(ip6s_fragtimeout); /* XXX in6_ifstat_inc(ifp, ifs6_reass_fail) */ - frag6_freef(q6->ip6q_prev, i); + frag6_freef(q6, i); } - } /* * If we are over the maximum number of fragments * (due to the limit being lowered), drain off @@ -876,10 +822,10 @@ while ((V_ip6_maxfragpackets == 0 || (V_ip6_maxfragpackets > 0 && V_ip6q[i].count > V_ip6_maxfragbucketsize)) && - head->ip6q_prev != head) { + (q6 = TAILQ_LAST(head, ip6qhead)) != NULL) { IP6STAT_INC(ip6s_fragoverflow); /* XXX in6_ifstat_inc(ifp, ifs6_reass_fail) */ - frag6_freef(head->ip6q_prev, i); + frag6_freef(q6, i); } IP6Q_UNLOCK(i); } @@ -892,11 +838,11 @@ atomic_load_int(&V_frag6_nfragpackets) > (u_int)V_ip6_maxfragpackets) { IP6Q_LOCK(i); - head = IP6Q_HEAD(i); - if (head->ip6q_prev != head) { + q6 = TAILQ_LAST(IP6Q_HEAD(i), ip6qhead); + if (q6 != NULL) { IP6STAT_INC(ip6s_fragoverflow); /* XXX in6_ifstat_inc(ifp, ifs6_reass_fail) */ - frag6_freef(head->ip6q_prev, i); + frag6_freef(q6, i); } IP6Q_UNLOCK(i); i = (i + 1) % IP6REASS_NHASH; @@ -913,7 +859,7 @@ frag6_drain(void) { VNET_ITERATOR_DECL(vnet_iter); - struct ip6q *head; + struct ip6q *q6; int i; VNET_LIST_RLOCK_NOSLEEP(); @@ -922,11 +868,10 @@ for (i = 0; i < IP6REASS_NHASH; i++) { if (IP6Q_TRYLOCK(i) == 0) continue; - head = IP6Q_HEAD(i); - while (head->ip6q_next != head) { + while ((q6 = TAILQ_FIRST(IP6Q_HEAD(i))) != NULL) { IP6STAT_INC(ip6s_fragdropped); /* XXX in6_ifstat_inc(ifp, ifs6_reass_fail) */ - frag6_freef(head->ip6q_next, i); + frag6_freef(q6, i); } IP6Q_UNLOCK(i); } Index: sys/netinet6/ip6_var.h =================================================================== --- sys/netinet6/ip6_var.h +++ sys/netinet6/ip6_var.h @@ -68,20 +68,22 @@ #include +#ifdef _KERNEL +struct ip6asfrag; +TAILQ_HEAD(ip6fraghead, ip6asfrag); + /* * IP6 reassembly queue structure. Each fragment * being reassembled is attached to one of these structures. */ struct ip6q { - struct ip6asfrag *ip6q_down; - struct ip6asfrag *ip6q_up; + struct ip6fraghead ip6q_frags; u_int32_t ip6q_ident; u_int8_t ip6q_nxt; u_int8_t ip6q_ecn; u_int8_t ip6q_ttl; struct in6_addr ip6q_src, ip6q_dst; - struct ip6q *ip6q_next; - struct ip6q *ip6q_prev; + TAILQ_ENTRY(ip6q) ip6q_tq; int ip6q_unfrglen; /* len of unfragmentable part */ #ifdef notyet u_char *ip6q_nxtp; @@ -89,18 +91,7 @@ int ip6q_nfrag; /* # of fragments */ struct label *ip6q_label; }; - -struct ip6asfrag { - struct ip6asfrag *ip6af_down; - struct ip6asfrag *ip6af_up; - struct mbuf *ip6af_m; - int ip6af_offset; /* offset in ip6af_m to next header */ - int ip6af_frglen; /* fragmentable part length */ - int ip6af_off; /* fragment offset */ - u_int16_t ip6af_mff; /* more fragment bit in frag off */ -}; - -#define IP6_REASS_MBUF(ip6af) (*(struct mbuf **)&((ip6af)->ip6af_m)) +#endif /* _KERNEL */ /* * IP6 reinjecting structure.