Explicitly set IP ECN header codepoint properly

[rscheff]

Summary

TCP RACK appears to cache the IP header when preparing
a new TCP packet for transmission - other than the base
stack.

As the IP ECN codepoint is only "OR"ed in, bits can
apparently only be set, but not reset, if the ECN
protocol needs them changed.

By masking out the IP ECN codepoint first in the IP TOS
byte, and then setting it subsequently, the correct
IP ECN codepoints get set in the IP header.

Test Plan

// Simultaneous Open

Establish a regular TCP ECN connection.
+0.0 echo 'parallel open: ECN'
+0.10 sysctl net.inet.tcp.ecn.enable=1
+0.40 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3
+0.01 fcntl(3, F_SETFL, O_RDWR|O_NONBLOCK) = 0
+0.01 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
+0.01 setsockopt(3, SOL_SOCKET, SO_LINGER, {onoff=1, linger=0}, 8) = 0
+0.01 setsockopt(3, SOL_SOCKET, SO_DEBUG, [1], 4) = 0
+0.04 connect(3, ..., ...) = -1 EINPROGRESS
+0.00 >[noecn] SEW 0:0(0) win 65535 <...>
+0.00 <[noecn] SEW 0:0(0) ack 1 win 65535 <mss 1000>
+0.00 >[noecn] SE. 0:0(0) ack 1 win 65535 <...>
+0.00 <[noecn] SE. 0:0(0) ack 1 win 65535 <mss 1000>
+0.00 >[noecn] . 1:1(0) ack 1 win 65535
+0.01 getsockopt(3, SOL_SOCKET, SO_ERROR, [0], [4]) = 0
+0.01 fcntl(3, F_SETFL, O_RDWR) = 0
+0.05 write(3, ..., 1000) = 1000
+0.00 >[ect0] PW. 1:1001(1000) ack 1 win 65535
+0.00 <[noecn] E. 1:1(0) ack 1001 win 65535
+0.05 write(3, ..., 1000) = 1000
+0.00 >[ect0] PW. 1001:2001(1000) ack 1 win 65535
+0.00 <[ce] . 1:2(1) ack 2001 win 65535

// rack: ect0 prior to this change
+0.00 >[noecn] E. 2001:2001(0) ack 2 win 65535
+0.00 close(3) = 0

sys/netinet/tcp_output.c

  *       1      the cookie is valid and the latest cookie has been returned
  */
 int
 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie,
     unsigned int len, uint64_t *latest_cookie)
 {
         struct rm_priotracker tracker;
         unsigned int i, key_index;
+        int rv;
         uint64_t cur_cookie;
 
         if (V_tcp_fastopen_acceptany) {
                 *latest_cookie = 0;
                 return (1);
         }
 
+        TCP_FASTOPEN_KEYS_RLOCK(&tracker);
         if (len != TCP_FASTOPEN_COOKIE_LEN) {
                 if (V_tcp_fastopen_numkeys > 0) {
                         *latest_cookie =
                             tcp_fastopen_make_cookie(
                                 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest],
                                 inc);
-                        return (0);
-                }
-                return (-1);
-        }
+                        rv = 0;
+                } else
+                        rv = -1;
+                goto out;
 
         /*
          * Check against each available key, from newest to oldest.
          */
-        TCP_FASTOPEN_KEYS_RLOCK(&tracker);
         key_index = V_tcp_fastopen_keys.newest;
         for (i = 0; i < V_tcp_fastopen_numkeys; i++) {
                 cur_cookie =
                     tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index],
                         inc);
                 if (i == 0)
                         *latest_cookie = cur_cookie;
                 if (memcmp(cookie, &cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0) {
-                        TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
-                        return (1);
+                        rv = 1;
+                        goto out;
                 }
                 if (key_index == 0)
                         key_index = TCP_FASTOPEN_MAX_KEYS - 1;
                 else
                         key_index--;
         }
-        TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
+        rv = 0;
 
-        return (0);
+out:
+        TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
+        return (rv);
 }
 
 static int
 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)
 {
         int error;
         unsigned int new;
 

sys/netinet/tcp_stacks/rack.c

  *       1      the cookie is valid and the latest cookie has been returned
  */
 int
 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie,
     unsigned int len, uint64_t *latest_cookie)
 {
         struct rm_priotracker tracker;
         unsigned int i, key_index;
+        int rv;
         uint64_t cur_cookie;
 
         if (V_tcp_fastopen_acceptany) {
                 *latest_cookie = 0;
                 return (1);
         }
 
+        TCP_FASTOPEN_KEYS_RLOCK(&tracker);
         if (len != TCP_FASTOPEN_COOKIE_LEN) {
                 if (V_tcp_fastopen_numkeys > 0) {
                         *latest_cookie =
                             tcp_fastopen_make_cookie(
                                 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest],
                                 inc);
-                        return (0);
-                }
-                return (-1);
-        }
+                        rv = 0;
+                } else
+                        rv = -1;
+                goto out;
 
         /*
          * Check against each available key, from newest to oldest.
          */
-        TCP_FASTOPEN_KEYS_RLOCK(&tracker);
         key_index = V_tcp_fastopen_keys.newest;
         for (i = 0; i < V_tcp_fastopen_numkeys; i++) {
                 cur_cookie =
                     tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index],
                         inc);
                 if (i == 0)
                         *latest_cookie = cur_cookie;
                 if (memcmp(cookie, &cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0) {
-                        TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
-                        return (1);
+                        rv = 1;
+                        goto out;
                 }
                 if (key_index == 0)
                         key_index = TCP_FASTOPEN_MAX_KEYS - 1;
                 else
                         key_index--;
         }
-        TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
+        rv = 0;
 
-        return (0);
+out:
+        TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
+        return (rv);
 }
 
 static int
 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)
 {
         int error;
         unsigned int new;