Index: stable/11/sys/netinet/cc/cc_cdg.c =================================================================== --- stable/11/sys/netinet/cc/cc_cdg.c (revision 342188) +++ stable/11/sys/netinet/cc/cc_cdg.c (revision 342189) @@ -1,699 +1,712 @@ /*- * Copyright (c) 2009-2013 * Swinburne University of Technology, Melbourne, Australia * All rights reserved. * * This software was developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by David Hayes, made * possible in part by a gift from The Cisco University Research Program Fund, * a corporate advised fund of Silicon Valley Community Foundation. Development * and testing were further assisted by a grant from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * CAIA Delay-Gradient (CDG) congestion control algorithm * * An implemention of the delay-gradient congestion control algorithm proposed * in the following paper: * * D. A. Hayes and G. Armitage, "Revisiting TCP Congestion Control using Delay * Gradients", in IFIP Networking, Valencia, Spain, 9-13 May 2011. * * Developed as part of the NewTCP research project at Swinburne University of * Technology's Centre for Advanced Internet Architectures, Melbourne, * Australia. More details are available at: * http://caia.swin.edu.au/urp/newtcp/ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define CDG_VERSION "0.1" -#define CAST_PTR_INT(X) (*((int*)(X))) - /* Private delay-gradient induced congestion control signal. */ #define CC_CDG_DELAY 0x01000000 /* NewReno window deflation factor on loss (as a percentage). */ #define RENO_BETA 50 /* Queue states. */ #define CDG_Q_EMPTY 1 #define CDG_Q_RISING 2 #define CDG_Q_FALLING 3 #define CDG_Q_FULL 4 #define CDG_Q_UNKNOWN 9999 /* Number of bit shifts used in probexp lookup table. */ #define EXP_PREC 15 /* Largest gradient represented in probexp lookup table. */ #define MAXGRAD 5 /* * Delay Precision Enhance - number of bit shifts used for qtrend related * integer arithmetic precision. */ #define D_P_E 7 struct qdiff_sample { long qdiff; STAILQ_ENTRY(qdiff_sample) qdiff_lnk; }; struct cdg { long max_qtrend; long min_qtrend; STAILQ_HEAD(minrtts_head, qdiff_sample) qdiffmin_q; STAILQ_HEAD(maxrtts_head, qdiff_sample) qdiffmax_q; long window_incr; /* rttcount for window increase when in congestion avoidance */ long rtt_count; /* maximum measured rtt within an rtt period */ int maxrtt_in_rtt; /* maximum measured rtt within prev rtt period */ int maxrtt_in_prevrtt; /* minimum measured rtt within an rtt period */ int minrtt_in_rtt; /* minimum measured rtt within prev rtt period */ int minrtt_in_prevrtt; /* consecutive congestion episode counter */ uint32_t consec_cong_cnt; /* when tracking a new reno type loss window */ uint32_t shadow_w; /* maximum number of samples in the moving average queue */ int sample_q_size; /* number of samples in the moving average queue */ int num_samples; /* estimate of the queue state of the path */ int queue_state; }; /* * Lookup table for: * (1 - exp(-x)) << EXP_PREC, where x = [0,MAXGRAD] in 2^-7 increments * * Note: probexp[0] is set to 10 (not 0) as a safety for very low increase * gradients. */ static const int probexp[641] = { 10,255,508,759,1008,1255,1501,1744,1985,2225,2463,2698,2932,3165,3395,3624, 3850,4075,4299,4520,4740,4958,5175,5389,5602,5814,6024,6232,6438,6643,6846, 7048,7248,7447,7644,7839,8033,8226,8417,8606,8794,8981,9166,9350,9532,9713, 9892,10070,10247,10422,10596,10769,10940,11110,11278,11445,11611,11776,11939, 12101,12262,12422,12580,12737,12893,13048,13201,13354,13505,13655,13803,13951, 14097,14243,14387,14530,14672,14813,14952,15091,15229,15365,15500,15635,15768, 15900,16032,16162,16291,16419,16547,16673,16798,16922,17046,17168,17289,17410, 17529,17648,17766,17882,17998,18113,18227,18340,18453,18564,18675,18784,18893, 19001,19108,19215,19320,19425,19529,19632,19734,19835,19936,20036,20135,20233, 20331,20427,20523,20619,20713,20807,20900,20993,21084,21175,21265,21355,21444, 21532,21619,21706,21792,21878,21962,22046,22130,22213,22295,22376,22457,22537, 22617,22696,22774,22852,22929,23006,23082,23157,23232,23306,23380,23453,23525, 23597,23669,23739,23810,23879,23949,24017,24085,24153,24220,24286,24352,24418, 24483,24547,24611,24675,24738,24800,24862,24924,24985,25045,25106,25165,25224, 25283,25341,25399,25456,25513,25570,25626,25681,25737,25791,25846,25899,25953, 26006,26059,26111,26163,26214,26265,26316,26366,26416,26465,26514,26563,26611, 26659,26707,26754,26801,26847,26893,26939,26984,27029,27074,27118,27162,27206, 27249,27292,27335,27377,27419,27460,27502,27543,27583,27624,27664,27703,27743, 27782,27821,27859,27897,27935,27973,28010,28047,28084,28121,28157,28193,28228, 28263,28299,28333,28368,28402,28436,28470,28503,28536,28569,28602,28634,28667, 28699,28730,28762,28793,28824,28854,28885,28915,28945,28975,29004,29034,29063, 29092,29120,29149,29177,29205,29232,29260,29287,29314,29341,29368,29394,29421, 29447,29472,29498,29524,29549,29574,29599,29623,29648,29672,29696,29720,29744, 29767,29791,29814,29837,29860,29882,29905,29927,29949,29971,29993,30014,30036, 30057,30078,30099,30120,30141,30161,30181,30201,30221,30241,30261,30280,30300, 30319,30338,30357,30376,30394,30413,30431,30449,30467,30485,30503,30521,30538, 30555,30573,30590,30607,30624,30640,30657,30673,30690,30706,30722,30738,30753, 30769,30785,30800,30815,30831,30846,30861,30876,30890,30905,30919,30934,30948, 30962,30976,30990,31004,31018,31031,31045,31058,31072,31085,31098,31111,31124, 31137,31149,31162,31174,31187,31199,31211,31223,31235,31247,31259,31271,31283, 31294,31306,31317,31328,31339,31351,31362,31373,31383,31394,31405,31416,31426, 31436,31447,31457,31467,31477,31487,31497,31507,31517,31527,31537,31546,31556, 31565,31574,31584,31593,31602,31611,31620,31629,31638,31647,31655,31664,31673, 31681,31690,31698,31706,31715,31723,31731,31739,31747,31755,31763,31771,31778, 31786,31794,31801,31809,31816,31824,31831,31838,31846,31853,31860,31867,31874, 31881,31888,31895,31902,31908,31915,31922,31928,31935,31941,31948,31954,31960, 31967,31973,31979,31985,31991,31997,32003,32009,32015,32021,32027,32033,32038, 32044,32050,32055,32061,32066,32072,32077,32083,32088,32093,32098,32104,32109, 32114,32119,32124,32129,32134,32139,32144,32149,32154,32158,32163,32168,32173, 32177,32182,32186,32191,32195,32200,32204,32209,32213,32217,32222,32226,32230, 32234,32238,32242,32247,32251,32255,32259,32263,32267,32270,32274,32278,32282, 32286,32290,32293,32297,32301,32304,32308,32311,32315,32318,32322,32325,32329, 32332,32336,32339,32342,32346,32349,32352,32356,32359,32362,32365,32368,32371, 32374,32377,32381,32384,32387,32389,32392,32395,32398,32401,32404,32407,32410, 32412,32415,32418,32421,32423,32426,32429,32431,32434,32437,32439,32442,32444, 32447,32449,32452,32454,32457,32459,32461,32464,32466,32469,32471,32473,32476, 32478,32480,32482,32485,32487,32489,32491,32493,32495,32497,32500,32502,32504, 32506,32508,32510,32512,32514,32516,32518,32520,32522,32524,32526,32527,32529, 32531,32533,32535,32537,32538,32540,32542,32544,32545,32547}; static uma_zone_t qdiffsample_zone; static MALLOC_DEFINE(M_CDG, "cdg data", "Per connection data required for the CDG congestion control algorithm"); static int ertt_id; static VNET_DEFINE(uint32_t, cdg_alpha_inc); static VNET_DEFINE(uint32_t, cdg_beta_delay); static VNET_DEFINE(uint32_t, cdg_beta_loss); static VNET_DEFINE(uint32_t, cdg_smoothing_factor); static VNET_DEFINE(uint32_t, cdg_exp_backoff_scale); static VNET_DEFINE(uint32_t, cdg_consec_cong); static VNET_DEFINE(uint32_t, cdg_hold_backoff); #define V_cdg_alpha_inc VNET(cdg_alpha_inc) #define V_cdg_beta_delay VNET(cdg_beta_delay) #define V_cdg_beta_loss VNET(cdg_beta_loss) #define V_cdg_smoothing_factor VNET(cdg_smoothing_factor) #define V_cdg_exp_backoff_scale VNET(cdg_exp_backoff_scale) #define V_cdg_consec_cong VNET(cdg_consec_cong) #define V_cdg_hold_backoff VNET(cdg_hold_backoff) /* Function prototypes. */ static int cdg_mod_init(void); static int cdg_mod_destroy(void); static void cdg_conn_init(struct cc_var *ccv); static int cdg_cb_init(struct cc_var *ccv); static void cdg_cb_destroy(struct cc_var *ccv); static void cdg_cong_signal(struct cc_var *ccv, uint32_t signal_type); static void cdg_ack_received(struct cc_var *ccv, uint16_t ack_type); struct cc_algo cdg_cc_algo = { .name = "cdg", .mod_init = cdg_mod_init, .ack_received = cdg_ack_received, .cb_destroy = cdg_cb_destroy, .cb_init = cdg_cb_init, .conn_init = cdg_conn_init, .cong_signal = cdg_cong_signal, .mod_destroy = cdg_mod_destroy }; /* Vnet created and being initialised. */ static void cdg_init_vnet(const void *unused __unused) { V_cdg_alpha_inc = 0; V_cdg_beta_delay = 70; V_cdg_beta_loss = 50; V_cdg_smoothing_factor = 8; V_cdg_exp_backoff_scale = 3; V_cdg_consec_cong = 5; V_cdg_hold_backoff = 5; } static int cdg_mod_init(void) { VNET_ITERATOR_DECL(v); ertt_id = khelp_get_id("ertt"); if (ertt_id <= 0) return (EINVAL); qdiffsample_zone = uma_zcreate("cdg_qdiffsample", sizeof(struct qdiff_sample), NULL, NULL, NULL, NULL, 0, 0); VNET_LIST_RLOCK(); VNET_FOREACH(v) { CURVNET_SET(v); cdg_init_vnet(NULL); CURVNET_RESTORE(); } VNET_LIST_RUNLOCK(); cdg_cc_algo.post_recovery = newreno_cc_algo.post_recovery; cdg_cc_algo.after_idle = newreno_cc_algo.after_idle; return (0); } static int cdg_mod_destroy(void) { uma_zdestroy(qdiffsample_zone); return (0); } static int cdg_cb_init(struct cc_var *ccv) { struct cdg *cdg_data; cdg_data = malloc(sizeof(struct cdg), M_CDG, M_NOWAIT); if (cdg_data == NULL) return (ENOMEM); cdg_data->shadow_w = 0; cdg_data->max_qtrend = 0; cdg_data->min_qtrend = 0; cdg_data->queue_state = CDG_Q_UNKNOWN; cdg_data->maxrtt_in_rtt = 0; cdg_data->maxrtt_in_prevrtt = 0; cdg_data->minrtt_in_rtt = INT_MAX; cdg_data->minrtt_in_prevrtt = 0; cdg_data->window_incr = 0; cdg_data->rtt_count = 0; cdg_data->consec_cong_cnt = 0; cdg_data->sample_q_size = V_cdg_smoothing_factor; cdg_data->num_samples = 0; STAILQ_INIT(&cdg_data->qdiffmin_q); STAILQ_INIT(&cdg_data->qdiffmax_q); ccv->cc_data = cdg_data; return (0); } static void cdg_conn_init(struct cc_var *ccv) { struct cdg *cdg_data = ccv->cc_data; /* * Initialise the shadow_cwnd in case we are competing with loss based * flows from the start */ cdg_data->shadow_w = CCV(ccv, snd_cwnd); } static void cdg_cb_destroy(struct cc_var *ccv) { struct cdg *cdg_data; struct qdiff_sample *qds, *qds_n; cdg_data = ccv->cc_data; qds = STAILQ_FIRST(&cdg_data->qdiffmin_q); while (qds != NULL) { qds_n = STAILQ_NEXT(qds, qdiff_lnk); uma_zfree(qdiffsample_zone,qds); qds = qds_n; } qds = STAILQ_FIRST(&cdg_data->qdiffmax_q); while (qds != NULL) { qds_n = STAILQ_NEXT(qds, qdiff_lnk); uma_zfree(qdiffsample_zone,qds); qds = qds_n; } free(ccv->cc_data, M_CDG); } static int cdg_beta_handler(SYSCTL_HANDLER_ARGS) { + int error; + uint32_t new; - if (req->newptr != NULL && - (CAST_PTR_INT(req->newptr) == 0 || CAST_PTR_INT(req->newptr) > 100)) - return (EINVAL); + new = *(uint32_t *)arg1; + error = sysctl_handle_int(oidp, &new, 0, req); + if (error == 0 && req->newptr != NULL) { + if (new == 0 || new > 100) + error = EINVAL; + else + *(uint32_t *)arg1 = new; + } - return (sysctl_handle_int(oidp, arg1, arg2, req)); + return (error); } static int cdg_exp_backoff_scale_handler(SYSCTL_HANDLER_ARGS) { + int error; + uint32_t new; - if (req->newptr != NULL && CAST_PTR_INT(req->newptr) < 1) - return (EINVAL); + new = *(uint32_t *)arg1; + error = sysctl_handle_int(oidp, &new, 0, req); + if (error == 0 && req->newptr != NULL) { + if (new < 1) + error = EINVAL; + else + *(uint32_t *)arg1 = new; + } - return (sysctl_handle_int(oidp, arg1, arg2, req)); + return (error); } static inline unsigned long cdg_window_decrease(struct cc_var *ccv, unsigned long owin, unsigned int beta) { return ((ulmin(CCV(ccv, snd_wnd), owin) * beta) / 100); } /* * Window increase function * This window increase function is independent of the initial window size * to ensure small window flows are not discriminated against (i.e. fairness). * It increases at 1pkt/rtt like Reno for alpha_inc rtts, and then 2pkts/rtt for * the next alpha_inc rtts, etc. */ static void cdg_window_increase(struct cc_var *ccv, int new_measurement) { struct cdg *cdg_data; int incr, s_w_incr; cdg_data = ccv->cc_data; incr = s_w_incr = 0; if (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh)) { /* Slow start. */ incr = CCV(ccv, t_maxseg); s_w_incr = incr; cdg_data->window_incr = cdg_data->rtt_count = 0; } else { /* Congestion avoidance. */ if (new_measurement) { s_w_incr = CCV(ccv, t_maxseg); if (V_cdg_alpha_inc == 0) { incr = CCV(ccv, t_maxseg); } else { if (++cdg_data->rtt_count >= V_cdg_alpha_inc) { cdg_data->window_incr++; cdg_data->rtt_count = 0; } incr = CCV(ccv, t_maxseg) * cdg_data->window_incr; } } } if (cdg_data->shadow_w > 0) cdg_data->shadow_w = ulmin(cdg_data->shadow_w + s_w_incr, TCP_MAXWIN << CCV(ccv, snd_scale)); CCV(ccv, snd_cwnd) = ulmin(CCV(ccv, snd_cwnd) + incr, TCP_MAXWIN << CCV(ccv, snd_scale)); } static void cdg_cong_signal(struct cc_var *ccv, uint32_t signal_type) { struct cdg *cdg_data = ccv->cc_data; switch(signal_type) { case CC_CDG_DELAY: CCV(ccv, snd_ssthresh) = cdg_window_decrease(ccv, CCV(ccv, snd_cwnd), V_cdg_beta_delay); CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); CCV(ccv, snd_recover) = CCV(ccv, snd_max); cdg_data->window_incr = cdg_data->rtt_count = 0; ENTER_CONGRECOVERY(CCV(ccv, t_flags)); break; case CC_NDUPACK: /* * If already responding to congestion OR we have guessed no * queue in the path is full. */ if (IN_CONGRECOVERY(CCV(ccv, t_flags)) || cdg_data->queue_state < CDG_Q_FULL) { CCV(ccv, snd_ssthresh) = CCV(ccv, snd_cwnd); CCV(ccv, snd_recover) = CCV(ccv, snd_max); } else { /* * Loss is likely to be congestion related. We have * inferred a queue full state, so have shadow window * react to loss as NewReno would. */ if (cdg_data->shadow_w > 0) cdg_data->shadow_w = cdg_window_decrease(ccv, cdg_data->shadow_w, RENO_BETA); CCV(ccv, snd_ssthresh) = ulmax(cdg_data->shadow_w, cdg_window_decrease(ccv, CCV(ccv, snd_cwnd), V_cdg_beta_loss)); cdg_data->window_incr = cdg_data->rtt_count = 0; } ENTER_RECOVERY(CCV(ccv, t_flags)); break; default: newreno_cc_algo.cong_signal(ccv, signal_type); break; } } /* * Using a negative exponential probabilistic backoff so that sources with * varying RTTs which share the same link will, on average, have the same * probability of backoff over time. * * Prob_backoff = 1 - exp(-qtrend / V_cdg_exp_backoff_scale), where * V_cdg_exp_backoff_scale is the average qtrend for the exponential backoff. */ static inline int prob_backoff(long qtrend) { int backoff, idx, p; backoff = (qtrend > ((MAXGRAD * V_cdg_exp_backoff_scale) << D_P_E)); if (!backoff) { if (V_cdg_exp_backoff_scale > 1) idx = (qtrend + V_cdg_exp_backoff_scale / 2) / V_cdg_exp_backoff_scale; else idx = qtrend; /* Backoff probability proportional to rate of queue growth. */ p = (INT_MAX / (1 << EXP_PREC)) * probexp[idx]; backoff = (random() < p); } return (backoff); } static inline void calc_moving_average(struct cdg *cdg_data, long qdiff_max, long qdiff_min) { struct qdiff_sample *qds; ++cdg_data->num_samples; if (cdg_data->num_samples > cdg_data->sample_q_size) { /* Minimum RTT. */ qds = STAILQ_FIRST(&cdg_data->qdiffmin_q); cdg_data->min_qtrend = cdg_data->min_qtrend + (qdiff_min - qds->qdiff) / cdg_data->sample_q_size; STAILQ_REMOVE_HEAD(&cdg_data->qdiffmin_q, qdiff_lnk); qds->qdiff = qdiff_min; STAILQ_INSERT_TAIL(&cdg_data->qdiffmin_q, qds, qdiff_lnk); /* Maximum RTT. */ qds = STAILQ_FIRST(&cdg_data->qdiffmax_q); cdg_data->max_qtrend = cdg_data->max_qtrend + (qdiff_max - qds->qdiff) / cdg_data->sample_q_size; STAILQ_REMOVE_HEAD(&cdg_data->qdiffmax_q, qdiff_lnk); qds->qdiff = qdiff_max; STAILQ_INSERT_TAIL(&cdg_data->qdiffmax_q, qds, qdiff_lnk); --cdg_data->num_samples; } else { qds = uma_zalloc(qdiffsample_zone, M_NOWAIT); if (qds != NULL) { cdg_data->min_qtrend = cdg_data->min_qtrend + qdiff_min / cdg_data->sample_q_size; qds->qdiff = qdiff_min; STAILQ_INSERT_TAIL(&cdg_data->qdiffmin_q, qds, qdiff_lnk); } qds = uma_zalloc(qdiffsample_zone, M_NOWAIT); if (qds) { cdg_data->max_qtrend = cdg_data->max_qtrend + qdiff_max / cdg_data->sample_q_size; qds->qdiff = qdiff_max; STAILQ_INSERT_TAIL(&cdg_data->qdiffmax_q, qds, qdiff_lnk); } } } static void cdg_ack_received(struct cc_var *ccv, uint16_t ack_type) { struct cdg *cdg_data; struct ertt *e_t; long qdiff_max, qdiff_min; int congestion, new_measurement, slowstart; cdg_data = ccv->cc_data; e_t = (struct ertt *)khelp_get_osd(CCV(ccv, osd), ertt_id); new_measurement = e_t->flags & ERTT_NEW_MEASUREMENT; congestion = 0; cdg_data->maxrtt_in_rtt = imax(e_t->rtt, cdg_data->maxrtt_in_rtt); cdg_data->minrtt_in_rtt = imin(e_t->rtt, cdg_data->minrtt_in_rtt); if (new_measurement) { slowstart = (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh)); /* * Update smoothed gradient measurements. Since we are only * using one measurement per RTT, use max or min rtt_in_rtt. * This is also less noisy than a sample RTT measurement. Max * RTT measurements can have trouble due to OS issues. */ if (cdg_data->maxrtt_in_prevrtt) { qdiff_max = ((long)(cdg_data->maxrtt_in_rtt - cdg_data->maxrtt_in_prevrtt) << D_P_E ); qdiff_min = ((long)(cdg_data->minrtt_in_rtt - cdg_data->minrtt_in_prevrtt) << D_P_E ); calc_moving_average(cdg_data, qdiff_max, qdiff_min); /* Probabilistic backoff with respect to gradient. */ if (slowstart && qdiff_min > 0) congestion = prob_backoff(qdiff_min); else if (cdg_data->min_qtrend > 0) congestion = prob_backoff(cdg_data->min_qtrend); else if (slowstart && qdiff_max > 0) congestion = prob_backoff(qdiff_max); else if (cdg_data->max_qtrend > 0) congestion = prob_backoff(cdg_data->max_qtrend); /* Update estimate of queue state. */ if (cdg_data->min_qtrend > 0 && cdg_data->max_qtrend <= 0) { cdg_data->queue_state = CDG_Q_FULL; } else if (cdg_data->min_qtrend >= 0 && cdg_data->max_qtrend < 0) { cdg_data->queue_state = CDG_Q_EMPTY; cdg_data->shadow_w = 0; } else if (cdg_data->min_qtrend > 0 && cdg_data->max_qtrend > 0) { cdg_data->queue_state = CDG_Q_RISING; } else if (cdg_data->min_qtrend < 0 && cdg_data->max_qtrend < 0) { cdg_data->queue_state = CDG_Q_FALLING; } if (cdg_data->min_qtrend < 0 || cdg_data->max_qtrend < 0) cdg_data->consec_cong_cnt = 0; } cdg_data->minrtt_in_prevrtt = cdg_data->minrtt_in_rtt; cdg_data->minrtt_in_rtt = INT_MAX; cdg_data->maxrtt_in_prevrtt = cdg_data->maxrtt_in_rtt; cdg_data->maxrtt_in_rtt = 0; e_t->flags &= ~ERTT_NEW_MEASUREMENT; } if (congestion) { cdg_data->consec_cong_cnt++; if (!IN_RECOVERY(CCV(ccv, t_flags))) { if (cdg_data->consec_cong_cnt <= V_cdg_consec_cong) cdg_cong_signal(ccv, CC_CDG_DELAY); else /* * We have been backing off but the queue is not * falling. Assume we are competing with * loss-based flows and don't back off for the * next V_cdg_hold_backoff RTT periods. */ if (cdg_data->consec_cong_cnt >= V_cdg_consec_cong + V_cdg_hold_backoff) cdg_data->consec_cong_cnt = 0; /* Won't see effect until 2nd RTT. */ cdg_data->maxrtt_in_prevrtt = 0; /* * Resync shadow window in case we are competing with a * loss based flow */ cdg_data->shadow_w = ulmax(CCV(ccv, snd_cwnd), cdg_data->shadow_w); } } else if (ack_type == CC_ACK) cdg_window_increase(ccv, new_measurement); } /* When a vnet is created and being initialised, init the per-stack CDG vars. */ VNET_SYSINIT(cdg_init_vnet, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST, cdg_init_vnet, NULL); SYSCTL_DECL(_net_inet_tcp_cc_cdg); SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, cdg, CTLFLAG_RW, NULL, "CAIA delay-gradient congestion control related settings"); SYSCTL_STRING(_net_inet_tcp_cc_cdg, OID_AUTO, version, CTLFLAG_RD, CDG_VERSION, sizeof(CDG_VERSION) - 1, "Current algorithm/implementation version number"); SYSCTL_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, alpha_inc, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(cdg_alpha_inc), 0, "Increment the window increase factor alpha by 1 MSS segment every " "alpha_inc RTTs during congestion avoidance mode."); SYSCTL_PROC(_net_inet_tcp_cc_cdg, OID_AUTO, beta_delay, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(cdg_beta_delay), 70, &cdg_beta_handler, "IU", "Delay-based window decrease factor as a percentage " "(on delay-based backoff, w = w * beta_delay / 100)"); SYSCTL_PROC(_net_inet_tcp_cc_cdg, OID_AUTO, beta_loss, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(cdg_beta_loss), 50, &cdg_beta_handler, "IU", "Loss-based window decrease factor as a percentage " "(on loss-based backoff, w = w * beta_loss / 100)"); SYSCTL_PROC(_net_inet_tcp_cc_cdg, OID_AUTO, exp_backoff_scale, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(cdg_exp_backoff_scale), 2, &cdg_exp_backoff_scale_handler, "IU", "Scaling parameter for the probabilistic exponential backoff"); SYSCTL_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, smoothing_factor, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(cdg_smoothing_factor), 8, "Number of samples used for moving average smoothing (0 = no smoothing)"); SYSCTL_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, loss_compete_consec_cong, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(cdg_consec_cong), 5, "Number of consecutive delay-gradient based congestion episodes which will " "trigger loss based CC compatibility"); SYSCTL_UINT(_net_inet_tcp_cc_cdg, OID_AUTO, loss_compete_hold_backoff, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(cdg_hold_backoff), 5, "Number of consecutive delay-gradient based congestion episodes to hold " "the window backoff for loss based CC compatibility"); DECLARE_CC_MODULE(cdg, &cdg_cc_algo); MODULE_DEPEND(cdg, ertt, 1, 1, 1); Index: stable/11/sys/netinet/cc/cc_chd.c =================================================================== --- stable/11/sys/netinet/cc/cc_chd.c (revision 342188) +++ stable/11/sys/netinet/cc/cc_chd.c (revision 342189) @@ -1,498 +1,495 @@ /*- * Copyright (c) 2009-2010 * Swinburne University of Technology, Melbourne, Australia * Copyright (c) 2010-2011 The FreeBSD Foundation * All rights reserved. * * This software was developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by David Hayes and * Lawrence Stewart, made possible in part by a grant from the Cisco University * Research Program Fund at Community Foundation Silicon Valley. * * Portions of this software were developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, Melbourne, Australia by * David Hayes under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * An implementation of the CAIA-Hamilton delay based congestion control * algorithm, based on "Improved coexistence and loss tolerance for delay based * TCP congestion control" by D. A. Hayes and G. Armitage., in 35th Annual IEEE * Conference on Local Computer Networks (LCN 2010), Denver, Colorado, USA, * 11-14 October 2010. * * Originally released as part of the NewTCP research project at Swinburne * University of Technology's Centre for Advanced Internet Architectures, * Melbourne, Australia, which was made possible in part by a grant from the * Cisco University Research Program Fund at Community Foundation Silicon * Valley. More details are available at: * http://caia.swin.edu.au/urp/newtcp/ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#define CAST_PTR_INT(X) (*((int*)(X))) - /* * Private signal type for rate based congestion signal. * See for appropriate bit-range to use for private signals. */ #define CC_CHD_DELAY 0x02000000 /* Largest possible number returned by random(). */ #define RANDOM_MAX INT_MAX static void chd_ack_received(struct cc_var *ccv, uint16_t ack_type); static void chd_cb_destroy(struct cc_var *ccv); static int chd_cb_init(struct cc_var *ccv); static void chd_cong_signal(struct cc_var *ccv, uint32_t signal_type); static void chd_conn_init(struct cc_var *ccv); static int chd_mod_init(void); struct chd { /* * Shadow window - keeps track of what the NewReno congestion window * would have been if delay-based cwnd backoffs had not been made. This * functionality aids coexistence with loss-based TCP flows which may be * sharing links along the path. */ unsigned long shadow_w; /* * Loss-based TCP compatibility flag - When set, it turns on the shadow * window functionality. */ int loss_compete; /* The maximum round trip time seen within a measured rtt period. */ int maxrtt_in_rtt; /* The previous qdly that caused cwnd to backoff. */ int prev_backoff_qdly; }; static int ertt_id; static VNET_DEFINE(uint32_t, chd_qmin) = 5; static VNET_DEFINE(uint32_t, chd_pmax) = 50; static VNET_DEFINE(uint32_t, chd_loss_fair) = 1; static VNET_DEFINE(uint32_t, chd_use_max) = 1; static VNET_DEFINE(uint32_t, chd_qthresh) = 20; #define V_chd_qthresh VNET(chd_qthresh) #define V_chd_qmin VNET(chd_qmin) #define V_chd_pmax VNET(chd_pmax) #define V_chd_loss_fair VNET(chd_loss_fair) #define V_chd_use_max VNET(chd_use_max) static MALLOC_DEFINE(M_CHD, "chd data", "Per connection data required for the CHD congestion control algorithm"); struct cc_algo chd_cc_algo = { .name = "chd", .ack_received = chd_ack_received, .cb_destroy = chd_cb_destroy, .cb_init = chd_cb_init, .cong_signal = chd_cong_signal, .conn_init = chd_conn_init, .mod_init = chd_mod_init }; static __inline void chd_window_decrease(struct cc_var *ccv) { unsigned long win; win = min(CCV(ccv, snd_wnd), CCV(ccv, snd_cwnd)) / CCV(ccv, t_maxseg); win -= max((win / 2), 1); CCV(ccv, snd_ssthresh) = max(win, 2) * CCV(ccv, t_maxseg); } /* * Probabilistic backoff function. Returns 1 if we should backoff or 0 * otherwise. The calculation of p is similar to the calculation of p in cc_hd. */ static __inline int should_backoff(int qdly, int maxqdly, struct chd *chd_data) { unsigned long p, rand; rand = random(); if (qdly < V_chd_qthresh) { chd_data->loss_compete = 0; p = (((RANDOM_MAX / 100) * V_chd_pmax) / (V_chd_qthresh - V_chd_qmin)) * (qdly - V_chd_qmin); } else { if (qdly > V_chd_qthresh) { p = (((RANDOM_MAX / 100) * V_chd_pmax) / (maxqdly - V_chd_qthresh)) * (maxqdly - qdly); if (V_chd_loss_fair && rand < p) chd_data->loss_compete = 1; } else { p = (RANDOM_MAX / 100) * V_chd_pmax; chd_data->loss_compete = 0; } } return (rand < p); } static __inline void chd_window_increase(struct cc_var *ccv, int new_measurement) { struct chd *chd_data; int incr; chd_data = ccv->cc_data; incr = 0; if (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh)) { /* Adapted from NewReno slow start. */ if (V_tcp_do_rfc3465) { /* In slow-start with ABC enabled. */ if (CCV(ccv, snd_nxt) == CCV(ccv, snd_max)) { /* Not due to RTO. */ incr = min(ccv->bytes_this_ack, V_tcp_abc_l_var * CCV(ccv, t_maxseg)); } else { /* Due to RTO. */ incr = min(ccv->bytes_this_ack, CCV(ccv, t_maxseg)); } } else incr = CCV(ccv, t_maxseg); } else { /* Congestion avoidance. */ if (V_tcp_do_rfc3465) { if (ccv->flags & CCF_ABC_SENTAWND) { ccv->flags &= ~CCF_ABC_SENTAWND; incr = CCV(ccv, t_maxseg); } } else if (new_measurement) incr = CCV(ccv, t_maxseg); } if (chd_data->shadow_w > 0) { /* Track NewReno window. */ chd_data->shadow_w = min(chd_data->shadow_w + incr, TCP_MAXWIN << CCV(ccv, snd_scale)); } CCV(ccv,snd_cwnd) = min(CCV(ccv, snd_cwnd) + incr, TCP_MAXWIN << CCV(ccv, snd_scale)); } /* * All ACK signals are used for timing measurements to determine delay-based * congestion. However, window increases are only performed when * ack_type == CC_ACK. */ static void chd_ack_received(struct cc_var *ccv, uint16_t ack_type) { struct chd *chd_data; struct ertt *e_t; int backoff, new_measurement, qdly, rtt; e_t = khelp_get_osd(CCV(ccv, osd), ertt_id); chd_data = ccv->cc_data; new_measurement = e_t->flags & ERTT_NEW_MEASUREMENT; backoff = qdly = 0; chd_data->maxrtt_in_rtt = imax(e_t->rtt, chd_data->maxrtt_in_rtt); if (new_measurement) { /* * There is a new per RTT measurement, so check to see if there * is delay based congestion. */ rtt = V_chd_use_max ? chd_data->maxrtt_in_rtt : e_t->rtt; chd_data->maxrtt_in_rtt = 0; if (rtt && e_t->minrtt && !IN_RECOVERY(CCV(ccv, t_flags))) { qdly = rtt - e_t->minrtt; if (qdly > V_chd_qmin) { /* * Probabilistic delay based congestion * indication. */ backoff = should_backoff(qdly, e_t->maxrtt - e_t->minrtt, chd_data); } else chd_data->loss_compete = 0; } /* Reset per RTT measurement flag to start a new measurement. */ e_t->flags &= ~ERTT_NEW_MEASUREMENT; } if (backoff) { /* * Update shadow_w before delay based backoff. */ if (chd_data->loss_compete || qdly > chd_data->prev_backoff_qdly) { /* * Delay is higher than when we backed off previously, * so it is possible that this flow is competing with * loss based flows. */ chd_data->shadow_w = max(CCV(ccv, snd_cwnd), chd_data->shadow_w); } else { /* * Reset shadow_w, as it is probable that this flow is * not competing with loss based flows at the moment. */ chd_data->shadow_w = 0; } chd_data->prev_backoff_qdly = qdly; /* * Send delay-based congestion signal to the congestion signal * handler. */ chd_cong_signal(ccv, CC_CHD_DELAY); } else if (ack_type == CC_ACK) chd_window_increase(ccv, new_measurement); } static void chd_cb_destroy(struct cc_var *ccv) { if (ccv->cc_data != NULL) free(ccv->cc_data, M_CHD); } static int chd_cb_init(struct cc_var *ccv) { struct chd *chd_data; chd_data = malloc(sizeof(struct chd), M_CHD, M_NOWAIT); if (chd_data == NULL) return (ENOMEM); chd_data->shadow_w = 0; ccv->cc_data = chd_data; return (0); } static void chd_cong_signal(struct cc_var *ccv, uint32_t signal_type) { struct ertt *e_t; struct chd *chd_data; int qdly; e_t = khelp_get_osd(CCV(ccv, osd), ertt_id); chd_data = ccv->cc_data; qdly = imax(e_t->rtt, chd_data->maxrtt_in_rtt) - e_t->minrtt; switch(signal_type) { case CC_CHD_DELAY: chd_window_decrease(ccv); /* Set new ssthresh. */ CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); CCV(ccv, snd_recover) = CCV(ccv, snd_max); ENTER_CONGRECOVERY(CCV(ccv, t_flags)); break; case CC_NDUPACK: /* Packet loss. */ /* * Only react to loss as a congestion signal if qdly > * V_chd_qthresh. If qdly is less than qthresh, presume that * this is a non congestion related loss. If qdly is greater * than qthresh, assume that we are competing with loss based * tcp flows and restore window from any unnecessary backoffs, * before the decrease. */ if (!IN_RECOVERY(CCV(ccv, t_flags)) && qdly > V_chd_qthresh) { if (chd_data->loss_compete) { CCV(ccv, snd_cwnd) = max(CCV(ccv, snd_cwnd), chd_data->shadow_w); } chd_window_decrease(ccv); } else { /* * This loss isn't congestion related, or already * recovering from congestion. */ CCV(ccv, snd_ssthresh) = CCV(ccv, snd_cwnd); CCV(ccv, snd_recover) = CCV(ccv, snd_max); } if (chd_data->shadow_w > 0) { chd_data->shadow_w = max(chd_data->shadow_w / CCV(ccv, t_maxseg) / 2, 2) * CCV(ccv, t_maxseg); } ENTER_FASTRECOVERY(CCV(ccv, t_flags)); break; default: newreno_cc_algo.cong_signal(ccv, signal_type); } } static void chd_conn_init(struct cc_var *ccv) { struct chd *chd_data; chd_data = ccv->cc_data; chd_data->prev_backoff_qdly = 0; chd_data->maxrtt_in_rtt = 0; chd_data->loss_compete = 0; /* * Initialise the shadow_cwnd to be equal to snd_cwnd in case we are * competing with loss based flows from the start. */ chd_data->shadow_w = CCV(ccv, snd_cwnd); } static int chd_mod_init(void) { ertt_id = khelp_get_id("ertt"); if (ertt_id <= 0) { printf("%s: h_ertt module not found\n", __func__); return (ENOENT); } chd_cc_algo.after_idle = newreno_cc_algo.after_idle; chd_cc_algo.post_recovery = newreno_cc_algo.post_recovery; return (0); } static int chd_loss_fair_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_chd_loss_fair; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) > 1) + if (new > 1) error = EINVAL; else V_chd_loss_fair = new; } return (error); } static int chd_pmax_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_chd_pmax; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) == 0 || - CAST_PTR_INT(req->newptr) > 100) + if (new == 0 || new > 100) error = EINVAL; else V_chd_pmax = new; } return (error); } static int chd_qthresh_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_chd_qthresh; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) <= V_chd_qmin) + if (new <= V_chd_qmin) error = EINVAL; else V_chd_qthresh = new; } return (error); } SYSCTL_DECL(_net_inet_tcp_cc_chd); SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, chd, CTLFLAG_RW, NULL, "CAIA Hamilton delay-based congestion control related settings"); SYSCTL_PROC(_net_inet_tcp_cc_chd, OID_AUTO, loss_fair, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(chd_loss_fair), 1, &chd_loss_fair_handler, "IU", "Flag to enable shadow window functionality."); SYSCTL_PROC(_net_inet_tcp_cc_chd, OID_AUTO, pmax, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(chd_pmax), 5, &chd_pmax_handler, "IU", "Per RTT maximum backoff probability as a percentage"); SYSCTL_PROC(_net_inet_tcp_cc_chd, OID_AUTO, queue_threshold, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(chd_qthresh), 20, &chd_qthresh_handler, "IU", "Queueing congestion threshold in ticks"); SYSCTL_UINT(_net_inet_tcp_cc_chd, OID_AUTO, queue_min, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(chd_qmin), 5, "Minimum queueing delay threshold in ticks"); SYSCTL_UINT(_net_inet_tcp_cc_chd, OID_AUTO, use_max, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(chd_use_max), 1, "Use the maximum RTT seen within the measurement period (RTT) " "as the basic delay measurement for the algorithm."); DECLARE_CC_MODULE(chd, &chd_cc_algo); MODULE_DEPEND(chd, ertt, 1, 1, 1); Index: stable/11/sys/netinet/cc/cc_dctcp.c =================================================================== --- stable/11/sys/netinet/cc/cc_dctcp.c (revision 342188) +++ stable/11/sys/netinet/cc/cc_dctcp.c (revision 342189) @@ -1,472 +1,470 @@ /*- * Copyright (c) 2007-2008 * Swinburne University of Technology, Melbourne, Australia * Copyright (c) 2009-2010 Lawrence Stewart * Copyright (c) 2014 Midori Kato * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * An implementation of the DCTCP algorithm for FreeBSD, based on * "Data Center TCP (DCTCP)" by M. Alizadeh, A. Greenberg, D. A. Maltz, * J. Padhye, P. Patel, B. Prabhakar, S. Sengupta, and M. Sridharan., * in ACM Conference on SIGCOMM 2010, New York, USA, * Originally released as the contribution of Microsoft Research project. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#define CAST_PTR_INT(X) (*((int*)(X))) - #define MAX_ALPHA_VALUE 1024 static VNET_DEFINE(uint32_t, dctcp_alpha) = 0; #define V_dctcp_alpha VNET(dctcp_alpha) static VNET_DEFINE(uint32_t, dctcp_shift_g) = 4; #define V_dctcp_shift_g VNET(dctcp_shift_g) static VNET_DEFINE(uint32_t, dctcp_slowstart) = 0; #define V_dctcp_slowstart VNET(dctcp_slowstart) struct dctcp { int bytes_ecn; /* # of marked bytes during a RTT */ int bytes_total; /* # of acked bytes during a RTT */ int alpha; /* the fraction of marked bytes */ int ce_prev; /* CE state of the last segment */ int save_sndnxt; /* end sequence number of the current window */ int ece_curr; /* ECE flag in this segment */ int ece_prev; /* ECE flag in the last segment */ uint32_t num_cong_events; /* # of congestion events */ }; static MALLOC_DEFINE(M_dctcp, "dctcp data", "Per connection data required for the dctcp algorithm"); static void dctcp_ack_received(struct cc_var *ccv, uint16_t type); static void dctcp_after_idle(struct cc_var *ccv); static void dctcp_cb_destroy(struct cc_var *ccv); static int dctcp_cb_init(struct cc_var *ccv); static void dctcp_cong_signal(struct cc_var *ccv, uint32_t type); static void dctcp_conn_init(struct cc_var *ccv); static void dctcp_post_recovery(struct cc_var *ccv); static void dctcp_ecnpkt_handler(struct cc_var *ccv); static void dctcp_update_alpha(struct cc_var *ccv); struct cc_algo dctcp_cc_algo = { .name = "dctcp", .ack_received = dctcp_ack_received, .cb_destroy = dctcp_cb_destroy, .cb_init = dctcp_cb_init, .cong_signal = dctcp_cong_signal, .conn_init = dctcp_conn_init, .post_recovery = dctcp_post_recovery, .ecnpkt_handler = dctcp_ecnpkt_handler, .after_idle = dctcp_after_idle, }; static void dctcp_ack_received(struct cc_var *ccv, uint16_t type) { struct dctcp *dctcp_data; int bytes_acked = 0; dctcp_data = ccv->cc_data; if (CCV(ccv, t_flags) & TF_ECN_PERMIT) { /* * DCTCP doesn't treat receipt of ECN marked packet as a * congestion event. Thus, DCTCP always executes the ACK * processing out of congestion recovery. */ if (IN_CONGRECOVERY(CCV(ccv, t_flags))) { EXIT_CONGRECOVERY(CCV(ccv, t_flags)); newreno_cc_algo.ack_received(ccv, type); ENTER_CONGRECOVERY(CCV(ccv, t_flags)); } else newreno_cc_algo.ack_received(ccv, type); if (type == CC_DUPACK) bytes_acked = CCV(ccv, t_maxseg); if (type == CC_ACK) bytes_acked = ccv->bytes_this_ack; /* Update total bytes. */ dctcp_data->bytes_total += bytes_acked; /* Update total marked bytes. */ if (dctcp_data->ece_curr) { if (!dctcp_data->ece_prev && bytes_acked > CCV(ccv, t_maxseg)) { dctcp_data->bytes_ecn += (bytes_acked - CCV(ccv, t_maxseg)); } else dctcp_data->bytes_ecn += bytes_acked; dctcp_data->ece_prev = 1; } else { if (dctcp_data->ece_prev && bytes_acked > CCV(ccv, t_maxseg)) dctcp_data->bytes_ecn += CCV(ccv, t_maxseg); dctcp_data->ece_prev = 0; } dctcp_data->ece_curr = 0; /* * Update the fraction of marked bytes at the end of * current window size. */ if ((IN_FASTRECOVERY(CCV(ccv, t_flags)) && SEQ_GEQ(ccv->curack, CCV(ccv, snd_recover))) || (!IN_FASTRECOVERY(CCV(ccv, t_flags)) && SEQ_GT(ccv->curack, dctcp_data->save_sndnxt))) dctcp_update_alpha(ccv); } else newreno_cc_algo.ack_received(ccv, type); } static void dctcp_after_idle(struct cc_var *ccv) { struct dctcp *dctcp_data; dctcp_data = ccv->cc_data; /* Initialize internal parameters after idle time */ dctcp_data->bytes_ecn = 0; dctcp_data->bytes_total = 0; dctcp_data->save_sndnxt = CCV(ccv, snd_nxt); dctcp_data->alpha = V_dctcp_alpha; dctcp_data->ece_curr = 0; dctcp_data->ece_prev = 0; dctcp_data->num_cong_events = 0; dctcp_cc_algo.after_idle = newreno_cc_algo.after_idle; } static void dctcp_cb_destroy(struct cc_var *ccv) { if (ccv->cc_data != NULL) free(ccv->cc_data, M_dctcp); } static int dctcp_cb_init(struct cc_var *ccv) { struct dctcp *dctcp_data; dctcp_data = malloc(sizeof(struct dctcp), M_dctcp, M_NOWAIT|M_ZERO); if (dctcp_data == NULL) return (ENOMEM); /* Initialize some key variables with sensible defaults. */ dctcp_data->bytes_ecn = 0; dctcp_data->bytes_total = 0; /* * When alpha is set to 0 in the beginning, DCTCP sender transfers as * much data as possible until the value converges which may expand the * queueing delay at the switch. When alpha is set to 1, queueing delay * is kept small. * Throughput-sensitive applications should have alpha = 0 * Latency-sensitive applications should have alpha = 1 * * Note: DCTCP draft suggests initial alpha to be 1 but we've decided to * keep it 0 as default. */ dctcp_data->alpha = V_dctcp_alpha; dctcp_data->save_sndnxt = 0; dctcp_data->ce_prev = 0; dctcp_data->ece_curr = 0; dctcp_data->ece_prev = 0; dctcp_data->num_cong_events = 0; ccv->cc_data = dctcp_data; return (0); } /* * Perform any necessary tasks before we enter congestion recovery. */ static void dctcp_cong_signal(struct cc_var *ccv, uint32_t type) { struct dctcp *dctcp_data; u_int win, mss; dctcp_data = ccv->cc_data; win = CCV(ccv, snd_cwnd); mss = CCV(ccv, t_maxseg); switch (type) { case CC_NDUPACK: if (!IN_FASTRECOVERY(CCV(ccv, t_flags))) { if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { CCV(ccv, snd_ssthresh) = mss * max(win / 2 / mss, 2); dctcp_data->num_cong_events++; } else { /* cwnd has already updated as congestion * recovery. Reverse cwnd value using * snd_cwnd_prev and recalculate snd_ssthresh */ win = CCV(ccv, snd_cwnd_prev); CCV(ccv, snd_ssthresh) = max(win / 2 / mss, 2) * mss; } ENTER_RECOVERY(CCV(ccv, t_flags)); } break; case CC_ECN: /* * Save current snd_cwnd when the host encounters both * congestion recovery and fast recovery. */ CCV(ccv, snd_cwnd_prev) = win; if (!IN_CONGRECOVERY(CCV(ccv, t_flags))) { if (V_dctcp_slowstart && dctcp_data->num_cong_events++ == 0) { CCV(ccv, snd_ssthresh) = mss * max(win / 2 / mss, 2); dctcp_data->alpha = MAX_ALPHA_VALUE; dctcp_data->bytes_ecn = 0; dctcp_data->bytes_total = 0; dctcp_data->save_sndnxt = CCV(ccv, snd_nxt); } else CCV(ccv, snd_ssthresh) = max((win - ((win * dctcp_data->alpha) >> 11)) / mss, 2) * mss; CCV(ccv, snd_cwnd) = CCV(ccv, snd_ssthresh); ENTER_CONGRECOVERY(CCV(ccv, t_flags)); } dctcp_data->ece_curr = 1; break; case CC_RTO: if (CCV(ccv, t_flags) & TF_ECN_PERMIT) { CCV(ccv, t_flags) |= TF_ECN_SND_CWR; dctcp_update_alpha(ccv); dctcp_data->save_sndnxt += CCV(ccv, t_maxseg); dctcp_data->num_cong_events++; } break; } } static void dctcp_conn_init(struct cc_var *ccv) { struct dctcp *dctcp_data; dctcp_data = ccv->cc_data; if (CCV(ccv, t_flags) & TF_ECN_PERMIT) dctcp_data->save_sndnxt = CCV(ccv, snd_nxt); } /* * Perform any necessary tasks before we exit congestion recovery. */ static void dctcp_post_recovery(struct cc_var *ccv) { dctcp_cc_algo.post_recovery = newreno_cc_algo.post_recovery; if (CCV(ccv, t_flags) & TF_ECN_PERMIT) dctcp_update_alpha(ccv); } /* * Execute an additional ECN processing using ECN field in IP header and the CWR * bit in TCP header. * * delay_ack == 0 - Delayed ACK disabled * delay_ack == 1 - Delayed ACK enabled */ static void dctcp_ecnpkt_handler(struct cc_var *ccv) { struct dctcp *dctcp_data; uint32_t ccflag; int delay_ack; dctcp_data = ccv->cc_data; ccflag = ccv->flags; delay_ack = 1; /* * DCTCP responses an ACK immediately when the CE state * in between this segment and the last segment is not same. */ if (ccflag & CCF_IPHDR_CE) { if (!dctcp_data->ce_prev && (ccflag & CCF_DELACK)) delay_ack = 0; dctcp_data->ce_prev = 1; CCV(ccv, t_flags) |= TF_ECN_SND_ECE; } else { if (dctcp_data->ce_prev && (ccflag & CCF_DELACK)) delay_ack = 0; dctcp_data->ce_prev = 0; CCV(ccv, t_flags) &= ~TF_ECN_SND_ECE; } /* DCTCP sets delayed ack when this segment sets the CWR flag. */ if ((ccflag & CCF_DELACK) && (ccflag & CCF_TCPHDR_CWR)) delay_ack = 1; if (delay_ack == 0) ccv->flags |= CCF_ACKNOW; else ccv->flags &= ~CCF_ACKNOW; } /* * Update the fraction of marked bytes represented as 'alpha'. * Also initialize several internal parameters at the end of this function. */ static void dctcp_update_alpha(struct cc_var *ccv) { struct dctcp *dctcp_data; int alpha_prev; dctcp_data = ccv->cc_data; alpha_prev = dctcp_data->alpha; dctcp_data->bytes_total = max(dctcp_data->bytes_total, 1); /* * Update alpha: alpha = (1 - g) * alpha + g * F. * Here: * g is weight factor * recommaded to be set to 1/16 * small g = slow convergence between competitive DCTCP flows * large g = impacts low utilization of bandwidth at switches * F is fraction of marked segments in last RTT * updated every RTT * Alpha must be round to 0 - MAX_ALPHA_VALUE. */ dctcp_data->alpha = min(alpha_prev - (alpha_prev >> V_dctcp_shift_g) + (dctcp_data->bytes_ecn << (10 - V_dctcp_shift_g)) / dctcp_data->bytes_total, MAX_ALPHA_VALUE); /* Initialize internal parameters for next alpha calculation */ dctcp_data->bytes_ecn = 0; dctcp_data->bytes_total = 0; dctcp_data->save_sndnxt = CCV(ccv, snd_nxt); } static int dctcp_alpha_handler(SYSCTL_HANDLER_ARGS) { uint32_t new; int error; new = V_dctcp_alpha; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) > 1) + if (new > 1) error = EINVAL; else { if (new > MAX_ALPHA_VALUE) V_dctcp_alpha = MAX_ALPHA_VALUE; else V_dctcp_alpha = new; } } return (error); } static int dctcp_shift_g_handler(SYSCTL_HANDLER_ARGS) { uint32_t new; int error; new = V_dctcp_shift_g; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) > 1) + if (new > 1) error = EINVAL; else V_dctcp_shift_g = new; } return (error); } static int dctcp_slowstart_handler(SYSCTL_HANDLER_ARGS) { uint32_t new; int error; new = V_dctcp_slowstart; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) > 1) + if (new > 1) error = EINVAL; else V_dctcp_slowstart = new; } return (error); } SYSCTL_DECL(_net_inet_tcp_cc_dctcp); SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, dctcp, CTLFLAG_RW, NULL, "dctcp congestion control related settings"); SYSCTL_PROC(_net_inet_tcp_cc_dctcp, OID_AUTO, alpha, CTLFLAG_VNET|CTLTYPE_UINT|CTLFLAG_RW, &VNET_NAME(dctcp_alpha), 0, &dctcp_alpha_handler, "IU", "dctcp alpha parameter"); SYSCTL_PROC(_net_inet_tcp_cc_dctcp, OID_AUTO, shift_g, CTLFLAG_VNET|CTLTYPE_UINT|CTLFLAG_RW, &VNET_NAME(dctcp_shift_g), 4, &dctcp_shift_g_handler, "IU", "dctcp shift parameter"); SYSCTL_PROC(_net_inet_tcp_cc_dctcp, OID_AUTO, slowstart, CTLFLAG_VNET|CTLTYPE_UINT|CTLFLAG_RW, &VNET_NAME(dctcp_slowstart), 0, &dctcp_slowstart_handler, "IU", "half CWND reduction after the first slow start"); DECLARE_CC_MODULE(dctcp, &dctcp_cc_algo); Index: stable/11/sys/netinet/cc/cc_hd.c =================================================================== --- stable/11/sys/netinet/cc/cc_hd.c (revision 342188) +++ stable/11/sys/netinet/cc/cc_hd.c (revision 342189) @@ -1,254 +1,250 @@ /*- * Copyright (c) 2009-2010 * Swinburne University of Technology, Melbourne, Australia * Copyright (c) 2010 Lawrence Stewart * Copyright (c) 2010-2011 The FreeBSD Foundation * All rights reserved. * * This software was developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by David Hayes and * Lawrence Stewart, made possible in part by a grant from the Cisco University * Research Program Fund at Community Foundation Silicon Valley. * * Portions of this software were developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, Melbourne, Australia by * David Hayes under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * An implementation of the Hamilton Institute's delay-based congestion control * algorithm for FreeBSD, based on "A strategy for fair coexistence of loss and * delay-based congestion control algorithms," by L. Budzisz, R. Stanojevic, R. * Shorten, and F. Baker, IEEE Commun. Lett., vol. 13, no. 7, pp. 555--557, Jul. * 2009. * * Originally released as part of the NewTCP research project at Swinburne * University of Technology's Centre for Advanced Internet Architectures, * Melbourne, Australia, which was made possible in part by a grant from the * Cisco University Research Program Fund at Community Foundation Silicon * Valley. More details are available at: * http://caia.swin.edu.au/urp/newtcp/ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#define CAST_PTR_INT(X) (*((int*)(X))) - /* Largest possible number returned by random(). */ #define RANDOM_MAX INT_MAX static void hd_ack_received(struct cc_var *ccv, uint16_t ack_type); static int hd_mod_init(void); static int ertt_id; static VNET_DEFINE(uint32_t, hd_qthresh) = 20; static VNET_DEFINE(uint32_t, hd_qmin) = 5; static VNET_DEFINE(uint32_t, hd_pmax) = 5; #define V_hd_qthresh VNET(hd_qthresh) #define V_hd_qmin VNET(hd_qmin) #define V_hd_pmax VNET(hd_pmax) struct cc_algo hd_cc_algo = { .name = "hd", .ack_received = hd_ack_received, .mod_init = hd_mod_init }; /* * Hamilton backoff function. Returns 1 if we should backoff or 0 otherwise. */ static __inline int should_backoff(int qdly, int maxqdly) { unsigned long p; if (qdly < V_hd_qthresh) { p = (((RANDOM_MAX / 100) * V_hd_pmax) / (V_hd_qthresh - V_hd_qmin)) * (qdly - V_hd_qmin); } else { if (qdly > V_hd_qthresh) p = (((RANDOM_MAX / 100) * V_hd_pmax) / (maxqdly - V_hd_qthresh)) * (maxqdly - qdly); else p = (RANDOM_MAX / 100) * V_hd_pmax; } return (random() < p); } /* * If the ack type is CC_ACK, and the inferred queueing delay is greater than * the Qmin threshold, cwnd is reduced probabilistically. When backing off due * to delay, HD behaves like NewReno when an ECN signal is received. HD behaves * as NewReno in all other circumstances. */ static void hd_ack_received(struct cc_var *ccv, uint16_t ack_type) { struct ertt *e_t; int qdly; if (ack_type == CC_ACK) { e_t = khelp_get_osd(CCV(ccv, osd), ertt_id); if (e_t->rtt && e_t->minrtt && V_hd_qthresh > 0) { qdly = e_t->rtt - e_t->minrtt; if (qdly > V_hd_qmin && !IN_RECOVERY(CCV(ccv, t_flags))) { /* Probabilistic backoff of cwnd. */ if (should_backoff(qdly, e_t->maxrtt - e_t->minrtt)) { /* * Update cwnd and ssthresh update to * half cwnd and behave like an ECN (ie * not a packet loss). */ newreno_cc_algo.cong_signal(ccv, CC_ECN); return; } } } } newreno_cc_algo.ack_received(ccv, ack_type); /* As for NewReno. */ } static int hd_mod_init(void) { ertt_id = khelp_get_id("ertt"); if (ertt_id <= 0) { printf("%s: h_ertt module not found\n", __func__); return (ENOENT); } hd_cc_algo.after_idle = newreno_cc_algo.after_idle; hd_cc_algo.cong_signal = newreno_cc_algo.cong_signal; hd_cc_algo.post_recovery = newreno_cc_algo.post_recovery; return (0); } static int hd_pmax_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_hd_pmax; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) == 0 || - CAST_PTR_INT(req->newptr) > 100) + if (new == 0 || new > 100) error = EINVAL; else V_hd_pmax = new; } return (error); } static int hd_qmin_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_hd_qmin; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) > V_hd_qthresh) + if (new > V_hd_qthresh) error = EINVAL; else V_hd_qmin = new; } return (error); } static int hd_qthresh_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_hd_qthresh; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) < 1 || - CAST_PTR_INT(req->newptr) < V_hd_qmin) + if (new == 0 || new < V_hd_qmin) error = EINVAL; else V_hd_qthresh = new; } return (error); } SYSCTL_DECL(_net_inet_tcp_cc_hd); SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, hd, CTLFLAG_RW, NULL, "Hamilton delay-based congestion control related settings"); SYSCTL_PROC(_net_inet_tcp_cc_hd, OID_AUTO, queue_threshold, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(hd_qthresh), 20, &hd_qthresh_handler, "IU", "queueing congestion threshold (qth) in ticks"); SYSCTL_PROC(_net_inet_tcp_cc_hd, OID_AUTO, pmax, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(hd_pmax), 5, &hd_pmax_handler, "IU", "per packet maximum backoff probability as a percentage"); SYSCTL_PROC(_net_inet_tcp_cc_hd, OID_AUTO, queue_min, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(hd_qmin), 5, &hd_qmin_handler, "IU", "minimum queueing delay threshold (qmin) in ticks"); DECLARE_CC_MODULE(hd, &hd_cc_algo); MODULE_DEPEND(hd, ertt, 1, 1, 1); Index: stable/11/sys/netinet/cc/cc_vegas.c =================================================================== --- stable/11/sys/netinet/cc/cc_vegas.c (revision 342188) +++ stable/11/sys/netinet/cc/cc_vegas.c (revision 342189) @@ -1,307 +1,303 @@ /*- * Copyright (c) 2009-2010 * Swinburne University of Technology, Melbourne, Australia * Copyright (c) 2010 Lawrence Stewart * Copyright (c) 2010-2011 The FreeBSD Foundation * All rights reserved. * * This software was developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, by David Hayes and * Lawrence Stewart, made possible in part by a grant from the Cisco University * Research Program Fund at Community Foundation Silicon Valley. * * Portions of this software were developed at the Centre for Advanced Internet * Architectures, Swinburne University of Technology, Melbourne, Australia by * David Hayes under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * An implementation of the Vegas congestion control algorithm for FreeBSD, * based on L. S. Brakmo and L. L. Peterson, "TCP Vegas: end to end congestion * avoidance on a global internet", IEEE J. Sel. Areas Commun., vol. 13, no. 8, * pp. 1465-1480, Oct. 1995. The original Vegas duplicate ack policy has not * been implemented, since clock ticks are not as coarse as they were (i.e. * 500ms) when Vegas was designed. Also, packets are timed once per RTT as in * the original paper. * * Originally released as part of the NewTCP research project at Swinburne * University of Technology's Centre for Advanced Internet Architectures, * Melbourne, Australia, which was made possible in part by a grant from the * Cisco University Research Program Fund at Community Foundation Silicon * Valley. More details are available at: * http://caia.swin.edu.au/urp/newtcp/ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#define CAST_PTR_INT(X) (*((int*)(X))) - /* * Private signal type for rate based congestion signal. * See for appropriate bit-range to use for private signals. */ #define CC_VEGAS_RATE 0x01000000 static void vegas_ack_received(struct cc_var *ccv, uint16_t ack_type); static void vegas_cb_destroy(struct cc_var *ccv); static int vegas_cb_init(struct cc_var *ccv); static void vegas_cong_signal(struct cc_var *ccv, uint32_t signal_type); static void vegas_conn_init(struct cc_var *ccv); static int vegas_mod_init(void); struct vegas { int slow_start_toggle; }; static int32_t ertt_id; static VNET_DEFINE(uint32_t, vegas_alpha) = 1; static VNET_DEFINE(uint32_t, vegas_beta) = 3; #define V_vegas_alpha VNET(vegas_alpha) #define V_vegas_beta VNET(vegas_beta) static MALLOC_DEFINE(M_VEGAS, "vegas data", "Per connection data required for the Vegas congestion control algorithm"); struct cc_algo vegas_cc_algo = { .name = "vegas", .ack_received = vegas_ack_received, .cb_destroy = vegas_cb_destroy, .cb_init = vegas_cb_init, .cong_signal = vegas_cong_signal, .conn_init = vegas_conn_init, .mod_init = vegas_mod_init }; /* * The vegas window adjustment is done once every RTT, as indicated by the * ERTT_NEW_MEASUREMENT flag. This flag is reset once the new measurment data * has been used. */ static void vegas_ack_received(struct cc_var *ccv, uint16_t ack_type) { struct ertt *e_t; struct vegas *vegas_data; long actual_tx_rate, expected_tx_rate, ndiff; e_t = khelp_get_osd(CCV(ccv, osd), ertt_id); vegas_data = ccv->cc_data; if (e_t->flags & ERTT_NEW_MEASUREMENT) { /* Once per RTT. */ if (e_t->minrtt && e_t->markedpkt_rtt) { expected_tx_rate = e_t->marked_snd_cwnd / e_t->minrtt; actual_tx_rate = e_t->bytes_tx_in_marked_rtt / e_t->markedpkt_rtt; ndiff = (expected_tx_rate - actual_tx_rate) * e_t->minrtt / CCV(ccv, t_maxseg); if (ndiff < V_vegas_alpha) { if (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh)) { vegas_data->slow_start_toggle = vegas_data->slow_start_toggle ? 0 : 1; } else { vegas_data->slow_start_toggle = 0; CCV(ccv, snd_cwnd) = min(CCV(ccv, snd_cwnd) + CCV(ccv, t_maxseg), TCP_MAXWIN << CCV(ccv, snd_scale)); } } else if (ndiff > V_vegas_beta) { /* Rate-based congestion. */ vegas_cong_signal(ccv, CC_VEGAS_RATE); vegas_data->slow_start_toggle = 0; } } e_t->flags &= ~ERTT_NEW_MEASUREMENT; } if (vegas_data->slow_start_toggle) newreno_cc_algo.ack_received(ccv, ack_type); } static void vegas_cb_destroy(struct cc_var *ccv) { if (ccv->cc_data != NULL) free(ccv->cc_data, M_VEGAS); } static int vegas_cb_init(struct cc_var *ccv) { struct vegas *vegas_data; vegas_data = malloc(sizeof(struct vegas), M_VEGAS, M_NOWAIT); if (vegas_data == NULL) return (ENOMEM); vegas_data->slow_start_toggle = 1; ccv->cc_data = vegas_data; return (0); } /* * If congestion has been triggered triggered by the Vegas measured rates, it is * handled here, otherwise it falls back to newreno's congestion handling. */ static void vegas_cong_signal(struct cc_var *ccv, uint32_t signal_type) { struct vegas *vegas_data; int presignalrecov; vegas_data = ccv->cc_data; if (IN_RECOVERY(CCV(ccv, t_flags))) presignalrecov = 1; else presignalrecov = 0; switch(signal_type) { case CC_VEGAS_RATE: if (!IN_RECOVERY(CCV(ccv, t_flags))) { CCV(ccv, snd_cwnd) = max(2 * CCV(ccv, t_maxseg), CCV(ccv, snd_cwnd) - CCV(ccv, t_maxseg)); if (CCV(ccv, snd_cwnd) < CCV(ccv, snd_ssthresh)) /* Exit slow start. */ CCV(ccv, snd_ssthresh) = CCV(ccv, snd_cwnd); } break; default: newreno_cc_algo.cong_signal(ccv, signal_type); } if (IN_RECOVERY(CCV(ccv, t_flags)) && !presignalrecov) vegas_data->slow_start_toggle = (CCV(ccv, snd_cwnd) < CCV(ccv, snd_ssthresh)) ? 1 : 0; } static void vegas_conn_init(struct cc_var *ccv) { struct vegas *vegas_data; vegas_data = ccv->cc_data; vegas_data->slow_start_toggle = 1; } static int vegas_mod_init(void) { ertt_id = khelp_get_id("ertt"); if (ertt_id <= 0) { printf("%s: h_ertt module not found\n", __func__); return (ENOENT); } vegas_cc_algo.after_idle = newreno_cc_algo.after_idle; vegas_cc_algo.post_recovery = newreno_cc_algo.post_recovery; return (0); } static int vegas_alpha_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_vegas_alpha; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) < 1 || - CAST_PTR_INT(req->newptr) > V_vegas_beta) + if (new == 0 || new > V_vegas_beta) error = EINVAL; else V_vegas_alpha = new; } return (error); } static int vegas_beta_handler(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = V_vegas_beta; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr != NULL) { - if (CAST_PTR_INT(req->newptr) < 1 || - CAST_PTR_INT(req->newptr) < V_vegas_alpha) + if (new == 0 || new < V_vegas_alpha) error = EINVAL; else V_vegas_beta = new; } return (error); } SYSCTL_DECL(_net_inet_tcp_cc_vegas); SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, vegas, CTLFLAG_RW, NULL, "Vegas related settings"); SYSCTL_PROC(_net_inet_tcp_cc_vegas, OID_AUTO, alpha, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(vegas_alpha), 1, &vegas_alpha_handler, "IU", "vegas alpha, specified as number of \"buffers\" (0 < alpha < beta)"); SYSCTL_PROC(_net_inet_tcp_cc_vegas, OID_AUTO, beta, CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, &VNET_NAME(vegas_beta), 3, &vegas_beta_handler, "IU", "vegas beta, specified as number of \"buffers\" (0 < alpha < beta)"); DECLARE_CC_MODULE(vegas, &vegas_cc_algo); MODULE_DEPEND(vegas, ertt, 1, 1, 1); Index: stable/11/sys/netinet/siftr.c =================================================================== --- stable/11/sys/netinet/siftr.c (revision 342188) +++ stable/11/sys/netinet/siftr.c (revision 342189) @@ -1,1566 +1,1564 @@ /*- * Copyright (c) 2007-2009 * Swinburne University of Technology, Melbourne, Australia. * Copyright (c) 2009-2010, The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed at the Centre for Advanced * Internet Architectures, Swinburne University of Technology, Melbourne, * Australia by Lawrence Stewart under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /****************************************************** * Statistical Information For TCP Research (SIFTR) * * A FreeBSD kernel module that adds very basic intrumentation to the * TCP stack, allowing internal stats to be recorded to a log file * for experimental, debugging and performance analysis purposes. * * SIFTR was first released in 2007 by James Healy and Lawrence Stewart whilst * working on the NewTCP research project at Swinburne University of * Technology's Centre for Advanced Internet Architectures, Melbourne, * Australia, which was made possible in part by a grant from the Cisco * University Research Program Fund at Community Foundation Silicon Valley. * More details are available at: * http://caia.swin.edu.au/urp/newtcp/ * * Work on SIFTR v1.2.x was sponsored by the FreeBSD Foundation as part of * the "Enhancing the FreeBSD TCP Implementation" project 2008-2009. * More details are available at: * http://www.freebsdfoundation.org/ * http://caia.swin.edu.au/freebsd/etcp09/ * * Lawrence Stewart is the current maintainer, and all contact regarding * SIFTR should be directed to him via email: lastewart@swin.edu.au * * Initial release date: June 2007 * Most recent update: September 2010 ******************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SIFTR_IPV6 #include #include #endif /* SIFTR_IPV6 */ #include /* * Three digit version number refers to X.Y.Z where: * X is the major version number * Y is bumped to mark backwards incompatible changes * Z is bumped to mark backwards compatible changes */ #define V_MAJOR 1 #define V_BACKBREAK 2 #define V_BACKCOMPAT 4 #define MODVERSION __CONCAT(V_MAJOR, __CONCAT(V_BACKBREAK, V_BACKCOMPAT)) #define MODVERSION_STR __XSTRING(V_MAJOR) "." __XSTRING(V_BACKBREAK) "." \ __XSTRING(V_BACKCOMPAT) #define HOOK 0 #define UNHOOK 1 #define SIFTR_EXPECTED_MAX_TCP_FLOWS 65536 #define SYS_NAME "FreeBSD" #define PACKET_TAG_SIFTR 100 #define PACKET_COOKIE_SIFTR 21749576 #define SIFTR_LOG_FILE_MODE 0644 #define SIFTR_DISABLE 0 #define SIFTR_ENABLE 1 /* * Hard upper limit on the length of log messages. Bump this up if you add new * data fields such that the line length could exceed the below value. */ #define MAX_LOG_MSG_LEN 200 /* XXX: Make this a sysctl tunable. */ #define SIFTR_ALQ_BUFLEN (1000*MAX_LOG_MSG_LEN) /* * 1 byte for IP version * IPv4: src/dst IP (4+4) + src/dst port (2+2) = 12 bytes * IPv6: src/dst IP (16+16) + src/dst port (2+2) = 36 bytes */ #ifdef SIFTR_IPV6 #define FLOW_KEY_LEN 37 #else #define FLOW_KEY_LEN 13 #endif #ifdef SIFTR_IPV6 #define SIFTR_IPMODE 6 #else #define SIFTR_IPMODE 4 #endif /* useful macros */ -#define CAST_PTR_INT(X) (*((int*)(X))) - #define UPPER_SHORT(X) (((X) & 0xFFFF0000) >> 16) #define LOWER_SHORT(X) ((X) & 0x0000FFFF) #define FIRST_OCTET(X) (((X) & 0xFF000000) >> 24) #define SECOND_OCTET(X) (((X) & 0x00FF0000) >> 16) #define THIRD_OCTET(X) (((X) & 0x0000FF00) >> 8) #define FOURTH_OCTET(X) ((X) & 0x000000FF) static MALLOC_DEFINE(M_SIFTR, "siftr", "dynamic memory used by SIFTR"); static MALLOC_DEFINE(M_SIFTR_PKTNODE, "siftr_pktnode", "SIFTR pkt_node struct"); static MALLOC_DEFINE(M_SIFTR_HASHNODE, "siftr_hashnode", "SIFTR flow_hash_node struct"); /* Used as links in the pkt manager queue. */ struct pkt_node { /* Timestamp of pkt as noted in the pfil hook. */ struct timeval tval; /* Direction pkt is travelling; either PFIL_IN or PFIL_OUT. */ uint8_t direction; /* IP version pkt_node relates to; either INP_IPV4 or INP_IPV6. */ uint8_t ipver; /* Hash of the pkt which triggered the log message. */ uint32_t hash; /* Local/foreign IP address. */ #ifdef SIFTR_IPV6 uint32_t ip_laddr[4]; uint32_t ip_faddr[4]; #else uint8_t ip_laddr[4]; uint8_t ip_faddr[4]; #endif /* Local TCP port. */ uint16_t tcp_localport; /* Foreign TCP port. */ uint16_t tcp_foreignport; /* Congestion Window (bytes). */ u_long snd_cwnd; /* Sending Window (bytes). */ u_long snd_wnd; /* Receive Window (bytes). */ u_long rcv_wnd; /* Unused (was: Bandwidth Controlled Window (bytes)). */ u_long snd_bwnd; /* Slow Start Threshold (bytes). */ u_long snd_ssthresh; /* Current state of the TCP FSM. */ int conn_state; /* Max Segment Size (bytes). */ u_int max_seg_size; /* * Smoothed RTT stored as found in the TCP control block * in units of (TCP_RTT_SCALE*hz). */ int smoothed_rtt; /* Is SACK enabled? */ u_char sack_enabled; /* Window scaling for snd window. */ u_char snd_scale; /* Window scaling for recv window. */ u_char rcv_scale; /* TCP control block flags. */ u_int flags; /* Retransmit timeout length. */ int rxt_length; /* Size of the TCP send buffer in bytes. */ u_int snd_buf_hiwater; /* Current num bytes in the send socket buffer. */ u_int snd_buf_cc; /* Size of the TCP receive buffer in bytes. */ u_int rcv_buf_hiwater; /* Current num bytes in the receive socket buffer. */ u_int rcv_buf_cc; /* Number of bytes inflight that we are waiting on ACKs for. */ u_int sent_inflight_bytes; /* Number of segments currently in the reassembly queue. */ int t_segqlen; /* Flowid for the connection. */ u_int flowid; /* Flow type for the connection. */ u_int flowtype; /* Link to next pkt_node in the list. */ STAILQ_ENTRY(pkt_node) nodes; }; struct flow_hash_node { uint16_t counter; uint8_t key[FLOW_KEY_LEN]; LIST_ENTRY(flow_hash_node) nodes; }; struct siftr_stats { /* # TCP pkts seen by the SIFTR PFIL hooks, including any skipped. */ uint64_t n_in; uint64_t n_out; /* # pkts skipped due to failed malloc calls. */ uint32_t nskip_in_malloc; uint32_t nskip_out_malloc; /* # pkts skipped due to failed mtx acquisition. */ uint32_t nskip_in_mtx; uint32_t nskip_out_mtx; /* # pkts skipped due to failed inpcb lookups. */ uint32_t nskip_in_inpcb; uint32_t nskip_out_inpcb; /* # pkts skipped due to failed tcpcb lookups. */ uint32_t nskip_in_tcpcb; uint32_t nskip_out_tcpcb; /* # pkts skipped due to stack reinjection. */ uint32_t nskip_in_dejavu; uint32_t nskip_out_dejavu; }; static DPCPU_DEFINE(struct siftr_stats, ss); static volatile unsigned int siftr_exit_pkt_manager_thread = 0; static unsigned int siftr_enabled = 0; static unsigned int siftr_pkts_per_log = 1; static unsigned int siftr_generate_hashes = 0; /* static unsigned int siftr_binary_log = 0; */ static char siftr_logfile[PATH_MAX] = "/var/log/siftr.log"; static char siftr_logfile_shadow[PATH_MAX] = "/var/log/siftr.log"; static u_long siftr_hashmask; STAILQ_HEAD(pkthead, pkt_node) pkt_queue = STAILQ_HEAD_INITIALIZER(pkt_queue); LIST_HEAD(listhead, flow_hash_node) *counter_hash; static int wait_for_pkt; static struct alq *siftr_alq = NULL; static struct mtx siftr_pkt_queue_mtx; static struct mtx siftr_pkt_mgr_mtx; static struct thread *siftr_pkt_manager_thr = NULL; /* * pfil.h defines PFIL_IN as 1 and PFIL_OUT as 2, * which we use as an index into this array. */ static char direction[3] = {'\0', 'i','o'}; /* Required function prototypes. */ static int siftr_sysctl_enabled_handler(SYSCTL_HANDLER_ARGS); static int siftr_sysctl_logfile_name_handler(SYSCTL_HANDLER_ARGS); /* Declare the net.inet.siftr sysctl tree and populate it. */ SYSCTL_DECL(_net_inet_siftr); SYSCTL_NODE(_net_inet, OID_AUTO, siftr, CTLFLAG_RW, NULL, "siftr related settings"); SYSCTL_PROC(_net_inet_siftr, OID_AUTO, enabled, CTLTYPE_UINT|CTLFLAG_RW, &siftr_enabled, 0, &siftr_sysctl_enabled_handler, "IU", "switch siftr module operations on/off"); SYSCTL_PROC(_net_inet_siftr, OID_AUTO, logfile, CTLTYPE_STRING|CTLFLAG_RW, &siftr_logfile_shadow, sizeof(siftr_logfile_shadow), &siftr_sysctl_logfile_name_handler, "A", "file to save siftr log messages to"); SYSCTL_UINT(_net_inet_siftr, OID_AUTO, ppl, CTLFLAG_RW, &siftr_pkts_per_log, 1, "number of packets between generating a log message"); SYSCTL_UINT(_net_inet_siftr, OID_AUTO, genhashes, CTLFLAG_RW, &siftr_generate_hashes, 0, "enable packet hash generation"); /* XXX: TODO SYSCTL_UINT(_net_inet_siftr, OID_AUTO, binary, CTLFLAG_RW, &siftr_binary_log, 0, "write log files in binary instead of ascii"); */ /* Begin functions. */ static void siftr_process_pkt(struct pkt_node * pkt_node) { struct flow_hash_node *hash_node; struct listhead *counter_list; struct siftr_stats *ss; struct ale *log_buf; uint8_t key[FLOW_KEY_LEN]; uint8_t found_match, key_offset; hash_node = NULL; ss = DPCPU_PTR(ss); found_match = 0; key_offset = 1; /* * Create the key that will be used to create a hash index * into our hash table. Our key consists of: * ipversion, localip, localport, foreignip, foreignport */ key[0] = pkt_node->ipver; memcpy(key + key_offset, &pkt_node->ip_laddr, sizeof(pkt_node->ip_laddr)); key_offset += sizeof(pkt_node->ip_laddr); memcpy(key + key_offset, &pkt_node->tcp_localport, sizeof(pkt_node->tcp_localport)); key_offset += sizeof(pkt_node->tcp_localport); memcpy(key + key_offset, &pkt_node->ip_faddr, sizeof(pkt_node->ip_faddr)); key_offset += sizeof(pkt_node->ip_faddr); memcpy(key + key_offset, &pkt_node->tcp_foreignport, sizeof(pkt_node->tcp_foreignport)); counter_list = counter_hash + (hash32_buf(key, sizeof(key), 0) & siftr_hashmask); /* * If the list is not empty i.e. the hash index has * been used by another flow previously. */ if (LIST_FIRST(counter_list) != NULL) { /* * Loop through the hash nodes in the list. * There should normally only be 1 hash node in the list, * except if there have been collisions at the hash index * computed by hash32_buf(). */ LIST_FOREACH(hash_node, counter_list, nodes) { /* * Check if the key for the pkt we are currently * processing is the same as the key stored in the * hash node we are currently processing. * If they are the same, then we've found the * hash node that stores the counter for the flow * the pkt belongs to. */ if (memcmp(hash_node->key, key, sizeof(key)) == 0) { found_match = 1; break; } } } /* If this flow hash hasn't been seen before or we have a collision. */ if (hash_node == NULL || !found_match) { /* Create a new hash node to store the flow's counter. */ hash_node = malloc(sizeof(struct flow_hash_node), M_SIFTR_HASHNODE, M_WAITOK); if (hash_node != NULL) { /* Initialise our new hash node list entry. */ hash_node->counter = 0; memcpy(hash_node->key, key, sizeof(key)); LIST_INSERT_HEAD(counter_list, hash_node, nodes); } else { /* Malloc failed. */ if (pkt_node->direction == PFIL_IN) ss->nskip_in_malloc++; else ss->nskip_out_malloc++; return; } } else if (siftr_pkts_per_log > 1) { /* * Taking the remainder of the counter divided * by the current value of siftr_pkts_per_log * and storing that in counter provides a neat * way to modulate the frequency of log * messages being written to the log file. */ hash_node->counter = (hash_node->counter + 1) % siftr_pkts_per_log; /* * If we have not seen enough packets since the last time * we wrote a log message for this connection, return. */ if (hash_node->counter > 0) return; } log_buf = alq_getn(siftr_alq, MAX_LOG_MSG_LEN, ALQ_WAITOK); if (log_buf == NULL) return; /* Should only happen if the ALQ is shutting down. */ #ifdef SIFTR_IPV6 pkt_node->ip_laddr[3] = ntohl(pkt_node->ip_laddr[3]); pkt_node->ip_faddr[3] = ntohl(pkt_node->ip_faddr[3]); if (pkt_node->ipver == INP_IPV6) { /* IPv6 packet */ pkt_node->ip_laddr[0] = ntohl(pkt_node->ip_laddr[0]); pkt_node->ip_laddr[1] = ntohl(pkt_node->ip_laddr[1]); pkt_node->ip_laddr[2] = ntohl(pkt_node->ip_laddr[2]); pkt_node->ip_faddr[0] = ntohl(pkt_node->ip_faddr[0]); pkt_node->ip_faddr[1] = ntohl(pkt_node->ip_faddr[1]); pkt_node->ip_faddr[2] = ntohl(pkt_node->ip_faddr[2]); /* Construct an IPv6 log message. */ log_buf->ae_bytesused = snprintf(log_buf->ae_data, MAX_LOG_MSG_LEN, "%c,0x%08x,%zd.%06ld,%x:%x:%x:%x:%x:%x:%x:%x,%u,%x:%x:%x:" "%x:%x:%x:%x:%x,%u,%ld,%ld,%ld,%ld,%ld,%u,%u,%u,%u,%u,%u," "%u,%d,%u,%u,%u,%u,%u,%u,%u,%u\n", direction[pkt_node->direction], pkt_node->hash, pkt_node->tval.tv_sec, pkt_node->tval.tv_usec, UPPER_SHORT(pkt_node->ip_laddr[0]), LOWER_SHORT(pkt_node->ip_laddr[0]), UPPER_SHORT(pkt_node->ip_laddr[1]), LOWER_SHORT(pkt_node->ip_laddr[1]), UPPER_SHORT(pkt_node->ip_laddr[2]), LOWER_SHORT(pkt_node->ip_laddr[2]), UPPER_SHORT(pkt_node->ip_laddr[3]), LOWER_SHORT(pkt_node->ip_laddr[3]), ntohs(pkt_node->tcp_localport), UPPER_SHORT(pkt_node->ip_faddr[0]), LOWER_SHORT(pkt_node->ip_faddr[0]), UPPER_SHORT(pkt_node->ip_faddr[1]), LOWER_SHORT(pkt_node->ip_faddr[1]), UPPER_SHORT(pkt_node->ip_faddr[2]), LOWER_SHORT(pkt_node->ip_faddr[2]), UPPER_SHORT(pkt_node->ip_faddr[3]), LOWER_SHORT(pkt_node->ip_faddr[3]), ntohs(pkt_node->tcp_foreignport), pkt_node->snd_ssthresh, pkt_node->snd_cwnd, pkt_node->snd_bwnd, pkt_node->snd_wnd, pkt_node->rcv_wnd, pkt_node->snd_scale, pkt_node->rcv_scale, pkt_node->conn_state, pkt_node->max_seg_size, pkt_node->smoothed_rtt, pkt_node->sack_enabled, pkt_node->flags, pkt_node->rxt_length, pkt_node->snd_buf_hiwater, pkt_node->snd_buf_cc, pkt_node->rcv_buf_hiwater, pkt_node->rcv_buf_cc, pkt_node->sent_inflight_bytes, pkt_node->t_segqlen, pkt_node->flowid, pkt_node->flowtype); } else { /* IPv4 packet */ pkt_node->ip_laddr[0] = FIRST_OCTET(pkt_node->ip_laddr[3]); pkt_node->ip_laddr[1] = SECOND_OCTET(pkt_node->ip_laddr[3]); pkt_node->ip_laddr[2] = THIRD_OCTET(pkt_node->ip_laddr[3]); pkt_node->ip_laddr[3] = FOURTH_OCTET(pkt_node->ip_laddr[3]); pkt_node->ip_faddr[0] = FIRST_OCTET(pkt_node->ip_faddr[3]); pkt_node->ip_faddr[1] = SECOND_OCTET(pkt_node->ip_faddr[3]); pkt_node->ip_faddr[2] = THIRD_OCTET(pkt_node->ip_faddr[3]); pkt_node->ip_faddr[3] = FOURTH_OCTET(pkt_node->ip_faddr[3]); #endif /* SIFTR_IPV6 */ /* Construct an IPv4 log message. */ log_buf->ae_bytesused = snprintf(log_buf->ae_data, MAX_LOG_MSG_LEN, "%c,0x%08x,%jd.%06ld,%u.%u.%u.%u,%u,%u.%u.%u.%u,%u,%ld,%ld," "%ld,%ld,%ld,%u,%u,%u,%u,%u,%u,%u,%d,%u,%u,%u,%u,%u,%u,%u,%u\n", direction[pkt_node->direction], pkt_node->hash, (intmax_t)pkt_node->tval.tv_sec, pkt_node->tval.tv_usec, pkt_node->ip_laddr[0], pkt_node->ip_laddr[1], pkt_node->ip_laddr[2], pkt_node->ip_laddr[3], ntohs(pkt_node->tcp_localport), pkt_node->ip_faddr[0], pkt_node->ip_faddr[1], pkt_node->ip_faddr[2], pkt_node->ip_faddr[3], ntohs(pkt_node->tcp_foreignport), pkt_node->snd_ssthresh, pkt_node->snd_cwnd, pkt_node->snd_bwnd, pkt_node->snd_wnd, pkt_node->rcv_wnd, pkt_node->snd_scale, pkt_node->rcv_scale, pkt_node->conn_state, pkt_node->max_seg_size, pkt_node->smoothed_rtt, pkt_node->sack_enabled, pkt_node->flags, pkt_node->rxt_length, pkt_node->snd_buf_hiwater, pkt_node->snd_buf_cc, pkt_node->rcv_buf_hiwater, pkt_node->rcv_buf_cc, pkt_node->sent_inflight_bytes, pkt_node->t_segqlen, pkt_node->flowid, pkt_node->flowtype); #ifdef SIFTR_IPV6 } #endif alq_post_flags(siftr_alq, log_buf, 0); } static void siftr_pkt_manager_thread(void *arg) { STAILQ_HEAD(pkthead, pkt_node) tmp_pkt_queue = STAILQ_HEAD_INITIALIZER(tmp_pkt_queue); struct pkt_node *pkt_node, *pkt_node_temp; uint8_t draining; draining = 2; mtx_lock(&siftr_pkt_mgr_mtx); /* draining == 0 when queue has been flushed and it's safe to exit. */ while (draining) { /* * Sleep until we are signalled to wake because thread has * been told to exit or until 1 tick has passed. */ mtx_sleep(&wait_for_pkt, &siftr_pkt_mgr_mtx, PWAIT, "pktwait", 1); /* Gain exclusive access to the pkt_node queue. */ mtx_lock(&siftr_pkt_queue_mtx); /* * Move pkt_queue to tmp_pkt_queue, which leaves * pkt_queue empty and ready to receive more pkt_nodes. */ STAILQ_CONCAT(&tmp_pkt_queue, &pkt_queue); /* * We've finished making changes to the list. Unlock it * so the pfil hooks can continue queuing pkt_nodes. */ mtx_unlock(&siftr_pkt_queue_mtx); /* * We can't hold a mutex whilst calling siftr_process_pkt * because ALQ might sleep waiting for buffer space. */ mtx_unlock(&siftr_pkt_mgr_mtx); /* Flush all pkt_nodes to the log file. */ STAILQ_FOREACH_SAFE(pkt_node, &tmp_pkt_queue, nodes, pkt_node_temp) { siftr_process_pkt(pkt_node); STAILQ_REMOVE_HEAD(&tmp_pkt_queue, nodes); free(pkt_node, M_SIFTR_PKTNODE); } KASSERT(STAILQ_EMPTY(&tmp_pkt_queue), ("SIFTR tmp_pkt_queue not empty after flush")); mtx_lock(&siftr_pkt_mgr_mtx); /* * If siftr_exit_pkt_manager_thread gets set during the window * where we are draining the tmp_pkt_queue above, there might * still be pkts in pkt_queue that need to be drained. * Allow one further iteration to occur after * siftr_exit_pkt_manager_thread has been set to ensure * pkt_queue is completely empty before we kill the thread. * * siftr_exit_pkt_manager_thread is set only after the pfil * hooks have been removed, so only 1 extra iteration * is needed to drain the queue. */ if (siftr_exit_pkt_manager_thread) draining--; } mtx_unlock(&siftr_pkt_mgr_mtx); /* Calls wakeup on this thread's struct thread ptr. */ kthread_exit(); } static uint32_t hash_pkt(struct mbuf *m, uint32_t offset) { uint32_t hash; hash = 0; while (m != NULL && offset > m->m_len) { /* * The IP packet payload does not start in this mbuf, so * need to figure out which mbuf it starts in and what offset * into the mbuf's data region the payload starts at. */ offset -= m->m_len; m = m->m_next; } while (m != NULL) { /* Ensure there is data in the mbuf */ if ((m->m_len - offset) > 0) hash = hash32_buf(m->m_data + offset, m->m_len - offset, hash); m = m->m_next; offset = 0; } return (hash); } /* * Check if a given mbuf has the SIFTR mbuf tag. If it does, log the fact that * it's a reinjected packet and return. If it doesn't, tag the mbuf and return. * Return value >0 means the caller should skip processing this mbuf. */ static inline int siftr_chkreinject(struct mbuf *m, int dir, struct siftr_stats *ss) { if (m_tag_locate(m, PACKET_COOKIE_SIFTR, PACKET_TAG_SIFTR, NULL) != NULL) { if (dir == PFIL_IN) ss->nskip_in_dejavu++; else ss->nskip_out_dejavu++; return (1); } else { struct m_tag *tag = m_tag_alloc(PACKET_COOKIE_SIFTR, PACKET_TAG_SIFTR, 0, M_NOWAIT); if (tag == NULL) { if (dir == PFIL_IN) ss->nskip_in_malloc++; else ss->nskip_out_malloc++; return (1); } m_tag_prepend(m, tag); } return (0); } /* * Look up an inpcb for a packet. Return the inpcb pointer if found, or NULL * otherwise. */ static inline struct inpcb * siftr_findinpcb(int ipver, struct ip *ip, struct mbuf *m, uint16_t sport, uint16_t dport, int dir, struct siftr_stats *ss) { struct inpcb *inp; /* We need the tcbinfo lock. */ INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); if (dir == PFIL_IN) inp = (ipver == INP_IPV4 ? in_pcblookup(&V_tcbinfo, ip->ip_src, sport, ip->ip_dst, dport, INPLOOKUP_RLOCKPCB, m->m_pkthdr.rcvif) : #ifdef SIFTR_IPV6 in6_pcblookup(&V_tcbinfo, &((struct ip6_hdr *)ip)->ip6_src, sport, &((struct ip6_hdr *)ip)->ip6_dst, dport, INPLOOKUP_RLOCKPCB, m->m_pkthdr.rcvif) #else NULL #endif ); else inp = (ipver == INP_IPV4 ? in_pcblookup(&V_tcbinfo, ip->ip_dst, dport, ip->ip_src, sport, INPLOOKUP_RLOCKPCB, m->m_pkthdr.rcvif) : #ifdef SIFTR_IPV6 in6_pcblookup(&V_tcbinfo, &((struct ip6_hdr *)ip)->ip6_dst, dport, &((struct ip6_hdr *)ip)->ip6_src, sport, INPLOOKUP_RLOCKPCB, m->m_pkthdr.rcvif) #else NULL #endif ); /* If we can't find the inpcb, bail. */ if (inp == NULL) { if (dir == PFIL_IN) ss->nskip_in_inpcb++; else ss->nskip_out_inpcb++; } return (inp); } static inline void siftr_siftdata(struct pkt_node *pn, struct inpcb *inp, struct tcpcb *tp, int ipver, int dir, int inp_locally_locked) { #ifdef SIFTR_IPV6 if (ipver == INP_IPV4) { pn->ip_laddr[3] = inp->inp_laddr.s_addr; pn->ip_faddr[3] = inp->inp_faddr.s_addr; #else *((uint32_t *)pn->ip_laddr) = inp->inp_laddr.s_addr; *((uint32_t *)pn->ip_faddr) = inp->inp_faddr.s_addr; #endif #ifdef SIFTR_IPV6 } else { pn->ip_laddr[0] = inp->in6p_laddr.s6_addr32[0]; pn->ip_laddr[1] = inp->in6p_laddr.s6_addr32[1]; pn->ip_laddr[2] = inp->in6p_laddr.s6_addr32[2]; pn->ip_laddr[3] = inp->in6p_laddr.s6_addr32[3]; pn->ip_faddr[0] = inp->in6p_faddr.s6_addr32[0]; pn->ip_faddr[1] = inp->in6p_faddr.s6_addr32[1]; pn->ip_faddr[2] = inp->in6p_faddr.s6_addr32[2]; pn->ip_faddr[3] = inp->in6p_faddr.s6_addr32[3]; } #endif pn->tcp_localport = inp->inp_lport; pn->tcp_foreignport = inp->inp_fport; pn->snd_cwnd = tp->snd_cwnd; pn->snd_wnd = tp->snd_wnd; pn->rcv_wnd = tp->rcv_wnd; pn->snd_bwnd = 0; /* Unused, kept for compat. */ pn->snd_ssthresh = tp->snd_ssthresh; pn->snd_scale = tp->snd_scale; pn->rcv_scale = tp->rcv_scale; pn->conn_state = tp->t_state; pn->max_seg_size = tp->t_maxseg; pn->smoothed_rtt = tp->t_srtt; pn->sack_enabled = (tp->t_flags & TF_SACK_PERMIT) != 0; pn->flags = tp->t_flags; pn->rxt_length = tp->t_rxtcur; pn->snd_buf_hiwater = inp->inp_socket->so_snd.sb_hiwat; pn->snd_buf_cc = sbused(&inp->inp_socket->so_snd); pn->rcv_buf_hiwater = inp->inp_socket->so_rcv.sb_hiwat; pn->rcv_buf_cc = sbused(&inp->inp_socket->so_rcv); pn->sent_inflight_bytes = tp->snd_max - tp->snd_una; pn->t_segqlen = tp->t_segqlen; pn->flowid = inp->inp_flowid; pn->flowtype = inp->inp_flowtype; /* We've finished accessing the tcb so release the lock. */ if (inp_locally_locked) INP_RUNLOCK(inp); pn->ipver = ipver; pn->direction = dir; /* * Significantly more accurate than using getmicrotime(), but slower! * Gives true microsecond resolution at the expense of a hit to * maximum pps throughput processing when SIFTR is loaded and enabled. */ microtime(&pn->tval); TCP_PROBE1(siftr, &pn); } /* * pfil hook that is called for each IPv4 packet making its way through the * stack in either direction. * The pfil subsystem holds a non-sleepable mutex somewhere when * calling our hook function, so we can't sleep at all. * It's very important to use the M_NOWAIT flag with all function calls * that support it so that they won't sleep, otherwise you get a panic. */ static int siftr_chkpkt(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, struct inpcb *inp) { struct pkt_node *pn; struct ip *ip; struct tcphdr *th; struct tcpcb *tp; struct siftr_stats *ss; unsigned int ip_hl; int inp_locally_locked; inp_locally_locked = 0; ss = DPCPU_PTR(ss); /* * m_pullup is not required here because ip_{input|output} * already do the heavy lifting for us. */ ip = mtod(*m, struct ip *); /* Only continue processing if the packet is TCP. */ if (ip->ip_p != IPPROTO_TCP) goto ret; /* * If a kernel subsystem reinjects packets into the stack, our pfil * hook will be called multiple times for the same packet. * Make sure we only process unique packets. */ if (siftr_chkreinject(*m, dir, ss)) goto ret; if (dir == PFIL_IN) ss->n_in++; else ss->n_out++; /* * Create a tcphdr struct starting at the correct offset * in the IP packet. ip->ip_hl gives the ip header length * in 4-byte words, so multiply it to get the size in bytes. */ ip_hl = (ip->ip_hl << 2); th = (struct tcphdr *)((caddr_t)ip + ip_hl); /* * If the pfil hooks don't provide a pointer to the * inpcb, we need to find it ourselves and lock it. */ if (!inp) { /* Find the corresponding inpcb for this pkt. */ inp = siftr_findinpcb(INP_IPV4, ip, *m, th->th_sport, th->th_dport, dir, ss); if (inp == NULL) goto ret; else inp_locally_locked = 1; } INP_LOCK_ASSERT(inp); /* Find the TCP control block that corresponds with this packet */ tp = intotcpcb(inp); /* * If we can't find the TCP control block (happens occasionaly for a * packet sent during the shutdown phase of a TCP connection), * or we're in the timewait state, bail */ if (tp == NULL || inp->inp_flags & INP_TIMEWAIT) { if (dir == PFIL_IN) ss->nskip_in_tcpcb++; else ss->nskip_out_tcpcb++; goto inp_unlock; } pn = malloc(sizeof(struct pkt_node), M_SIFTR_PKTNODE, M_NOWAIT|M_ZERO); if (pn == NULL) { if (dir == PFIL_IN) ss->nskip_in_malloc++; else ss->nskip_out_malloc++; goto inp_unlock; } siftr_siftdata(pn, inp, tp, INP_IPV4, dir, inp_locally_locked); if (siftr_generate_hashes) { if ((*m)->m_pkthdr.csum_flags & CSUM_TCP) { /* * For outbound packets, the TCP checksum isn't * calculated yet. This is a problem for our packet * hashing as the receiver will calc a different hash * to ours if we don't include the correct TCP checksum * in the bytes being hashed. To work around this * problem, we manually calc the TCP checksum here in * software. We unset the CSUM_TCP flag so the lower * layers don't recalc it. */ (*m)->m_pkthdr.csum_flags &= ~CSUM_TCP; /* * Calculate the TCP checksum in software and assign * to correct TCP header field, which will follow the * packet mbuf down the stack. The trick here is that * tcp_output() sets th->th_sum to the checksum of the * pseudo header for us already. Because of the nature * of the checksumming algorithm, we can sum over the * entire IP payload (i.e. TCP header and data), which * will include the already calculated pseduo header * checksum, thus giving us the complete TCP checksum. * * To put it in simple terms, if checksum(1,2,3,4)=10, * then checksum(1,2,3,4,5) == checksum(10,5). * This property is what allows us to "cheat" and * checksum only the IP payload which has the TCP * th_sum field populated with the pseudo header's * checksum, and not need to futz around checksumming * pseudo header bytes and TCP header/data in one hit. * Refer to RFC 1071 for more info. * * NB: in_cksum_skip(struct mbuf *m, int len, int skip) * in_cksum_skip 2nd argument is NOT the number of * bytes to read from the mbuf at "skip" bytes offset * from the start of the mbuf (very counter intuitive!). * The number of bytes to read is calculated internally * by the function as len-skip i.e. to sum over the IP * payload (TCP header + data) bytes, it is INCORRECT * to call the function like this: * in_cksum_skip(at, ip->ip_len - offset, offset) * Rather, it should be called like this: * in_cksum_skip(at, ip->ip_len, offset) * which means read "ip->ip_len - offset" bytes from * the mbuf cluster "at" at offset "offset" bytes from * the beginning of the "at" mbuf's data pointer. */ th->th_sum = in_cksum_skip(*m, ntohs(ip->ip_len), ip_hl); } /* * XXX: Having to calculate the checksum in software and then * hash over all bytes is really inefficient. Would be nice to * find a way to create the hash and checksum in the same pass * over the bytes. */ pn->hash = hash_pkt(*m, ip_hl); } mtx_lock(&siftr_pkt_queue_mtx); STAILQ_INSERT_TAIL(&pkt_queue, pn, nodes); mtx_unlock(&siftr_pkt_queue_mtx); goto ret; inp_unlock: if (inp_locally_locked) INP_RUNLOCK(inp); ret: /* Returning 0 ensures pfil will not discard the pkt */ return (0); } #ifdef SIFTR_IPV6 static int siftr_chkpkt6(void *arg, struct mbuf **m, struct ifnet *ifp, int dir, struct inpcb *inp) { struct pkt_node *pn; struct ip6_hdr *ip6; struct tcphdr *th; struct tcpcb *tp; struct siftr_stats *ss; unsigned int ip6_hl; int inp_locally_locked; inp_locally_locked = 0; ss = DPCPU_PTR(ss); /* * m_pullup is not required here because ip6_{input|output} * already do the heavy lifting for us. */ ip6 = mtod(*m, struct ip6_hdr *); /* * Only continue processing if the packet is TCP * XXX: We should follow the next header fields * as shown on Pg 6 RFC 2460, but right now we'll * only check pkts that have no extension headers. */ if (ip6->ip6_nxt != IPPROTO_TCP) goto ret6; /* * If a kernel subsystem reinjects packets into the stack, our pfil * hook will be called multiple times for the same packet. * Make sure we only process unique packets. */ if (siftr_chkreinject(*m, dir, ss)) goto ret6; if (dir == PFIL_IN) ss->n_in++; else ss->n_out++; ip6_hl = sizeof(struct ip6_hdr); /* * Create a tcphdr struct starting at the correct offset * in the ipv6 packet. ip->ip_hl gives the ip header length * in 4-byte words, so multiply it to get the size in bytes. */ th = (struct tcphdr *)((caddr_t)ip6 + ip6_hl); /* * For inbound packets, the pfil hooks don't provide a pointer to the * inpcb, so we need to find it ourselves and lock it. */ if (!inp) { /* Find the corresponding inpcb for this pkt. */ inp = siftr_findinpcb(INP_IPV6, (struct ip *)ip6, *m, th->th_sport, th->th_dport, dir, ss); if (inp == NULL) goto ret6; else inp_locally_locked = 1; } /* Find the TCP control block that corresponds with this packet. */ tp = intotcpcb(inp); /* * If we can't find the TCP control block (happens occasionaly for a * packet sent during the shutdown phase of a TCP connection), * or we're in the timewait state, bail. */ if (tp == NULL || inp->inp_flags & INP_TIMEWAIT) { if (dir == PFIL_IN) ss->nskip_in_tcpcb++; else ss->nskip_out_tcpcb++; goto inp_unlock6; } pn = malloc(sizeof(struct pkt_node), M_SIFTR_PKTNODE, M_NOWAIT|M_ZERO); if (pn == NULL) { if (dir == PFIL_IN) ss->nskip_in_malloc++; else ss->nskip_out_malloc++; goto inp_unlock6; } siftr_siftdata(pn, inp, tp, INP_IPV6, dir, inp_locally_locked); /* XXX: Figure out how to generate hashes for IPv6 packets. */ mtx_lock(&siftr_pkt_queue_mtx); STAILQ_INSERT_TAIL(&pkt_queue, pn, nodes); mtx_unlock(&siftr_pkt_queue_mtx); goto ret6; inp_unlock6: if (inp_locally_locked) INP_RUNLOCK(inp); ret6: /* Returning 0 ensures pfil will not discard the pkt. */ return (0); } #endif /* #ifdef SIFTR_IPV6 */ static int siftr_pfil(int action) { struct pfil_head *pfh_inet; #ifdef SIFTR_IPV6 struct pfil_head *pfh_inet6; #endif VNET_ITERATOR_DECL(vnet_iter); VNET_LIST_RLOCK(); VNET_FOREACH(vnet_iter) { CURVNET_SET(vnet_iter); pfh_inet = pfil_head_get(PFIL_TYPE_AF, AF_INET); #ifdef SIFTR_IPV6 pfh_inet6 = pfil_head_get(PFIL_TYPE_AF, AF_INET6); #endif if (action == HOOK) { pfil_add_hook(siftr_chkpkt, NULL, PFIL_IN | PFIL_OUT | PFIL_WAITOK, pfh_inet); #ifdef SIFTR_IPV6 pfil_add_hook(siftr_chkpkt6, NULL, PFIL_IN | PFIL_OUT | PFIL_WAITOK, pfh_inet6); #endif } else if (action == UNHOOK) { pfil_remove_hook(siftr_chkpkt, NULL, PFIL_IN | PFIL_OUT | PFIL_WAITOK, pfh_inet); #ifdef SIFTR_IPV6 pfil_remove_hook(siftr_chkpkt6, NULL, PFIL_IN | PFIL_OUT | PFIL_WAITOK, pfh_inet6); #endif } CURVNET_RESTORE(); } VNET_LIST_RUNLOCK(); return (0); } static int siftr_sysctl_logfile_name_handler(SYSCTL_HANDLER_ARGS) { struct alq *new_alq; int error; error = sysctl_handle_string(oidp, arg1, arg2, req); /* Check for error or same filename */ if (error != 0 || req->newptr == NULL || strncmp(siftr_logfile, arg1, arg2) == 0) goto done; /* Filname changed */ error = alq_open(&new_alq, arg1, curthread->td_ucred, SIFTR_LOG_FILE_MODE, SIFTR_ALQ_BUFLEN, 0); if (error != 0) goto done; /* * If disabled, siftr_alq == NULL so we simply close * the alq as we've proved it can be opened. * If enabled, close the existing alq and switch the old * for the new. */ if (siftr_alq == NULL) { alq_close(new_alq); } else { alq_close(siftr_alq); siftr_alq = new_alq; } /* Update filename upon success */ strlcpy(siftr_logfile, arg1, arg2); done: return (error); } static int siftr_manage_ops(uint8_t action) { struct siftr_stats totalss; struct timeval tval; struct flow_hash_node *counter, *tmp_counter; struct sbuf *s; int i, key_index, ret, error; uint32_t bytes_to_write, total_skipped_pkts; uint16_t lport, fport; uint8_t *key, ipver; #ifdef SIFTR_IPV6 uint32_t laddr[4]; uint32_t faddr[4]; #else uint8_t laddr[4]; uint8_t faddr[4]; #endif error = 0; total_skipped_pkts = 0; /* Init an autosizing sbuf that initially holds 200 chars. */ if ((s = sbuf_new(NULL, NULL, 200, SBUF_AUTOEXTEND)) == NULL) return (-1); if (action == SIFTR_ENABLE) { /* * Create our alq * XXX: We should abort if alq_open fails! */ alq_open(&siftr_alq, siftr_logfile, curthread->td_ucred, SIFTR_LOG_FILE_MODE, SIFTR_ALQ_BUFLEN, 0); STAILQ_INIT(&pkt_queue); DPCPU_ZERO(ss); siftr_exit_pkt_manager_thread = 0; ret = kthread_add(&siftr_pkt_manager_thread, NULL, NULL, &siftr_pkt_manager_thr, RFNOWAIT, 0, "siftr_pkt_manager_thr"); siftr_pfil(HOOK); microtime(&tval); sbuf_printf(s, "enable_time_secs=%jd\tenable_time_usecs=%06ld\t" "siftrver=%s\thz=%u\ttcp_rtt_scale=%u\tsysname=%s\t" "sysver=%u\tipmode=%u\n", (intmax_t)tval.tv_sec, tval.tv_usec, MODVERSION_STR, hz, TCP_RTT_SCALE, SYS_NAME, __FreeBSD_version, SIFTR_IPMODE); sbuf_finish(s); alq_writen(siftr_alq, sbuf_data(s), sbuf_len(s), ALQ_WAITOK); } else if (action == SIFTR_DISABLE && siftr_pkt_manager_thr != NULL) { /* * Remove the pfil hook functions. All threads currently in * the hook functions are allowed to exit before siftr_pfil() * returns. */ siftr_pfil(UNHOOK); /* This will block until the pkt manager thread unlocks it. */ mtx_lock(&siftr_pkt_mgr_mtx); /* Tell the pkt manager thread that it should exit now. */ siftr_exit_pkt_manager_thread = 1; /* * Wake the pkt_manager thread so it realises that * siftr_exit_pkt_manager_thread == 1 and exits gracefully. * The wakeup won't be delivered until we unlock * siftr_pkt_mgr_mtx so this isn't racy. */ wakeup(&wait_for_pkt); /* Wait for the pkt_manager thread to exit. */ mtx_sleep(siftr_pkt_manager_thr, &siftr_pkt_mgr_mtx, PWAIT, "thrwait", 0); siftr_pkt_manager_thr = NULL; mtx_unlock(&siftr_pkt_mgr_mtx); totalss.n_in = DPCPU_VARSUM(ss, n_in); totalss.n_out = DPCPU_VARSUM(ss, n_out); totalss.nskip_in_malloc = DPCPU_VARSUM(ss, nskip_in_malloc); totalss.nskip_out_malloc = DPCPU_VARSUM(ss, nskip_out_malloc); totalss.nskip_in_mtx = DPCPU_VARSUM(ss, nskip_in_mtx); totalss.nskip_out_mtx = DPCPU_VARSUM(ss, nskip_out_mtx); totalss.nskip_in_tcpcb = DPCPU_VARSUM(ss, nskip_in_tcpcb); totalss.nskip_out_tcpcb = DPCPU_VARSUM(ss, nskip_out_tcpcb); totalss.nskip_in_inpcb = DPCPU_VARSUM(ss, nskip_in_inpcb); totalss.nskip_out_inpcb = DPCPU_VARSUM(ss, nskip_out_inpcb); total_skipped_pkts = totalss.nskip_in_malloc + totalss.nskip_out_malloc + totalss.nskip_in_mtx + totalss.nskip_out_mtx + totalss.nskip_in_tcpcb + totalss.nskip_out_tcpcb + totalss.nskip_in_inpcb + totalss.nskip_out_inpcb; microtime(&tval); sbuf_printf(s, "disable_time_secs=%jd\tdisable_time_usecs=%06ld\t" "num_inbound_tcp_pkts=%ju\tnum_outbound_tcp_pkts=%ju\t" "total_tcp_pkts=%ju\tnum_inbound_skipped_pkts_malloc=%u\t" "num_outbound_skipped_pkts_malloc=%u\t" "num_inbound_skipped_pkts_mtx=%u\t" "num_outbound_skipped_pkts_mtx=%u\t" "num_inbound_skipped_pkts_tcpcb=%u\t" "num_outbound_skipped_pkts_tcpcb=%u\t" "num_inbound_skipped_pkts_inpcb=%u\t" "num_outbound_skipped_pkts_inpcb=%u\t" "total_skipped_tcp_pkts=%u\tflow_list=", (intmax_t)tval.tv_sec, tval.tv_usec, (uintmax_t)totalss.n_in, (uintmax_t)totalss.n_out, (uintmax_t)(totalss.n_in + totalss.n_out), totalss.nskip_in_malloc, totalss.nskip_out_malloc, totalss.nskip_in_mtx, totalss.nskip_out_mtx, totalss.nskip_in_tcpcb, totalss.nskip_out_tcpcb, totalss.nskip_in_inpcb, totalss.nskip_out_inpcb, total_skipped_pkts); /* * Iterate over the flow hash, printing a summary of each * flow seen and freeing any malloc'd memory. * The hash consists of an array of LISTs (man 3 queue). */ for (i = 0; i <= siftr_hashmask; i++) { LIST_FOREACH_SAFE(counter, counter_hash + i, nodes, tmp_counter) { key = counter->key; key_index = 1; ipver = key[0]; memcpy(laddr, key + key_index, sizeof(laddr)); key_index += sizeof(laddr); memcpy(&lport, key + key_index, sizeof(lport)); key_index += sizeof(lport); memcpy(faddr, key + key_index, sizeof(faddr)); key_index += sizeof(faddr); memcpy(&fport, key + key_index, sizeof(fport)); #ifdef SIFTR_IPV6 laddr[3] = ntohl(laddr[3]); faddr[3] = ntohl(faddr[3]); if (ipver == INP_IPV6) { laddr[0] = ntohl(laddr[0]); laddr[1] = ntohl(laddr[1]); laddr[2] = ntohl(laddr[2]); faddr[0] = ntohl(faddr[0]); faddr[1] = ntohl(faddr[1]); faddr[2] = ntohl(faddr[2]); sbuf_printf(s, "%x:%x:%x:%x:%x:%x:%x:%x;%u-" "%x:%x:%x:%x:%x:%x:%x:%x;%u,", UPPER_SHORT(laddr[0]), LOWER_SHORT(laddr[0]), UPPER_SHORT(laddr[1]), LOWER_SHORT(laddr[1]), UPPER_SHORT(laddr[2]), LOWER_SHORT(laddr[2]), UPPER_SHORT(laddr[3]), LOWER_SHORT(laddr[3]), ntohs(lport), UPPER_SHORT(faddr[0]), LOWER_SHORT(faddr[0]), UPPER_SHORT(faddr[1]), LOWER_SHORT(faddr[1]), UPPER_SHORT(faddr[2]), LOWER_SHORT(faddr[2]), UPPER_SHORT(faddr[3]), LOWER_SHORT(faddr[3]), ntohs(fport)); } else { laddr[0] = FIRST_OCTET(laddr[3]); laddr[1] = SECOND_OCTET(laddr[3]); laddr[2] = THIRD_OCTET(laddr[3]); laddr[3] = FOURTH_OCTET(laddr[3]); faddr[0] = FIRST_OCTET(faddr[3]); faddr[1] = SECOND_OCTET(faddr[3]); faddr[2] = THIRD_OCTET(faddr[3]); faddr[3] = FOURTH_OCTET(faddr[3]); #endif sbuf_printf(s, "%u.%u.%u.%u;%u-%u.%u.%u.%u;%u,", laddr[0], laddr[1], laddr[2], laddr[3], ntohs(lport), faddr[0], faddr[1], faddr[2], faddr[3], ntohs(fport)); #ifdef SIFTR_IPV6 } #endif free(counter, M_SIFTR_HASHNODE); } LIST_INIT(counter_hash + i); } sbuf_printf(s, "\n"); sbuf_finish(s); i = 0; do { bytes_to_write = min(SIFTR_ALQ_BUFLEN, sbuf_len(s)-i); alq_writen(siftr_alq, sbuf_data(s)+i, bytes_to_write, ALQ_WAITOK); i += bytes_to_write; } while (i < sbuf_len(s)); alq_close(siftr_alq); siftr_alq = NULL; } sbuf_delete(s); /* * XXX: Should be using ret to check if any functions fail * and set error appropriately */ return (error); } static int siftr_sysctl_enabled_handler(SYSCTL_HANDLER_ARGS) { - if (req->newptr == NULL) - goto skip; + int error; + uint32_t new; - /* If the value passed in isn't 0 or 1, return an error. */ - if (CAST_PTR_INT(req->newptr) != 0 && CAST_PTR_INT(req->newptr) != 1) - return (1); - - /* If we are changing state (0 to 1 or 1 to 0). */ - if (CAST_PTR_INT(req->newptr) != siftr_enabled ) - if (siftr_manage_ops(CAST_PTR_INT(req->newptr))) { - siftr_manage_ops(SIFTR_DISABLE); - return (1); + new = siftr_enabled; + error = sysctl_handle_int(oidp, &new, 0, req); + if (error != 0 && req->newptr != NULL) { + if (new > 1) + return (EINVAL); + else if (new != siftr_enabled) { + error = siftr_manage_ops(new); + if (error != 0) + siftr_manage_ops(SIFTR_DISABLE); } + } -skip: - return (sysctl_handle_int(oidp, arg1, arg2, req)); + return (error); } static void siftr_shutdown_handler(void *arg) { siftr_manage_ops(SIFTR_DISABLE); } /* * Module is being unloaded or machine is shutting down. Take care of cleanup. */ static int deinit_siftr(void) { /* Cleanup. */ siftr_manage_ops(SIFTR_DISABLE); hashdestroy(counter_hash, M_SIFTR, siftr_hashmask); mtx_destroy(&siftr_pkt_queue_mtx); mtx_destroy(&siftr_pkt_mgr_mtx); return (0); } /* * Module has just been loaded into the kernel. */ static int init_siftr(void) { EVENTHANDLER_REGISTER(shutdown_pre_sync, siftr_shutdown_handler, NULL, SHUTDOWN_PRI_FIRST); /* Initialise our flow counter hash table. */ counter_hash = hashinit(SIFTR_EXPECTED_MAX_TCP_FLOWS, M_SIFTR, &siftr_hashmask); mtx_init(&siftr_pkt_queue_mtx, "siftr_pkt_queue_mtx", NULL, MTX_DEF); mtx_init(&siftr_pkt_mgr_mtx, "siftr_pkt_mgr_mtx", NULL, MTX_DEF); /* Print message to the user's current terminal. */ uprintf("\nStatistical Information For TCP Research (SIFTR) %s\n" " http://caia.swin.edu.au/urp/newtcp\n\n", MODVERSION_STR); return (0); } /* * This is the function that is called to load and unload the module. * When the module is loaded, this function is called once with * "what" == MOD_LOAD * When the module is unloaded, this function is called twice with * "what" = MOD_QUIESCE first, followed by "what" = MOD_UNLOAD second * When the system is shut down e.g. CTRL-ALT-DEL or using the shutdown command, * this function is called once with "what" = MOD_SHUTDOWN * When the system is shut down, the handler isn't called until the very end * of the shutdown sequence i.e. after the disks have been synced. */ static int siftr_load_handler(module_t mod, int what, void *arg) { int ret; switch (what) { case MOD_LOAD: ret = init_siftr(); break; case MOD_QUIESCE: case MOD_SHUTDOWN: ret = deinit_siftr(); break; case MOD_UNLOAD: ret = 0; break; default: ret = EINVAL; break; } return (ret); } static moduledata_t siftr_mod = { .name = "siftr", .evhand = siftr_load_handler, }; /* * Param 1: name of the kernel module * Param 2: moduledata_t struct containing info about the kernel module * and the execution entry point for the module * Param 3: From sysinit_sub_id enumeration in /usr/include/sys/kernel.h * Defines the module initialisation order * Param 4: From sysinit_elem_order enumeration in /usr/include/sys/kernel.h * Defines the initialisation order of this kld relative to others * within the same subsystem as defined by param 3 */ DECLARE_MODULE(siftr, siftr_mod, SI_SUB_LAST, SI_ORDER_ANY); MODULE_DEPEND(siftr, alq, 1, 1, 1); MODULE_VERSION(siftr, MODVERSION); Index: stable/11 =================================================================== --- stable/11 (revision 342188) +++ stable/11 (revision 342189) Property changes on: stable/11 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r342125