Index: head/sbin/ipfw/dummynet.c =================================================================== --- head/sbin/ipfw/dummynet.c (revision 300778) +++ head/sbin/ipfw/dummynet.c (revision 300779) @@ -1,1410 +1,1988 @@ /* + * Codel/FQ_Codel and PIE/FQ_PIE Code: + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * * Copyright (c) 2002-2003,2010 Luigi Rizzo * * Redistribution and use in source forms, with and without modification, * are permitted provided that this entire comment appears intact. * * Redistribution in binary form may occur without any restrictions. * Obviously, it would be nice if you gave credit where credit is due * but requiring it would be too onerous. * * This software is provided ``AS IS'' without any warranties of any kind. * * $FreeBSD$ * * dummynet support */ +#define NEW_AQM #include #include /* XXX there are several sysctl leftover here */ #include #include "ipfw2.h" +#ifdef NEW_AQM +#include +#endif + #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* inet_ntoa */ static struct _s_x dummynet_params[] = { { "plr", TOK_PLR }, { "noerror", TOK_NOERROR }, { "buckets", TOK_BUCKETS }, { "dst-ip", TOK_DSTIP }, { "src-ip", TOK_SRCIP }, { "dst-port", TOK_DSTPORT }, { "src-port", TOK_SRCPORT }, { "proto", TOK_PROTO }, { "weight", TOK_WEIGHT }, { "lmax", TOK_LMAX }, { "maxlen", TOK_LMAX }, { "all", TOK_ALL }, { "mask", TOK_MASK }, /* alias for both */ { "sched_mask", TOK_SCHED_MASK }, { "flow_mask", TOK_FLOW_MASK }, { "droptail", TOK_DROPTAIL }, { "ecn", TOK_ECN }, { "red", TOK_RED }, { "gred", TOK_GRED }, +#ifdef NEW_AQM + { "codel", TOK_CODEL}, /* Codel AQM */ + { "fq_codel", TOK_FQ_CODEL}, /* FQ-Codel */ + { "pie", TOK_PIE}, /* PIE AQM */ + { "fq_pie", TOK_FQ_PIE}, /* FQ-PIE */ +#endif { "bw", TOK_BW }, { "bandwidth", TOK_BW }, { "delay", TOK_DELAY }, { "link", TOK_LINK }, { "pipe", TOK_PIPE }, { "queue", TOK_QUEUE }, { "flowset", TOK_FLOWSET }, { "sched", TOK_SCHED }, { "pri", TOK_PRI }, { "priority", TOK_PRI }, { "type", TOK_TYPE }, { "flow-id", TOK_FLOWID}, { "dst-ipv6", TOK_DSTIP6}, { "dst-ip6", TOK_DSTIP6}, { "src-ipv6", TOK_SRCIP6}, { "src-ip6", TOK_SRCIP6}, { "profile", TOK_PROFILE}, { "burst", TOK_BURST}, { "dummynet-params", TOK_NULL }, { NULL, 0 } /* terminator */ }; +#ifdef NEW_AQM +/* AQM/extra sched parameters tokens*/ +static struct _s_x aqm_params[] = { + { "target", TOK_TARGET}, + { "interval", TOK_INTERVAL}, + { "limit", TOK_LIMIT}, + { "flows", TOK_FLOWS}, + { "quantum", TOK_QUANTUM}, + { "ecn", TOK_ECN}, + { "noecn", TOK_NO_ECN}, + { "tupdate", TOK_TUPDATE}, + { "max_burst", TOK_MAX_BURST}, + { "max_ecnth", TOK_MAX_ECNTH}, + { "alpha", TOK_ALPHA}, + { "beta", TOK_BETA}, + { "capdrop", TOK_CAPDROP}, + { "nocapdrop", TOK_NO_CAPDROP}, + { "onoff", TOK_ONOFF}, + { "dre", TOK_DRE}, + { "ts", TOK_TS}, + { "derand", TOK_DERAND}, + { "noderand", TOK_NO_DERAND}, + { NULL, 0 } /* terminator */ +}; +#endif + #define O_NEXT(p, len) ((void *)((char *)p + len)) static void oid_fill(struct dn_id *oid, int len, int type, uintptr_t id) { oid->len = len; oid->type = type; oid->subtype = 0; oid->id = id; } /* make room in the buffer and move the pointer forward */ static void * o_next(struct dn_id **o, int len, int type) { struct dn_id *ret = *o; oid_fill(ret, len, type, 0); *o = O_NEXT(*o, len); return ret; } +#ifdef NEW_AQM + +/* Codel flags */ +enum { + CODEL_ECN_ENABLED = 1 +}; + +/* PIE flags, from PIE kernel module */ +enum { + PIE_ECN_ENABLED = 1, + PIE_CAPDROP_ENABLED = 2, + PIE_ON_OFF_MODE_ENABLED = 4, + PIE_DEPRATEEST_ENABLED = 8, + PIE_DERAND_ENABLED = 16 +}; + +#define PIE_FIX_POINT_BITS 13 +#define PIE_SCALE (1L<15) + return -1; + for (i = 0; ioid, l, DN_CMD_GET, DN_API_VERSION); + ep->oid.len = l; + ep->oid.subtype = subtype; + ep->nr = nr; + + ret = do_cmd(-IP_DUMMYNET3, ep, (uintptr_t)&l); + if (ret) { + free(ep); + errx(EX_DATAERR, "Error getting extra parameters\n"); + } + + switch (subtype) { + case DN_AQM_PARAMS: + if( !strcasecmp(ep->name, "codel")) { + us_to_time(ep->par[0], strt1); + us_to_time(ep->par[1], strt2); + l = sprintf(out, " AQM CoDel target %s interval %s", + strt1, strt2); + if (ep->par[2] & CODEL_ECN_ENABLED) + l = sprintf(out + l, " ECN"); + else + l += sprintf(out + l, " NoECN"); + } else if( !strcasecmp(ep->name, "pie")) { + us_to_time(ep->par[0], strt1); + us_to_time(ep->par[1], strt2); + us_to_time(ep->par[2], strt3); + l = sprintf(out, " AQM type PIE target %s tupdate %s alpha " + "%g beta %g max_burst %s max_ecnth %.3g", + strt1, + strt2, + ep->par[4] / (float) PIE_SCALE, + ep->par[5] / (float) PIE_SCALE, + strt3, + ep->par[3] / (float) PIE_SCALE + ); + + if (ep->par[6] & PIE_ECN_ENABLED) + l += sprintf(out + l, " ECN"); + else + l += sprintf(out + l, " NoECN"); + if (ep->par[6] & PIE_CAPDROP_ENABLED) + l += sprintf(out + l, " CapDrop"); + else + l += sprintf(out + l, " NoCapDrop"); + if (ep->par[6] & PIE_ON_OFF_MODE_ENABLED) + l += sprintf(out + l, " OnOff"); + if (ep->par[6] & PIE_DEPRATEEST_ENABLED) + l += sprintf(out + l, " DRE"); + else + l += sprintf(out + l, " TS"); + if (ep->par[6] & PIE_DERAND_ENABLED) + l += sprintf(out + l, " Derand"); + else + l += sprintf(out + l, " NoDerand"); + } + break; + + case DN_SCH_PARAMS: + if (!strcasecmp(ep->name,"FQ_CODEL")) { + us_to_time(ep->par[0], strt1); + us_to_time(ep->par[1], strt2); + l = sprintf(out," FQ_CODEL target %s interval %s" + " quantum %jd limit %jd flows %jd", + strt1, strt2, + (intmax_t) ep->par[3], + (intmax_t) ep->par[4], + (intmax_t) ep->par[5] + ); + if (ep->par[2] & CODEL_ECN_ENABLED) + l += sprintf(out + l, " ECN"); + else + l += sprintf(out + l, " NoECN"); + l += sprintf(out + l, "\n"); + } else if (!strcasecmp(ep->name,"FQ_PIE")) { + us_to_time(ep->par[0], strt1); + us_to_time(ep->par[1], strt2); + us_to_time(ep->par[2], strt3); + l = sprintf(out, " FQ_PIE target %s tupdate %s alpha " + "%g beta %g max_burst %s max_ecnth %.3g" + " quantum %jd limit %jd flows %jd", + strt1, + strt2, + ep->par[4] / (float) PIE_SCALE, + ep->par[5] / (float) PIE_SCALE, + strt3, + ep->par[3] / (float) PIE_SCALE, + (intmax_t) ep->par[7], + (intmax_t) ep->par[8], + (intmax_t) ep->par[9] + ); + + if (ep->par[6] & PIE_ECN_ENABLED) + l += sprintf(out + l, " ECN"); + else + l += sprintf(out + l, " NoECN"); + if (ep->par[6] & PIE_CAPDROP_ENABLED) + l += sprintf(out + l, " CapDrop"); + else + l += sprintf(out + l, " NoCapDrop"); + if (ep->par[6] & PIE_ON_OFF_MODE_ENABLED) + l += sprintf(out + l, " OnOff"); + if (ep->par[6] & PIE_DEPRATEEST_ENABLED) + l += sprintf(out + l, " DRE"); + else + l += sprintf(out + l, " TS"); + if (ep->par[6] & PIE_DERAND_ENABLED) + l += sprintf(out + l, " Derand"); + else + l += sprintf(out + l, " NoDerand"); + l += sprintf(out + l, "\n"); + } + break; + } + + free(ep); +} +#endif + + #if 0 static int sort_q(void *arg, const void *pa, const void *pb) { int rev = (co.do_sort < 0); int field = rev ? -co.do_sort : co.do_sort; long long res = 0; const struct dn_flow_queue *a = pa; const struct dn_flow_queue *b = pb; switch (field) { case 1: /* pkts */ res = a->len - b->len; break; case 2: /* bytes */ res = a->len_bytes - b->len_bytes; break; case 3: /* tot pkts */ res = a->tot_pkts - b->tot_pkts; break; case 4: /* tot bytes */ res = a->tot_bytes - b->tot_bytes; break; } if (res < 0) res = -1; if (res > 0) res = 1; return (int)(rev ? res : -res); } #endif /* print a mask and header for the subsequent list of flows */ static void print_mask(struct ipfw_flow_id *id) { if (!IS_IP6_FLOW_ID(id)) { printf(" " "mask: %s 0x%02x 0x%08x/0x%04x -> 0x%08x/0x%04x\n", id->extra ? "queue," : "", id->proto, id->src_ip, id->src_port, id->dst_ip, id->dst_port); } else { char buf[255]; printf("\n mask: %sproto: 0x%02x, flow_id: 0x%08x, ", id->extra ? "queue," : "", id->proto, id->flow_id6); inet_ntop(AF_INET6, &(id->src_ip6), buf, sizeof(buf)); printf("%s/0x%04x -> ", buf, id->src_port); inet_ntop(AF_INET6, &(id->dst_ip6), buf, sizeof(buf)); printf("%s/0x%04x\n", buf, id->dst_port); } } static void print_header(struct ipfw_flow_id *id) { if (!IS_IP6_FLOW_ID(id)) printf("BKT Prot ___Source IP/port____ " "____Dest. IP/port____ " "Tot_pkt/bytes Pkt/Byte Drp\n"); else printf("BKT ___Prot___ _flow-id_ " "______________Source IPv6/port_______________ " "_______________Dest. IPv6/port_______________ " "Tot_pkt/bytes Pkt/Byte Drp\n"); } static void list_flow(struct buf_pr *bp, struct dn_flow *ni) { char buff[255]; struct protoent *pe = NULL; struct in_addr ina; struct ipfw_flow_id *id = &ni->fid; pe = getprotobynumber(id->proto); /* XXX: Should check for IPv4 flows */ bprintf(bp, "%3u%c", (ni->oid.id) & 0xff, id->extra ? '*' : ' '); if (!IS_IP6_FLOW_ID(id)) { if (pe) bprintf(bp, "%-4s ", pe->p_name); else bprintf(bp, "%4u ", id->proto); ina.s_addr = htonl(id->src_ip); bprintf(bp, "%15s/%-5d ", inet_ntoa(ina), id->src_port); ina.s_addr = htonl(id->dst_ip); bprintf(bp, "%15s/%-5d ", inet_ntoa(ina), id->dst_port); } else { /* Print IPv6 flows */ if (pe != NULL) bprintf(bp, "%9s ", pe->p_name); else bprintf(bp, "%9u ", id->proto); bprintf(bp, "%7d %39s/%-5d ", id->flow_id6, inet_ntop(AF_INET6, &(id->src_ip6), buff, sizeof(buff)), id->src_port); bprintf(bp, " %39s/%-5d ", inet_ntop(AF_INET6, &(id->dst_ip6), buff, sizeof(buff)), id->dst_port); } pr_u64(bp, &ni->tot_pkts, 4); pr_u64(bp, &ni->tot_bytes, 8); bprintf(bp, "%2u %4u %3u", ni->length, ni->len_bytes, ni->drops); } static void print_flowset_parms(struct dn_fs *fs, char *prefix) { int l; char qs[30]; char plr[30]; - char red[90]; /* Display RED parameters */ + char red[200]; /* Display RED parameters */ l = fs->qsize; if (fs->flags & DN_QSIZE_BYTES) { if (l >= 8192) sprintf(qs, "%d KB", l / 1024); else sprintf(qs, "%d B", l); } else sprintf(qs, "%3d sl.", l); if (fs->plr) sprintf(plr, "plr %f", 1.0 * fs->plr / (double)(0x7fffffff)); else plr[0] = '\0'; if (fs->flags & DN_IS_RED) { /* RED parameters */ sprintf(red, "\n\t %cRED w_q %f min_th %d max_th %d max_p %f", (fs->flags & DN_IS_GENTLE_RED) ? 'G' : ' ', 1.0 * fs->w_q / (double)(1 << SCALE_RED), fs->min_th, fs->max_th, 1.0 * fs->max_p / (double)(1 << SCALE_RED)); if (fs->flags & DN_IS_ECN) strncat(red, " (ecn)", 6); +#ifdef NEW_AQM + /* get AQM parameters */ + } else if (fs->flags & DN_IS_AQM) { + get_extra_parms(fs->fs_nr, red, DN_AQM_PARAMS); +#endif } else sprintf(red, "droptail"); if (prefix[0]) { printf("%s %s%s %d queues (%d buckets) %s\n", prefix, qs, plr, fs->oid.id, fs->buckets, red); prefix[0] = '\0'; } else { printf("q%05d %s%s %d flows (%d buckets) sched %d " "weight %d lmax %d pri %d %s\n", fs->fs_nr, qs, plr, fs->oid.id, fs->buckets, fs->sched_nr, fs->par[0], fs->par[1], fs->par[2], red); if (fs->flags & DN_HAVE_MASK) print_mask(&fs->flow_mask); } } static void print_extra_delay_parms(struct dn_profile *p) { double loss; if (p->samples_no <= 0) return; loss = p->loss_level; loss /= p->samples_no; printf("\t profile: name \"%s\" loss %f samples %d\n", p->name, loss, p->samples_no); } static void flush_buf(char *buf) { if (buf[0]) printf("%s\n", buf); buf[0] = '\0'; } /* * generic list routine. We expect objects in a specific order, i.e. * PIPES AND SCHEDULERS: * link; scheduler; internal flowset if any; instances * we can tell a pipe from the number. * * FLOWSETS: * flowset; queues; * link i (int queue); scheduler i; si(i) { flowsets() : queues } */ static void list_pipes(struct dn_id *oid, struct dn_id *end) { char buf[160]; /* pending buffer */ int toPrint = 1; /* print header */ struct buf_pr bp; buf[0] = '\0'; bp_alloc(&bp, 4096); for (; oid != end; oid = O_NEXT(oid, oid->len)) { if (oid->len < sizeof(*oid)) errx(1, "invalid oid len %d\n", oid->len); switch (oid->type) { default: flush_buf(buf); printf("unrecognized object %d size %d\n", oid->type, oid->len); break; case DN_TEXT: /* list of attached flowsets */ { int i, l; struct { struct dn_id id; uint32_t p[0]; } *d = (void *)oid; l = (oid->len - sizeof(*oid))/sizeof(d->p[0]); if (l == 0) break; printf(" Children flowsets: "); for (i = 0; i < l; i++) printf("%u ", d->p[i]); printf("\n"); break; } case DN_CMD_GET: if (co.verbose) printf("answer for cmd %d, len %d\n", oid->type, oid->id); break; case DN_SCH: { struct dn_sch *s = (struct dn_sch *)oid; flush_buf(buf); printf(" sched %d type %s flags 0x%x %d buckets %d active\n", s->sched_nr, s->name, s->flags, s->buckets, s->oid.id); +#ifdef NEW_AQM + char parms[200]; + get_extra_parms(s->sched_nr, parms, DN_SCH_PARAMS); + printf("%s",parms); +#endif if (s->flags & DN_HAVE_MASK) print_mask(&s->sched_mask); } break; case DN_FLOW: if (toPrint != 0) { print_header(&((struct dn_flow *)oid)->fid); toPrint = 0; } list_flow(&bp, (struct dn_flow *)oid); printf("%s\n", bp.buf); break; case DN_LINK: { struct dn_link *p = (struct dn_link *)oid; double b = p->bandwidth; char bwbuf[30]; char burst[5 + 7]; /* This starts a new object so flush buffer */ flush_buf(buf); /* data rate */ if (b == 0) sprintf(bwbuf, "unlimited "); else if (b >= 1000000) sprintf(bwbuf, "%7.3f Mbit/s", b/1000000); else if (b >= 1000) sprintf(bwbuf, "%7.3f Kbit/s", b/1000); else sprintf(bwbuf, "%7.3f bit/s ", b); if (humanize_number(burst, sizeof(burst), p->burst, "", HN_AUTOSCALE, 0) < 0 || co.verbose) sprintf(burst, "%d", (int)p->burst); sprintf(buf, "%05d: %s %4d ms burst %s", p->link_nr % DN_MAX_ID, bwbuf, p->delay, burst); } break; case DN_FS: print_flowset_parms((struct dn_fs *)oid, buf); break; case DN_PROFILE: flush_buf(buf); print_extra_delay_parms((struct dn_profile *)oid); } flush_buf(buf); // XXX does it really go here ? } bp_free(&bp); } /* * Delete pipe, queue or scheduler i */ int ipfw_delete_pipe(int do_pipe, int i) { struct { struct dn_id oid; uintptr_t a[1]; /* add more if we want a list */ } cmd; oid_fill((void *)&cmd, sizeof(cmd), DN_CMD_DELETE, DN_API_VERSION); cmd.oid.subtype = (do_pipe == 1) ? DN_LINK : ( (do_pipe == 2) ? DN_FS : DN_SCH); cmd.a[0] = i; i = do_cmd(IP_DUMMYNET3, &cmd, cmd.oid.len); if (i) { i = 1; warn("rule %u: setsockopt(IP_DUMMYNET_DEL)", i); } return i; } /* * Code to parse delay profiles. * * Some link types introduce extra delays in the transmission * of a packet, e.g. because of MAC level framing, contention on * the use of the channel, MAC level retransmissions and so on. * From our point of view, the channel is effectively unavailable * for this extra time, which is constant or variable depending * on the link type. Additionally, packets may be dropped after this * time (e.g. on a wireless link after too many retransmissions). * We can model the additional delay with an empirical curve * that represents its distribution. * * cumulative probability * 1.0 ^ * | * L +-- loss-level x * | ****** * | * * | ***** * | * * | ** * | * * +-------*-------------------> * delay * * The empirical curve may have both vertical and horizontal lines. * Vertical lines represent constant delay for a range of * probabilities; horizontal lines correspond to a discontinuty * in the delay distribution: the link will use the largest delay * for a given probability. * * To pass the curve to dummynet, we must store the parameters * in a file as described below, and issue the command * * ipfw pipe config ... bw XXX profile ... * * The file format is the following, with whitespace acting as * a separator and '#' indicating the beginning a comment: * * samples N * the number of samples used in the internal * representation (2..1024; default 100); * * loss-level L * The probability above which packets are lost. * (0.0 <= L <= 1.0, default 1.0 i.e. no loss); * * name identifier * Optional a name (listed by "ipfw pipe show") * to identify the distribution; * * "delay prob" | "prob delay" * One of these two lines is mandatory and defines * the format of the following lines with data points. * * XXX YYY * 2 or more lines representing points in the curve, * with either delay or probability first, according * to the chosen format. * The unit for delay is milliseconds. * * Data points does not need to be ordered or equal to the number * specified in the "samples" line. ipfw will sort and interpolate * the curve as needed. * * Example of a profile file: name bla_bla_bla samples 100 loss-level 0.86 prob delay 0 200 # minimum overhead is 200ms 0.5 200 0.5 300 0.8 1000 0.9 1300 1 1300 * Internally, we will convert the curve to a fixed number of * samples, and when it is time to transmit a packet we will * model the extra delay as extra bits in the packet. * */ #define ED_MAX_LINE_LEN 256+ED_MAX_NAME_LEN #define ED_TOK_SAMPLES "samples" #define ED_TOK_LOSS "loss-level" #define ED_TOK_NAME "name" #define ED_TOK_DELAY "delay" #define ED_TOK_PROB "prob" #define ED_TOK_BW "bw" #define ED_SEPARATORS " \t\n" #define ED_MIN_SAMPLES_NO 2 /* * returns 1 if s is a non-negative number, with at least one '.' */ static int is_valid_number(const char *s) { int i, dots_found = 0; int len = strlen(s); for (i = 0; i 1)) return 0; return 1; } /* * Take as input a string describing a bandwidth value * and return the numeric bandwidth value. * set clocking interface or bandwidth value */ static void read_bandwidth(char *arg, int *bandwidth, char *if_name, int namelen) { if (*bandwidth != -1) warnx("duplicate token, override bandwidth value!"); if (arg[0] >= 'a' && arg[0] <= 'z') { if (!if_name) { errx(1, "no if support"); } if (namelen >= IFNAMSIZ) warn("interface name truncated"); namelen--; /* interface name */ strncpy(if_name, arg, namelen); if_name[namelen] = '\0'; *bandwidth = 0; } else { /* read bandwidth value */ int bw; char *end = NULL; bw = strtoul(arg, &end, 0); if (*end == 'K' || *end == 'k') { end++; bw *= 1000; } else if (*end == 'M' || *end == 'm') { end++; bw *= 1000000; } if ((*end == 'B' && _substrcmp2(end, "Bi", "Bit/s") != 0) || _substrcmp2(end, "by", "bytes") == 0) bw *= 8; if (bw < 0) errx(EX_DATAERR, "bandwidth too large"); *bandwidth = bw; if (if_name) if_name[0] = '\0'; } } struct point { double prob; double delay; }; static int compare_points(const void *vp1, const void *vp2) { const struct point *p1 = vp1; const struct point *p2 = vp2; double res = 0; res = p1->prob - p2->prob; if (res == 0) res = p1->delay - p2->delay; if (res < 0) return -1; else if (res > 0) return 1; else return 0; } #define ED_EFMT(s) EX_DATAERR,"error in %s at line %d: "#s,filename,lineno static void load_extra_delays(const char *filename, struct dn_profile *p, struct dn_link *link) { char line[ED_MAX_LINE_LEN]; FILE *f; int lineno = 0; int i; int samples = -1; double loss = -1.0; char profile_name[ED_MAX_NAME_LEN]; int delay_first = -1; int do_points = 0; struct point points[ED_MAX_SAMPLES_NO]; int points_no = 0; /* XXX link never NULL? */ p->link_nr = link->link_nr; profile_name[0] = '\0'; f = fopen(filename, "r"); if (f == NULL) err(EX_UNAVAILABLE, "fopen: %s", filename); while (fgets(line, ED_MAX_LINE_LEN, f)) { /* read commands */ char *s, *cur = line, *name = NULL, *arg = NULL; ++lineno; /* parse the line */ while (cur) { s = strsep(&cur, ED_SEPARATORS); if (s == NULL || *s == '#') break; if (*s == '\0') continue; if (arg) errx(ED_EFMT("too many arguments")); if (name == NULL) name = s; else arg = s; } if (name == NULL) /* empty line */ continue; if (arg == NULL) errx(ED_EFMT("missing arg for %s"), name); if (!strcasecmp(name, ED_TOK_SAMPLES)) { if (samples > 0) errx(ED_EFMT("duplicate ``samples'' line")); if (atoi(arg) <=0) errx(ED_EFMT("invalid number of samples")); samples = atoi(arg); if (samples>ED_MAX_SAMPLES_NO) errx(ED_EFMT("too many samples, maximum is %d"), ED_MAX_SAMPLES_NO); do_points = 0; } else if (!strcasecmp(name, ED_TOK_BW)) { char buf[IFNAMSIZ]; read_bandwidth(arg, &link->bandwidth, buf, sizeof(buf)); } else if (!strcasecmp(name, ED_TOK_LOSS)) { if (loss != -1.0) errx(ED_EFMT("duplicated token: %s"), name); if (!is_valid_number(arg)) errx(ED_EFMT("invalid %s"), arg); loss = atof(arg); if (loss > 1) errx(ED_EFMT("%s greater than 1.0"), name); do_points = 0; } else if (!strcasecmp(name, ED_TOK_NAME)) { if (profile_name[0] != '\0') errx(ED_EFMT("duplicated token: %s"), name); strncpy(profile_name, arg, sizeof(profile_name) - 1); profile_name[sizeof(profile_name)-1] = '\0'; do_points = 0; } else if (!strcasecmp(name, ED_TOK_DELAY)) { if (do_points) errx(ED_EFMT("duplicated token: %s"), name); delay_first = 1; do_points = 1; } else if (!strcasecmp(name, ED_TOK_PROB)) { if (do_points) errx(ED_EFMT("duplicated token: %s"), name); delay_first = 0; do_points = 1; } else if (do_points) { if (!is_valid_number(name) || !is_valid_number(arg)) errx(ED_EFMT("invalid point found")); if (delay_first) { points[points_no].delay = atof(name); points[points_no].prob = atof(arg); } else { points[points_no].delay = atof(arg); points[points_no].prob = atof(name); } if (points[points_no].prob > 1.0) errx(ED_EFMT("probability greater than 1.0")); ++points_no; } else { errx(ED_EFMT("unrecognised command '%s'"), name); } } fclose (f); if (samples == -1) { warnx("'%s' not found, assuming 100", ED_TOK_SAMPLES); samples = 100; } if (loss == -1.0) { warnx("'%s' not found, assuming no loss", ED_TOK_LOSS); loss = 1; } /* make sure that there are enough points. */ if (points_no < ED_MIN_SAMPLES_NO) errx(ED_EFMT("too few samples, need at least %d"), ED_MIN_SAMPLES_NO); qsort(points, points_no, sizeof(struct point), compare_points); /* interpolation */ for (i = 0; isamples[ix] = x1; } else { double m = (y2-y1)/(x2-x1); double c = y1 - m*x1; for (; ixsamples[ix] = (ix - c)/m; } } p->samples_no = samples; p->loss_level = loss * samples; strncpy(p->name, profile_name, sizeof(p->name)); } +#ifdef NEW_AQM + +/* Parse AQM/extra scheduler parameters */ +static int +process_extra_parms(int *ac, char **av, struct dn_extra_parms *ep, + uint16_t type) +{ + int i; + + /* use kernel defaults */ + for (i=0; ipar[i] = -1; + + switch(type) { + case TOK_CODEL: + case TOK_FQ_CODEL: + /* Codel + * 0- target, 1- interval, 2- flags, + * FQ_CODEL + * 3- quantum, 4- limit, 5- flows + */ + if (type==TOK_CODEL) + ep->par[2] = 0; + else + ep->par[2] = CODEL_ECN_ENABLED; + + while (*ac > 0) { + int tok = match_token(aqm_params, *av); + (*ac)--; av++; + switch(tok) { + case TOK_TARGET: + if (*ac <= 0 || time_to_us(av[0]) < 0) + errx(EX_DATAERR, "target needs time\n"); + + ep->par[0] = time_to_us(av[0]); + (*ac)--; av++; + break; + + case TOK_INTERVAL: + if (*ac <= 0 || time_to_us(av[0]) < 0) + errx(EX_DATAERR, "interval needs time\n"); + + ep->par[1] = time_to_us(av[0]); + (*ac)--; av++; + break; + + case TOK_ECN: + ep->par[2] = CODEL_ECN_ENABLED; + break; + case TOK_NO_ECN: + ep->par[2] &= ~CODEL_ECN_ENABLED; + break; + /* Config fq_codel parameters */ + case TOK_QUANTUM: + if (type != TOK_FQ_CODEL) + errx(EX_DATAERR, "quantum is not for codel\n"); + if (*ac <= 0 || !is_valid_number(av[0])) + errx(EX_DATAERR, "quantum needs number\n"); + + ep->par[3]= atoi(av[0]); + (*ac)--; av++; + break; + + case TOK_LIMIT: + if (type != TOK_FQ_CODEL) + errx(EX_DATAERR, "limit is not for codel, use queue instead\n"); + if (*ac <= 0 || !is_valid_number(av[0])) + errx(EX_DATAERR, "limit needs number\n"); + + ep->par[4] = atoi(av[0]); + (*ac)--; av++; + break; + + case TOK_FLOWS: + if (type != TOK_FQ_CODEL) + errx(EX_DATAERR, "flows is not for codel\n"); + if (*ac <= 0 || !is_valid_number(av[0])) + errx(EX_DATAERR, "flows needs number\n"); + + ep->par[5] = atoi(av[0]); + (*ac)--; av++; + break; + + default: + printf("%s is Invalid parameter\n", av[-1]); + } + } + break; + case TOK_PIE: + case TOK_FQ_PIE: + /* PIE + * 0- target , 1- tupdate, 2- max_burst, + * 3- max_ecnth, 4- alpha, + * 5- beta, 6- flags + * FQ_CODEL + * 7- quantum, 8- limit, 9- flows + */ + + if ( type == TOK_PIE) + ep->par[6] = PIE_CAPDROP_ENABLED | PIE_DEPRATEEST_ENABLED + | PIE_DERAND_ENABLED; + else + /* for FQ-PIE, use TS mode */ + ep->par[6] = PIE_CAPDROP_ENABLED | PIE_DERAND_ENABLED + | PIE_ECN_ENABLED; + + while (*ac > 0) { + int tok = match_token(aqm_params, *av); + (*ac)--; av++; + switch(tok) { + case TOK_TARGET: + if (*ac <= 0 || time_to_us(av[0]) < 0) + errx(EX_DATAERR, "target needs time\n"); + + ep->par[0] = time_to_us(av[0]); + (*ac)--; av++; + break; + + case TOK_TUPDATE: + if (*ac <= 0 || time_to_us(av[0]) < 0) + errx(EX_DATAERR, "tupdate needs time\n"); + + ep->par[1] = time_to_us(av[0]); + (*ac)--; av++; + break; + + case TOK_MAX_BURST: + if (*ac <= 0 || time_to_us(av[0]) < 0) + errx(EX_DATAERR, "max_burst needs time\n"); + + ep->par[2] = time_to_us(av[0]); + (*ac)--; av++; + break; + + case TOK_MAX_ECNTH: + if (*ac <= 0 || !is_valid_number(av[0])) + errx(EX_DATAERR, "max_ecnth needs number\n"); + + ep->par[3] = atof(av[0]) * PIE_SCALE; + (*ac)--; av++; + break; + + case TOK_ALPHA: + if (*ac <= 0 || !is_valid_number(av[0])) + errx(EX_DATAERR, "alpha needs number\n"); + + ep->par[4] = atof(av[0]) * PIE_SCALE; + (*ac)--; av++; + break; + + case TOK_BETA: + if (*ac <= 0 || !is_valid_number(av[0])) + errx(EX_DATAERR, "beta needs number\n"); + + ep->par[5] = atof(av[0]) * PIE_SCALE; + (*ac)--; av++; + break; + + case TOK_ECN: + ep->par[6] |= PIE_ECN_ENABLED; + break; + case TOK_NO_ECN: + ep->par[6] &= ~PIE_ECN_ENABLED; + break; + + case TOK_CAPDROP: + ep->par[6] |= PIE_CAPDROP_ENABLED; + break; + case TOK_NO_CAPDROP: + ep->par[6] &= ~PIE_CAPDROP_ENABLED; + break; + + case TOK_ONOFF: + ep->par[6] |= PIE_ON_OFF_MODE_ENABLED; + break; + + case TOK_DRE: + ep->par[6] |= PIE_DEPRATEEST_ENABLED; + break; + + case TOK_TS: + ep->par[6] &= ~PIE_DEPRATEEST_ENABLED; + break; + + case TOK_DERAND: + ep->par[6] |= PIE_DERAND_ENABLED; + break; + case TOK_NO_DERAND: + ep->par[6] &= ~PIE_DERAND_ENABLED; + break; + + /* Config fq_pie parameters */ + case TOK_QUANTUM: + if (type != TOK_FQ_PIE) + errx(EX_DATAERR, "quantum is not for pie\n"); + if (*ac <= 0 || !is_valid_number(av[0])) + errx(EX_DATAERR, "quantum needs number\n"); + + ep->par[7]= atoi(av[0]); + (*ac)--; av++; + break; + + case TOK_LIMIT: + if (type != TOK_FQ_PIE) + errx(EX_DATAERR, "limit is not for pie, use queue instead\n"); + if (*ac <= 0 || !is_valid_number(av[0])) + errx(EX_DATAERR, "limit needs number\n"); + + ep->par[8] = atoi(av[0]); + (*ac)--; av++; + break; + + case TOK_FLOWS: + if (type != TOK_FQ_PIE) + errx(EX_DATAERR, "flows is not for pie\n"); + if (*ac <= 0 || !is_valid_number(av[0])) + errx(EX_DATAERR, "flows needs number\n"); + + ep->par[9] = atoi(av[0]); + (*ac)--; av++; + break; + + + default: + printf("%s is invalid parameter\n", av[-1]); + } + } + break; + } + + return 0; +} + +#endif + + /* * configuration of pipes, schedulers, flowsets. * When we configure a new scheduler, an empty pipe is created, so: * * do_pipe = 1 -> "pipe N config ..." only for backward compatibility * sched N+Delta type fifo sched_mask ... * pipe N+Delta * flowset N+Delta pipe N+Delta (no parameters) * sched N type wf2q+ sched_mask ... * pipe N * * do_pipe = 2 -> flowset N config * flowset N parameters * * do_pipe = 3 -> sched N config * sched N parameters (default no pipe) * optional Pipe N config ... * pipe ==> */ void ipfw_config_pipe(int ac, char **av) { int i; u_int j; char *end; struct dn_id *buf, *base; struct dn_sch *sch = NULL; struct dn_link *p = NULL; struct dn_fs *fs = NULL; struct dn_profile *pf = NULL; struct ipfw_flow_id *mask = NULL; +#ifdef NEW_AQM + struct dn_extra_parms *aqm_extra; + struct dn_extra_parms *sch_extra; + int lmax_extra; +#endif + int lmax; uint32_t _foo = 0, *flags = &_foo , *buckets = &_foo; /* * allocate space for 1 header, * 1 scheduler, 1 link, 1 flowset, 1 profile */ lmax = sizeof(struct dn_id); /* command header */ lmax += sizeof(struct dn_sch) + sizeof(struct dn_link) + sizeof(struct dn_fs) + sizeof(struct dn_profile); +#ifdef NEW_AQM + /* Extra Params */ + lmax_extra = sizeof(struct dn_extra_parms); + /* two lmax_extra because one for AQM params and another + * sch params + */ + lmax += lmax_extra*2; +#endif + av++; ac--; /* Pipe number */ if (ac && isdigit(**av)) { i = atoi(*av); av++; ac--; } else i = -1; if (i <= 0) errx(EX_USAGE, "need a pipe/flowset/sched number"); base = buf = safe_calloc(1, lmax); /* all commands start with a 'CONFIGURE' and a version */ o_next(&buf, sizeof(struct dn_id), DN_CMD_CONFIG); base->id = DN_API_VERSION; switch (co.do_pipe) { case 1: /* "pipe N config ..." */ /* Allocate space for the WF2Q+ scheduler, its link * and the FIFO flowset. Set the number, but leave * the scheduler subtype and other parameters to 0 * so the kernel will use appropriate defaults. * XXX todo: add a flag to record if a parameter * is actually configured. * If we do a 'pipe config' mask -> sched_mask. * The FIFO scheduler and link are derived from the * WF2Q+ one in the kernel. */ +#ifdef NEW_AQM + sch_extra = o_next(&buf, lmax_extra, DN_TEXT); + sch_extra ->oid.subtype = 0; /* don't configure scheduler */ +#endif sch = o_next(&buf, sizeof(*sch), DN_SCH); p = o_next(&buf, sizeof(*p), DN_LINK); +#ifdef NEW_AQM + aqm_extra = o_next(&buf, lmax_extra, DN_TEXT); + aqm_extra ->oid.subtype = 0; /* don't configure AQM */ +#endif fs = o_next(&buf, sizeof(*fs), DN_FS); sch->sched_nr = i; sch->oid.subtype = 0; /* defaults to WF2Q+ */ mask = &sch->sched_mask; flags = &sch->flags; buckets = &sch->buckets; *flags |= DN_PIPE_CMD; p->link_nr = i; /* This flowset is only for the FIFO scheduler */ fs->fs_nr = i + 2*DN_MAX_ID; fs->sched_nr = i + DN_MAX_ID; break; case 2: /* "queue N config ... " */ +#ifdef NEW_AQM + aqm_extra = o_next(&buf, lmax_extra, DN_TEXT); + aqm_extra ->oid.subtype = 0; +#endif fs = o_next(&buf, sizeof(*fs), DN_FS); fs->fs_nr = i; mask = &fs->flow_mask; flags = &fs->flags; buckets = &fs->buckets; break; case 3: /* "sched N config ..." */ +#ifdef NEW_AQM + sch_extra = o_next(&buf, lmax_extra, DN_TEXT); + sch_extra ->oid.subtype = 0; +#endif sch = o_next(&buf, sizeof(*sch), DN_SCH); +#ifdef NEW_AQM + aqm_extra = o_next(&buf, lmax_extra, DN_TEXT); + aqm_extra ->oid.subtype = 0; +#endif fs = o_next(&buf, sizeof(*fs), DN_FS); sch->sched_nr = i; mask = &sch->sched_mask; flags = &sch->flags; buckets = &sch->buckets; /* fs is used only with !MULTIQUEUE schedulers */ fs->fs_nr = i + DN_MAX_ID; fs->sched_nr = i; break; } /* set to -1 those fields for which we want to reuse existing * values from the kernel. * Also, *_nr and subtype = 0 mean reuse the value from the kernel. * XXX todo: support reuse of the mask. */ if (p) p->bandwidth = -1; for (j = 0; j < sizeof(fs->par)/sizeof(fs->par[0]); j++) fs->par[j] = -1; while (ac > 0) { double d; int tok = match_token(dummynet_params, *av); ac--; av++; switch(tok) { case TOK_NOERROR: NEED(fs, "noerror is only for pipes"); fs->flags |= DN_NOERROR; break; case TOK_PLR: NEED(fs, "plr is only for pipes"); NEED1("plr needs argument 0..1\n"); d = strtod(av[0], NULL); if (d > 1) d = 1; else if (d < 0) d = 0; fs->plr = (int)(d*0x7fffffff); ac--; av++; break; case TOK_QUEUE: NEED(fs, "queue is only for pipes or flowsets"); NEED1("queue needs queue size\n"); end = NULL; fs->qsize = strtoul(av[0], &end, 0); if (*end == 'K' || *end == 'k') { fs->flags |= DN_QSIZE_BYTES; fs->qsize *= 1024; } else if (*end == 'B' || _substrcmp2(end, "by", "bytes") == 0) { fs->flags |= DN_QSIZE_BYTES; } ac--; av++; break; case TOK_BUCKETS: NEED(fs, "buckets is only for pipes or flowsets"); NEED1("buckets needs argument\n"); *buckets = strtoul(av[0], NULL, 0); ac--; av++; break; case TOK_FLOW_MASK: case TOK_SCHED_MASK: case TOK_MASK: NEED(mask, "tok_mask"); NEED1("mask needs mask specifier\n"); /* * per-flow queue, mask is dst_ip, dst_port, * src_ip, src_port, proto measured in bits */ bzero(mask, sizeof(*mask)); end = NULL; while (ac >= 1) { uint32_t *p32 = NULL; uint16_t *p16 = NULL; uint32_t *p20 = NULL; struct in6_addr *pa6 = NULL; uint32_t a; tok = match_token(dummynet_params, *av); ac--; av++; switch(tok) { case TOK_ALL: /* * special case, all bits significant * except 'extra' (the queue number) */ mask->dst_ip = ~0; mask->src_ip = ~0; mask->dst_port = ~0; mask->src_port = ~0; mask->proto = ~0; n2mask(&mask->dst_ip6, 128); n2mask(&mask->src_ip6, 128); mask->flow_id6 = ~0; *flags |= DN_HAVE_MASK; goto end_mask; case TOK_QUEUE: mask->extra = ~0; *flags |= DN_HAVE_MASK; goto end_mask; case TOK_DSTIP: mask->addr_type = 4; p32 = &mask->dst_ip; break; case TOK_SRCIP: mask->addr_type = 4; p32 = &mask->src_ip; break; case TOK_DSTIP6: mask->addr_type = 6; pa6 = &mask->dst_ip6; break; case TOK_SRCIP6: mask->addr_type = 6; pa6 = &mask->src_ip6; break; case TOK_FLOWID: mask->addr_type = 6; p20 = &mask->flow_id6; break; case TOK_DSTPORT: p16 = &mask->dst_port; break; case TOK_SRCPORT: p16 = &mask->src_port; break; case TOK_PROTO: break; default: ac++; av--; /* backtrack */ goto end_mask; } if (ac < 1) errx(EX_USAGE, "mask: value missing"); if (*av[0] == '/') { a = strtoul(av[0]+1, &end, 0); if (pa6 == NULL) a = (a == 32) ? ~0 : (1 << a) - 1; } else a = strtoul(av[0], &end, 0); if (p32 != NULL) *p32 = a; else if (p16 != NULL) { if (a > 0xFFFF) errx(EX_DATAERR, "port mask must be 16 bit"); *p16 = (uint16_t)a; } else if (p20 != NULL) { if (a > 0xfffff) errx(EX_DATAERR, "flow_id mask must be 20 bit"); *p20 = (uint32_t)a; } else if (pa6 != NULL) { if (a > 128) errx(EX_DATAERR, "in6addr invalid mask len"); else n2mask(pa6, a); } else { if (a > 0xFF) errx(EX_DATAERR, "proto mask must be 8 bit"); mask->proto = (uint8_t)a; } if (a != 0) *flags |= DN_HAVE_MASK; ac--; av++; } /* end while, config masks */ end_mask: break; +#ifdef NEW_AQM + case TOK_CODEL: + case TOK_PIE: + NEED(fs, "codel/pie is only for flowsets"); + fs->flags &= ~(DN_IS_RED|DN_IS_GENTLE_RED); + fs->flags |= DN_IS_AQM; + + strcpy(aqm_extra->name,av[-1]); + aqm_extra->oid.subtype = DN_AQM_PARAMS; + + process_extra_parms(&ac, av, aqm_extra, tok); + break; + + case TOK_FQ_CODEL: + case TOK_FQ_PIE: + if (!strcmp(av[-1],"type")) + errx(EX_DATAERR, "use type before fq_codel/fq_pie"); + + NEED(sch, "fq_codel/fq_pie is only for schd"); + strcpy(sch_extra->name,av[-1]); + sch_extra->oid.subtype = DN_SCH_PARAMS; + process_extra_parms(&ac, av, sch_extra, tok); + break; +#endif case TOK_RED: case TOK_GRED: NEED1("red/gred needs w_q/min_th/max_th/max_p\n"); fs->flags |= DN_IS_RED; if (tok == TOK_GRED) fs->flags |= DN_IS_GENTLE_RED; /* * the format for parameters is w_q/min_th/max_th/max_p */ if ((end = strsep(&av[0], "/"))) { double w_q = strtod(end, NULL); if (w_q > 1 || w_q <= 0) errx(EX_DATAERR, "0 < w_q <= 1"); fs->w_q = (int) (w_q * (1 << SCALE_RED)); } if ((end = strsep(&av[0], "/"))) { fs->min_th = strtoul(end, &end, 0); if (*end == 'K' || *end == 'k') fs->min_th *= 1024; } if ((end = strsep(&av[0], "/"))) { fs->max_th = strtoul(end, &end, 0); if (*end == 'K' || *end == 'k') fs->max_th *= 1024; } if ((end = strsep(&av[0], "/"))) { double max_p = strtod(end, NULL); if (max_p > 1 || max_p < 0) errx(EX_DATAERR, "0 <= max_p <= 1"); fs->max_p = (int)(max_p * (1 << SCALE_RED)); } ac--; av++; break; case TOK_ECN: fs->flags |= DN_IS_ECN; break; case TOK_DROPTAIL: NEED(fs, "droptail is only for flowsets"); fs->flags &= ~(DN_IS_RED|DN_IS_GENTLE_RED); break; case TOK_BW: NEED(p, "bw is only for links"); NEED1("bw needs bandwidth or interface\n"); read_bandwidth(av[0], &p->bandwidth, NULL, 0); ac--; av++; break; case TOK_DELAY: NEED(p, "delay is only for links"); NEED1("delay needs argument 0..10000ms\n"); p->delay = strtoul(av[0], NULL, 0); ac--; av++; break; case TOK_TYPE: { int l; NEED(sch, "type is only for schedulers"); NEED1("type needs a string"); l = strlen(av[0]); if (l == 0 || l > 15) errx(1, "type %s too long\n", av[0]); strcpy(sch->name, av[0]); sch->oid.subtype = 0; /* use string */ - ac--; av++; +#ifdef NEW_AQM + /* if fq_codel is selected, consider all tokens after it + * as parameters + */ + if (!strcasecmp(av[0],"fq_codel") || !strcasecmp(av[0],"fq_pie")){ + strcpy(sch_extra->name,av[0]); + sch_extra->oid.subtype = DN_SCH_PARAMS; + process_extra_parms(&ac, av, sch_extra, tok); + } else { + ac--;av++; + } +#else + ac--;av++; +#endif break; } case TOK_WEIGHT: NEED(fs, "weight is only for flowsets"); NEED1("weight needs argument\n"); fs->par[0] = strtol(av[0], &end, 0); ac--; av++; break; case TOK_LMAX: NEED(fs, "lmax is only for flowsets"); NEED1("lmax needs argument\n"); fs->par[1] = strtol(av[0], &end, 0); ac--; av++; break; case TOK_PRI: NEED(fs, "priority is only for flowsets"); NEED1("priority needs argument\n"); fs->par[2] = strtol(av[0], &end, 0); ac--; av++; break; case TOK_SCHED: case TOK_PIPE: NEED(fs, "pipe/sched"); NEED1("pipe/link/sched needs number\n"); fs->sched_nr = strtoul(av[0], &end, 0); ac--; av++; break; case TOK_PROFILE: NEED((!pf), "profile already set"); NEED(p, "profile"); { NEED1("extra delay needs the file name\n"); pf = o_next(&buf, sizeof(*pf), DN_PROFILE); load_extra_delays(av[0], pf, p); //XXX can't fail? --ac; ++av; } break; case TOK_BURST: NEED(p, "burst"); NEED1("burst needs argument\n"); errno = 0; if (expand_number(av[0], &p->burst) < 0) if (errno != ERANGE) errx(EX_DATAERR, "burst: invalid argument"); if (errno || p->burst > (1ULL << 48) - 1) errx(EX_DATAERR, "burst: out of range (0..2^48-1)"); ac--; av++; break; default: errx(EX_DATAERR, "unrecognised option ``%s''", av[-1]); } } /* check validity of parameters */ if (p) { if (p->delay > 10000) errx(EX_DATAERR, "delay must be < 10000"); if (p->bandwidth == -1) p->bandwidth = 0; } if (fs) { /* XXX accept a 0 scheduler to keep the default */ if (fs->flags & DN_QSIZE_BYTES) { size_t len; long limit; len = sizeof(limit); if (sysctlbyname("net.inet.ip.dummynet.pipe_byte_limit", &limit, &len, NULL, 0) == -1) limit = 1024*1024; if (fs->qsize > limit) errx(EX_DATAERR, "queue size must be < %ldB", limit); } else { size_t len; long limit; len = sizeof(limit); if (sysctlbyname("net.inet.ip.dummynet.pipe_slot_limit", &limit, &len, NULL, 0) == -1) limit = 100; if (fs->qsize > limit) errx(EX_DATAERR, "2 <= queue size <= %ld", limit); } +#ifdef NEW_AQM + if ((fs->flags & DN_IS_ECN) && !((fs->flags & DN_IS_RED)|| + (fs->flags & DN_IS_AQM))) + errx(EX_USAGE, "ECN can be used with red/gred/" + "codel/fq_codel only!"); +#else if ((fs->flags & DN_IS_ECN) && !(fs->flags & DN_IS_RED)) errx(EX_USAGE, "enable red/gred for ECN"); + +#endif if (fs->flags & DN_IS_RED) { size_t len; int lookup_depth, avg_pkt_size; if (!(fs->flags & DN_IS_ECN) && (fs->min_th >= fs->max_th)) errx(EX_DATAERR, "min_th %d must be < than max_th %d", fs->min_th, fs->max_th); else if ((fs->flags & DN_IS_ECN) && (fs->min_th > fs->max_th)) errx(EX_DATAERR, "min_th %d must be =< than max_th %d", fs->min_th, fs->max_th); if (fs->max_th == 0) errx(EX_DATAERR, "max_th must be > 0"); len = sizeof(int); if (sysctlbyname("net.inet.ip.dummynet.red_lookup_depth", &lookup_depth, &len, NULL, 0) == -1) lookup_depth = 256; if (lookup_depth == 0) errx(EX_DATAERR, "net.inet.ip.dummynet.red_lookup_depth" " must be greater than zero"); len = sizeof(int); if (sysctlbyname("net.inet.ip.dummynet.red_avg_pkt_size", &avg_pkt_size, &len, NULL, 0) == -1) avg_pkt_size = 512; if (avg_pkt_size == 0) errx(EX_DATAERR, "net.inet.ip.dummynet.red_avg_pkt_size must" " be greater than zero"); #if 0 /* the following computation is now done in the kernel */ /* * Ticks needed for sending a medium-sized packet. * Unfortunately, when we are configuring a WF2Q+ queue, we * do not have bandwidth information, because that is stored * in the parent pipe, and also we have multiple queues * competing for it. So we set s=0, which is not very * correct. But on the other hand, why do we want RED with * WF2Q+ ? */ if (p.bandwidth==0) /* this is a WF2Q+ queue */ s = 0; else s = (double)ck.hz * avg_pkt_size * 8 / p.bandwidth; /* * max idle time (in ticks) before avg queue size becomes 0. * NOTA: (3/w_q) is approx the value x so that * (1-w_q)^x < 10^-3. */ w_q = ((double)fs->w_q) / (1 << SCALE_RED); idle = s * 3. / w_q; fs->lookup_step = (int)idle / lookup_depth; if (!fs->lookup_step) fs->lookup_step = 1; weight = 1 - w_q; for (t = fs->lookup_step; t > 1; --t) weight *= 1 - w_q; fs->lookup_weight = (int)(weight * (1 << SCALE_RED)); #endif /* code moved in the kernel */ } } i = do_cmd(IP_DUMMYNET3, base, (char *)buf - (char *)base); if (i) err(1, "setsockopt(%s)", "IP_DUMMYNET_CONFIGURE"); } void dummynet_flush(void) { struct dn_id oid; oid_fill(&oid, sizeof(oid), DN_CMD_FLUSH, DN_API_VERSION); do_cmd(IP_DUMMYNET3, &oid, oid.len); } /* Parse input for 'ipfw [pipe|sched|queue] show [range list]' * Returns the number of ranges, and possibly stores them * in the array v of size len. */ static int parse_range(int ac, char *av[], uint32_t *v, int len) { int n = 0; char *endptr, *s; uint32_t base[2]; if (v == NULL || len < 2) { v = base; len = 2; } for (s = *av; s != NULL; av++, ac--) { v[0] = strtoul(s, &endptr, 10); v[1] = (*endptr != '-') ? v[0] : strtoul(endptr+1, &endptr, 10); if (*endptr == '\0') { /* prepare for next round */ s = (ac > 0) ? *(av+1) : NULL; } else { if (*endptr != ',') { warn("invalid number: %s", s); s = ++endptr; continue; } /* continue processing from here */ s = ++endptr; ac++; av--; } if (v[1] < v[0] || v[1] >= DN_MAX_ID-1 || v[1] >= DN_MAX_ID-1) { continue; /* invalid entry */ } n++; /* translate if 'pipe list' */ if (co.do_pipe == 1) { v[0] += DN_MAX_ID; v[1] += DN_MAX_ID; } v = (n*2 < len) ? v + 2 : base; } return n; } /* main entry point for dummynet list functions. co.do_pipe indicates * which function we want to support. * av may contain filtering arguments, either individual entries * or ranges, or lists (space or commas are valid separators). * Format for a range can be n1-n2 or n3 n4 n5 ... * In a range n1 must be <= n2, otherwise the range is ignored. * A number 'n4' is translate in a range 'n4-n4' * All number must be > 0 and < DN_MAX_ID-1 */ void dummynet_list(int ac, char *av[], int show_counters) { struct dn_id *oid, *x = NULL; int ret, i; int n; /* # of ranges */ u_int buflen, l; u_int max_size; /* largest obj passed up */ (void)show_counters; // XXX unused, but we should use it. ac--; av++; /* skip 'list' | 'show' word */ n = parse_range(ac, av, NULL, 0); /* Count # of ranges. */ /* Allocate space to store ranges */ l = sizeof(*oid) + sizeof(uint32_t) * n * 2; oid = safe_calloc(1, l); oid_fill(oid, l, DN_CMD_GET, DN_API_VERSION); if (n > 0) /* store ranges in idx */ parse_range(ac, av, (uint32_t *)(oid + 1), n*2); /* * Compute the size of the largest object returned. If the * response leaves at least this much spare space in the * buffer, then surely the response is complete; otherwise * there might be a risk of truncation and we will need to * retry with a larger buffer. * XXX don't bother with smaller structs. */ max_size = sizeof(struct dn_fs); if (max_size < sizeof(struct dn_sch)) max_size = sizeof(struct dn_sch); if (max_size < sizeof(struct dn_flow)) max_size = sizeof(struct dn_flow); switch (co.do_pipe) { case 1: oid->subtype = DN_LINK; /* list pipe */ break; case 2: oid->subtype = DN_FS; /* list queue */ break; case 3: oid->subtype = DN_SCH; /* list sched */ break; } /* * Ask the kernel an estimate of the required space (result * in oid.id), unless we are requesting a subset of objects, * in which case the kernel does not give an exact answer. * In any case, space might grow in the meantime due to the * creation of new queues, so we must be prepared to retry. */ if (n > 0) { buflen = 4*1024; } else { ret = do_cmd(-IP_DUMMYNET3, oid, (uintptr_t)&l); if (ret != 0 || oid->id <= sizeof(*oid)) goto done; buflen = oid->id + max_size; oid->len = sizeof(*oid); /* restore */ } /* Try a few times, until the buffer fits */ for (i = 0; i < 20; i++) { l = buflen; x = safe_realloc(x, l); bcopy(oid, x, oid->len); ret = do_cmd(-IP_DUMMYNET3, x, (uintptr_t)&l); if (ret != 0 || x->id <= sizeof(*oid)) goto done; /* no response */ if (l + max_size <= buflen) break; /* ok */ buflen *= 2; /* double for next attempt */ } list_pipes(x, O_NEXT(x, l)); done: if (x) free(x); free(oid); } Index: head/sbin/ipfw/ipfw2.h =================================================================== --- head/sbin/ipfw/ipfw2.h (revision 300778) +++ head/sbin/ipfw/ipfw2.h (revision 300779) @@ -1,354 +1,379 @@ /* * Copyright (c) 2002-2003 Luigi Rizzo * Copyright (c) 1996 Alex Nash, Paul Traina, Poul-Henning Kamp * Copyright (c) 1994 Ugen J.S.Antsilevich * * Idea and grammar partially left from: * Copyright (c) 1993 Daniel Boulet * * Redistribution and use in source forms, with and without modification, * are permitted provided that this entire comment appears intact. * * Redistribution in binary form may occur without any restrictions. * Obviously, it would be nice if you gave credit where credit is due * but requiring it would be too onerous. * * This software is provided ``AS IS'' without any warranties of any kind. * * NEW command line interface for IP firewall facility * * $FreeBSD$ */ /* * Options that can be set on the command line. * When reading commands from a file, a subset of the options can also * be applied globally by specifying them before the file name. * After that, each line can contain its own option that changes * the global value. * XXX The context is not restored after each line. */ struct cmdline_opts { /* boolean options: */ int do_value_as_ip; /* show table value as IP */ int do_resolv; /* try to resolve all ip to names */ int do_time; /* Show time stamps */ int do_quiet; /* Be quiet in add and flush */ int do_pipe; /* this cmd refers to a pipe/queue/sched */ int do_nat; /* this cmd refers to a nat config */ int do_dynamic; /* display dynamic rules */ int do_expired; /* display expired dynamic rules */ int do_compact; /* show rules in compact mode */ int do_force; /* do not ask for confirmation */ int show_sets; /* display the set each rule belongs to */ int test_only; /* only check syntax */ int comment_only; /* only print action and comment */ int verbose; /* be verbose on some commands */ /* The options below can have multiple values. */ int do_sort; /* field to sort results (0 = no) */ /* valid fields are 1 and above */ int use_set; /* work with specified set number */ /* 0 means all sets, otherwise apply to set use_set - 1 */ }; extern struct cmdline_opts co; /* * _s_x is a structure that stores a string <-> token pairs, used in * various places in the parser. Entries are stored in arrays, * with an entry with s=NULL as terminator. * The search routines are match_token() and match_value(). * Often, an element with x=0 contains an error string. * */ struct _s_x { char const *s; int x; }; extern struct _s_x f_ipdscp[]; enum tokens { TOK_NULL=0, TOK_OR, TOK_NOT, TOK_STARTBRACE, TOK_ENDBRACE, TOK_ACCEPT, TOK_COUNT, TOK_EACTION, TOK_PIPE, TOK_LINK, TOK_QUEUE, TOK_FLOWSET, TOK_SCHED, TOK_DIVERT, TOK_TEE, TOK_NETGRAPH, TOK_NGTEE, TOK_FORWARD, TOK_SKIPTO, TOK_DENY, TOK_REJECT, TOK_RESET, TOK_UNREACH, TOK_CHECKSTATE, TOK_NAT, TOK_REASS, TOK_CALL, TOK_RETURN, TOK_ALTQ, TOK_LOG, TOK_TAG, TOK_UNTAG, TOK_TAGGED, TOK_UID, TOK_GID, TOK_JAIL, TOK_IN, TOK_LIMIT, TOK_KEEPSTATE, TOK_LAYER2, TOK_OUT, TOK_DIVERTED, TOK_DIVERTEDLOOPBACK, TOK_DIVERTEDOUTPUT, TOK_XMIT, TOK_RECV, TOK_VIA, TOK_FRAG, TOK_IPOPTS, TOK_IPLEN, TOK_IPID, TOK_IPPRECEDENCE, TOK_DSCP, TOK_IPTOS, TOK_IPTTL, TOK_IPVER, TOK_ESTAB, TOK_SETUP, TOK_TCPDATALEN, TOK_TCPFLAGS, TOK_TCPOPTS, TOK_TCPSEQ, TOK_TCPACK, TOK_TCPWIN, TOK_ICMPTYPES, TOK_MAC, TOK_MACTYPE, TOK_VERREVPATH, TOK_VERSRCREACH, TOK_ANTISPOOF, TOK_IPSEC, TOK_COMMENT, TOK_PLR, TOK_NOERROR, TOK_BUCKETS, TOK_DSTIP, TOK_SRCIP, TOK_DSTPORT, TOK_SRCPORT, TOK_ALL, TOK_MASK, TOK_FLOW_MASK, TOK_SCHED_MASK, TOK_BW, TOK_DELAY, TOK_PROFILE, TOK_BURST, TOK_RED, TOK_GRED, TOK_ECN, TOK_DROPTAIL, TOK_PROTO, +#ifdef NEW_AQM + /* AQM tokens*/ + TOK_NO_ECN, + TOK_CODEL, + TOK_FQ_CODEL, + TOK_TARGET, + TOK_INTERVAL, + TOK_FLOWS, + TOK_QUANTUM, + + TOK_PIE, + TOK_FQ_PIE, + TOK_TUPDATE, + TOK_MAX_BURST, + TOK_MAX_ECNTH, + TOK_ALPHA, + TOK_BETA, + TOK_CAPDROP, + TOK_NO_CAPDROP, + TOK_ONOFF, + TOK_DRE, + TOK_TS, + TOK_DERAND, + TOK_NO_DERAND, +#endif /* dummynet tokens */ TOK_WEIGHT, TOK_LMAX, TOK_PRI, TOK_TYPE, TOK_SLOTSIZE, TOK_IP, TOK_IF, TOK_ALOG, TOK_DENY_INC, TOK_SAME_PORTS, TOK_UNREG_ONLY, TOK_SKIP_GLOBAL, TOK_RESET_ADDR, TOK_ALIAS_REV, TOK_PROXY_ONLY, TOK_REDIR_ADDR, TOK_REDIR_PORT, TOK_REDIR_PROTO, TOK_IPV6, TOK_FLOWID, TOK_ICMP6TYPES, TOK_EXT6HDR, TOK_DSTIP6, TOK_SRCIP6, TOK_IPV4, TOK_UNREACH6, TOK_RESET6, TOK_FIB, TOK_SETFIB, TOK_LOOKUP, TOK_SOCKARG, TOK_SETDSCP, TOK_FLOW, TOK_IFLIST, /* Table tokens */ TOK_CREATE, TOK_DESTROY, TOK_LIST, TOK_INFO, TOK_DETAIL, TOK_MODIFY, TOK_FLUSH, TOK_SWAP, TOK_ADD, TOK_DEL, TOK_VALTYPE, TOK_ALGO, TOK_TALIST, TOK_ATOMIC, TOK_LOCK, TOK_UNLOCK, TOK_VLIST, TOK_OLIST, }; /* * the following macro returns an error message if we run out of * arguments. */ #define NEED(_p, msg) {if (!_p) errx(EX_USAGE, msg);} #define NEED1(msg) {if (!(*av)) errx(EX_USAGE, msg);} struct buf_pr { char *buf; /* allocated buffer */ char *ptr; /* current pointer */ size_t size; /* total buffer size */ size_t avail; /* available storage */ size_t needed; /* length needed */ }; int pr_u64(struct buf_pr *bp, uint64_t *pd, int width); int bp_alloc(struct buf_pr *b, size_t size); void bp_free(struct buf_pr *b); int bprintf(struct buf_pr *b, char *format, ...); /* memory allocation support */ void *safe_calloc(size_t number, size_t size); void *safe_realloc(void *ptr, size_t size); /* string comparison functions used for historical compatibility */ int _substrcmp(const char *str1, const char* str2); int _substrcmp2(const char *str1, const char* str2, const char* str3); int stringnum_cmp(const char *a, const char *b); /* utility functions */ int match_token(struct _s_x *table, const char *string); int match_token_relaxed(struct _s_x *table, const char *string); int get_token(struct _s_x *table, const char *string, const char *errbase); char const *match_value(struct _s_x *p, int value); size_t concat_tokens(char *buf, size_t bufsize, struct _s_x *table, char *delimiter); int fill_flags(struct _s_x *flags, char *p, char **e, uint32_t *set, uint32_t *clear); void print_flags_buffer(char *buf, size_t sz, struct _s_x *list, uint32_t set); struct _ip_fw3_opheader; int do_cmd(int optname, void *optval, uintptr_t optlen); int do_set3(int optname, struct _ip_fw3_opheader *op3, uintptr_t optlen); int do_get3(int optname, struct _ip_fw3_opheader *op3, size_t *optlen); struct in6_addr; void n2mask(struct in6_addr *mask, int n); int contigmask(uint8_t *p, int len); /* * Forward declarations to avoid include way too many headers. * C does not allow duplicated typedefs, so we use the base struct * that the typedef points to. * Should the typedefs use a different type, the compiler will * still detect the change when compiling the body of the * functions involved, so we do not lose error checking. */ struct _ipfw_insn; struct _ipfw_insn_altq; struct _ipfw_insn_u32; struct _ipfw_insn_ip6; struct _ipfw_insn_icmp6; /* * The reserved set numer. This is a constant in ip_fw.h * but we store it in a variable so other files do not depend * in that header just for one constant. */ extern int resvd_set_number; /* first-level command handlers */ void ipfw_add(char *av[]); void ipfw_show_nat(int ac, char **av); void ipfw_config_pipe(int ac, char **av); void ipfw_config_nat(int ac, char **av); void ipfw_sets_handler(char *av[]); void ipfw_table_handler(int ac, char *av[]); void ipfw_sysctl_handler(char *av[], int which); void ipfw_delete(char *av[]); void ipfw_flush(int force); void ipfw_zero(int ac, char *av[], int optname); void ipfw_list(int ac, char *av[], int show_counters); void ipfw_internal_handler(int ac, char *av[]); int ipfw_check_object_name(const char *name); #ifdef PF /* altq.c */ void altq_set_enabled(int enabled); u_int32_t altq_name_to_qid(const char *name); void print_altq_cmd(struct buf_pr *bp, struct _ipfw_insn_altq *altqptr); #else #define NO_ALTQ #endif /* dummynet.c */ void dummynet_list(int ac, char *av[], int show_counters); void dummynet_flush(void); int ipfw_delete_pipe(int pipe_or_queue, int n); /* ipv6.c */ void print_unreach6_code(struct buf_pr *bp, uint16_t code); void print_ip6(struct buf_pr *bp, struct _ipfw_insn_ip6 *cmd, char const *s); void print_flow6id(struct buf_pr *bp, struct _ipfw_insn_u32 *cmd); void print_icmp6types(struct buf_pr *bp, struct _ipfw_insn_u32 *cmd); void print_ext6hdr(struct buf_pr *bp, struct _ipfw_insn *cmd ); struct _ipfw_insn *add_srcip6(struct _ipfw_insn *cmd, char *av, int cblen); struct _ipfw_insn *add_dstip6(struct _ipfw_insn *cmd, char *av, int cblen); void fill_flow6(struct _ipfw_insn_u32 *cmd, char *av, int cblen); void fill_unreach6_code(u_short *codep, char *str); void fill_icmp6types(struct _ipfw_insn_icmp6 *cmd, char *av, int cblen); int fill_ext6hdr(struct _ipfw_insn *cmd, char *av); /* tables.c */ struct _ipfw_obj_ctlv; int table_check_name(const char *tablename); void ipfw_list_ta(int ac, char *av[]); void ipfw_list_values(int ac, char *av[]); Index: head/sys/modules/dummynet/Makefile =================================================================== --- head/sys/modules/dummynet/Makefile (revision 300778) +++ head/sys/modules/dummynet/Makefile (revision 300779) @@ -1,11 +1,12 @@ # $FreeBSD$ .PATH: ${.CURDIR}/../../netpfil/ipfw KMOD= dummynet SRCS= ip_dummynet.c SRCS+= ip_dn_glue.c ip_dn_io.c +SRCS+= dn_aqm_codel.c dn_aqm_pie.c SRCS+= dn_heap.c dn_sched_fifo.c dn_sched_qfq.c dn_sched_rr.c dn_sched_wf2q.c -SRCS+= dn_sched_prio.c +SRCS+= dn_sched_prio.c dn_sched_fq_codel.c dn_sched_fq_pie.c SRCS+= opt_inet6.h .include Index: head/sys/netinet/ip_dummynet.h =================================================================== --- head/sys/netinet/ip_dummynet.h (revision 300778) +++ head/sys/netinet/ip_dummynet.h (revision 300779) @@ -1,264 +1,285 @@ /*- * Copyright (c) 1998-2010 Luigi Rizzo, Universita` di Pisa * Portions Copyright (c) 2000 Akamba Corp. * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _IP_DUMMYNET_H #define _IP_DUMMYNET_H - +#define NEW_AQM /* * Definition of the kernel-userland API for dummynet. * * Setsockopt() and getsockopt() pass a batch of objects, each * of them starting with a "struct dn_id" which should fully identify * the object and its relation with others in the sequence. * The first object in each request should have * type= DN_CMD_*, id = DN_API_VERSION. * For other objects, type and subtype specify the object, len indicates * the total length including the header, and 'id' identifies the specific * object. * * Most objects are numbered with an identifier in the range 1..65535. * DN_MAX_ID indicates the first value outside the range. */ #define DN_API_VERSION 12500000 #define DN_MAX_ID 0x10000 struct dn_id { uint16_t len; /* total obj len including this header */ uint8_t type; uint8_t subtype; uint32_t id; /* generic id */ }; /* * These values are in the type field of struct dn_id. * To preserve the ABI, never rearrange the list or delete * entries with the exception of DN_LAST */ enum { DN_NONE = 0, DN_LINK = 1, DN_FS, DN_SCH, DN_SCH_I, DN_QUEUE, DN_DELAY_LINE, DN_PROFILE, DN_FLOW, /* struct dn_flow */ DN_TEXT, /* opaque text is the object */ DN_CMD_CONFIG = 0x80, /* objects follow */ DN_CMD_DELETE, /* subtype + list of entries */ DN_CMD_GET, /* subtype + list of entries */ DN_CMD_FLUSH, /* for compatibility with FreeBSD 7.2/8 */ DN_COMPAT_PIPE, DN_COMPAT_QUEUE, DN_GET_COMPAT, /* special commands for emulation of sysctl variables */ DN_SYSCTL_GET, DN_SYSCTL_SET, - +#ifdef NEW_AQM + /* subtypes used for setting/getting extra parameters. + * these subtypes used with IP_DUMMYNET3 command (get) + * and DN_TEXT (set). */ + DN_AQM_PARAMS, /* AQM extra params */ + DN_SCH_PARAMS, /* scheduler extra params */ +#endif DN_LAST, }; enum { /* subtype for schedulers, flowset and the like */ DN_SCHED_UNKNOWN = 0, DN_SCHED_FIFO = 1, DN_SCHED_WF2QP = 2, /* others are in individual modules */ }; enum { /* user flags */ DN_HAVE_MASK = 0x0001, /* fs or sched has a mask */ DN_NOERROR = 0x0002, /* do not report errors */ DN_QHT_HASH = 0x0004, /* qht is a hash table */ DN_QSIZE_BYTES = 0x0008, /* queue size is in bytes */ DN_HAS_PROFILE = 0x0010, /* a link has a profile */ DN_IS_RED = 0x0020, DN_IS_GENTLE_RED= 0x0040, DN_IS_ECN = 0x0080, + #ifdef NEW_AQM + DN_IS_AQM = 0x0100, /* AQMs: e.g Codel & PIE */ + #endif DN_PIPE_CMD = 0x1000, /* pipe config... */ }; /* * link template. */ struct dn_link { struct dn_id oid; /* * Userland sets bw and delay in bits/s and milliseconds. * The kernel converts this back and forth to bits/tick and ticks. * XXX what about burst ? */ int32_t link_nr; int bandwidth; /* bit/s or bits/tick. */ int delay; /* ms and ticks */ uint64_t burst; /* scaled. bits*Hz XXX */ }; /* * A flowset, which is a template for flows. Contains parameters * from the command line: id, target scheduler, queue sizes, plr, * flow masks, buckets for the flow hash, and possibly scheduler- * specific parameters (weight, quantum and so on). */ struct dn_fs { struct dn_id oid; uint32_t fs_nr; /* the flowset number */ uint32_t flags; /* userland flags */ int qsize; /* queue size in slots or bytes */ int32_t plr; /* PLR, pkt loss rate (2^31-1 means 100%) */ uint32_t buckets; /* buckets used for the queue hash table */ struct ipfw_flow_id flow_mask; uint32_t sched_nr; /* the scheduler we attach to */ /* generic scheduler parameters. Leave them at -1 if unset. * Now we use 0: weight, 1: lmax, 2: priority */ int par[4]; /* RED/GRED parameters. * weight and probabilities are in the range 0..1 represented * in fixed point arithmetic with SCALE_RED decimal bits. */ #define SCALE_RED 16 #define SCALE(x) ( (x) << SCALE_RED ) #define SCALE_VAL(x) ( (x) >> SCALE_RED ) #define SCALE_MUL(x,y) ( ( (x) * (y) ) >> SCALE_RED ) int w_q ; /* queue weight (scaled) */ int max_th ; /* maximum threshold for queue (scaled) */ int min_th ; /* minimum threshold for queue (scaled) */ int max_p ; /* maximum value for p_b (scaled) */ }; /* * dn_flow collects flow_id and stats for queues and scheduler * instances, and is used to pass these info to userland. * oid.type/oid.subtype describe the object, oid.id is number * of the parent object. */ struct dn_flow { struct dn_id oid; struct ipfw_flow_id fid; uint64_t tot_pkts; /* statistics counters */ uint64_t tot_bytes; uint32_t length; /* Queue length, in packets */ uint32_t len_bytes; /* Queue length, in bytes */ uint32_t drops; }; /* * Scheduler template, mostly indicating the name, number, * sched_mask and buckets. */ struct dn_sch { struct dn_id oid; uint32_t sched_nr; /* N, scheduler number */ uint32_t buckets; /* number of buckets for the instances */ uint32_t flags; /* have_mask, ... */ char name[16]; /* null terminated */ /* mask to select the appropriate scheduler instance */ struct ipfw_flow_id sched_mask; /* M */ }; /* A delay profile is attached to a link. * Note that a profile, as any other object, cannot be longer than 2^16 */ #define ED_MAX_SAMPLES_NO 1024 struct dn_profile { struct dn_id oid; /* fields to simulate a delay profile */ #define ED_MAX_NAME_LEN 32 char name[ED_MAX_NAME_LEN]; int link_nr; int loss_level; int bandwidth; // XXX use link bandwidth? int samples_no; /* actual len of samples[] */ int samples[ED_MAX_SAMPLES_NO]; /* may be shorter */ }; - +#ifdef NEW_AQM +/* Extra parameters for AQM and scheduler. + * This struct is used to pass and retrieve parameters (configurations) + * to/from AQM and Scheduler. + */ +struct dn_extra_parms { + struct dn_id oid; + char name[16]; + uint32_t nr; +#define DN_MAX_EXTRA_PARM 10 + int64_t par[DN_MAX_EXTRA_PARM]; +}; +#endif /* * Overall structure of dummynet In dummynet, packets are selected with the firewall rules, and passed to two different objects: PIPE or QUEUE (bad name). A QUEUE defines a classifier, which groups packets into flows according to a 'mask', puts them into independent queues (one per flow) with configurable size and queue management policy, and passes flows to a scheduler: (flow_mask|sched_mask) sched_mask +---------+ weight Wx +-------------+ | |->-[flow]-->--| |-+ -->--| QUEUE x | ... | | | | |->-[flow]-->--| SCHEDuler N | | +---------+ | | | ... | +--[LINK N]-->-- +---------+ weight Wy | | +--[LINK N]-->-- | |->-[flow]-->--| | | -->--| QUEUE y | ... | | | | |->-[flow]-->--| | | +---------+ +-------------+ | +-------------+ Many QUEUE objects can connect to the same scheduler, each QUEUE object can have its own set of parameters. In turn, the SCHEDuler 'forks' multiple instances according to a 'sched_mask', each instance manages its own set of queues and transmits on a private instance of a configurable LINK. A PIPE is a simplified version of the above, where there is no flow_mask, and each scheduler instance handles a single queue. The following data structures (visible from userland) describe the objects used by dummynet: + dn_link, contains the main configuration parameters related to delay and bandwidth; + dn_profile describes a delay profile; + dn_flow describes the flow status (flow id, statistics) + dn_sch describes a scheduler + dn_fs describes a flowset (msk, weight, queue parameters) * */ #endif /* _IP_DUMMYNET_H */ Index: head/sys/netpfil/ipfw/dn_aqm.h =================================================================== --- head/sys/netpfil/ipfw/dn_aqm.h (nonexistent) +++ head/sys/netpfil/ipfw/dn_aqm.h (revision 300779) @@ -0,0 +1,167 @@ +/*- + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * API for writting an Active Queue Management algorithm for Dummynet + * + * $FreeBSD$ + */ + +#ifndef _IP_DN_AQM_H +#define _IP_DN_AQM_H + + +/* NOW is the current time in millisecond*/ +#define NOW ((dn_cfg.curr_time * tick) / 1000) + +#define AQM_UNOW (dn_cfg.curr_time * tick) +#define AQM_TIME_1US ((aqm_time_t)(1)) +#define AQM_TIME_1MS ((aqm_time_t)(1000)) +#define AQM_TIME_1S ((aqm_time_t)(AQM_TIME_1MS * 1000)) + +/* aqm time allows to store up to 4294 seconds */ +typedef uint32_t aqm_time_t; +typedef int32_t aqm_stime_t; + +#define DN_AQM_MTAG_TS 55345 + +/* Macro for variable bounding */ +#define BOUND_VAR(x,l,h) (x < l? l : x > h? h : x) + +/* sysctl variable to count number of droped packets */ +extern unsigned long io_pkt_drop; + +/* + * Structure for holding data and function pointers that together represent a + * AQM algorithm. + */ + struct dn_aqm { +#define DN_AQM_NAME_MAX 50 + char name[DN_AQM_NAME_MAX]; /* name of AQM algorithm */ + uint32_t type; /* AQM type number */ + + /* Methods implemented by AQM algorithm: + * + * enqueue enqueue packet 'm' on queue 'q'. + * Return 0 on success, 1 on drop. + * + * dequeue dequeue a packet from queue 'q'. + * Return a packet, NULL if no packet available. + * + * config configure AQM algorithm + * If required, this function should allocate space to store + * the configurations and set 'fs->aqmcfg' to point to this space. + * 'dn_extra_parms' includes array of parameters send + * from ipfw userland command. + * Return 0 on success, non-zero otherwise. + * + * deconfig deconfigure AQM algorithm. + * The allocated configuration memory space should be freed here. + * Return 0 on success, non-zero otherwise. + * + * init initialise AQM status variables of queue 'q' + * This function is used to allocate space and init AQM status for a + * queue and q->aqm_status to point to this space. + * Return 0 on success, non-zero otherwise. + * + * cleanup cleanup AQM status variables of queue 'q' + * The allocated memory space for AQM status should be freed here. + * Return 0 on success, non-zero otherwise. + * + * getconfig retrieve AQM configurations + * This function is used to return AQM parameters to userland + * command. The function should fill 'dn_extra_parms' struct with + * the AQM configurations using 'par' array. + * + */ + + int (*enqueue)(struct dn_queue *, struct mbuf *); + struct mbuf * (*dequeue)(struct dn_queue *); + int (*config)(struct dn_fsk *, struct dn_extra_parms *ep, int); + int (*deconfig)(struct dn_fsk *); + int (*init)(struct dn_queue *); + int (*cleanup)(struct dn_queue *); + int (*getconfig)(struct dn_fsk *, struct dn_extra_parms *); + + int ref_count; /*Number of queues instances in the system */ + int cfg_ref_count; /*Number of AQM instances in the system */ + SLIST_ENTRY (dn_aqm) next; /* Next AQM in the list */ +}; + +/* Helper function to update queue and scheduler statistics. + * negative len + drop -> drop + * negative len -> dequeue + * positive len -> enqueue + * positive len + drop -> drop during enqueue + */ +__inline static void +update_stats(struct dn_queue *q, int len, int drop) +{ + int inc = 0; + struct dn_flow *sni; + struct dn_flow *qni; + + sni = &q->_si->ni; + qni = &q->ni; + + if (len < 0) + inc = -1; + else if(len > 0) + inc = 1; + + if (drop) { + qni->drops++; + sni->drops++; + io_pkt_drop++; + } else { + /*update queue stats */ + qni->length += inc; + qni->len_bytes += len; + + /*update scheduler instance stats */ + sni->length += inc; + sni->len_bytes += len; + } + /* tot_pkts is updated in dn_enqueue function */ +} + + +/* kernel module related function */ +int +dn_aqm_modevent(module_t mod, int cmd, void *arg); + +#define DECLARE_DNAQM_MODULE(name, dnaqm) \ + static moduledata_t name##_mod = { \ + #name, dn_aqm_modevent, dnaqm \ + }; \ + DECLARE_MODULE(name, name##_mod, \ + SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY); \ + MODULE_DEPEND(name, dummynet, 3, 3, 3) + +#endif Property changes on: head/sys/netpfil/ipfw/dn_aqm.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/netpfil/ipfw/dn_aqm_codel.c =================================================================== --- head/sys/netpfil/ipfw/dn_aqm_codel.c (nonexistent) +++ head/sys/netpfil/ipfw/dn_aqm_codel.c (revision 300779) @@ -0,0 +1,444 @@ +/* + * Codel - The Controlled-Delay Active Queue Management algorithm. + * + * $FreeBSD$ + * + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ +#include +#include + +#include +#include /* ip_len, ip_off */ +#include /* ip_output(), IP_FORWARDING */ +#include +#include +#include /* various ether_* routines */ +#include /* for ip6_input, ip6_output prototypes */ +#include +#include + +#ifdef NEW_AQM +#include +#include +#include +#include +#include + +#define DN_AQM_CODEL 1 + +static struct dn_aqm codel_desc; + +/* default codel parameters */ +struct dn_aqm_codel_parms codel_sysctl = {5000 * AQM_TIME_1US, + 100000 * AQM_TIME_1US, 0}; + +static int +codel_sysctl_interval_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + value = codel_sysctl.interval; + value /= AQM_TIME_1US; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 1 || value > 100 * AQM_TIME_1S) + return (EINVAL); + codel_sysctl.interval = value * AQM_TIME_1US ; + return (0); +} + +static int +codel_sysctl_target_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + value = codel_sysctl.target; + value /= AQM_TIME_1US; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + D("%ld", value); + if (value < 1 || value > 5 * AQM_TIME_1S) + return (EINVAL); + codel_sysctl.target = value * AQM_TIME_1US ; + return (0); +} + +/* defining Codel sysctl variables */ +SYSBEGIN(f4) + +SYSCTL_DECL(_net_inet); +SYSCTL_DECL(_net_inet_ip); +SYSCTL_DECL(_net_inet_ip_dummynet); +static SYSCTL_NODE(_net_inet_ip_dummynet, OID_AUTO, + codel, CTLFLAG_RW, 0, "CODEL"); + +#ifdef SYSCTL_NODE +SYSCTL_PROC(_net_inet_ip_dummynet_codel, OID_AUTO, target, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0,codel_sysctl_target_handler, "L", + "CoDel target in microsecond"); + +SYSCTL_PROC(_net_inet_ip_dummynet_codel, OID_AUTO, interval, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, codel_sysctl_interval_handler, "L", + "CoDel interval in microsecond"); +#endif + +/* This function computes codel_interval/sqrt(count) + * Newton's method of approximation is used to compute 1/sqrt(count). + * http://betterexplained.com/articles/ + * understanding-quakes-fast-inverse-square-root/ + */ +aqm_time_t +control_law(struct codel_status *cst, struct dn_aqm_codel_parms *cprms, + aqm_time_t t) +{ + uint32_t count; + uint64_t temp; + count = cst->count; + + /* we don't calculate isqrt(1) to get more accurate result*/ + if (count == 1) { + /* prepare isqrt (old guess) for the next iteration i.e. 1/sqrt(2)*/ + cst->isqrt = (1UL<< FIX_POINT_BITS) * 7/10; + /* return time + isqrt(1)*interval */ + return t + cprms->interval; + } + + /* newguess = g(1.5 - 0.5*c*g^2) + * Multiplying both sides by 2 to make all the constants intergers + * newguess * 2 = g(3 - c*g^2) g=old guess, c=count + * So, newguess = newguess /2 + * Fixed point operations are used here. + */ + + /* Calculate g^2 */ + temp = (uint32_t) cst->isqrt * cst->isqrt; + /* Calculate (3 - c*g^2) i.e. (3 - c * temp) */ + temp = (3ULL<< (FIX_POINT_BITS*2)) - (count * temp); + + /* + * Divide by 2 because we multiplied the original equation by two + * Also, we shift the result by 8 bits to prevent overflow. + * */ + temp >>= (1 + 8); + + /* Now, temp = (1.5 - 0.5*c*g^2) + * Calculate g (1.5 - 0.5*c*g^2) i.e. g * temp + */ + temp = (cst->isqrt * temp) >> (FIX_POINT_BITS + FIX_POINT_BITS - 8); + cst->isqrt = temp; + + /* calculate codel_interval/sqrt(count) */ + return t + ((cprms->interval * temp) >> FIX_POINT_BITS); +} + +/* + * Extract a packet from the head of queue 'q' + * Return a packet or NULL if the queue is empty. + * Also extract packet's timestamp from mtag. + */ +struct mbuf * +codel_extract_head(struct dn_queue *q, aqm_time_t *pkt_ts) +{ + struct m_tag *mtag; + struct mbuf *m = q->mq.head; + + if (m == NULL) + return m; + q->mq.head = m->m_nextpkt; + + /* Update stats */ + update_stats(q, -m->m_pkthdr.len, 0); + + if (q->ni.length == 0) /* queue is now idle */ + q->q_time = dn_cfg.curr_time; + + /* extract packet TS*/ + mtag = m_tag_locate(m, MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, NULL); + if (mtag == NULL) { + D("Codel timestamp mtag not found!"); + *pkt_ts = 0; + } else { + *pkt_ts = *(aqm_time_t *)(mtag + 1); + m_tag_delete(m,mtag); + } + + return m; +} + +/* + * Enqueue a packet 'm' in queue 'q' + */ +static int +aqm_codel_enqueue(struct dn_queue *q, struct mbuf *m) +{ + struct dn_fs *f; + uint64_t len; + struct codel_status *cst; /*codel status variables */ + struct m_tag *mtag; + + f = &(q->fs->fs); + len = m->m_pkthdr.len; + cst = q->aqm_status; + if(!cst) { + D("Codel queue is not initialized\n"); + goto drop; + } + + /* Finding maximum packet size */ + // XXX we can get MTU from driver instead + if (len > cst->maxpkt_size) + cst->maxpkt_size = len; + + /* check for queue size and drop the tail if exceed queue limit*/ + if (f->flags & DN_QSIZE_BYTES) { + if ( q->ni.len_bytes > f->qsize) + goto drop; + } + else { + if ( q->ni.length >= f->qsize) + goto drop; + } + + /* Add timestamp as mtag */ + mtag = m_tag_locate(m, MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, NULL); + if (mtag == NULL) + mtag = m_tag_alloc(MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, + sizeof(aqm_time_t), M_NOWAIT); + if (mtag == NULL) { + m_freem(m); + goto drop; + } + + *(aqm_time_t *)(mtag + 1) = AQM_UNOW; + m_tag_prepend(m, mtag); + + mq_append(&q->mq, m); + update_stats(q, len, 0); + return (0); + +drop: + update_stats(q, 0, 1); + FREE_PKT(m); + return (1); +} + + +/* Dequeue a pcaket from queue q */ +static struct mbuf * +aqm_codel_dequeue(struct dn_queue *q) +{ + return codel_dequeue(q); +} + +/* + * initialize Codel for queue 'q' + * First allocate memory for codel status. + */ +static int +aqm_codel_init(struct dn_queue *q) +{ + struct codel_status *cst; + + if (!q->fs->aqmcfg) { + D("Codel is not configure!d"); + return EINVAL; + } + + q->aqm_status = malloc(sizeof(struct codel_status), + M_DUMMYNET, M_NOWAIT | M_ZERO); + if (q->aqm_status == NULL) { + D("Cannot allocate AQM_codel private data"); + return ENOMEM ; + } + + /* init codel status variables */ + cst = q->aqm_status; + cst->dropping=0; + cst->first_above_time=0; + cst->drop_next_time=0; + cst->count=0; + cst->maxpkt_size = 500; + + /* increase reference counters */ + codel_desc.ref_count++; + + return 0; +} + +/* + * Clean up Codel status for queue 'q' + * Destroy memory allocated for codel status. + */ +static int +aqm_codel_cleanup(struct dn_queue *q) +{ + + if (q && q->aqm_status) { + free(q->aqm_status, M_DUMMYNET); + q->aqm_status = NULL; + /* decrease reference counters */ + codel_desc.ref_count--; + } + else + D("Codel already cleaned up"); + return 0; +} + +/* + * Config codel parameters + * also allocate memory for codel configurations + */ +static int +aqm_codel_config(struct dn_fsk* fs, struct dn_extra_parms *ep, int len) +{ + struct dn_aqm_codel_parms *ccfg; + + int l = sizeof(struct dn_extra_parms); + if (len < l) { + D("invalid sched parms length got %d need %d", len, l); + return EINVAL; + } + /* we free the old cfg because maybe the original allocation + * not the same size as the new one (different AQM type). + */ + if (fs->aqmcfg) { + free(fs->aqmcfg, M_DUMMYNET); + fs->aqmcfg = NULL; + } + + fs->aqmcfg = malloc(sizeof(struct dn_aqm_codel_parms), + M_DUMMYNET, M_NOWAIT | M_ZERO); + if (fs->aqmcfg== NULL) { + D("cannot allocate AQM_codel configuration parameters"); + return ENOMEM; + } + + /* configure codel parameters */ + ccfg = fs->aqmcfg; + + if (ep->par[0] < 0) + ccfg->target = codel_sysctl.target; + else + ccfg->target = ep->par[0] * AQM_TIME_1US; + + if (ep->par[1] < 0) + ccfg->interval = codel_sysctl.interval; + else + ccfg->interval = ep->par[1] * AQM_TIME_1US; + + if (ep->par[2] < 0) + ccfg->flags = 0; + else + ccfg->flags = ep->par[2]; + + /* bound codel configurations */ + ccfg->target = BOUND_VAR(ccfg->target,1, 5 * AQM_TIME_1S); + ccfg->interval = BOUND_VAR(ccfg->interval,1, 5 * AQM_TIME_1S); + /* increase config reference counter */ + codel_desc.cfg_ref_count++; + + return 0; +} + +/* + * Deconfigure Codel and free memory allocation + */ +static int +aqm_codel_deconfig(struct dn_fsk* fs) +{ + + if (fs && fs->aqmcfg) { + free(fs->aqmcfg, M_DUMMYNET); + fs->aqmcfg = NULL; + fs->aqmfp = NULL; + /* decrease config reference counter */ + codel_desc.cfg_ref_count--; + } + + return 0; +} + +/* + * Retrieve Codel configuration parameters. + */ +static int +aqm_codel_getconfig(struct dn_fsk *fs, struct dn_extra_parms * ep) +{ + struct dn_aqm_codel_parms *ccfg; + + if (fs->aqmcfg) { + strcpy(ep->name, codel_desc.name); + ccfg = fs->aqmcfg; + ep->par[0] = ccfg->target / AQM_TIME_1US; + ep->par[1] = ccfg->interval / AQM_TIME_1US; + ep->par[2] = ccfg->flags; + return 0; + } + return 1; +} + +static struct dn_aqm codel_desc = { + _SI( .type = ) DN_AQM_CODEL, + _SI( .name = ) "CODEL", + _SI( .enqueue = ) aqm_codel_enqueue, + _SI( .dequeue = ) aqm_codel_dequeue, + _SI( .config = ) aqm_codel_config, + _SI( .getconfig = ) aqm_codel_getconfig, + _SI( .deconfig = ) aqm_codel_deconfig, + _SI( .init = ) aqm_codel_init, + _SI( .cleanup = ) aqm_codel_cleanup, +}; + +DECLARE_DNAQM_MODULE(dn_aqm_codel, &codel_desc); + + +#endif Property changes on: head/sys/netpfil/ipfw/dn_aqm_codel.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/netpfil/ipfw/dn_aqm_codel.h =================================================================== --- head/sys/netpfil/ipfw/dn_aqm_codel.h (nonexistent) +++ head/sys/netpfil/ipfw/dn_aqm_codel.h (revision 300779) @@ -0,0 +1,222 @@ +/* + * Codel - The Controlled-Delay Active Queue Management algorithm. + * + * $FreeBSD$ + * + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * + * Copyright (C) 2011-2014 Kathleen Nichols . + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * o Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * + * o Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * o The names of the authors may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * Alternatively, provided that this notice is retained in full, this + * software may be distributed under the terms of the GNU General Public + * License ("GPL") version 2, in which case the provisions of the GPL + * apply INSTEAD OF those given above. + + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _IP_DN_AQM_CODEL_H +#define _IP_DN_AQM_CODEL_H + + +// XXX How to choose MTAG? +#define FIX_POINT_BITS 16 + +enum { + CODEL_ECN_ENABLED = 1 +}; + +/* Codel parameters */ +struct dn_aqm_codel_parms { + aqm_time_t target; + aqm_time_t interval; + uint32_t flags; +}; + +/* codel status variables */ +struct codel_status { + uint32_t count; /* number of dropped pkts since entering drop state */ + uint16_t dropping; /* dropping state */ + aqm_time_t drop_next_time; /* time for next drop */ + aqm_time_t first_above_time; /* time for first ts over target we observed */ + uint16_t isqrt; /* last isqrt for control low */ + uint16_t maxpkt_size; /* max packet size seen so far */ +}; + +struct mbuf *codel_extract_head(struct dn_queue *, aqm_time_t *); +aqm_time_t control_law(struct codel_status *, + struct dn_aqm_codel_parms *, aqm_time_t ); + +__inline static struct mbuf * +codel_dodequeue(struct dn_queue *q, aqm_time_t now, uint16_t *ok_to_drop) +{ + struct mbuf * m; + struct dn_aqm_codel_parms *cprms; + struct codel_status *cst; + aqm_time_t pkt_ts, sojourn_time; + + *ok_to_drop = 0; + m = codel_extract_head(q, &pkt_ts); + + cst = q->aqm_status; + + if (m == NULL) { + /* queue is empty - we can't be above target */ + cst->first_above_time= 0; + return m; + } + + cprms = q->fs->aqmcfg; + + /* To span a large range of bandwidths, CoDel runs two + * different AQMs in parallel. One is sojourn-time-based + * and takes effect when the time to send an MTU-sized + * packet is less than target. The 1st term of the "if" + * below does this. The other is backlog-based and takes + * effect when the time to send an MTU-sized packet is >= + * target. The goal here is to keep the output link + * utilization high by never allowing the queue to get + * smaller than the amount that arrives in a typical + * interarrival time (MTU-sized packets arriving spaced + * by the amount of time it takes to send such a packet on + * the bottleneck). The 2nd term of the "if" does this. + */ + sojourn_time = now - pkt_ts; + if (sojourn_time < cprms->target || q->ni.len_bytes <= cst->maxpkt_size) { + /* went below - stay below for at least interval */ + cst->first_above_time = 0; + } else { + if (cst->first_above_time == 0) { + /* just went above from below. if still above at + * first_above_time, will say it's ok to drop. */ + cst->first_above_time = now + cprms->interval; + } else if (now >= cst->first_above_time) { + *ok_to_drop = 1; + } + } + return m; +} + +/* + * Dequeue a packet from queue 'q' + */ +__inline static struct mbuf * +codel_dequeue(struct dn_queue *q) +{ + struct mbuf *m; + struct dn_aqm_codel_parms *cprms; + struct codel_status *cst; + aqm_time_t now; + uint16_t ok_to_drop; + + cst = q->aqm_status;; + cprms = q->fs->aqmcfg; + now = AQM_UNOW; + + m = codel_dodequeue(q, now, &ok_to_drop); + if (cst->dropping) { + if (!ok_to_drop) { + /* sojourn time below target - leave dropping state */ + cst->dropping = false; + } + /* + * Time for the next drop. Drop current packet and dequeue + * next. If the dequeue doesn't take us out of dropping + * state, schedule the next drop. A large backlog might + * result in drop rates so high that the next drop should + * happen now, hence the 'while' loop. + */ + while (now >= cst->drop_next_time && cst->dropping) { + + /* mark the packet */ + if (cprms->flags & CODEL_ECN_ENABLED && ecn_mark(m)) { + cst->count++; + /* schedule the next mark. */ + cst->drop_next_time = control_law(cst, cprms, + cst->drop_next_time); + return m; + } + + /* drop the packet */ + update_stats(q, 0, 1); + FREE_PKT(m); + m = codel_dodequeue(q, now, &ok_to_drop); + + if (!ok_to_drop) { + /* leave dropping state */ + cst->dropping = false; + } else { + cst->count++; + /* schedule the next drop. */ + cst->drop_next_time = control_law(cst, cprms, + cst->drop_next_time); + } + } + /* If we get here we're not in dropping state. The 'ok_to_drop' + * return from dodequeue means that the sojourn time has been + * above 'target' for 'interval' so enter dropping state. + */ + } else if (ok_to_drop) { + + /* if ECN option is disabled or the packet cannot be marked, + * drop the packet and extract another. + */ + if (!(cprms->flags & CODEL_ECN_ENABLED) || !ecn_mark(m)) { + update_stats(q, 0, 1); + FREE_PKT(m); + m = codel_dodequeue(q, now, &ok_to_drop); + } + + cst->dropping = true; + + /* If min went above target close to when it last went + * below, assume that the drop rate that controlled the + * queue on the last cycle is a good starting point to + * control it now. ('drop_next' will be at most 'interval' + * later than the time of the last drop so 'now - drop_next' + * is a good approximation of the time from the last drop + * until now.) + */ + cst->count = (cst->count > 2 && ((aqm_stime_t)now - + (aqm_stime_t)cst->drop_next_time) < 8* cprms->interval)? + cst->count - 2 : 1; + /* we don't have to set initial guess for Newton's method isqrt as + * we initilaize isqrt in control_law function when count == 1 */ + cst->drop_next_time = control_law(cst, cprms, now); + } + + return m; +} + +#endif Property changes on: head/sys/netpfil/ipfw/dn_aqm_codel.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/netpfil/ipfw/dn_aqm_pie.c =================================================================== --- head/sys/netpfil/ipfw/dn_aqm_pie.c (nonexistent) +++ head/sys/netpfil/ipfw/dn_aqm_pie.c (revision 300779) @@ -0,0 +1,793 @@ +/* + * PIE - Proportional Integral controller Enhanced AQM algorithm. + * + * $FreeBSD$ + * + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ +#include +#include + +#include +#include /* ip_len, ip_off */ +#include /* ip_output(), IP_FORWARDING */ +#include +#include +#include /* various ether_* routines */ +#include /* for ip6_input, ip6_output prototypes */ +#include +#include + +#ifdef NEW_AQM +#include +#include +#include +#include +#include + +/* for debugging */ +#include + +static struct dn_aqm pie_desc; + +/* PIE defaults + * target=15ms, tupdate=15ms, max_burst=150ms, + * max_ecnth=0.1, alpha=0.125, beta=1.25, + */ +struct dn_aqm_pie_parms pie_sysctl = + { 15 * AQM_TIME_1MS, 15 * AQM_TIME_1MS, 150 * AQM_TIME_1MS, + PIE_SCALE/10 , PIE_SCALE * 0.125, PIE_SCALE * 1.25 , + PIE_CAPDROP_ENABLED | PIE_DEPRATEEST_ENABLED | PIE_DERAND_ENABLED }; + +static int +pie_sysctl_alpha_beta_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + if (!strcmp(oidp->oid_name,"alpha")) + value = pie_sysctl.alpha; + else + value = pie_sysctl.beta; + + value = value * 1000 / PIE_SCALE; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 1 || value > 7 * PIE_SCALE) + return (EINVAL); + value = (value * PIE_SCALE) / 1000; + if (!strcmp(oidp->oid_name,"alpha")) + pie_sysctl.alpha = value; + else + pie_sysctl.beta = value; + return (0); +} + +static int +pie_sysctl_target_tupdate_maxb_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + if (!strcmp(oidp->oid_name,"target")) + value = pie_sysctl.qdelay_ref; + else if (!strcmp(oidp->oid_name,"tupdate")) + value = pie_sysctl.tupdate; + else + value = pie_sysctl.max_burst; + + value = value / AQM_TIME_1US; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 1 || value > 10 * AQM_TIME_1S) + return (EINVAL); + value = value * AQM_TIME_1US; + + if (!strcmp(oidp->oid_name,"target")) + pie_sysctl.qdelay_ref = value; + else if (!strcmp(oidp->oid_name,"tupdate")) + pie_sysctl.tupdate = value; + else + pie_sysctl.max_burst = value; + return (0); +} + +static int +pie_sysctl_max_ecnth_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + value = pie_sysctl.max_ecnth; + value = value * 1000 / PIE_SCALE; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 1 || value > PIE_SCALE) + return (EINVAL); + value = (value * PIE_SCALE) / 1000; + pie_sysctl.max_ecnth = value; + return (0); +} + +/* define PIE sysctl variables */ +SYSBEGIN(f4) +SYSCTL_DECL(_net_inet); +SYSCTL_DECL(_net_inet_ip); +SYSCTL_DECL(_net_inet_ip_dummynet); +static SYSCTL_NODE(_net_inet_ip_dummynet, OID_AUTO, + pie, CTLFLAG_RW, 0, "PIE"); + +#ifdef SYSCTL_NODE +SYSCTL_PROC(_net_inet_ip_dummynet_pie, OID_AUTO, target, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + pie_sysctl_target_tupdate_maxb_handler, "L", + "queue target in microsecond"); +SYSCTL_PROC(_net_inet_ip_dummynet_pie, OID_AUTO, tupdate, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + pie_sysctl_target_tupdate_maxb_handler, "L", + "the frequency of drop probability calculation in microsecond"); +SYSCTL_PROC(_net_inet_ip_dummynet_pie, OID_AUTO, max_burst, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + pie_sysctl_target_tupdate_maxb_handler, "L", + "Burst allowance interval in microsecond"); + +SYSCTL_PROC(_net_inet_ip_dummynet_pie, OID_AUTO, max_ecnth, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + pie_sysctl_max_ecnth_handler, "L", + "ECN safeguard threshold scaled by 1000"); + +SYSCTL_PROC(_net_inet_ip_dummynet_pie, OID_AUTO, alpha, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + pie_sysctl_alpha_beta_handler, "L", + "PIE alpha scaled by 1000"); +SYSCTL_PROC(_net_inet_ip_dummynet_pie, OID_AUTO, beta, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + pie_sysctl_alpha_beta_handler, "L", + "beta scaled by 1000"); +#endif + + +/* + * Callout function for drop probability calculation + * This function is called over tupdate ms and takes pointer of PIE + * status variables as an argument + */ +static void +calculate_drop_prob(void *x) +{ + int64_t p, prob, oldprob; + struct dn_aqm_pie_parms *pprms; + struct pie_status *pst = (struct pie_status *) x; + + /* dealing with race condition */ + if (callout_pending(&pst->aqm_pie_callout)) { + /* callout was reset */ + mtx_unlock(&pst->lock_mtx); + return; + } + + if (!callout_active(&pst->aqm_pie_callout)) { + /* callout was stopped */ + mtx_unlock(&pst->lock_mtx); + mtx_destroy(&pst->lock_mtx); + free(x, M_DUMMYNET); + //pst->pq->aqm_status = NULL; + pie_desc.ref_count--; + return; + } + callout_deactivate(&pst->aqm_pie_callout); + + pprms = pst->parms; + prob = pst->drop_prob; + + /* calculate current qdelay */ + if (pprms->flags & PIE_DEPRATEEST_ENABLED) { + pst->current_qdelay = ((uint64_t)pst->pq->ni.len_bytes * + pst->avg_dq_time) >> PIE_DQ_THRESHOLD_BITS; + } + + /* calculate drop probability */ + p = (int64_t)pprms->alpha * + ((int64_t)pst->current_qdelay - (int64_t)pprms->qdelay_ref); + p +=(int64_t) pprms->beta * + ((int64_t)pst->current_qdelay - (int64_t)pst->qdelay_old); + + /* We PIE_MAX_PROB shift by 12-bits to increase the division precision */ + p *= (PIE_MAX_PROB << 12) / AQM_TIME_1S; + + /* auto-tune drop probability */ + if (prob< PIE_MAX_PROB * 0.000001) + p >>= 11 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.00001) + p >>= 9 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.0001) + p >>= 7 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.001) + p >>= 5 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.01) + p >>= 3 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.1) + p >>= 1 + PIE_FIX_POINT_BITS+12; + else + p >>= PIE_FIX_POINT_BITS+12; + + oldprob = prob; + + /* Cap Drop adjustment */ + if ((pprms->flags & PIE_CAPDROP_ENABLED) && prob >= PIE_MAX_PROB / 10 + && p > PIE_MAX_PROB / 50 ) + p = PIE_MAX_PROB / 50; + + prob = prob + p; + + /* decay the drop probability exponentially */ + if (pst->current_qdelay == 0 && pst->qdelay_old == 0) + /* 0.98 ~= 1- 1/64 */ + prob = prob - (prob >> 6); + + + /* check for multiplication overflow/underflow */ + if (p>0) { + if (proboldprob) { + prob= 0; + D("underflow"); + } + + /* make drop probability between 0 and PIE_MAX_PROB*/ + if (prob < 0) + prob = 0; + else if (prob > PIE_MAX_PROB) + prob = PIE_MAX_PROB; + + pst->drop_prob = prob; + + /* store current queue delay value in old queue delay*/ + pst->qdelay_old = pst->current_qdelay; + + /* update burst allowance */ + if ((pst->sflags & PIE_ACTIVE) && pst->burst_allowance>0) { + + if (pst->burst_allowance > pprms->tupdate ) + pst->burst_allowance -= pprms->tupdate; + else + pst->burst_allowance = 0; + } + + /* reschedule calculate_drop_prob function */ + if (pst->sflags & PIE_ACTIVE) + callout_reset_sbt(&pst->aqm_pie_callout, + (uint64_t)pprms->tupdate * SBT_1US, 0, calculate_drop_prob, pst, 0); + + mtx_unlock(&pst->lock_mtx); +} + +/* + * Extract a packet from the head of queue 'q' + * Return a packet or NULL if the queue is empty. + * If getts is set, also extract packet's timestamp from mtag. + */ +static struct mbuf * +pie_extract_head(struct dn_queue *q, aqm_time_t *pkt_ts, int getts) +{ + struct m_tag *mtag; + struct mbuf *m = q->mq.head; + + if (m == NULL) + return m; + q->mq.head = m->m_nextpkt; + + /* Update stats */ + update_stats(q, -m->m_pkthdr.len, 0); + + if (q->ni.length == 0) /* queue is now idle */ + q->q_time = dn_cfg.curr_time; + + if (getts) { + /* extract packet TS*/ + mtag = m_tag_locate(m, MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, NULL); + if (mtag == NULL) { + D("PIE timestamp mtag not found!"); + *pkt_ts = 0; + } else { + *pkt_ts = *(aqm_time_t *)(mtag + 1); + m_tag_delete(m,mtag); + } + } + return m; +} + +/* + * Initiate PIE variable and optionally activate it + */ +__inline static void +init_activate_pie(struct pie_status *pst, int resettimer) +{ + struct dn_aqm_pie_parms *pprms; + + mtx_lock(&pst->lock_mtx); + pprms = pst->parms; + pst->drop_prob = 0; + pst->qdelay_old = 0; + pst->burst_allowance = pprms->max_burst; + pst->accu_prob = 0; + pst->dq_count = 0; + pst->avg_dq_time = 0; + pst->sflags = PIE_INMEASUREMENT; + pst->measurement_start = AQM_UNOW; + + if (resettimer) { + pst->sflags |= PIE_ACTIVE; + callout_reset_sbt(&pst->aqm_pie_callout, + (uint64_t)pprms->tupdate * SBT_1US, + 0, calculate_drop_prob, pst, 0); + } + //DX(2, "PIE Activated"); + mtx_unlock(&pst->lock_mtx); +} + +/* + * Deactivate PIE and stop probe update callout + */ +__inline static void +deactivate_pie(struct pie_status *pst) +{ + mtx_lock(&pst->lock_mtx); + pst->sflags &= ~(PIE_ACTIVE | PIE_INMEASUREMENT); + callout_stop(&pst->aqm_pie_callout); + //D("PIE Deactivated"); + mtx_unlock(&pst->lock_mtx); +} + +/* + * Dequeue and return a pcaket from queue 'q' or NULL if 'q' is empty. + * Also, caculate depature time or queue delay using timestamp + */ +static struct mbuf * +aqm_pie_dequeue(struct dn_queue *q) +{ + struct mbuf *m; + struct dn_flow *ni; /* stats for scheduler instance */ + struct dn_aqm_pie_parms *pprms; + struct pie_status *pst; + aqm_time_t now; + aqm_time_t pkt_ts, dq_time; + int32_t w; + + pst = q->aqm_status; + pprms = pst->parms; + ni = &q->_si->ni; + + /*we extarct packet ts only when Departure Rate Estimation dis not used*/ + m = pie_extract_head(q, &pkt_ts, !(pprms->flags & PIE_DEPRATEEST_ENABLED)); + + if (!m || !(pst->sflags & PIE_ACTIVE)) + return m; + + now = AQM_UNOW; + if (pprms->flags & PIE_DEPRATEEST_ENABLED) { + /* calculate average depature time */ + if(pst->sflags & PIE_INMEASUREMENT) { + pst->dq_count += m->m_pkthdr.len; + + if (pst->dq_count >= PIE_DQ_THRESHOLD) { + dq_time = now - pst->measurement_start; + + /* + * if we don't have old avg dq_time i.e PIE is (re)initialized, + * don't use weight to calculate new avg_dq_time + */ + if(pst->avg_dq_time == 0) + pst->avg_dq_time = dq_time; + else { + /* + * weight = PIE_DQ_THRESHOLD/2^6, but we scaled + * weight by 2^8. Thus, scaled + * weight = PIE_DQ_THRESHOLD /2^8 + * */ + w = PIE_DQ_THRESHOLD >> 8; + pst->avg_dq_time = (dq_time* w + + (pst->avg_dq_time * ((1L << 8) - w))) >> 8; + pst->sflags &= ~PIE_INMEASUREMENT; + } + } + } + + /* + * Start new measurment cycle when the queue has + * PIE_DQ_THRESHOLD worth of bytes. + */ + if(!(pst->sflags & PIE_INMEASUREMENT) && + q->ni.len_bytes >= PIE_DQ_THRESHOLD) { + pst->sflags |= PIE_INMEASUREMENT; + pst->measurement_start = now; + pst->dq_count = 0; + } + } + /* Optionally, use packet timestamp to estimate queue delay */ + else + pst->current_qdelay = now - pkt_ts; + + return m; +} + +/* + * Enqueue a packet in q, subject to space and PIE queue management policy + * (whose parameters are in q->fs). + * Update stats for the queue and the scheduler. + * Return 0 on success, 1 on drop. The packet is consumed anyways. + */ +static int +aqm_pie_enqueue(struct dn_queue *q, struct mbuf* m) +{ + struct dn_fs *f; + uint64_t len; + uint32_t qlen; + struct pie_status *pst; + struct dn_aqm_pie_parms *pprms; + int t; + + len = m->m_pkthdr.len; + pst = q->aqm_status; + if(!pst) { + DX(2, "PIE queue is not initialized\n"); + update_stats(q, 0, 1); + FREE_PKT(m); + return 1; + } + + f = &(q->fs->fs); + pprms = pst->parms; + t = ENQUE; + + /* get current queue length in bytes or packets*/ + qlen = (f->flags & DN_QSIZE_BYTES) ? + q->ni.len_bytes : q->ni.length; + + /* check for queue size and drop the tail if exceed queue limit*/ + if (qlen >= f->qsize) + t = DROP; + /* drop/mark the packet when PIE is active and burst time elapsed */ + else if ((pst->sflags & PIE_ACTIVE) && pst->burst_allowance==0 + && drop_early(pst, q->ni.len_bytes) == DROP) { + /* + * if drop_prob over ECN threshold, drop the packet + * otherwise mark and enqueue it. + */ + if ((pprms->flags & PIE_ECN_ENABLED) && pst->drop_prob < + (pprms->max_ecnth << (PIE_PROB_BITS - PIE_FIX_POINT_BITS)) + && ecn_mark(m)) + t = ENQUE; + else + t = DROP; + } + + /* Turn PIE on when 1/3 of the queue is full */ + if (!(pst->sflags & PIE_ACTIVE) && qlen >= pst->one_third_q_size) { + init_activate_pie(pst, 1); + } + + /* Reset burst tolerance and optinally turn PIE off*/ + if ((pst->sflags & PIE_ACTIVE) && pst->drop_prob == 0 && + pst->current_qdelay < (pprms->qdelay_ref >> 1) && + pst->qdelay_old < (pprms->qdelay_ref >> 1)) { + + pst->burst_allowance = pprms->max_burst; + if ((pprms->flags & PIE_ON_OFF_MODE_ENABLED) && qlen<=0) + deactivate_pie(pst); + } + + /* Timestamp the packet if Departure Rate Estimation is disabled */ + if (t != DROP && !(pprms->flags & PIE_DEPRATEEST_ENABLED)) { + /* Add TS to mbuf as a TAG */ + struct m_tag *mtag; + mtag = m_tag_locate(m, MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, NULL); + if (mtag == NULL) + mtag = m_tag_alloc(MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, + sizeof(aqm_time_t), M_NOWAIT); + if (mtag == NULL) { + m_freem(m); + t = DROP; + } + *(aqm_time_t *)(mtag + 1) = AQM_UNOW; + m_tag_prepend(m, mtag); + } + + if (t != DROP) { + mq_append(&q->mq, m); + update_stats(q, len, 0); + return (0); + } else { + update_stats(q, 0, 1); + + /* reset accu_prob after packet drop */ + pst->accu_prob = 0; + FREE_PKT(m); + return 1; + } + return 0; +} + +/* + * initialize PIE for queue 'q' + * First allocate memory for PIE status. + */ +static int +aqm_pie_init(struct dn_queue *q) +{ + struct pie_status *pst; + struct dn_aqm_pie_parms *pprms; + int err = 0; + + pprms = q->fs->aqmcfg; + + do { /* exit with break when error occurs*/ + if (!pprms){ + D("AQM_PIE is not configured"); + err = EINVAL; + break; + } + + q->aqm_status = malloc(sizeof(struct pie_status), + M_DUMMYNET, M_NOWAIT | M_ZERO); + if (q->aqm_status == NULL) { + D("cannot allocate PIE private data"); + err = ENOMEM ; + break; + } + + pst = q->aqm_status; + /* increase reference count for PIE module */ + pie_desc.ref_count++; + + pst->pq = q; + pst->parms = pprms; + + /* For speed optimization, we caculate 1/3 queue size once here */ + // we can use x/3 = (x >>2) + (x >>4) + (x >>7) + pst->one_third_q_size = q->fs->fs.qsize/3; + + mtx_init(&pst->lock_mtx, "mtx_pie", NULL, MTX_DEF); + callout_init_mtx(&pst->aqm_pie_callout, &pst->lock_mtx, + CALLOUT_RETURNUNLOCKED); + + pst->current_qdelay = 0; + init_activate_pie(pst, !(pprms->flags & PIE_ON_OFF_MODE_ENABLED)); + + //DX(2, "aqm_PIE_init"); + + } while(0); + + return err; +} + +/* + * Clean up PIE status for queue 'q' + * Destroy memory allocated for PIE status. + */ +static int +aqm_pie_cleanup(struct dn_queue *q) +{ + + if(!q) { + D("q is null"); + return 0; + } + struct pie_status *pst = q->aqm_status; + if(!pst) { + //D("queue is already cleaned up"); + return 0; + } + if(!q->fs || !q->fs->aqmcfg) { + D("fs is null or no cfg"); + return 1; + } + if (q->fs->aqmfp && q->fs->aqmfp->type !=DN_AQM_PIE) { + D("Not PIE fs (%d)", q->fs->fs.fs_nr); + return 1; + } + + mtx_lock(&pst->lock_mtx); + + /* stop callout timer */ + if (callout_stop(&pst->aqm_pie_callout) || !(pst->sflags & PIE_ACTIVE)) { + mtx_unlock(&pst->lock_mtx); + mtx_destroy(&pst->lock_mtx); + free(q->aqm_status, M_DUMMYNET); + q->aqm_status = NULL; + pie_desc.ref_count--; + return 0; + } else { + q->aqm_status = NULL; + mtx_unlock(&pst->lock_mtx); + DX(2, "PIE callout has not been stoped from cleanup!"); + return EBUSY; + } + return 0; +} + +/* + * Config PIE parameters + * also allocate memory for PIE configurations + */ +static int +aqm_pie_config(struct dn_fsk* fs, struct dn_extra_parms *ep, int len) +{ + struct dn_aqm_pie_parms *pcfg; + + int l = sizeof(struct dn_extra_parms); + if (len < l) { + D("invalid sched parms length got %d need %d", len, l); + return EINVAL; + } + /* we free the old cfg because maybe the orignal allocation + * was used for diffirent AQM type. + */ + if (fs->aqmcfg) { + free(fs->aqmcfg, M_DUMMYNET); + fs->aqmcfg = NULL; + } + + fs->aqmcfg = malloc(sizeof(struct dn_aqm_pie_parms), + M_DUMMYNET, M_NOWAIT | M_ZERO); + if (fs->aqmcfg== NULL) { + D("cannot allocate PIE configuration parameters"); + return ENOMEM; + } + + /* par array contains pie configuration as follow + * 0- qdelay_ref,1- tupdate, 2- max_burst + * 3- max_ecnth, 4- alpha, 5- beta, 6- flags + */ + + /* configure PIE parameters */ + pcfg = fs->aqmcfg; + + if (ep->par[0] < 0) + pcfg->qdelay_ref = pie_sysctl.qdelay_ref * AQM_TIME_1US; + else + pcfg->qdelay_ref = ep->par[0]; + if (ep->par[1] < 0) + pcfg->tupdate = pie_sysctl.tupdate * AQM_TIME_1US; + else + pcfg->tupdate = ep->par[1]; + if (ep->par[2] < 0) + pcfg->max_burst = pie_sysctl.max_burst * AQM_TIME_1US; + else + pcfg->max_burst = ep->par[2]; + if (ep->par[3] < 0) + pcfg->max_ecnth = pie_sysctl.max_ecnth; + else + pcfg->max_ecnth = ep->par[3]; + if (ep->par[4] < 0) + pcfg->alpha = pie_sysctl.alpha; + else + pcfg->alpha = ep->par[4]; + if (ep->par[5] < 0) + pcfg->beta = pie_sysctl.beta; + else + pcfg->beta = ep->par[5]; + if (ep->par[6] < 0) + pcfg->flags = pie_sysctl.flags; + else + pcfg->flags = ep->par[6]; + + /* bound PIE configurations */ + pcfg->qdelay_ref = BOUND_VAR(pcfg->qdelay_ref, 1, 10 * AQM_TIME_1S); + pcfg->tupdate = BOUND_VAR(pcfg->tupdate, 1, 10 * AQM_TIME_1S); + pcfg->max_burst = BOUND_VAR(pcfg->max_burst, 0, 10 * AQM_TIME_1S); + pcfg->max_ecnth = BOUND_VAR(pcfg->max_ecnth, 0, PIE_SCALE); + pcfg->alpha = BOUND_VAR(pcfg->alpha, 0, 7 * PIE_SCALE); + pcfg->beta = BOUND_VAR(pcfg->beta, 0 , 7 * PIE_SCALE); + + pie_desc.cfg_ref_count++; + //D("pie cfg_ref_count=%d", pie_desc.cfg_ref_count); + return 0; +} + +/* + * Deconfigure PIE and free memory allocation + */ +static int +aqm_pie_deconfig(struct dn_fsk* fs) +{ + if (fs && fs->aqmcfg) { + free(fs->aqmcfg, M_DUMMYNET); + fs->aqmcfg = NULL; + pie_desc.cfg_ref_count--; + } + return 0; +} + +/* + * Retrieve PIE configuration parameters. + */ +static int +aqm_pie_getconfig (struct dn_fsk *fs, struct dn_extra_parms * ep) +{ + struct dn_aqm_pie_parms *pcfg; + if (fs->aqmcfg) { + strcpy(ep->name, pie_desc.name); + pcfg = fs->aqmcfg; + ep->par[0] = pcfg->qdelay_ref / AQM_TIME_1US; + ep->par[1] = pcfg->tupdate / AQM_TIME_1US; + ep->par[2] = pcfg->max_burst / AQM_TIME_1US; + ep->par[3] = pcfg->max_ecnth; + ep->par[4] = pcfg->alpha; + ep->par[5] = pcfg->beta; + ep->par[6] = pcfg->flags; + + return 0; + } + return 1; +} + +static struct dn_aqm pie_desc = { + _SI( .type = ) DN_AQM_PIE, + _SI( .name = ) "PIE", + _SI( .ref_count = ) 0, + _SI( .cfg_ref_count = ) 0, + _SI( .enqueue = ) aqm_pie_enqueue, + _SI( .dequeue = ) aqm_pie_dequeue, + _SI( .config = ) aqm_pie_config, + _SI( .deconfig = ) aqm_pie_deconfig, + _SI( .getconfig = ) aqm_pie_getconfig, + _SI( .init = ) aqm_pie_init, + _SI( .cleanup = ) aqm_pie_cleanup, +}; + +DECLARE_DNAQM_MODULE(dn_aqm_pie, &pie_desc); +#endif Property changes on: head/sys/netpfil/ipfw/dn_aqm_pie.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/netpfil/ipfw/dn_aqm_pie.h =================================================================== --- head/sys/netpfil/ipfw/dn_aqm_pie.h (nonexistent) +++ head/sys/netpfil/ipfw/dn_aqm_pie.h (revision 300779) @@ -0,0 +1,151 @@ +/* + * PIE - Proportional Integral controller Enhanced AQM algorithm. + * + * $FreeBSD$ + * + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _IP_DN_AQM_PIE_H +#define _IP_DN_AQM_PIE_H + +#define DN_AQM_PIE 2 +#define PIE_DQ_THRESHOLD_BITS 14 +/* 2^14 =16KB */ +#define PIE_DQ_THRESHOLD (1UL << PIE_DQ_THRESHOLD_BITS) +#define MEAN_PKTSIZE 800 + +/* 31-bits because random() generates range from 0->(2**31)-1 */ +#define PIE_PROB_BITS 31 +#define PIE_MAX_PROB ((1ULL<parms; + + /* queue is not congested */ + + if ((pst->qdelay_old < (pprms->qdelay_ref >> 1) + && pst->drop_prob < PIE_MAX_PROB / 5 ) + || qlen <= 2 * MEAN_PKTSIZE) + return ENQUE; + + + if (pst->drop_prob == 0) + pst->accu_prob = 0; + + /* increment accu_prob */ + if (pprms->flags & PIE_DERAND_ENABLED) + pst->accu_prob += pst->drop_prob; + + /* De-randomize option + * if accu_prob < 0.85 -> enqueue + * if accu_prob>8.5 ->drop + * between 0.85 and 8.5 || !De-randomize --> drop on prob + */ + if (pprms->flags & PIE_DERAND_ENABLED) { + if(pst->accu_prob < (uint64_t) (PIE_MAX_PROB * 0.85)) + return ENQUE; + if( pst->accu_prob >= (uint64_t) (PIE_MAX_PROB * 8.5)) + return DROP; + } + + if (random() < pst->drop_prob) { + pst->accu_prob = 0; + return DROP; + } + + return ENQUE; +} + +#endif Property changes on: head/sys/netpfil/ipfw/dn_aqm_pie.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/netpfil/ipfw/dn_sched.h =================================================================== --- head/sys/netpfil/ipfw/dn_sched.h (revision 300778) +++ head/sys/netpfil/ipfw/dn_sched.h (revision 300779) @@ -1,192 +1,201 @@ /* * Copyright (c) 2010 Riccardo Panicucci, Luigi Rizzo, Universita` di Pisa * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * The API to write a packet scheduling algorithm for dummynet. * * $FreeBSD$ */ #ifndef _DN_SCHED_H #define _DN_SCHED_H #define DN_MULTIQUEUE 0x01 /* * Descriptor for a scheduling algorithm. * Contains all function pointers for a given scheduler * This is typically created when a module is loaded, and stored * in a global list of schedulers. */ struct dn_alg { uint32_t type; /* the scheduler type */ const char *name; /* scheduler name */ uint32_t flags; /* DN_MULTIQUEUE if supports multiple queues */ /* * The following define the size of 3 optional data structures * that may need to be allocated at runtime, and are appended * to each of the base data structures: scheduler, sched.inst, * and queue. We don't have a per-flowset structure. */ /* + parameters attached to the template, e.g. * default queue sizes, weights, quantum size, and so on; */ size_t schk_datalen; /* + per-instance parameters, such as timestamps, * containers for queues, etc; */ size_t si_datalen; size_t q_datalen; /* per-queue parameters (e.g. S,F) */ /* * Methods implemented by the scheduler: * enqueue enqueue packet 'm' on scheduler 's', queue 'q'. * q is NULL for !MULTIQUEUE. * Return 0 on success, 1 on drop (packet consumed anyways). * Note that q should be interpreted only as a hint * on the flow that the mbuf belongs to: while a * scheduler will normally enqueue m into q, it is ok * to leave q alone and put the mbuf elsewhere. * This function is called in two cases: * - when a new packet arrives to the scheduler; * - when a scheduler is reconfigured. In this case the * call is issued by the new_queue callback, with a * non empty queue (q) and m pointing to the first * mbuf in the queue. For this reason, the function * should internally check for (m != q->mq.head) * before calling dn_enqueue(). * * dequeue Called when scheduler instance 's' can * dequeue a packet. Return NULL if none are available. * XXX what about non work-conserving ? * * config called on 'sched X config ...', normally writes * in the area of size sch_arg * * destroy called on 'sched delete', frees everything * in sch_arg (other parts are handled by more specific * functions) * * new_sched called when a new instance is created, e.g. * to create the local queue for !MULTIQUEUE, set V or * copy parameters for WFQ, and so on. * * free_sched called when deleting an instance, cleans * extra data in the per-instance area. * * new_fsk called when a flowset is linked to a scheduler, * e.g. to validate parameters such as weights etc. * free_fsk when a flowset is unlinked from a scheduler. * (probably unnecessary) * * new_queue called to set the per-queue parameters, * e.g. S and F, adjust sum of weights in the parent, etc. * * The new_queue callback is normally called from when * creating a new queue. In some cases (such as a * scheduler change or reconfiguration) it can be called * with a non empty queue. In this case, the queue * In case of non empty queue, the new_queue callback could * need to call the enqueue function. In this case, * the callback should eventually call enqueue() passing * as m the first element in the queue. * * free_queue actions related to a queue removal, e.g. undo * all the above. If the queue has data in it, also remove * from the scheduler. This can e.g. happen during a reconfigure. */ int (*enqueue)(struct dn_sch_inst *, struct dn_queue *, struct mbuf *); struct mbuf * (*dequeue)(struct dn_sch_inst *); int (*config)(struct dn_schk *); int (*destroy)(struct dn_schk*); int (*new_sched)(struct dn_sch_inst *); int (*free_sched)(struct dn_sch_inst *); int (*new_fsk)(struct dn_fsk *f); int (*free_fsk)(struct dn_fsk *f); int (*new_queue)(struct dn_queue *q); int (*free_queue)(struct dn_queue *q); +#ifdef NEW_AQM + /* Getting scheduler extra parameters */ + int (*getconfig)(struct dn_schk *, struct dn_extra_parms *); +#endif /* run-time fields */ int ref_count; /* XXX number of instances in the system */ SLIST_ENTRY(dn_alg) next; /* Next scheduler in the list */ }; /* MSVC does not support initializers so we need this ugly macro */ #ifdef _WIN32 #define _SI(fld) #else #define _SI(fld) fld #endif /* * Additionally, dummynet exports some functions and macros * to be used by schedulers: */ void dn_free_pkts(struct mbuf *mnext); int dn_enqueue(struct dn_queue *q, struct mbuf* m, int drop); /* bound a variable between min and max */ int ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg); /* * Extract the head of a queue, update stats. Must be the very last * thing done on a dequeue as the queue itself may go away. */ static __inline struct mbuf* dn_dequeue(struct dn_queue *q) { struct mbuf *m = q->mq.head; if (m == NULL) return NULL; +#ifdef NEW_AQM + /* Call AQM dequeue function */ + if (q->fs->aqmfp && q->fs->aqmfp->dequeue ) + return q->fs->aqmfp->dequeue(q); +#endif q->mq.head = m->m_nextpkt; q->mq.count--; /* Update stats for the queue */ q->ni.length--; q->ni.len_bytes -= m->m_pkthdr.len; if (q->_si) { q->_si->ni.length--; q->_si->ni.len_bytes -= m->m_pkthdr.len; } if (q->ni.length == 0) /* queue is now idle */ q->q_time = dn_cfg.curr_time; return m; } int dn_sched_modevent(module_t mod, int cmd, void *arg); #define DECLARE_DNSCHED_MODULE(name, dnsched) \ static moduledata_t name##_mod = { \ #name, dn_sched_modevent, dnsched \ }; \ DECLARE_MODULE(name, name##_mod, \ SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY); \ MODULE_DEPEND(name, dummynet, 3, 3, 3) #endif /* _DN_SCHED_H */ Index: head/sys/netpfil/ipfw/dn_sched_fifo.c =================================================================== --- head/sys/netpfil/ipfw/dn_sched_fifo.c (revision 300778) +++ head/sys/netpfil/ipfw/dn_sched_fifo.c (revision 300779) @@ -1,121 +1,127 @@ /* * Copyright (c) 2010 Riccardo Panicucci, Universita` di Pisa * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ */ #ifdef _KERNEL #include #include #include #include #include #include #include /* IFNAMSIZ */ #include #include /* ipfw_rule_ref */ #include /* flow_id */ #include #include #include +#ifdef NEW_AQM +#include +#endif #include #else #include #endif /* * This file implements a FIFO scheduler for a single queue. * The queue is allocated as part of the scheduler instance, * and there is a single flowset is in the template which stores * queue size and policy. * Enqueue and dequeue use the default library functions. */ static int fifo_enqueue(struct dn_sch_inst *si, struct dn_queue *q, struct mbuf *m) { /* XXX if called with q != NULL and m=NULL, this is a * re-enqueue from an existing scheduler, which we should * handle. */ (void)q; return dn_enqueue((struct dn_queue *)(si+1), m, 0); } static struct mbuf * fifo_dequeue(struct dn_sch_inst *si) { return dn_dequeue((struct dn_queue *)(si + 1)); } static int fifo_new_sched(struct dn_sch_inst *si) { /* This scheduler instance contains the queue */ struct dn_queue *q = (struct dn_queue *)(si + 1); set_oid(&q->ni.oid, DN_QUEUE, sizeof(*q)); q->_si = si; q->fs = si->sched->fs; return 0; } static int fifo_free_sched(struct dn_sch_inst *si) { struct dn_queue *q = (struct dn_queue *)(si + 1); dn_free_pkts(q->mq.head); bzero(q, sizeof(*q)); return 0; } /* * FIFO scheduler descriptor * contains the type of the scheduler, the name, the size of extra * data structures, and function pointers. */ static struct dn_alg fifo_desc = { _SI( .type = ) DN_SCHED_FIFO, _SI( .name = ) "FIFO", _SI( .flags = ) 0, _SI( .schk_datalen = ) 0, _SI( .si_datalen = ) sizeof(struct dn_queue), _SI( .q_datalen = ) 0, _SI( .enqueue = ) fifo_enqueue, _SI( .dequeue = ) fifo_dequeue, _SI( .config = ) NULL, _SI( .destroy = ) NULL, _SI( .new_sched = ) fifo_new_sched, _SI( .free_sched = ) fifo_free_sched, _SI( .new_fsk = ) NULL, _SI( .free_fsk = ) NULL, _SI( .new_queue = ) NULL, _SI( .free_queue = ) NULL, +#ifdef NEW_AQM + _SI( .getconfig = ) NULL, +#endif }; DECLARE_DNSCHED_MODULE(dn_fifo, &fifo_desc); Index: head/sys/netpfil/ipfw/dn_sched_fq_codel.c =================================================================== --- head/sys/netpfil/ipfw/dn_sched_fq_codel.c (nonexistent) +++ head/sys/netpfil/ipfw/dn_sched_fq_codel.c (revision 300779) @@ -0,0 +1,617 @@ +/* + * FQ_Codel - The FlowQueue-Codel scheduler/AQM + * + * $FreeBSD$ + * + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifdef _KERNEL +#include +#include +//#include +#include +#include +#include +#include /* IFNAMSIZ */ +#include +#include /* ipfw_rule_ref */ +#include /* flow_id */ +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#else +#include +#endif + +/* NOTE: In fq_codel module, we reimplements CoDel AQM functions + * because fq_codel use different flows (sub-queues) structure and + * dn_queue includes many variables not needed by a flow (sub-queue + * )i.e. avoid extra overhead (88 bytes vs 208 bytes). + * Also, CoDel functions manages stats of sub-queues as well as the main queue. + */ + +#define DN_SCHED_FQ_CODEL 6 + +static struct dn_alg fq_codel_desc; + +/* fq_codel default parameters including codel */ +struct dn_sch_fq_codel_parms +fq_codel_sysctl = {{5000 * AQM_TIME_1US, 100000 * AQM_TIME_1US, + CODEL_ECN_ENABLED}, 1024, 10240, 1514}; + +static int +fqcodel_sysctl_interval_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + value = fq_codel_sysctl.ccfg.interval; + value /= AQM_TIME_1US; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 1 || value > 100 * AQM_TIME_1S) + return (EINVAL); + fq_codel_sysctl.ccfg.interval = value * AQM_TIME_1US ; + + return (0); +} + +static int +fqcodel_sysctl_target_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + value = fq_codel_sysctl.ccfg.target; + value /= AQM_TIME_1US; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 1 || value > 5 * AQM_TIME_1S) + return (EINVAL); + fq_codel_sysctl.ccfg.target = value * AQM_TIME_1US ; + + return (0); +} + + +SYSBEGIN(f4) + +SYSCTL_DECL(_net_inet); +SYSCTL_DECL(_net_inet_ip); +SYSCTL_DECL(_net_inet_ip_dummynet); +static SYSCTL_NODE(_net_inet_ip_dummynet, OID_AUTO, fqcodel, + CTLFLAG_RW, 0, "FQ_CODEL"); + +#ifdef SYSCTL_NODE + +SYSCTL_PROC(_net_inet_ip_dummynet_fqcodel, OID_AUTO, target, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, fqcodel_sysctl_target_handler, "L", + "FQ_CoDel target in microsecond"); +SYSCTL_PROC(_net_inet_ip_dummynet_fqcodel, OID_AUTO, interval, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, fqcodel_sysctl_interval_handler, "L", + "FQ_CoDel interval in microsecond"); + +SYSCTL_UINT(_net_inet_ip_dummynet_fqcodel, OID_AUTO, quantum, + CTLFLAG_RW, &fq_codel_sysctl.quantum, 1514, "FQ_CoDel quantum"); +SYSCTL_UINT(_net_inet_ip_dummynet_fqcodel, OID_AUTO, flows, + CTLFLAG_RW, &fq_codel_sysctl.flows_cnt, 1024, + "Number of queues for FQ_CoDel"); +SYSCTL_UINT(_net_inet_ip_dummynet_fqcodel, OID_AUTO, limit, + CTLFLAG_RW, &fq_codel_sysctl.limit, 10240, "FQ_CoDel queues size limit"); +#endif + +/* Drop a packet form the head of codel queue */ +static void +codel_drop_head(struct fq_codel_flow *q, struct fq_codel_si *si) +{ + struct mbuf *m = q->mq.head; + + if (m == NULL) + return; + q->mq.head = m->m_nextpkt; + + fq_update_stats(q, si, -m->m_pkthdr.len, 1); + + if (si->main_q.ni.length == 0) /* queue is now idle */ + si->main_q.q_time = dn_cfg.curr_time; + + FREE_PKT(m); +} + +/* Enqueue a packet 'm' to a queue 'q' and add timestamp to that packet. + * Return 1 when unable to add timestamp, otherwise return 0 + */ +static int +codel_enqueue(struct fq_codel_flow *q, struct mbuf *m, struct fq_codel_si *si) +{ + uint64_t len; + + len = m->m_pkthdr.len; + /* finding maximum packet size */ + if (len > q->cst.maxpkt_size) + q->cst.maxpkt_size = len; + + /* Add timestamp to mbuf as MTAG */ + struct m_tag *mtag; + mtag = m_tag_locate(m, MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, NULL); + if (mtag == NULL) + mtag = m_tag_alloc(MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, sizeof(aqm_time_t), + M_NOWAIT); + if (mtag == NULL) { + m_freem(m); + goto drop; + } + *(aqm_time_t *)(mtag + 1) = AQM_UNOW; + m_tag_prepend(m, mtag); + + mq_append(&q->mq, m); + fq_update_stats(q, si, len, 0); + return 0; + +drop: + fq_update_stats(q, si, len, 1); + m_freem(m); + return 1; +} + +/* + * Classify a packet to queue number using Jenkins hash function. + * Return: queue number + * the input of the hash are protocol no, perturbation, src IP, dst IP, + * src port, dst port, + */ +static inline int +fq_codel_classify_flow(struct mbuf *m, uint16_t fcount, struct fq_codel_si *si) +{ + struct ip *ip; + struct tcphdr *th; + struct udphdr *uh; + uint8_t tuple[41]; + uint16_t hash=0; + +//#ifdef INET6 + struct ip6_hdr *ip6; + int isip6; + isip6 = (mtod(m, struct ip *)->ip_v == 6) ? 1 : 0; + + if(isip6) { + ip6 = mtod(m, struct ip6_hdr *); + *((uint8_t *) &tuple[0]) = ip6->ip6_nxt; + *((uint32_t *) &tuple[1]) = si->perturbation; + memcpy(&tuple[5], ip6->ip6_src.s6_addr, 16); + memcpy(&tuple[21], ip6->ip6_dst.s6_addr, 16); + + switch (ip6->ip6_nxt) { + case IPPROTO_TCP: + th = (struct tcphdr *)(ip6 + 1); + *((uint16_t *) &tuple[37]) = th->th_dport; + *((uint16_t *) &tuple[39]) = th->th_sport; + break; + + case IPPROTO_UDP: + uh = (struct udphdr *)(ip6 + 1); + *((uint16_t *) &tuple[37]) = uh->uh_dport; + *((uint16_t *) &tuple[39]) = uh->uh_sport; + break; + default: + memset(&tuple[37], 0, 4); + + } + + hash = jenkins_hash(tuple, 41, HASHINIT) % fcount; + return hash; + } +//#endif + + /* IPv4 */ + ip = mtod(m, struct ip *); + *((uint8_t *) &tuple[0]) = ip->ip_p; + *((uint32_t *) &tuple[1]) = si->perturbation; + *((uint32_t *) &tuple[5]) = ip->ip_src.s_addr; + *((uint32_t *) &tuple[9]) = ip->ip_dst.s_addr; + + switch (ip->ip_p) { + case IPPROTO_TCP: + th = (struct tcphdr *)(ip + 1); + *((uint16_t *) &tuple[13]) = th->th_dport; + *((uint16_t *) &tuple[15]) = th->th_sport; + break; + + case IPPROTO_UDP: + uh = (struct udphdr *)(ip + 1); + *((uint16_t *) &tuple[13]) = uh->uh_dport; + *((uint16_t *) &tuple[15]) = uh->uh_sport; + break; + default: + memset(&tuple[13], 0, 4); + + } + hash = jenkins_hash(tuple, 17, HASHINIT) % fcount; + + return hash; +} + +/* + * Enqueue a packet into an appropriate queue according to + * FQ_CODEL algorithm. + */ +static int +fq_codel_enqueue(struct dn_sch_inst *_si, struct dn_queue *_q, + struct mbuf *m) +{ + struct fq_codel_si *si; + struct fq_codel_schk *schk; + struct dn_sch_fq_codel_parms *param; + struct dn_queue *mainq; + int idx, drop, i, maxidx; + + mainq = (struct dn_queue *)(_si + 1); + si = (struct fq_codel_si *)_si; + schk = (struct fq_codel_schk *)(si->_si.sched+1); + param = &schk->cfg; + + /* classify a packet to queue number*/ + idx = fq_codel_classify_flow(m, param->flows_cnt, si); + /* enqueue packet into appropriate queue using CoDel AQM. + * Note: 'codel_enqueue' function returns 1 only when it unable to + * add timestamp to packet (no limit check)*/ + drop = codel_enqueue(&si->flows[idx], m, si); + + /* codel unable to timestamp a packet */ + if (drop) + return 1; + + /* If the flow (sub-queue) is not active ,then add it to the tail of + * new flows list, initialize and activate it. + */ + if (!si->flows[idx].active ) { + STAILQ_INSERT_TAIL(&si->newflows, &si->flows[idx], flowchain); + si->flows[idx].deficit = param->quantum; + si->flows[idx].cst.dropping = false; + si->flows[idx].cst.first_above_time = 0; + si->flows[idx].active = 1; + //D("activate %d",idx); + } + + /* check the limit for all queues and remove a packet from the + * largest one + */ + if (mainq->ni.length > schk->cfg.limit) { D("over limit"); + /* find first active flow */ + for (maxidx = 0; maxidx < schk->cfg.flows_cnt; maxidx++) + if (si->flows[maxidx].active) + break; + if (maxidx < schk->cfg.flows_cnt) { + /* find the largest sub- queue */ + for (i = maxidx + 1; i < schk->cfg.flows_cnt; i++) + if (si->flows[i].active && si->flows[i].stats.length > + si->flows[maxidx].stats.length) + maxidx = i; + codel_drop_head(&si->flows[maxidx], si); + D("maxidx = %d",maxidx); + drop = 1; + } + } + + return drop; +} + +/* + * Dequeue a packet from an appropriate queue according to + * FQ_CODEL algorithm. + */ +static struct mbuf * +fq_codel_dequeue(struct dn_sch_inst *_si) +{ + struct fq_codel_si *si; + struct fq_codel_schk *schk; + struct dn_sch_fq_codel_parms *param; + struct fq_codel_flow *f; + struct mbuf *mbuf; + struct fq_codel_list *fq_codel_flowlist; + + si = (struct fq_codel_si *)_si; + schk = (struct fq_codel_schk *)(si->_si.sched+1); + param = &schk->cfg; + + do { + /* select a list to start with */ + if (STAILQ_EMPTY(&si->newflows)) + fq_codel_flowlist = &si->oldflows; + else + fq_codel_flowlist = &si->newflows; + + /* Both new and old queue lists are empty, return NULL */ + if (STAILQ_EMPTY(fq_codel_flowlist)) + return NULL; + + f = STAILQ_FIRST(fq_codel_flowlist); + while (f != NULL) { + /* if there is no flow(sub-queue) deficit, increase deficit + * by quantum, move the flow to the tail of old flows list + * and try another flow. + * Otherwise, the flow will be used for dequeue. + */ + if (f->deficit < 0) { + f->deficit += param->quantum; + STAILQ_REMOVE_HEAD(fq_codel_flowlist, flowchain); + STAILQ_INSERT_TAIL(&si->oldflows, f, flowchain); + } else + break; + + f = STAILQ_FIRST(fq_codel_flowlist); + } + + /* the new flows list is empty, try old flows list */ + if (STAILQ_EMPTY(fq_codel_flowlist)) + continue; + + /* Dequeue a packet from the selected flow */ + mbuf = fqc_codel_dequeue(f, si); + + /* Codel did not return a packet */ + if (!mbuf) { + /* If the selected flow belongs to new flows list, then move + * it to the tail of old flows list. Otherwise, deactivate it and + * remove it from the old list and + */ + if (fq_codel_flowlist == &si->newflows) { + STAILQ_REMOVE_HEAD(fq_codel_flowlist, flowchain); + STAILQ_INSERT_TAIL(&si->oldflows, f, flowchain); + } else { + f->active = 0; + STAILQ_REMOVE_HEAD(fq_codel_flowlist, flowchain); + } + /* start again */ + continue; + } + + /* we have a packet to return, + * update flow deficit and return the packet*/ + f->deficit -= mbuf->m_pkthdr.len; + return mbuf; + + } while (1); + + /* unreachable point */ + return NULL; +} + +/* + * Initialize fq_codel scheduler instance. + * also, allocate memory for flows array. + */ +static int +fq_codel_new_sched(struct dn_sch_inst *_si) +{ + struct fq_codel_si *si; + struct dn_queue *q; + struct fq_codel_schk *schk; + int i; + + si = (struct fq_codel_si *)_si; + schk = (struct fq_codel_schk *)(_si->sched+1); + + if(si->flows) { + D("si already configured!"); + return 0; + } + + /* init the main queue */ + q = &si->main_q; + set_oid(&q->ni.oid, DN_QUEUE, sizeof(*q)); + q->_si = _si; + q->fs = _si->sched->fs; + + /* allocate memory for flows array */ + si->flows = malloc(schk->cfg.flows_cnt * sizeof(struct fq_codel_flow), + M_DUMMYNET, M_NOWAIT | M_ZERO); + if (si->flows == NULL) { + D("cannot allocate memory for fq_codel configuration parameters"); + return ENOMEM ; + } + + /* init perturbation for this si */ + si->perturbation = random(); + + /* init the old and new flows lists */ + STAILQ_INIT(&si->newflows); + STAILQ_INIT(&si->oldflows); + + /* init the flows (sub-queues) */ + for (i = 0; i < schk->cfg.flows_cnt; i++) { + /* init codel */ + si->flows[i].cst.maxpkt_size = 500; + } + + fq_codel_desc.ref_count++; + return 0; +} + +/* + * Free fq_codel scheduler instance. + */ +static int +fq_codel_free_sched(struct dn_sch_inst *_si) +{ + struct fq_codel_si *si = (struct fq_codel_si *)_si ; + + /* free the flows array */ + free(si->flows , M_DUMMYNET); + si->flows = NULL; + fq_codel_desc.ref_count--; + + return 0; +} + +/* + * Configure fq_codel scheduler. + * the configurations for the scheduler is passed from userland. + */ +static int +fq_codel_config(struct dn_schk *_schk) +{ + struct fq_codel_schk *schk; + struct dn_extra_parms *ep; + struct dn_sch_fq_codel_parms *fqc_cfg; + + schk = (struct fq_codel_schk *)(_schk+1); + ep = (struct dn_extra_parms *) _schk->cfg; + + /* par array contains fq_codel configuration as follow + * Codel: 0- target,1- interval, 2- flags + * FQ_CODEL: 3- quantum, 4- limit, 5- flows + */ + if (ep && ep->oid.len ==sizeof(*ep) && + ep->oid.subtype == DN_SCH_PARAMS) { + + fqc_cfg = &schk->cfg; + if (ep->par[0] < 0) + fqc_cfg->ccfg.target = fq_codel_sysctl.ccfg.target; + else + fqc_cfg->ccfg.target = ep->par[0] * AQM_TIME_1US; + + if (ep->par[1] < 0) + fqc_cfg->ccfg.interval = fq_codel_sysctl.ccfg.interval; + else + fqc_cfg->ccfg.interval = ep->par[1] * AQM_TIME_1US; + + if (ep->par[2] < 0) + fqc_cfg->ccfg.flags = 0; + else + fqc_cfg->ccfg.flags = ep->par[2]; + + /* FQ configurations */ + if (ep->par[3] < 0) + fqc_cfg->quantum = fq_codel_sysctl.quantum; + else + fqc_cfg->quantum = ep->par[3]; + + if (ep->par[4] < 0) + fqc_cfg->limit = fq_codel_sysctl.limit; + else + fqc_cfg->limit = ep->par[4]; + + if (ep->par[5] < 0) + fqc_cfg->flows_cnt = fq_codel_sysctl.flows_cnt; + else + fqc_cfg->flows_cnt = ep->par[5]; + + /* Bound the configurations */ + fqc_cfg->ccfg.target = BOUND_VAR(fqc_cfg->ccfg.target, 1 , + 5 * AQM_TIME_1S); ; + fqc_cfg->ccfg.interval = BOUND_VAR(fqc_cfg->ccfg.interval, 1, + 100 * AQM_TIME_1S); + + fqc_cfg->quantum = BOUND_VAR(fqc_cfg->quantum,1, 9000); + fqc_cfg->limit= BOUND_VAR(fqc_cfg->limit,1,20480); + fqc_cfg->flows_cnt= BOUND_VAR(fqc_cfg->flows_cnt,1,65536); + } + else + return 1; + + return 0; +} + +/* + * Return fq_codel scheduler configurations + * the configurations for the scheduler is passed to userland. + */ +static int +fq_codel_getconfig (struct dn_schk *_schk, struct dn_extra_parms *ep) { + + struct fq_codel_schk *schk = (struct fq_codel_schk *)(_schk+1); + struct dn_sch_fq_codel_parms *fqc_cfg; + + fqc_cfg = &schk->cfg; + + strcpy(ep->name, fq_codel_desc.name); + ep->par[0] = fqc_cfg->ccfg.target / AQM_TIME_1US; + ep->par[1] = fqc_cfg->ccfg.interval / AQM_TIME_1US; + ep->par[2] = fqc_cfg->ccfg.flags; + + ep->par[3] = fqc_cfg->quantum; + ep->par[4] = fqc_cfg->limit; + ep->par[5] = fqc_cfg->flows_cnt; + + return 0; +} + +/* + * fq_codel scheduler descriptor + * contains the type of the scheduler, the name, the size of extra + * data structures, and function pointers. + */ +static struct dn_alg fq_codel_desc = { + _SI( .type = ) DN_SCHED_FQ_CODEL, + _SI( .name = ) "FQ_CODEL", + _SI( .flags = ) 0, + + _SI( .schk_datalen = ) sizeof(struct fq_codel_schk), + _SI( .si_datalen = ) sizeof(struct fq_codel_si) - sizeof(struct dn_sch_inst), + _SI( .q_datalen = ) 0, + + _SI( .enqueue = ) fq_codel_enqueue, + _SI( .dequeue = ) fq_codel_dequeue, + _SI( .config = ) fq_codel_config, /* new sched i.e. sched X config ...*/ + _SI( .destroy = ) NULL, /*sched x delete */ + _SI( .new_sched = ) fq_codel_new_sched, /* new schd instance */ + _SI( .free_sched = ) fq_codel_free_sched, /* delete schd instance */ + _SI( .new_fsk = ) NULL, + _SI( .free_fsk = ) NULL, + _SI( .new_queue = ) NULL, + _SI( .free_queue = ) NULL, + _SI( .getconfig = ) fq_codel_getconfig, + _SI( .ref_count = ) 0 +}; + +DECLARE_DNSCHED_MODULE(dn_fq_codel, &fq_codel_desc); Property changes on: head/sys/netpfil/ipfw/dn_sched_fq_codel.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/netpfil/ipfw/dn_sched_fq_codel.h =================================================================== --- head/sys/netpfil/ipfw/dn_sched_fq_codel.h (nonexistent) +++ head/sys/netpfil/ipfw/dn_sched_fq_codel.h (revision 300779) @@ -0,0 +1,167 @@ +/*- + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * FQ_Codel Structures and helper functions + * + * $FreeBSD$ + */ + +#ifndef _IP_DN_SCHED_FQ_CODEL_H +#define _IP_DN_SCHED_FQ_CODEL_H + +/* list of queues */ +STAILQ_HEAD(fq_codel_list, fq_codel_flow) ; + +/* fq_codel parameters including codel */ +struct dn_sch_fq_codel_parms { + struct dn_aqm_codel_parms ccfg; /* CoDel Parameters */ + /* FQ_CODEL Parameters */ + uint32_t flows_cnt; /* number of flows */ + uint32_t limit; /* hard limit of fq_codel queue size*/ + uint32_t quantum; +}; /* defaults */ + +/* flow (sub-queue) stats */ +struct flow_stats { + uint64_t tot_pkts; /* statistics counters */ + uint64_t tot_bytes; + uint32_t length; /* Queue length, in packets */ + uint32_t len_bytes; /* Queue length, in bytes */ + uint32_t drops; +}; + +/* A flow of packets (sub-queue).*/ +struct fq_codel_flow { + struct mq mq; /* list of packets */ + struct flow_stats stats; /* statistics */ + int deficit; + int active; /* 1: flow is active (in a list) */ + struct codel_status cst; + STAILQ_ENTRY(fq_codel_flow) flowchain; +}; + +/* extra fq_codel scheduler configurations */ +struct fq_codel_schk { + struct dn_sch_fq_codel_parms cfg; +}; + +/* fq_codel scheduler instance */ +struct fq_codel_si { + struct dn_sch_inst _si; /* standard scheduler instance */ + struct dn_queue main_q; /* main queue is after si directly */ + + struct fq_codel_flow *flows; /* array of flows (queues) */ + uint32_t perturbation; /* random value */ + struct fq_codel_list newflows; /* list of new queues */ + struct fq_codel_list oldflows; /* list of old queues */ +}; + +/* Helper function to update queue&main-queue and scheduler statistics. + * negative len + drop -> drop + * negative len -> dequeue + * positive len -> enqueue + * positive len + drop -> drop during enqueue + */ +__inline static void +fq_update_stats(struct fq_codel_flow *q, struct fq_codel_si *si, int len, + int drop) +{ + int inc = 0; + + if (len < 0) + inc = -1; + else if (len > 0) + inc = 1; + + if (drop) { + si->main_q.ni.drops ++; + q->stats.drops ++; + si->_si.ni.drops ++; + io_pkt_drop ++; + } + + if (!drop || (drop && len < 0)) { + /* Update stats for the main queue */ + si->main_q.ni.length += inc; + si->main_q.ni.len_bytes += len; + + /*update sub-queue stats */ + q->stats.length += inc; + q->stats.len_bytes += len; + + /*update scheduler instance stats */ + si->_si.ni.length += inc; + si->_si.ni.len_bytes += len; + } + + if (inc > 0) { + si->main_q.ni.tot_bytes += len; + si->main_q.ni.tot_pkts ++; + + q->stats.tot_bytes +=len; + q->stats.tot_pkts++; + + si->_si.ni.tot_bytes +=len; + si->_si.ni.tot_pkts ++; + } + +} + +/* extract the head of fq_codel sub-queue */ +__inline static struct mbuf * +fq_codel_extract_head(struct fq_codel_flow *q, aqm_time_t *pkt_ts, struct fq_codel_si *si) +{ + struct mbuf *m = q->mq.head; + + if (m == NULL) + return m; + q->mq.head = m->m_nextpkt; + + fq_update_stats(q, si, -m->m_pkthdr.len, 0); + + if (si->main_q.ni.length == 0) /* queue is now idle */ + si->main_q.q_time = dn_cfg.curr_time; + + /* extract packet timestamp*/ + struct m_tag *mtag; + mtag = m_tag_locate(m, MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, NULL); + if (mtag == NULL){ + D("timestamp tag is not found!"); + *pkt_ts = 0; + } else { + *pkt_ts = *(aqm_time_t *)(mtag + 1); + m_tag_delete(m,mtag); + } + + return m; +} + + +#endif Property changes on: head/sys/netpfil/ipfw/dn_sched_fq_codel.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/netpfil/ipfw/dn_sched_fq_codel_helper.h =================================================================== --- head/sys/netpfil/ipfw/dn_sched_fq_codel_helper.h (nonexistent) +++ head/sys/netpfil/ipfw/dn_sched_fq_codel_helper.h (revision 300779) @@ -0,0 +1,187 @@ +/* + * Codel - The Controlled-Delay Active Queue Management algorithm. + * + * $FreeBSD$ + * + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * + * Copyright (C) 2011-2014 Kathleen Nichols . + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * o Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * + * o Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * o The names of the authors may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * Alternatively, provided that this notice is retained in full, this + * software may be distributed under the terms of the GNU General Public + * License ("GPL") version 2, in which case the provisions of the GPL + * apply INSTEAD OF those given above. + + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _IP_DN_SCHED_FQ_CODEL_HELPER_H +#define _IP_DN_SCHED_FQ_CODEL_HELPER_H + +__inline static struct mbuf * +fqc_dodequeue(struct fq_codel_flow *q, aqm_time_t now, uint16_t *ok_to_drop, + struct fq_codel_si *si) +{ + struct mbuf * m; + struct fq_codel_schk *schk = (struct fq_codel_schk *)(si->_si.sched+1); + aqm_time_t pkt_ts, sojourn_time; + + *ok_to_drop = 0; + m = fq_codel_extract_head(q, &pkt_ts, si); + + if (m == NULL) { + /*queue is empty - we can't be above target*/ + q->cst.first_above_time= 0; + return m; + } + + /* To span a large range of bandwidths, CoDel runs two + * different AQMs in parallel. One is sojourn-time-based + * and takes effect when the time to send an MTU-sized + * packet is less than target. The 1st term of the "if" + * below does this. The other is backlog-based and takes + * effect when the time to send an MTU-sized packet is >= + * target. The goal here is to keep the output link + * utilization high by never allowing the queue to get + * smaller than the amount that arrives in a typical + * interarrival time (MTU-sized packets arriving spaced + * by the amount of time it takes to send such a packet on + * the bottleneck). The 2nd term of the "if" does this. + */ + sojourn_time = now - pkt_ts; + if (sojourn_time < schk->cfg.ccfg.target || q->stats.len_bytes <= q->cst.maxpkt_size) { + /* went below - stay below for at least interval */ + q->cst.first_above_time = 0; + } else { + if (q->cst.first_above_time == 0) { + /* just went above from below. if still above at + * first_above_time, will say it's ok to drop. */ + q->cst.first_above_time = now + schk->cfg.ccfg.interval; + } else if (now >= q->cst.first_above_time) { + *ok_to_drop = 1; + } + } + return m; +} + +/* Codel dequeue function */ +__inline static struct mbuf * +fqc_codel_dequeue(struct fq_codel_flow *q, struct fq_codel_si *si) +{ + struct mbuf *m; + struct dn_aqm_codel_parms *cprms; + struct codel_status *cst; + aqm_time_t now; + uint16_t ok_to_drop; + struct fq_codel_schk *schk = (struct fq_codel_schk *)(si->_si.sched+1); + + cst = &q->cst; + cprms = &schk->cfg.ccfg; + + now = AQM_UNOW; + m = fqc_dodequeue(q, now, &ok_to_drop, si); + + if (cst->dropping) { + if (!ok_to_drop) { + /* sojourn time below target - leave dropping state */ + cst->dropping = false; + } + + /* Time for the next drop. Drop current packet and dequeue + * next. If the dequeue doesn't take us out of dropping + * state, schedule the next drop. A large backlog might + * result in drop rates so high that the next drop should + * happen now, hence the 'while' loop. + */ + while (now >= cst->drop_next_time && cst->dropping) { + + /* mark the packet */ + if (cprms->flags & CODEL_ECN_ENABLED && ecn_mark(m)) { + cst->count++; + /* schedule the next mark. */ + cst->drop_next_time = control_law(cst, cprms, cst->drop_next_time); + return m; + } + + /* drop the packet */ + fq_update_stats(q, si, 0, 1); + m_freem(m); + m = fqc_dodequeue(q, now, &ok_to_drop, si); + + if (!ok_to_drop) { + /* leave dropping state */ + cst->dropping = false; + } else { + cst->count++; + /* schedule the next drop. */ + cst->drop_next_time = control_law(cst, cprms, cst->drop_next_time); + } + } + /* If we get here we're not in dropping state. The 'ok_to_drop' + * return from dodequeue means that the sojourn time has been + * above 'target' for 'interval' so enter dropping state. + */ + } else if (ok_to_drop) { + + /* if ECN option is disabled or the packet cannot be marked, + * drop the packet and extract another. + */ + if (!(cprms->flags & CODEL_ECN_ENABLED) || !ecn_mark(m)) { + fq_update_stats(q, si, 0, 1); + m_freem(m); + m = fqc_dodequeue(q, now, &ok_to_drop,si); + } + + cst->dropping = true; + + /* If min went above target close to when it last went + * below, assume that the drop rate that controlled the + * queue on the last cycle is a good starting point to + * control it now. ('drop_next' will be at most 'interval' + * later than the time of the last drop so 'now - drop_next' + * is a good approximation of the time from the last drop + * until now.) + */ + cst->count = (cst->count > 2 && ((aqm_stime_t)now - + (aqm_stime_t)cst->drop_next_time) < 8* cprms->interval)? cst->count - 2 : 1; + + /* we don't have to set initial guess for Newton's method isqrt as + * we initilaize isqrt in control_law function when count == 1 */ + cst->drop_next_time = control_law(cst, cprms, now); + } + + return m; +} + +#endif Property changes on: head/sys/netpfil/ipfw/dn_sched_fq_codel_helper.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/netpfil/ipfw/dn_sched_fq_pie.c =================================================================== --- head/sys/netpfil/ipfw/dn_sched_fq_pie.c (nonexistent) +++ head/sys/netpfil/ipfw/dn_sched_fq_pie.c (revision 300779) @@ -0,0 +1,1262 @@ +/* + * FQ_PIE - The FlowQueue-PIE scheduler/AQM + * + * $FreeBSD$ + * + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* Important note: + * As there is no an office document for FQ-PIE specification, we used + * FQ-CoDel algorithm with some modifications to implement FQ-PIE. + * This FQ-PIE implementation is a beta version and have not been tested + * extensively. Our FQ-PIE uses stand-alone PIE AQM per sub-queue. By + * default, timestamp is used to calculate queue delay instead of departure + * rate estimation method. Although departure rate estimation is available + * as testing option, the results could be incorrect. Moreover, turning PIE on + * and off option is available but it does not work properly in this version. + */ + + +#ifdef _KERNEL +#include +#include +#include +#include +#include +#include +#include +#include /* IFNAMSIZ */ +#include +#include /* ipfw_rule_ref */ +#include /* flow_id */ +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#else +#include +#endif + +#define DN_SCHED_FQ_PIE 7 + +/* list of queues */ +STAILQ_HEAD(fq_pie_list, fq_pie_flow) ; + +/* FQ_PIE parameters including PIE */ +struct dn_sch_fq_pie_parms { + struct dn_aqm_pie_parms pcfg; /* PIE configuration Parameters */ + /* FQ_PIE Parameters */ + uint32_t flows_cnt; /* number of flows */ + uint32_t limit; /* hard limit of FQ_PIE queue size*/ + uint32_t quantum; +}; + +/* flow (sub-queue) stats */ +struct flow_stats { + uint64_t tot_pkts; /* statistics counters */ + uint64_t tot_bytes; + uint32_t length; /* Queue length, in packets */ + uint32_t len_bytes; /* Queue length, in bytes */ + uint32_t drops; +}; + +/* A flow of packets (sub-queue)*/ +struct fq_pie_flow { + struct mq mq; /* list of packets */ + struct flow_stats stats; /* statistics */ + int deficit; + int active; /* 1: flow is active (in a list) */ + struct pie_status pst; /* pie status variables */ + struct fq_pie_si *psi; /* parent scheduler instance */ + STAILQ_ENTRY(fq_pie_flow) flowchain; +}; + +/* extra fq_pie scheduler configurations */ +struct fq_pie_schk { + struct dn_sch_fq_pie_parms cfg; +}; + +/* fq_pie scheduler instance */ +struct fq_pie_si { + struct dn_sch_inst _si; /* standard scheduler instance */ + struct dn_queue main_q; /* main queue is after si directly */ + uint32_t nr_active_q; + struct fq_pie_flow *flows; /* array of flows (queues) */ + uint32_t perturbation; /* random value */ + struct fq_pie_list newflows; /* list of new queues */ + struct fq_pie_list oldflows; /* list of old queues */ +}; + + +struct mem_to_free { + void *mem_flows; + void *mem_callout; +}; +static struct mtx freemem_mtx; +static struct dn_alg fq_pie_desc; + +/* Default FQ-PIE parameters including PIE */ +/* PIE defaults + * target=15ms, max_burst=150ms, max_ecnth=0.1, + * alpha=0.125, beta=1.25, tupdate=15ms + * FQ- + * flows=1024, limit=10240, quantum =1514 + */ +struct dn_sch_fq_pie_parms + fq_pie_sysctl = {{15000 * AQM_TIME_1US, 15000 * AQM_TIME_1US, + 150000 * AQM_TIME_1US, PIE_SCALE * 0.1, PIE_SCALE * 0.125, + PIE_SCALE * 1.25, PIE_CAPDROP_ENABLED | PIE_DERAND_ENABLED}, + 1024, 10240, 1514}; + +static int +fqpie_sysctl_alpha_beta_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + if (!strcmp(oidp->oid_name,"alpha")) + value = fq_pie_sysctl.pcfg.alpha; + else + value = fq_pie_sysctl.pcfg.beta; + + value = value * 1000 / PIE_SCALE; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 1 || value > 7 * PIE_SCALE) + return (EINVAL); + value = (value * PIE_SCALE) / 1000; + if (!strcmp(oidp->oid_name,"alpha")) + fq_pie_sysctl.pcfg.alpha = value; + else + fq_pie_sysctl.pcfg.beta = value; + return (0); +} + +static int +fqpie_sysctl_target_tupdate_maxb_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + if (!strcmp(oidp->oid_name,"target")) + value = fq_pie_sysctl.pcfg.qdelay_ref; + else if (!strcmp(oidp->oid_name,"tupdate")) + value = fq_pie_sysctl.pcfg.tupdate; + else + value = fq_pie_sysctl.pcfg.max_burst; + + value = value / AQM_TIME_1US; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 1 || value > 10 * AQM_TIME_1S) + return (EINVAL); + value = value * AQM_TIME_1US; + + if (!strcmp(oidp->oid_name,"target")) + fq_pie_sysctl.pcfg.qdelay_ref = value; + else if (!strcmp(oidp->oid_name,"tupdate")) + fq_pie_sysctl.pcfg.tupdate = value; + else + fq_pie_sysctl.pcfg.max_burst = value; + return (0); +} + +static int +fqpie_sysctl_max_ecnth_handler(SYSCTL_HANDLER_ARGS) +{ + int error; + long value; + + value = fq_pie_sysctl.pcfg.max_ecnth; + value = value * 1000 / PIE_SCALE; + error = sysctl_handle_long(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 1 || value > PIE_SCALE) + return (EINVAL); + value = (value * PIE_SCALE) / 1000; + fq_pie_sysctl.pcfg.max_ecnth = value; + return (0); +} + +/* define FQ- PIE sysctl variables */ +SYSBEGIN(f4) +SYSCTL_DECL(_net_inet); +SYSCTL_DECL(_net_inet_ip); +SYSCTL_DECL(_net_inet_ip_dummynet); +static SYSCTL_NODE(_net_inet_ip_dummynet, OID_AUTO, fqpie, + CTLFLAG_RW, 0, "FQ_PIE"); + +#ifdef SYSCTL_NODE + +SYSCTL_PROC(_net_inet_ip_dummynet_fqpie, OID_AUTO, target, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + fqpie_sysctl_target_tupdate_maxb_handler, "L", + "queue target in microsecond"); + +SYSCTL_PROC(_net_inet_ip_dummynet_fqpie, OID_AUTO, tupdate, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + fqpie_sysctl_target_tupdate_maxb_handler, "L", + "the frequency of drop probability calculation in microsecond"); + +SYSCTL_PROC(_net_inet_ip_dummynet_fqpie, OID_AUTO, max_burst, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + fqpie_sysctl_target_tupdate_maxb_handler, "L", + "Burst allowance interval in microsecond"); + +SYSCTL_PROC(_net_inet_ip_dummynet_fqpie, OID_AUTO, max_ecnth, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + fqpie_sysctl_max_ecnth_handler, "L", + "ECN safeguard threshold scaled by 1000"); + +SYSCTL_PROC(_net_inet_ip_dummynet_fqpie, OID_AUTO, alpha, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + fqpie_sysctl_alpha_beta_handler, "L", "PIE alpha scaled by 1000"); + +SYSCTL_PROC(_net_inet_ip_dummynet_fqpie, OID_AUTO, beta, + CTLTYPE_LONG | CTLFLAG_RW, NULL, 0, + fqpie_sysctl_alpha_beta_handler, "L", "beta scaled by 1000"); + +SYSCTL_UINT(_net_inet_ip_dummynet_fqpie, OID_AUTO, quantum, + CTLFLAG_RW, &fq_pie_sysctl.quantum, 1514, "quantum for FQ_PIE"); +SYSCTL_UINT(_net_inet_ip_dummynet_fqpie, OID_AUTO, flows, + CTLFLAG_RW, &fq_pie_sysctl.flows_cnt, 1024, "Number of queues for FQ_PIE"); +SYSCTL_UINT(_net_inet_ip_dummynet_fqpie, OID_AUTO, limit, + CTLFLAG_RW, &fq_pie_sysctl.limit, 10240, "limit for FQ_PIE"); +#endif + +/* Helper function to update queue&main-queue and scheduler statistics. + * negative len & drop -> drop + * negative len -> dequeue + * positive len -> enqueue + * positive len + drop -> drop during enqueue + */ +__inline static void +fq_update_stats(struct fq_pie_flow *q, struct fq_pie_si *si, int len, + int drop) +{ + int inc = 0; + + if (len < 0) + inc = -1; + else if (len > 0) + inc = 1; + + if (drop) { + si->main_q.ni.drops ++; + q->stats.drops ++; + si->_si.ni.drops ++; + io_pkt_drop ++; + } + + if (!drop || (drop && len < 0)) { + /* Update stats for the main queue */ + si->main_q.ni.length += inc; + si->main_q.ni.len_bytes += len; + + /*update sub-queue stats */ + q->stats.length += inc; + q->stats.len_bytes += len; + + /*update scheduler instance stats */ + si->_si.ni.length += inc; + si->_si.ni.len_bytes += len; + } + + if (inc > 0) { + si->main_q.ni.tot_bytes += len; + si->main_q.ni.tot_pkts ++; + + q->stats.tot_bytes +=len; + q->stats.tot_pkts++; + + si->_si.ni.tot_bytes +=len; + si->_si.ni.tot_pkts ++; + } + +} + +/* + * Extract a packet from the head of sub-queue 'q' + * Return a packet or NULL if the queue is empty. + * If getts is set, also extract packet's timestamp from mtag. + */ +__inline static struct mbuf * +fq_pie_extract_head(struct fq_pie_flow *q, aqm_time_t *pkt_ts, + struct fq_pie_si *si, int getts) +{ + struct mbuf *m = q->mq.head; + + if (m == NULL) + return m; + q->mq.head = m->m_nextpkt; + + fq_update_stats(q, si, -m->m_pkthdr.len, 0); + + if (si->main_q.ni.length == 0) /* queue is now idle */ + si->main_q.q_time = dn_cfg.curr_time; + + if (getts) { + /* extract packet timestamp*/ + struct m_tag *mtag; + mtag = m_tag_locate(m, MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, NULL); + if (mtag == NULL){ + D("PIE timestamp mtag not found!"); + *pkt_ts = 0; + } else { + *pkt_ts = *(aqm_time_t *)(mtag + 1); + m_tag_delete(m,mtag); + } + } + return m; +} + +/* + * Callout function for drop probability calculation + * This function is called over tupdate ms and takes pointer of FQ-PIE + * flow as an argument + */ +static void +fq_calculate_drop_prob(void *x) +{ + struct fq_pie_flow *q = (struct fq_pie_flow *) x; + struct pie_status *pst = &q->pst; + struct dn_aqm_pie_parms *pprms; + int64_t p, prob, oldprob; + aqm_time_t now; + + /* dealing with race condition */ + if (callout_pending(&pst->aqm_pie_callout)) { + /* callout was reset */ + mtx_unlock(&pst->lock_mtx); + return; + } + + if (!callout_active(&pst->aqm_pie_callout)) { + /* callout was stopped */ + mtx_unlock(&pst->lock_mtx); + mtx_destroy(&pst->lock_mtx); + q->psi->nr_active_q--; + return; + } + callout_deactivate(&pst->aqm_pie_callout); + + now = AQM_UNOW; + pprms = pst->parms; + prob = pst->drop_prob; + + /* calculate current qdelay */ + if (pprms->flags & PIE_DEPRATEEST_ENABLED) { + pst->current_qdelay = ((uint64_t)q->stats.len_bytes * pst->avg_dq_time) + >> PIE_DQ_THRESHOLD_BITS; + } + + /* calculate drop probability */ + p = (int64_t)pprms->alpha * + ((int64_t)pst->current_qdelay - (int64_t)pprms->qdelay_ref); + p +=(int64_t) pprms->beta * + ((int64_t)pst->current_qdelay - (int64_t)pst->qdelay_old); + + /* We PIE_MAX_PROB shift by 12-bits to increase the division precision */ + p *= (PIE_MAX_PROB << 12) / AQM_TIME_1S; + + /* auto-tune drop probability */ + if (prob< PIE_MAX_PROB * 0.000001) + p >>= 11 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.00001) + p >>= 9 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.0001) + p >>= 7 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.001) + p >>= 5 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.01) + p >>= 3 + PIE_FIX_POINT_BITS+12; + else if (prob < PIE_MAX_PROB * 0.1) + p >>= 1 + PIE_FIX_POINT_BITS+12; + else + p >>= PIE_FIX_POINT_BITS+12; + + oldprob = prob; + + /* Cap Drop adjustment */ + if ((pprms->flags & PIE_CAPDROP_ENABLED) && prob >= PIE_MAX_PROB / 10 + && p > PIE_MAX_PROB / 50 ) + p = PIE_MAX_PROB / 50; + + prob = prob + p; + + /* decay the drop probability exponentially */ + if (pst->current_qdelay == 0 && pst->qdelay_old == 0) + /* 0.98 ~= 1- 1/64 */ + prob = prob - (prob >> 6); + + + /* check for multiplication over/under flow */ + if (p>0) { + if (proboldprob) { + prob= 0; + D("underflow"); + } + + /* make drop probability between 0 and PIE_MAX_PROB*/ + if (prob < 0) + prob = 0; + else if (prob > PIE_MAX_PROB) + prob = PIE_MAX_PROB; + + pst->drop_prob = prob; + + /* store current delay value */ + pst->qdelay_old = pst->current_qdelay; + + /* update burst allowance */ + if ((pst->sflags & PIE_ACTIVE) && pst->burst_allowance) { + if (pst->burst_allowance > pprms->tupdate) + pst->burst_allowance -= pprms->tupdate; + else + pst->burst_allowance = 0; + } + + if (pst->sflags & PIE_ACTIVE) + callout_reset_sbt(&pst->aqm_pie_callout, + (uint64_t)pprms->tupdate * SBT_1US, + 0, fq_calculate_drop_prob, q, 0); + + mtx_unlock(&pst->lock_mtx); +} + +/* + * Reset PIE variables & activate the queue + */ +__inline static void +fq_activate_pie(struct fq_pie_flow *q) +{ + struct pie_status *pst = &q->pst; + struct dn_aqm_pie_parms *pprms; + + mtx_lock(&pst->lock_mtx); + pprms = pst->parms; + + pprms = pst->parms; + pst->drop_prob = 0; + pst->qdelay_old = 0; + pst->burst_allowance = pprms->max_burst; + pst->accu_prob = 0; + pst->dq_count = 0; + pst->avg_dq_time = 0; + pst->sflags = PIE_INMEASUREMENT | PIE_ACTIVE; + pst->measurement_start = AQM_UNOW; + + callout_reset_sbt(&pst->aqm_pie_callout, + (uint64_t)pprms->tupdate * SBT_1US, + 0, fq_calculate_drop_prob, q, 0); + + mtx_unlock(&pst->lock_mtx); +} + + + /* + * Deactivate PIE and stop probe update callout + */ +__inline static void +fq_deactivate_pie(struct pie_status *pst) +{ + mtx_lock(&pst->lock_mtx); + pst->sflags &= ~(PIE_ACTIVE | PIE_INMEASUREMENT); + callout_stop(&pst->aqm_pie_callout); + //D("PIE Deactivated"); + mtx_unlock(&pst->lock_mtx); +} + + /* + * Initialize PIE for sub-queue 'q' + */ +static int +pie_init(struct fq_pie_flow *q) +{ + struct pie_status *pst=&q->pst; + struct dn_aqm_pie_parms *pprms = pst->parms; + struct fq_pie_schk *fqpie_schk; + + fqpie_schk = (struct fq_pie_schk *)(q->psi->_si.sched+1); + int err = 0; + + if (!pprms){ + D("AQM_PIE is not configured"); + err = EINVAL; + } else { + q->psi->nr_active_q++; + + /* For speed optimization, we caculate 1/3 queue size once here */ + // XXX limit divided by number of queues divided by 3 ??? + pst->one_third_q_size = (fqpie_schk->cfg.limit / + fqpie_schk->cfg.flows_cnt) / 3; + + mtx_init(&pst->lock_mtx, "mtx_pie", NULL, MTX_DEF); + callout_init_mtx(&pst->aqm_pie_callout, &pst->lock_mtx, + CALLOUT_RETURNUNLOCKED); + } + + return err; +} + +/* + * Clean up PIE status for sub-queue 'q' + * Stop callout timer and destroy mtx + */ +static int +pie_cleanup(struct fq_pie_flow *q) +{ + struct pie_status *pst = &q->pst; + + mtx_lock(&pst->lock_mtx); + if (callout_stop(&pst->aqm_pie_callout) || !(pst->sflags & PIE_ACTIVE)) { + mtx_unlock(&pst->lock_mtx); + mtx_destroy(&pst->lock_mtx); + q->psi->nr_active_q--; + } else { + mtx_unlock(&pst->lock_mtx); + return EBUSY; + } + return 0; +} + +/* + * Dequeue and return a pcaket from sub-queue 'q' or NULL if 'q' is empty. + * Also, caculate depature time or queue delay using timestamp + */ + static struct mbuf * +pie_dequeue(struct fq_pie_flow *q, struct fq_pie_si *si) +{ + struct mbuf *m; + struct dn_aqm_pie_parms *pprms; + struct pie_status *pst; + aqm_time_t now; + aqm_time_t pkt_ts, dq_time; + int32_t w; + + pst = &q->pst; + pprms = q->pst.parms; + + /*we extarct packet ts only when Departure Rate Estimation dis not used*/ + m = fq_pie_extract_head(q, &pkt_ts, si, + !(pprms->flags & PIE_DEPRATEEST_ENABLED)); + + if (!m || !(pst->sflags & PIE_ACTIVE)) + return m; + + now = AQM_UNOW; + if (pprms->flags & PIE_DEPRATEEST_ENABLED) { + /* calculate average depature time */ + if(pst->sflags & PIE_INMEASUREMENT) { + pst->dq_count += m->m_pkthdr.len; + + if (pst->dq_count >= PIE_DQ_THRESHOLD) { + dq_time = now - pst->measurement_start; + + /* + * if we don't have old avg dq_time i.e PIE is (re)initialized, + * don't use weight to calculate new avg_dq_time + */ + if(pst->avg_dq_time == 0) + pst->avg_dq_time = dq_time; + else { + /* + * weight = PIE_DQ_THRESHOLD/2^6, but we scaled + * weight by 2^8. Thus, scaled + * weight = PIE_DQ_THRESHOLD /2^8 + * */ + w = PIE_DQ_THRESHOLD >> 8; + pst->avg_dq_time = (dq_time* w + + (pst->avg_dq_time * ((1L << 8) - w))) >> 8; + pst->sflags &= ~PIE_INMEASUREMENT; + } + } + } + + /* + * Start new measurment cycle when the queue has + * PIE_DQ_THRESHOLD worth of bytes. + */ + if(!(pst->sflags & PIE_INMEASUREMENT) && + q->stats.len_bytes >= PIE_DQ_THRESHOLD) { + pst->sflags |= PIE_INMEASUREMENT; + pst->measurement_start = now; + pst->dq_count = 0; + } + } + /* Optionally, use packet timestamp to estimate queue delay */ + else + pst->current_qdelay = now - pkt_ts; + + return m; +} + + + /* + * Enqueue a packet in q, subject to space and FQ-PIE queue management policy + * (whose parameters are in q->fs). + * Update stats for the queue and the scheduler. + * Return 0 on success, 1 on drop. The packet is consumed anyways. + */ +static int +pie_enqueue(struct fq_pie_flow *q, struct mbuf* m, struct fq_pie_si *si) +{ + uint64_t len; + struct pie_status *pst; + struct dn_aqm_pie_parms *pprms; + int t; + + len = m->m_pkthdr.len; + pst = &q->pst; + pprms = pst->parms; + t = ENQUE; + + /* drop/mark the packet when PIE is active and burst time elapsed */ + if (pst->sflags & PIE_ACTIVE && pst->burst_allowance == 0 + && drop_early(pst, q->stats.len_bytes) == DROP) { + /* + * if drop_prob over ECN threshold, drop the packet + * otherwise mark and enqueue it. + */ + if (pprms->flags & PIE_ECN_ENABLED && pst->drop_prob < + (pprms->max_ecnth << (PIE_PROB_BITS - PIE_FIX_POINT_BITS)) + && ecn_mark(m)) + t = ENQUE; + else + t = DROP; + } + + /* Turn PIE on when 1/3 of the queue is full */ + if (!(pst->sflags & PIE_ACTIVE) && q->stats.len_bytes >= + pst->one_third_q_size) { + fq_activate_pie(q); + } + + /* reset burst tolerance and optinally turn PIE off*/ + if (pst->drop_prob == 0 && pst->current_qdelay < (pprms->qdelay_ref >> 1) + && pst->qdelay_old < (pprms->qdelay_ref >> 1)) { + + pst->burst_allowance = pprms->max_burst; + if (pprms->flags & PIE_ON_OFF_MODE_ENABLED && q->stats.len_bytes<=0) + fq_deactivate_pie(pst); + } + + /* Use timestamp if Departure Rate Estimation mode is disabled */ + if (t != DROP && !(pprms->flags & PIE_DEPRATEEST_ENABLED)) { + /* Add TS to mbuf as a TAG */ + struct m_tag *mtag; + mtag = m_tag_locate(m, MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, NULL); + if (mtag == NULL) + mtag = m_tag_alloc(MTAG_ABI_COMPAT, DN_AQM_MTAG_TS, + sizeof(aqm_time_t), M_NOWAIT); + if (mtag == NULL) { + m_freem(m); + t = DROP; + } + *(aqm_time_t *)(mtag + 1) = AQM_UNOW; + m_tag_prepend(m, mtag); + } + + if (t != DROP) { + mq_append(&q->mq, m); + fq_update_stats(q, si, len, 0); + return 0; + } else { + fq_update_stats(q, si, len, 1); + pst->accu_prob = 0; + FREE_PKT(m); + return 1; + } + + return 0; +} + +/* Drop a packet form the head of FQ-PIE sub-queue */ +static void +pie_drop_head(struct fq_pie_flow *q, struct fq_pie_si *si) +{ + struct mbuf *m = q->mq.head; + + if (m == NULL) + return; + q->mq.head = m->m_nextpkt; + + fq_update_stats(q, si, -m->m_pkthdr.len, 1); + + if (si->main_q.ni.length == 0) /* queue is now idle */ + si->main_q.q_time = dn_cfg.curr_time; + /* reset accu_prob after packet drop */ + q->pst.accu_prob = 0; + + FREE_PKT(m); +} + +/* + * Classify a packet to queue number using Jenkins hash function. + * Return: queue number + * the input of the hash are protocol no, perturbation, src IP, dst IP, + * src port, dst port, + */ +static inline int +fq_pie_classify_flow(struct mbuf *m, uint16_t fcount, struct fq_pie_si *si) +{ + struct ip *ip; + struct tcphdr *th; + struct udphdr *uh; + uint8_t tuple[41]; + uint16_t hash=0; + +//#ifdef INET6 + struct ip6_hdr *ip6; + int isip6; + isip6 = (mtod(m, struct ip *)->ip_v == 6) ? 1 : 0; + + if(isip6) { + ip6 = mtod(m, struct ip6_hdr *); + *((uint8_t *) &tuple[0]) = ip6->ip6_nxt; + *((uint32_t *) &tuple[1]) = si->perturbation; + memcpy(&tuple[5], ip6->ip6_src.s6_addr, 16); + memcpy(&tuple[21], ip6->ip6_dst.s6_addr, 16); + + switch (ip6->ip6_nxt) { + case IPPROTO_TCP: + th = (struct tcphdr *)(ip6 + 1); + *((uint16_t *) &tuple[37]) = th->th_dport; + *((uint16_t *) &tuple[39]) = th->th_sport; + break; + + case IPPROTO_UDP: + uh = (struct udphdr *)(ip6 + 1); + *((uint16_t *) &tuple[37]) = uh->uh_dport; + *((uint16_t *) &tuple[39]) = uh->uh_sport; + break; + default: + memset(&tuple[37], 0, 4); + } + + hash = jenkins_hash(tuple, 41, HASHINIT) % fcount; + return hash; + } +//#endif + + /* IPv4 */ + ip = mtod(m, struct ip *); + *((uint8_t *) &tuple[0]) = ip->ip_p; + *((uint32_t *) &tuple[1]) = si->perturbation; + *((uint32_t *) &tuple[5]) = ip->ip_src.s_addr; + *((uint32_t *) &tuple[9]) = ip->ip_dst.s_addr; + + switch (ip->ip_p) { + case IPPROTO_TCP: + th = (struct tcphdr *)(ip + 1); + *((uint16_t *) &tuple[13]) = th->th_dport; + *((uint16_t *) &tuple[15]) = th->th_sport; + break; + + case IPPROTO_UDP: + uh = (struct udphdr *)(ip + 1); + *((uint16_t *) &tuple[13]) = uh->uh_dport; + *((uint16_t *) &tuple[15]) = uh->uh_sport; + break; + default: + memset(&tuple[13], 0, 4); + } + hash = jenkins_hash(tuple, 17, HASHINIT) % fcount; + + return hash; +} + +/* + * Enqueue a packet into an appropriate queue according to + * FQ-CoDe; algorithm. + */ +static int +fq_pie_enqueue(struct dn_sch_inst *_si, struct dn_queue *_q, + struct mbuf *m) +{ + struct fq_pie_si *si; + struct fq_pie_schk *schk; + struct dn_sch_fq_pie_parms *param; + struct dn_queue *mainq; + int idx, drop, i, maxidx; + + mainq = (struct dn_queue *)(_si + 1); + si = (struct fq_pie_si *)_si; + schk = (struct fq_pie_schk *)(si->_si.sched+1); + param = &schk->cfg; + + /* classify a packet to queue number*/ + idx = fq_pie_classify_flow(m, param->flows_cnt, si); + + /* enqueue packet into appropriate queue using PIE AQM. + * Note: 'pie_enqueue' function returns 1 only when it unable to + * add timestamp to packet (no limit check)*/ + drop = pie_enqueue(&si->flows[idx], m, si); + + /* pie unable to timestamp a packet */ + if (drop) + return 1; + + /* If the flow (sub-queue) is not active ,then add it to tail of + * new flows list, initialize and activate it. + */ + if (!si->flows[idx].active) { + STAILQ_INSERT_TAIL(&si->newflows, &si->flows[idx], flowchain); + si->flows[idx].deficit = param->quantum; + fq_activate_pie(&si->flows[idx]); + si->flows[idx].active = 1; + } + + /* check the limit for all queues and remove a packet from the + * largest one + */ + if (mainq->ni.length > schk->cfg.limit) { + /* find first active flow */ + for (maxidx = 0; maxidx < schk->cfg.flows_cnt; maxidx++) + if (si->flows[maxidx].active) + break; + if (maxidx < schk->cfg.flows_cnt) { + /* find the largest sub- queue */ + for (i = maxidx + 1; i < schk->cfg.flows_cnt; i++) + if (si->flows[i].active && si->flows[i].stats.length > + si->flows[maxidx].stats.length) + maxidx = i; + pie_drop_head(&si->flows[maxidx], si); + drop = 1; + } + } + + return drop; +} + +/* + * Dequeue a packet from an appropriate queue according to + * FQ-CoDel algorithm. + */ +static struct mbuf * +fq_pie_dequeue(struct dn_sch_inst *_si) +{ + struct fq_pie_si *si; + struct fq_pie_schk *schk; + struct dn_sch_fq_pie_parms *param; + struct fq_pie_flow *f; + struct mbuf *mbuf; + struct fq_pie_list *fq_pie_flowlist; + + si = (struct fq_pie_si *)_si; + schk = (struct fq_pie_schk *)(si->_si.sched+1); + param = &schk->cfg; + + do { + /* select a list to start with */ + if (STAILQ_EMPTY(&si->newflows)) + fq_pie_flowlist = &si->oldflows; + else + fq_pie_flowlist = &si->newflows; + + /* Both new and old queue lists are empty, return NULL */ + if (STAILQ_EMPTY(fq_pie_flowlist)) + return NULL; + + f = STAILQ_FIRST(fq_pie_flowlist); + while (f != NULL) { + /* if there is no flow(sub-queue) deficit, increase deficit + * by quantum, move the flow to the tail of old flows list + * and try another flow. + * Otherwise, the flow will be used for dequeue. + */ + if (f->deficit < 0) { + f->deficit += param->quantum; + STAILQ_REMOVE_HEAD(fq_pie_flowlist, flowchain); + STAILQ_INSERT_TAIL(&si->oldflows, f, flowchain); + } else + break; + + f = STAILQ_FIRST(fq_pie_flowlist); + } + + /* the new flows list is empty, try old flows list */ + if (STAILQ_EMPTY(fq_pie_flowlist)) + continue; + + /* Dequeue a packet from the selected flow */ + mbuf = pie_dequeue(f, si); + + /* pie did not return a packet */ + if (!mbuf) { + /* If the selected flow belongs to new flows list, then move + * it to the tail of old flows list. Otherwise, deactivate it and + * remove it from the old list and + */ + if (fq_pie_flowlist == &si->newflows) { + STAILQ_REMOVE_HEAD(fq_pie_flowlist, flowchain); + STAILQ_INSERT_TAIL(&si->oldflows, f, flowchain); + } else { + f->active = 0; + fq_deactivate_pie(&f->pst); + STAILQ_REMOVE_HEAD(fq_pie_flowlist, flowchain); + } + /* start again */ + continue; + } + + /* we have a packet to return, + * update flow deficit and return the packet*/ + f->deficit -= mbuf->m_pkthdr.len; + return mbuf; + + } while (1); + + /* unreachable point */ + return NULL; +} + +/* + * Initialize fq_pie scheduler instance. + * also, allocate memory for flows array. + */ +static int +fq_pie_new_sched(struct dn_sch_inst *_si) +{ + struct fq_pie_si *si; + struct dn_queue *q; + struct fq_pie_schk *schk; + int i; + + si = (struct fq_pie_si *)_si; + schk = (struct fq_pie_schk *)(_si->sched+1); + + if(si->flows) { + D("si already configured!"); + return 0; + } + + /* init the main queue */ + q = &si->main_q; + set_oid(&q->ni.oid, DN_QUEUE, sizeof(*q)); + q->_si = _si; + q->fs = _si->sched->fs; + + /* allocate memory for flows array */ + si->flows = malloc(schk->cfg.flows_cnt * sizeof(struct fq_pie_flow), + M_DUMMYNET, M_NOWAIT | M_ZERO); + if (si->flows == NULL) { + D("cannot allocate memory for fq_pie configuration parameters"); + return ENOMEM ; + } + + /* init perturbation for this si */ + si->perturbation = random(); + si->nr_active_q = 0; + + /* init the old and new flows lists */ + STAILQ_INIT(&si->newflows); + STAILQ_INIT(&si->oldflows); + + /* init the flows (sub-queues) */ + for (i = 0; i < schk->cfg.flows_cnt; i++) { + si->flows[i].pst.parms = &schk->cfg.pcfg; + si->flows[i].psi = si; + pie_init(&si->flows[i]); + } + + /* init mtx lock and callout function for free memory */ + if (!fq_pie_desc.ref_count) { + mtx_init(&freemem_mtx, "mtx_pie", NULL, MTX_DEF); + } + + mtx_lock(&freemem_mtx); + fq_pie_desc.ref_count++; + mtx_unlock(&freemem_mtx); + + return 0; +} + +/* + * Free FQ-PIE flows memory callout function. + * This function is scheduled when a flow or more still active and + * the scheduer is about to be destroyed, to prevent memory leak. + */ +static void +free_flows(void *_mem) +{ + struct mem_to_free *mem = _mem; + + free(mem->mem_flows, M_DUMMYNET); + free(mem->mem_callout, M_DUMMYNET); + free(_mem, M_DUMMYNET); + + fq_pie_desc.ref_count--; + if (!fq_pie_desc.ref_count) { + mtx_unlock(&freemem_mtx); + mtx_destroy(&freemem_mtx); + } else + mtx_unlock(&freemem_mtx); + //D("mem freed ok!"); +} + +/* + * Free fq_pie scheduler instance. + */ +static int +fq_pie_free_sched(struct dn_sch_inst *_si) +{ + struct fq_pie_si *si; + struct fq_pie_schk *schk; + int i; + + si = (struct fq_pie_si *)_si; + schk = (struct fq_pie_schk *)(_si->sched+1); + + for (i = 0; i < schk->cfg.flows_cnt; i++) { + pie_cleanup(&si->flows[i]); + } + + /* if there are still some queues have a callout going to start, + * we cannot free flows memory. If we do so, a panic can happen + * as prob calculate callout function uses flows memory. + */ + if (!si->nr_active_q) { + /* free the flows array */ + free(si->flows , M_DUMMYNET); + si->flows = NULL; + mtx_lock(&freemem_mtx); + fq_pie_desc.ref_count--; + if (!fq_pie_desc.ref_count) { + mtx_unlock(&freemem_mtx); + mtx_destroy(&freemem_mtx); + } else + mtx_unlock(&freemem_mtx); + //D("ok!"); + return 0; + } else { + /* memory leak happens here. So, we register a callout function to free + * flows memory later. + */ + D("unable to stop all fq_pie sub-queues!"); + mtx_lock(&freemem_mtx); + + struct callout *mem_callout; + struct mem_to_free *mem; + + mem = malloc(sizeof(*mem), M_DUMMYNET, + M_NOWAIT | M_ZERO); + mem_callout = malloc(sizeof(*mem_callout), M_DUMMYNET, + M_NOWAIT | M_ZERO); + + callout_init_mtx(mem_callout, &freemem_mtx, + CALLOUT_RETURNUNLOCKED); + + mem->mem_flows = si->flows; + mem->mem_callout = mem_callout; + callout_reset_sbt(mem_callout, + (uint64_t)(si->flows[0].pst.parms->tupdate + 1000) * SBT_1US, + 0, free_flows, mem, 0); + + si->flows = NULL; + mtx_unlock(&freemem_mtx); + + return EBUSY; + } +} + +/* + * Configure FQ-PIE scheduler. + * the configurations for the scheduler is passed fromipfw userland. + */ +static int +fq_pie_config(struct dn_schk *_schk) +{ + struct fq_pie_schk *schk; + struct dn_extra_parms *ep; + struct dn_sch_fq_pie_parms *fqp_cfg; + + schk = (struct fq_pie_schk *)(_schk+1); + ep = (struct dn_extra_parms *) _schk->cfg; + + /* par array contains fq_pie configuration as follow + * PIE: 0- qdelay_ref,1- tupdate, 2- max_burst + * 3- max_ecnth, 4- alpha, 5- beta, 6- flags + * FQ_PIE: 7- quantum, 8- limit, 9- flows + */ + if (ep && ep->oid.len ==sizeof(*ep) && + ep->oid.subtype == DN_SCH_PARAMS) { + + fqp_cfg = &schk->cfg; + if (ep->par[0] < 0) + fqp_cfg->pcfg.qdelay_ref = fq_pie_sysctl.pcfg.qdelay_ref; + else + fqp_cfg->pcfg.qdelay_ref = ep->par[0]; + if (ep->par[1] < 0) + fqp_cfg->pcfg.tupdate = fq_pie_sysctl.pcfg.tupdate; + else + fqp_cfg->pcfg.tupdate = ep->par[1]; + if (ep->par[2] < 0) + fqp_cfg->pcfg.max_burst = fq_pie_sysctl.pcfg.max_burst; + else + fqp_cfg->pcfg.max_burst = ep->par[2]; + if (ep->par[3] < 0) + fqp_cfg->pcfg.max_ecnth = fq_pie_sysctl.pcfg.max_ecnth; + else + fqp_cfg->pcfg.max_ecnth = ep->par[3]; + if (ep->par[4] < 0) + fqp_cfg->pcfg.alpha = fq_pie_sysctl.pcfg.alpha; + else + fqp_cfg->pcfg.alpha = ep->par[4]; + if (ep->par[5] < 0) + fqp_cfg->pcfg.beta = fq_pie_sysctl.pcfg.beta; + else + fqp_cfg->pcfg.beta = ep->par[5]; + if (ep->par[6] < 0) + fqp_cfg->pcfg.flags = 0; + else + fqp_cfg->pcfg.flags = ep->par[6]; + + /* FQ configurations */ + if (ep->par[7] < 0) + fqp_cfg->quantum = fq_pie_sysctl.quantum; + else + fqp_cfg->quantum = ep->par[7]; + if (ep->par[8] < 0) + fqp_cfg->limit = fq_pie_sysctl.limit; + else + fqp_cfg->limit = ep->par[8]; + if (ep->par[9] < 0) + fqp_cfg->flows_cnt = fq_pie_sysctl.flows_cnt; + else + fqp_cfg->flows_cnt = ep->par[9]; + + /* Bound the configurations */ + fqp_cfg->pcfg.qdelay_ref = BOUND_VAR(fqp_cfg->pcfg.qdelay_ref, + 1, 5 * AQM_TIME_1S); + fqp_cfg->pcfg.tupdate = BOUND_VAR(fqp_cfg->pcfg.tupdate, + 1, 5 * AQM_TIME_1S); + fqp_cfg->pcfg.max_burst = BOUND_VAR(fqp_cfg->pcfg.max_burst, + 0, 5 * AQM_TIME_1S); + fqp_cfg->pcfg.max_ecnth = BOUND_VAR(fqp_cfg->pcfg.max_ecnth, + 0, PIE_SCALE); + fqp_cfg->pcfg.alpha = BOUND_VAR(fqp_cfg->pcfg.alpha, 0, 7 * PIE_SCALE); + fqp_cfg->pcfg.beta = BOUND_VAR(fqp_cfg->pcfg.beta, 0, 7 * PIE_SCALE); + + fqp_cfg->quantum = BOUND_VAR(fqp_cfg->quantum,1,9000); + fqp_cfg->limit= BOUND_VAR(fqp_cfg->limit,1,20480); + fqp_cfg->flows_cnt= BOUND_VAR(fqp_cfg->flows_cnt,1,65536); + } + else { + D("Wrong parameters for fq_pie scheduler"); + return 1; + } + + return 0; +} + +/* + * Return FQ-PIE scheduler configurations + * the configurations for the scheduler is passed to userland. + */ +static int +fq_pie_getconfig (struct dn_schk *_schk, struct dn_extra_parms *ep) { + + struct fq_pie_schk *schk = (struct fq_pie_schk *)(_schk+1); + struct dn_sch_fq_pie_parms *fqp_cfg; + + fqp_cfg = &schk->cfg; + + strcpy(ep->name, fq_pie_desc.name); + ep->par[0] = fqp_cfg->pcfg.qdelay_ref; + ep->par[1] = fqp_cfg->pcfg.tupdate; + ep->par[2] = fqp_cfg->pcfg.max_burst; + ep->par[3] = fqp_cfg->pcfg.max_ecnth; + ep->par[4] = fqp_cfg->pcfg.alpha; + ep->par[5] = fqp_cfg->pcfg.beta; + ep->par[6] = fqp_cfg->pcfg.flags; + + ep->par[7] = fqp_cfg->quantum; + ep->par[8] = fqp_cfg->limit; + ep->par[9] = fqp_cfg->flows_cnt; + + return 0; +} + +/* + * FQ-PIE scheduler descriptor + * contains the type of the scheduler, the name, the size of extra + * data structures, and function pointers. + */ +static struct dn_alg fq_pie_desc = { + _SI( .type = ) DN_SCHED_FQ_PIE, + _SI( .name = ) "FQ_PIE", + _SI( .flags = ) 0, + + _SI( .schk_datalen = ) sizeof(struct fq_pie_schk), + _SI( .si_datalen = ) sizeof(struct fq_pie_si) - sizeof(struct dn_sch_inst), + _SI( .q_datalen = ) 0, + + _SI( .enqueue = ) fq_pie_enqueue, + _SI( .dequeue = ) fq_pie_dequeue, + _SI( .config = ) fq_pie_config, /* new sched i.e. sched X config ...*/ + _SI( .destroy = ) NULL, /*sched x delete */ + _SI( .new_sched = ) fq_pie_new_sched, /* new schd instance */ + _SI( .free_sched = ) fq_pie_free_sched, /* delete schd instance */ + _SI( .new_fsk = ) NULL, + _SI( .free_fsk = ) NULL, + _SI( .new_queue = ) NULL, + _SI( .free_queue = ) NULL, + _SI( .getconfig = ) fq_pie_getconfig, + _SI( .ref_count = ) 0 +}; + +DECLARE_DNSCHED_MODULE(dn_fq_pie, &fq_pie_desc); Property changes on: head/sys/netpfil/ipfw/dn_sched_fq_pie.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/netpfil/ipfw/dn_sched_prio.c =================================================================== --- head/sys/netpfil/ipfw/dn_sched_prio.c (revision 300778) +++ head/sys/netpfil/ipfw/dn_sched_prio.c (revision 300779) @@ -1,229 +1,235 @@ /* * Copyright (c) 2010 Riccardo Panicucci, Universita` di Pisa * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ */ #ifdef _KERNEL #include #include #include #include #include #include #include /* IFNAMSIZ */ #include #include /* ipfw_rule_ref */ #include /* flow_id */ #include #include #include +#ifdef NEW_AQM +#include +#endif #include #else #include #endif #define DN_SCHED_PRIO 5 //XXX #if !defined(_KERNEL) || !defined(__linux__) #define test_bit(ix, pData) ((*pData) & (1<<(ix))) #define __set_bit(ix, pData) (*pData) |= (1<<(ix)) #define __clear_bit(ix, pData) (*pData) &= ~(1<<(ix)) #endif #ifdef __MIPSEL__ #define __clear_bit(ix, pData) (*pData) &= ~(1<<(ix)) #endif /* Size of the array of queues pointers. */ #define BITMAP_T unsigned long #define MAXPRIO (sizeof(BITMAP_T) * 8) /* * The scheduler instance contains an array of pointers to queues, * one for each priority, and a bitmap listing backlogged queues. */ struct prio_si { BITMAP_T bitmap; /* array bitmap */ struct dn_queue *q_array[MAXPRIO]; /* Array of queues pointers */ }; /* * If a queue with the same priority is already backlogged, use * that one instead of the queue passed as argument. */ static int prio_enqueue(struct dn_sch_inst *_si, struct dn_queue *q, struct mbuf *m) { struct prio_si *si = (struct prio_si *)(_si + 1); int prio = q->fs->fs.par[0]; if (test_bit(prio, &si->bitmap) == 0) { /* No queue with this priority, insert */ __set_bit(prio, &si->bitmap); si->q_array[prio] = q; } else { /* use the existing queue */ q = si->q_array[prio]; } if (dn_enqueue(q, m, 0)) return 1; return 0; } /* * Packets are dequeued only from the highest priority queue. * The function ffs() return the lowest bit in the bitmap that rapresent * the array index (-1) which contains the pointer to the highest priority * queue. * After the dequeue, if this queue become empty, it is index is removed * from the bitmap. * Scheduler is idle if the bitmap is empty * * NOTE: highest priority is 0, lowest is sched->max_prio_q */ static struct mbuf * prio_dequeue(struct dn_sch_inst *_si) { struct prio_si *si = (struct prio_si *)(_si + 1); struct mbuf *m; struct dn_queue *q; int prio; if (si->bitmap == 0) /* scheduler idle */ return NULL; prio = ffs(si->bitmap) - 1; /* Take the highest priority queue in the scheduler */ q = si->q_array[prio]; // assert(q) m = dn_dequeue(q); if (q->mq.head == NULL) { /* Queue is now empty, remove from scheduler * and mark it */ si->q_array[prio] = NULL; __clear_bit(prio, &si->bitmap); } return m; } static int prio_new_sched(struct dn_sch_inst *_si) { struct prio_si *si = (struct prio_si *)(_si + 1); bzero(si->q_array, sizeof(si->q_array)); si->bitmap = 0; return 0; } static int prio_new_fsk(struct dn_fsk *fs) { /* Check if the prioritiy is between 0 and MAXPRIO-1 */ ipdn_bound_var(&fs->fs.par[0], 0, 0, MAXPRIO - 1, "PRIO priority"); return 0; } static int prio_new_queue(struct dn_queue *q) { struct prio_si *si = (struct prio_si *)(q->_si + 1); int prio = q->fs->fs.par[0]; struct dn_queue *oldq; q->ni.oid.subtype = DN_SCHED_PRIO; if (q->mq.head == NULL) return 0; /* Queue already full, must insert in the scheduler or append * mbufs to existing queue. This partly duplicates prio_enqueue */ if (test_bit(prio, &si->bitmap) == 0) { /* No queue with this priority, insert */ __set_bit(prio, &si->bitmap); si->q_array[prio] = q; } else if ( (oldq = si->q_array[prio]) != q) { /* must append to the existing queue. * can simply append q->mq.head to q2->... * and add the counters to those of q2 */ oldq->mq.tail->m_nextpkt = q->mq.head; oldq->mq.tail = q->mq.tail; oldq->ni.length += q->ni.length; q->ni.length = 0; oldq->ni.len_bytes += q->ni.len_bytes; q->ni.len_bytes = 0; q->mq.tail = q->mq.head = NULL; } return 0; } static int prio_free_queue(struct dn_queue *q) { int prio = q->fs->fs.par[0]; struct prio_si *si = (struct prio_si *)(q->_si + 1); if (si->q_array[prio] == q) { si->q_array[prio] = NULL; __clear_bit(prio, &si->bitmap); } return 0; } static struct dn_alg prio_desc = { _SI( .type = ) DN_SCHED_PRIO, _SI( .name = ) "PRIO", _SI( .flags = ) DN_MULTIQUEUE, /* we need extra space in the si and the queue */ _SI( .schk_datalen = ) 0, _SI( .si_datalen = ) sizeof(struct prio_si), _SI( .q_datalen = ) 0, _SI( .enqueue = ) prio_enqueue, _SI( .dequeue = ) prio_dequeue, _SI( .config = ) NULL, _SI( .destroy = ) NULL, _SI( .new_sched = ) prio_new_sched, _SI( .free_sched = ) NULL, _SI( .new_fsk = ) prio_new_fsk, _SI( .free_fsk = ) NULL, _SI( .new_queue = ) prio_new_queue, _SI( .free_queue = ) prio_free_queue, +#ifdef NEW_AQM + _SI( .getconfig = ) NULL, +#endif }; DECLARE_DNSCHED_MODULE(dn_prio, &prio_desc); Index: head/sys/netpfil/ipfw/dn_sched_qfq.c =================================================================== --- head/sys/netpfil/ipfw/dn_sched_qfq.c (revision 300778) +++ head/sys/netpfil/ipfw/dn_sched_qfq.c (revision 300779) @@ -1,877 +1,883 @@ /* * Copyright (c) 2010 Fabio Checconi, Luigi Rizzo, Paolo Valente * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ */ #ifdef _KERNEL #include #include #include #include #include #include #include /* IFNAMSIZ */ #include #include /* ipfw_rule_ref */ #include /* flow_id */ #include #include #include +#ifdef NEW_AQM +#include +#endif #include #else #include #endif #ifdef QFQ_DEBUG #define _P64 unsigned long long /* cast for printing uint64_t */ struct qfq_sched; static void dump_sched(struct qfq_sched *q, const char *msg); #define NO(x) x #else #define NO(x) #endif #define DN_SCHED_QFQ 4 // XXX Where? typedef unsigned long bitmap; /* * bitmaps ops are critical. Some linux versions have __fls * and the bitmap ops. Some machines have ffs * NOTE: fls() returns 1 for the least significant bit, * __fls() returns 0 for the same case. * We use the base-0 version __fls() to match the description in * the ToN QFQ paper */ #if defined(_WIN32) || (defined(__MIPSEL__) && defined(LINUX_24)) int fls(unsigned int n) { int i = 0; for (i = 0; n > 0; n >>= 1, i++) ; return i; } #endif #if !defined(_KERNEL) || defined( __FreeBSD__ ) || defined(_WIN32) || (defined(__MIPSEL__) && defined(LINUX_24)) static inline unsigned long __fls(unsigned long word) { return fls(word) - 1; } #endif #if !defined(_KERNEL) || !defined(__linux__) #ifdef QFQ_DEBUG static int test_bit(int ix, bitmap *p) { if (ix < 0 || ix > 31) D("bad index %d", ix); return *p & (1< 31) D("bad index %d", ix); *p |= (1< 31) D("bad index %d", ix); *p &= ~(1<index = 0 *.__grp->slot_shift where MIN_SLOT_SHIFT is derived by difference from the others. The max group index corresponds to Lmax/w_min, where Lmax=1<group mapping. Class weights are * in the range [1, QFQ_MAX_WEIGHT], we to map each class i to the * group with the smallest index that can support the L_i / r_i * configured for the class. * * grp->index is the index of the group; and grp->slot_shift * is the shift for the corresponding (scaled) sigma_i. * * When computing the group index, we do (len< 0; } /* Round a precise timestamp to its slotted value. */ static inline uint64_t qfq_round_down(uint64_t ts, unsigned int shift) { return ts & ~((1ULL << shift) - 1); } /* return the pointer to the group with lowest index in the bitmap */ static inline struct qfq_group *qfq_ffs(struct qfq_sched *q, unsigned long bitmap) { int index = ffs(bitmap) - 1; // zero-based return &q->groups[index]; } /* * Calculate a flow index, given its weight and maximum packet length. * index = log_2(maxlen/weight) but we need to apply the scaling. * This is used only once at flow creation. */ static int qfq_calc_index(uint32_t inv_w, unsigned int maxlen) { uint64_t slot_size = (uint64_t)maxlen *inv_w; unsigned long size_map; int index = 0; size_map = (unsigned long)(slot_size >> QFQ_MIN_SLOT_SHIFT); if (!size_map) goto out; index = __fls(size_map) + 1; // basically a log_2() index -= !(slot_size - (1ULL << (index + QFQ_MIN_SLOT_SHIFT - 1))); if (index < 0) index = 0; out: ND("W = %d, L = %d, I = %d\n", ONE_FP/inv_w, maxlen, index); return index; } /*---- end support functions ----*/ /*-------- API calls --------------------------------*/ /* * Validate and copy parameters from flowset. */ static int qfq_new_queue(struct dn_queue *_q) { struct qfq_sched *q = (struct qfq_sched *)(_q->_si + 1); struct qfq_class *cl = (struct qfq_class *)_q; int i; uint32_t w; /* approximated weight */ /* import parameters from the flowset. They should be correct * already. */ w = _q->fs->fs.par[0]; cl->lmax = _q->fs->fs.par[1]; if (!w || w > QFQ_MAX_WEIGHT) { w = 1; D("rounding weight to 1"); } cl->inv_w = ONE_FP/w; w = ONE_FP/cl->inv_w; if (q->wsum + w > QFQ_MAX_WSUM) return EINVAL; i = qfq_calc_index(cl->inv_w, cl->lmax); cl->grp = &q->groups[i]; q->wsum += w; q->iwsum = ONE_FP / q->wsum; /* XXX note theory */ // XXX cl->S = q->V; ? return 0; } /* remove an empty queue */ static int qfq_free_queue(struct dn_queue *_q) { struct qfq_sched *q = (struct qfq_sched *)(_q->_si + 1); struct qfq_class *cl = (struct qfq_class *)_q; if (cl->inv_w) { q->wsum -= ONE_FP/cl->inv_w; if (q->wsum != 0) q->iwsum = ONE_FP / q->wsum; cl->inv_w = 0; /* reset weight to avoid run twice */ } return 0; } /* Calculate a mask to mimic what would be ffs_from(). */ static inline unsigned long mask_from(unsigned long bitmap, int from) { return bitmap & ~((1UL << from) - 1); } /* * The state computation relies on ER=0, IR=1, EB=2, IB=3 * First compute eligibility comparing grp->S, q->V, * then check if someone is blocking us and possibly add EB */ static inline unsigned int qfq_calc_state(struct qfq_sched *q, struct qfq_group *grp) { /* if S > V we are not eligible */ unsigned int state = qfq_gt(grp->S, q->V); unsigned long mask = mask_from(q->bitmaps[ER], grp->index); struct qfq_group *next; if (mask) { next = qfq_ffs(q, mask); if (qfq_gt(grp->F, next->F)) state |= EB; } return state; } /* * In principle * q->bitmaps[dst] |= q->bitmaps[src] & mask; * q->bitmaps[src] &= ~mask; * but we should make sure that src != dst */ static inline void qfq_move_groups(struct qfq_sched *q, unsigned long mask, int src, int dst) { q->bitmaps[dst] |= q->bitmaps[src] & mask; q->bitmaps[src] &= ~mask; } static inline void qfq_unblock_groups(struct qfq_sched *q, int index, uint64_t old_finish) { unsigned long mask = mask_from(q->bitmaps[ER], index + 1); struct qfq_group *next; if (mask) { next = qfq_ffs(q, mask); if (!qfq_gt(next->F, old_finish)) return; } mask = (1UL << index) - 1; qfq_move_groups(q, mask, EB, ER); qfq_move_groups(q, mask, IB, IR); } /* * perhaps * old_V ^= q->V; old_V >>= QFQ_MIN_SLOT_SHIFT; if (old_V) { ... } * */ static inline void qfq_make_eligible(struct qfq_sched *q, uint64_t old_V) { unsigned long mask, vslot, old_vslot; vslot = q->V >> QFQ_MIN_SLOT_SHIFT; old_vslot = old_V >> QFQ_MIN_SLOT_SHIFT; if (vslot != old_vslot) { /* must be 2ULL, see ToN QFQ article fig.5, we use base-0 fls */ mask = (2ULL << (__fls(vslot ^ old_vslot))) - 1; qfq_move_groups(q, mask, IR, ER); qfq_move_groups(q, mask, IB, EB); } } /* * XXX we should make sure that slot becomes less than 32. * This is guaranteed by the input values. * roundedS is always cl->S rounded on grp->slot_shift bits. */ static inline void qfq_slot_insert(struct qfq_group *grp, struct qfq_class *cl, uint64_t roundedS) { uint64_t slot = (roundedS - grp->S) >> grp->slot_shift; unsigned int i = (grp->front + slot) % QFQ_MAX_SLOTS; cl->next = grp->slots[i]; grp->slots[i] = cl; __set_bit(slot, &grp->full_slots); } /* * remove the entry from the slot */ static inline void qfq_front_slot_remove(struct qfq_group *grp) { struct qfq_class **h = &grp->slots[grp->front]; *h = (*h)->next; if (!*h) __clear_bit(0, &grp->full_slots); } /* * Returns the first full queue in a group. As a side effect, * adjust the bucket list so the first non-empty bucket is at * position 0 in full_slots. */ static inline struct qfq_class * qfq_slot_scan(struct qfq_group *grp) { int i; ND("grp %d full %x", grp->index, grp->full_slots); if (!grp->full_slots) return NULL; i = ffs(grp->full_slots) - 1; // zero-based if (i > 0) { grp->front = (grp->front + i) % QFQ_MAX_SLOTS; grp->full_slots >>= i; } return grp->slots[grp->front]; } /* * adjust the bucket list. When the start time of a group decreases, * we move the index down (modulo QFQ_MAX_SLOTS) so we don't need to * move the objects. The mask of occupied slots must be shifted * because we use ffs() to find the first non-empty slot. * This covers decreases in the group's start time, but what about * increases of the start time ? * Here too we should make sure that i is less than 32 */ static inline void qfq_slot_rotate(struct qfq_sched *q, struct qfq_group *grp, uint64_t roundedS) { unsigned int i = (grp->S - roundedS) >> grp->slot_shift; (void)q; grp->full_slots <<= i; grp->front = (grp->front - i) % QFQ_MAX_SLOTS; } static inline void qfq_update_eligible(struct qfq_sched *q, uint64_t old_V) { bitmap ineligible; ineligible = q->bitmaps[IR] | q->bitmaps[IB]; if (ineligible) { if (!q->bitmaps[ER]) { struct qfq_group *grp; grp = qfq_ffs(q, ineligible); if (qfq_gt(grp->S, q->V)) q->V = grp->S; } qfq_make_eligible(q, old_V); } } /* * Updates the class, returns true if also the group needs to be updated. */ static inline int qfq_update_class(struct qfq_sched *q, struct qfq_group *grp, struct qfq_class *cl) { (void)q; cl->S = cl->F; if (cl->_q.mq.head == NULL) { qfq_front_slot_remove(grp); } else { unsigned int len; uint64_t roundedS; len = cl->_q.mq.head->m_pkthdr.len; cl->F = cl->S + (uint64_t)len * cl->inv_w; roundedS = qfq_round_down(cl->S, grp->slot_shift); if (roundedS == grp->S) return 0; qfq_front_slot_remove(grp); qfq_slot_insert(grp, cl, roundedS); } return 1; } static struct mbuf * qfq_dequeue(struct dn_sch_inst *si) { struct qfq_sched *q = (struct qfq_sched *)(si + 1); struct qfq_group *grp; struct qfq_class *cl; struct mbuf *m; uint64_t old_V; NO(q->loops++;) if (!q->bitmaps[ER]) { NO(if (q->queued) dump_sched(q, "start dequeue");) return NULL; } grp = qfq_ffs(q, q->bitmaps[ER]); cl = grp->slots[grp->front]; /* extract from the first bucket in the bucket list */ m = dn_dequeue(&cl->_q); if (!m) { D("BUG/* non-workconserving leaf */"); return NULL; } NO(q->queued--;) old_V = q->V; q->V += (uint64_t)m->m_pkthdr.len * q->iwsum; ND("m is %p F 0x%llx V now 0x%llx", m, cl->F, q->V); if (qfq_update_class(q, grp, cl)) { uint64_t old_F = grp->F; cl = qfq_slot_scan(grp); if (!cl) { /* group gone, remove from ER */ __clear_bit(grp->index, &q->bitmaps[ER]); // grp->S = grp->F + 1; // XXX debugging only } else { uint64_t roundedS = qfq_round_down(cl->S, grp->slot_shift); unsigned int s; if (grp->S == roundedS) goto skip_unblock; grp->S = roundedS; grp->F = roundedS + (2ULL << grp->slot_shift); /* remove from ER and put in the new set */ __clear_bit(grp->index, &q->bitmaps[ER]); s = qfq_calc_state(q, grp); __set_bit(grp->index, &q->bitmaps[s]); } /* we need to unblock even if the group has gone away */ qfq_unblock_groups(q, grp->index, old_F); } skip_unblock: qfq_update_eligible(q, old_V); NO(if (!q->bitmaps[ER] && q->queued) dump_sched(q, "end dequeue");) return m; } /* * Assign a reasonable start time for a new flow k in group i. * Admissible values for \hat(F) are multiples of \sigma_i * no greater than V+\sigma_i . Larger values mean that * we had a wraparound so we consider the timestamp to be stale. * * If F is not stale and F >= V then we set S = F. * Otherwise we should assign S = V, but this may violate * the ordering in ER. So, if we have groups in ER, set S to * the F_j of the first group j which would be blocking us. * We are guaranteed not to move S backward because * otherwise our group i would still be blocked. */ static inline void qfq_update_start(struct qfq_sched *q, struct qfq_class *cl) { unsigned long mask; uint64_t limit, roundedF; int slot_shift = cl->grp->slot_shift; roundedF = qfq_round_down(cl->F, slot_shift); limit = qfq_round_down(q->V, slot_shift) + (1ULL << slot_shift); if (!qfq_gt(cl->F, q->V) || qfq_gt(roundedF, limit)) { /* timestamp was stale */ mask = mask_from(q->bitmaps[ER], cl->grp->index); if (mask) { struct qfq_group *next = qfq_ffs(q, mask); if (qfq_gt(roundedF, next->F)) { /* from pv 71261956973ba9e0637848a5adb4a5819b4bae83 */ if (qfq_gt(limit, next->F)) cl->S = next->F; else /* preserve timestamp correctness */ cl->S = limit; return; } } cl->S = q->V; } else { /* timestamp is not stale */ cl->S = cl->F; } } static int qfq_enqueue(struct dn_sch_inst *si, struct dn_queue *_q, struct mbuf *m) { struct qfq_sched *q = (struct qfq_sched *)(si + 1); struct qfq_group *grp; struct qfq_class *cl = (struct qfq_class *)_q; uint64_t roundedS; int s; NO(q->loops++;) DX(4, "len %d flow %p inv_w 0x%x grp %d", m->m_pkthdr.len, _q, cl->inv_w, cl->grp->index); /* XXX verify that the packet obeys the parameters */ if (m != _q->mq.head) { if (dn_enqueue(_q, m, 0)) /* packet was dropped */ return 1; NO(q->queued++;) if (m != _q->mq.head) return 0; } /* If reach this point, queue q was idle */ grp = cl->grp; qfq_update_start(q, cl); /* adjust start time */ /* compute new finish time and rounded start. */ cl->F = cl->S + (uint64_t)(m->m_pkthdr.len) * cl->inv_w; roundedS = qfq_round_down(cl->S, grp->slot_shift); /* * insert cl in the correct bucket. * If cl->S >= grp->S we don't need to adjust the * bucket list and simply go to the insertion phase. * Otherwise grp->S is decreasing, we must make room * in the bucket list, and also recompute the group state. * Finally, if there were no flows in this group and nobody * was in ER make sure to adjust V. */ if (grp->full_slots) { if (!qfq_gt(grp->S, cl->S)) goto skip_update; /* create a slot for this cl->S */ qfq_slot_rotate(q, grp, roundedS); /* group was surely ineligible, remove */ __clear_bit(grp->index, &q->bitmaps[IR]); __clear_bit(grp->index, &q->bitmaps[IB]); } else if (!q->bitmaps[ER] && qfq_gt(roundedS, q->V)) q->V = roundedS; grp->S = roundedS; grp->F = roundedS + (2ULL << grp->slot_shift); // i.e. 2\sigma_i s = qfq_calc_state(q, grp); __set_bit(grp->index, &q->bitmaps[s]); ND("new state %d 0x%x", s, q->bitmaps[s]); ND("S %llx F %llx V %llx", cl->S, cl->F, q->V); skip_update: qfq_slot_insert(grp, cl, roundedS); return 0; } #if 0 static inline void qfq_slot_remove(struct qfq_sched *q, struct qfq_group *grp, struct qfq_class *cl, struct qfq_class **pprev) { unsigned int i, offset; uint64_t roundedS; roundedS = qfq_round_down(cl->S, grp->slot_shift); offset = (roundedS - grp->S) >> grp->slot_shift; i = (grp->front + offset) % QFQ_MAX_SLOTS; #ifdef notyet if (!pprev) { pprev = &grp->slots[i]; while (*pprev && *pprev != cl) pprev = &(*pprev)->next; } #endif *pprev = cl->next; if (!grp->slots[i]) __clear_bit(offset, &grp->full_slots); } /* * called to forcibly destroy a queue. * If the queue is not in the front bucket, or if it has * other queues in the front bucket, we can simply remove * the queue with no other side effects. * Otherwise we must propagate the event up. * XXX description to be completed. */ static void qfq_deactivate_class(struct qfq_sched *q, struct qfq_class *cl, struct qfq_class **pprev) { struct qfq_group *grp = &q->groups[cl->index]; unsigned long mask; uint64_t roundedS; int s; cl->F = cl->S; // not needed if the class goes away. qfq_slot_remove(q, grp, cl, pprev); if (!grp->full_slots) { /* nothing left in the group, remove from all sets. * Do ER last because if we were blocking other groups * we must unblock them. */ __clear_bit(grp->index, &q->bitmaps[IR]); __clear_bit(grp->index, &q->bitmaps[EB]); __clear_bit(grp->index, &q->bitmaps[IB]); if (test_bit(grp->index, &q->bitmaps[ER]) && !(q->bitmaps[ER] & ~((1UL << grp->index) - 1))) { mask = q->bitmaps[ER] & ((1UL << grp->index) - 1); if (mask) mask = ~((1UL << __fls(mask)) - 1); else mask = ~0UL; qfq_move_groups(q, mask, EB, ER); qfq_move_groups(q, mask, IB, IR); } __clear_bit(grp->index, &q->bitmaps[ER]); } else if (!grp->slots[grp->front]) { cl = qfq_slot_scan(grp); roundedS = qfq_round_down(cl->S, grp->slot_shift); if (grp->S != roundedS) { __clear_bit(grp->index, &q->bitmaps[ER]); __clear_bit(grp->index, &q->bitmaps[IR]); __clear_bit(grp->index, &q->bitmaps[EB]); __clear_bit(grp->index, &q->bitmaps[IB]); grp->S = roundedS; grp->F = roundedS + (2ULL << grp->slot_shift); s = qfq_calc_state(q, grp); __set_bit(grp->index, &q->bitmaps[s]); } } qfq_update_eligible(q, q->V); } #endif static int qfq_new_fsk(struct dn_fsk *f) { ipdn_bound_var(&f->fs.par[0], 1, 1, QFQ_MAX_WEIGHT, "qfq weight"); ipdn_bound_var(&f->fs.par[1], 1500, 1, 2000, "qfq maxlen"); ND("weight %d len %d\n", f->fs.par[0], f->fs.par[1]); return 0; } /* * initialize a new scheduler instance */ static int qfq_new_sched(struct dn_sch_inst *si) { struct qfq_sched *q = (struct qfq_sched *)(si + 1); struct qfq_group *grp; int i; for (i = 0; i <= QFQ_MAX_INDEX; i++) { grp = &q->groups[i]; grp->index = i; grp->slot_shift = QFQ_MTU_SHIFT + FRAC_BITS - (QFQ_MAX_INDEX - i); } return 0; } /* * QFQ scheduler descriptor */ static struct dn_alg qfq_desc = { _SI( .type = ) DN_SCHED_QFQ, _SI( .name = ) "QFQ", _SI( .flags = ) DN_MULTIQUEUE, _SI( .schk_datalen = ) 0, _SI( .si_datalen = ) sizeof(struct qfq_sched), _SI( .q_datalen = ) sizeof(struct qfq_class) - sizeof(struct dn_queue), _SI( .enqueue = ) qfq_enqueue, _SI( .dequeue = ) qfq_dequeue, _SI( .config = ) NULL, _SI( .destroy = ) NULL, _SI( .new_sched = ) qfq_new_sched, _SI( .free_sched = ) NULL, _SI( .new_fsk = ) qfq_new_fsk, _SI( .free_fsk = ) NULL, _SI( .new_queue = ) qfq_new_queue, _SI( .free_queue = ) qfq_free_queue, +#ifdef NEW_AQM + _SI( .getconfig = ) NULL, +#endif }; DECLARE_DNSCHED_MODULE(dn_qfq, &qfq_desc); #ifdef QFQ_DEBUG static void dump_groups(struct qfq_sched *q, uint32_t mask) { int i, j; for (i = 0; i < QFQ_MAX_INDEX + 1; i++) { struct qfq_group *g = &q->groups[i]; if (0 == (mask & (1<slots[j]) D(" bucket %d %p", j, g->slots[j]); } D("full_slots 0x%llx", (_P64)g->full_slots); D(" %2d S 0x%20llx F 0x%llx %c", i, (_P64)g->S, (_P64)g->F, mask & (1<loops, q->queued, (_P64)q->V); D(" ER 0x%08x", (unsigned)q->bitmaps[ER]); D(" EB 0x%08x", (unsigned)q->bitmaps[EB]); D(" IR 0x%08x", (unsigned)q->bitmaps[IR]); D(" IB 0x%08x", (unsigned)q->bitmaps[IB]); dump_groups(q, 0xffffffff); }; #endif /* QFQ_DEBUG */ Index: head/sys/netpfil/ipfw/dn_sched_rr.c =================================================================== --- head/sys/netpfil/ipfw/dn_sched_rr.c (revision 300778) +++ head/sys/netpfil/ipfw/dn_sched_rr.c (revision 300779) @@ -1,315 +1,321 @@ /* * Copyright (c) 2010 Riccardo Panicucci, Universita` di Pisa * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ */ #ifdef _KERNEL #include #include #include #include #include #include #include /* IFNAMSIZ */ #include #include /* ipfw_rule_ref */ #include /* flow_id */ #include #include #include +#ifdef NEW_AQM +#include +#endif #include #else #include #endif #define DN_SCHED_RR 3 // XXX Where? struct rr_queue { struct dn_queue q; /* Standard queue */ int status; /* 1: queue is in the list */ uint32_t credit; /* max bytes we can transmit */ uint32_t quantum; /* quantum * weight */ struct rr_queue *qnext; /* */ }; /* struct rr_schk contains global config parameters * and is right after dn_schk */ struct rr_schk { uint32_t min_q; /* Min quantum */ uint32_t max_q; /* Max quantum */ uint32_t q_bytes; /* default quantum in bytes */ }; /* per-instance round robin list, right after dn_sch_inst */ struct rr_si { struct rr_queue *head, *tail; /* Pointer to current queue */ }; /* Append a queue to the rr list */ static inline void rr_append(struct rr_queue *q, struct rr_si *si) { q->status = 1; /* mark as in-rr_list */ q->credit = q->quantum; /* initialize credit */ /* append to the tail */ if (si->head == NULL) si->head = q; else si->tail->qnext = q; si->tail = q; /* advance the tail pointer */ q->qnext = si->head; /* make it circular */ } /* Remove the head queue from circular list. */ static inline void rr_remove_head(struct rr_si *si) { if (si->head == NULL) return; /* empty queue */ si->head->status = 0; if (si->head == si->tail) { si->head = si->tail = NULL; return; } si->head = si->head->qnext; si->tail->qnext = si->head; } /* Remove a queue from circular list. * XXX see if ti can be merge with remove_queue() */ static inline void remove_queue_q(struct rr_queue *q, struct rr_si *si) { struct rr_queue *prev; if (q->status != 1) return; if (q == si->head) { rr_remove_head(si); return; } for (prev = si->head; prev; prev = prev->qnext) { if (prev->qnext != q) continue; prev->qnext = q->qnext; if (q == si->tail) si->tail = prev; q->status = 0; break; } } static inline void next_pointer(struct rr_si *si) { if (si->head == NULL) return; /* empty queue */ si->head = si->head->qnext; si->tail = si->tail->qnext; } static int rr_enqueue(struct dn_sch_inst *_si, struct dn_queue *q, struct mbuf *m) { struct rr_si *si; struct rr_queue *rrq; if (m != q->mq.head) { if (dn_enqueue(q, m, 0)) /* packet was dropped */ return 1; if (m != q->mq.head) return 0; } /* If reach this point, queue q was idle */ si = (struct rr_si *)(_si + 1); rrq = (struct rr_queue *)q; if (rrq->status == 1) /* Queue is already in the queue list */ return 0; /* Insert the queue in the queue list */ rr_append(rrq, si); return 0; } static struct mbuf * rr_dequeue(struct dn_sch_inst *_si) { /* Access scheduler instance private data */ struct rr_si *si = (struct rr_si *)(_si + 1); struct rr_queue *rrq; uint64_t len; while ( (rrq = si->head) ) { struct mbuf *m = rrq->q.mq.head; if ( m == NULL) { /* empty queue, remove from list */ rr_remove_head(si); continue; } len = m->m_pkthdr.len; if (len > rrq->credit) { /* Packet too big */ rrq->credit += rrq->quantum; /* Try next queue */ next_pointer(si); } else { rrq->credit -= len; return dn_dequeue(&rrq->q); } } /* no packet to dequeue*/ return NULL; } static int rr_config(struct dn_schk *_schk) { struct rr_schk *schk = (struct rr_schk *)(_schk + 1); ND("called"); /* use reasonable quantums (64..2k bytes, default 1500) */ schk->min_q = 64; schk->max_q = 2048; schk->q_bytes = 1500; /* quantum */ return 0; } static int rr_new_sched(struct dn_sch_inst *_si) { struct rr_si *si = (struct rr_si *)(_si + 1); ND("called"); si->head = si->tail = NULL; return 0; } static int rr_free_sched(struct dn_sch_inst *_si) { (void)_si; ND("called"); /* Nothing to do? */ return 0; } static int rr_new_fsk(struct dn_fsk *fs) { struct rr_schk *schk = (struct rr_schk *)(fs->sched + 1); /* par[0] is the weight, par[1] is the quantum step */ /* make sure the product fits an uint32_t */ ipdn_bound_var(&fs->fs.par[0], 1, 1, 65536, "RR weight"); ipdn_bound_var(&fs->fs.par[1], schk->q_bytes, schk->min_q, schk->max_q, "RR quantum"); return 0; } static int rr_new_queue(struct dn_queue *_q) { struct rr_queue *q = (struct rr_queue *)_q; uint64_t quantum; _q->ni.oid.subtype = DN_SCHED_RR; quantum = (uint64_t)_q->fs->fs.par[0] * _q->fs->fs.par[1]; if (quantum >= (1ULL<< 32)) { D("quantum too large, truncating to 4G - 1"); quantum = (1ULL<< 32) - 1; } q->quantum = quantum; ND("called, q->quantum %d", q->quantum); q->credit = q->quantum; q->status = 0; if (_q->mq.head != NULL) { /* Queue NOT empty, insert in the queue list */ rr_append(q, (struct rr_si *)(_q->_si + 1)); } return 0; } static int rr_free_queue(struct dn_queue *_q) { struct rr_queue *q = (struct rr_queue *)_q; ND("called"); if (q->status == 1) { struct rr_si *si = (struct rr_si *)(_q->_si + 1); remove_queue_q(q, si); } return 0; } /* * RR scheduler descriptor * contains the type of the scheduler, the name, the size of the * structures and function pointers. */ static struct dn_alg rr_desc = { _SI( .type = ) DN_SCHED_RR, _SI( .name = ) "RR", _SI( .flags = ) DN_MULTIQUEUE, _SI( .schk_datalen = ) sizeof(struct rr_schk), _SI( .si_datalen = ) sizeof(struct rr_si), _SI( .q_datalen = ) sizeof(struct rr_queue) - sizeof(struct dn_queue), _SI( .enqueue = ) rr_enqueue, _SI( .dequeue = ) rr_dequeue, _SI( .config = ) rr_config, _SI( .destroy = ) NULL, _SI( .new_sched = ) rr_new_sched, _SI( .free_sched = ) rr_free_sched, _SI( .new_fsk = ) rr_new_fsk, _SI( .free_fsk = ) NULL, _SI( .new_queue = ) rr_new_queue, _SI( .free_queue = ) rr_free_queue, +#ifdef NEW_AQM + _SI( .getconfig = ) NULL, +#endif }; DECLARE_DNSCHED_MODULE(dn_rr, &rr_desc); Index: head/sys/netpfil/ipfw/dn_sched_wf2q.c =================================================================== --- head/sys/netpfil/ipfw/dn_sched_wf2q.c (revision 300778) +++ head/sys/netpfil/ipfw/dn_sched_wf2q.c (revision 300779) @@ -1,373 +1,380 @@ /* * Copyright (c) 2010 Riccardo Panicucci, Universita` di Pisa * Copyright (c) 2000-2002 Luigi Rizzo, Universita` di Pisa * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ */ #ifdef _KERNEL #include #include #include #include #include #include #include /* IFNAMSIZ */ #include #include /* ipfw_rule_ref */ #include /* flow_id */ #include #include #include +#ifdef NEW_AQM +#include +#endif #include #else #include #endif #ifndef MAX64 #define MAX64(x,y) (( (int64_t) ( (y)-(x) )) > 0 ) ? (y) : (x) #endif /* * timestamps are computed on 64 bit using fixed point arithmetic. * LMAX_BITS, WMAX_BITS are the max number of bits for the packet len * and sum of weights, respectively. FRAC_BITS is the number of * fractional bits. We want FRAC_BITS >> WMAX_BITS to avoid too large * errors when computing the inverse, FRAC_BITS < 32 so we can do 1/w * using an unsigned 32-bit division, and to avoid wraparounds we need * LMAX_BITS + WMAX_BITS + FRAC_BITS << 64 * As an example * FRAC_BITS = 26, LMAX_BITS=14, WMAX_BITS = 19 */ #ifndef FRAC_BITS #define FRAC_BITS 28 /* shift for fixed point arithmetic */ #define ONE_FP (1UL << FRAC_BITS) #endif /* * Private information for the scheduler instance: * sch_heap (key is Finish time) returns the next queue to serve * ne_heap (key is Start time) stores not-eligible queues * idle_heap (key=start/finish time) stores idle flows. It must * support extract-from-middle. * A flow is only in 1 of the three heaps. * XXX todo: use a more efficient data structure, e.g. a tree sorted * by F with min_subtree(S) in each node */ struct wf2qp_si { struct dn_heap sch_heap; /* top extract - key Finish time */ struct dn_heap ne_heap; /* top extract - key Start time */ struct dn_heap idle_heap; /* random extract - key Start=Finish time */ uint64_t V; /* virtual time */ uint32_t inv_wsum; /* inverse of sum of weights */ uint32_t wsum; /* sum of weights */ }; struct wf2qp_queue { struct dn_queue _q; uint64_t S, F; /* start time, finish time */ uint32_t inv_w; /* ONE_FP / weight */ int32_t heap_pos; /* position (index) of struct in heap */ }; /* * This file implements a WF2Q+ scheduler as it has been in dummynet * since 2000. * The scheduler supports per-flow queues and has O(log N) complexity. * * WF2Q+ needs to drain entries from the idle heap so that we * can keep the sum of weights up to date. We can do it whenever * we get a chance, or periodically, or following some other * strategy. The function idle_check() drains at most N elements * from the idle heap. */ static void idle_check(struct wf2qp_si *si, int n, int force) { struct dn_heap *h = &si->idle_heap; while (n-- > 0 && h->elements > 0 && (force || DN_KEY_LT(HEAP_TOP(h)->key, si->V))) { struct dn_queue *q = HEAP_TOP(h)->object; struct wf2qp_queue *alg_fq = (struct wf2qp_queue *)q; heap_extract(h, NULL); /* XXX to let the flowset delete the queue we should * mark it as 'unused' by the scheduler. */ alg_fq->S = alg_fq->F + 1; /* Mark timestamp as invalid. */ si->wsum -= q->fs->fs.par[0]; /* adjust sum of weights */ if (si->wsum > 0) si->inv_wsum = ONE_FP/si->wsum; } } static int wf2qp_enqueue(struct dn_sch_inst *_si, struct dn_queue *q, struct mbuf *m) { struct dn_fsk *fs = q->fs; struct wf2qp_si *si = (struct wf2qp_si *)(_si + 1); struct wf2qp_queue *alg_fq; uint64_t len = m->m_pkthdr.len; if (m != q->mq.head) { if (dn_enqueue(q, m, 0)) /* packet was dropped */ return 1; if (m != q->mq.head) /* queue was already busy */ return 0; } /* If reach this point, queue q was idle */ alg_fq = (struct wf2qp_queue *)q; if (DN_KEY_LT(alg_fq->F, alg_fq->S)) { /* Fbrand new queue. */ alg_fq->S = si->V; /* init start time */ si->wsum += fs->fs.par[0]; /* add weight of new queue. */ si->inv_wsum = ONE_FP/si->wsum; } else { /* if it was idle then it was in the idle heap */ heap_extract(&si->idle_heap, q); alg_fq->S = MAX64(alg_fq->F, si->V); /* compute new S */ } alg_fq->F = alg_fq->S + len * alg_fq->inv_w; /* if nothing is backlogged, make sure this flow is eligible */ if (si->ne_heap.elements == 0 && si->sch_heap.elements == 0) si->V = MAX64(alg_fq->S, si->V); /* * Look at eligibility. A flow is not eligibile if S>V (when * this happens, it means that there is some other flow already * scheduled for the same pipe, so the sch_heap cannot be * empty). If the flow is not eligible we just store it in the * ne_heap. Otherwise, we store in the sch_heap. * Note that for all flows in sch_heap (SCH), S_i <= V, * and for all flows in ne_heap (NEH), S_i > V. * So when we need to compute max(V, min(S_i)) forall i in * SCH+NEH, we only need to look into NEH. */ if (DN_KEY_LT(si->V, alg_fq->S)) { /* S>V means flow Not eligible. */ if (si->sch_heap.elements == 0) D("++ ouch! not eligible but empty scheduler!"); heap_insert(&si->ne_heap, alg_fq->S, q); } else { heap_insert(&si->sch_heap, alg_fq->F, q); } return 0; } /* XXX invariant: sch > 0 || V >= min(S in neh) */ static struct mbuf * wf2qp_dequeue(struct dn_sch_inst *_si) { /* Access scheduler instance private data */ struct wf2qp_si *si = (struct wf2qp_si *)(_si + 1); struct mbuf *m; struct dn_queue *q; struct dn_heap *sch = &si->sch_heap; struct dn_heap *neh = &si->ne_heap; struct wf2qp_queue *alg_fq; if (sch->elements == 0 && neh->elements == 0) { /* we have nothing to do. We could kill the idle heap * altogether and reset V */ idle_check(si, 0x7fffffff, 1); si->V = 0; si->wsum = 0; /* should be set already */ return NULL; /* quick return if nothing to do */ } idle_check(si, 1, 0); /* drain something from the idle heap */ /* make sure at least one element is eligible, bumping V * and moving entries that have become eligible. * We need to repeat the first part twice, before and * after extracting the candidate, or enqueue() will * find the data structure in a wrong state. */ m = NULL; for(;;) { /* * Compute V = max(V, min(S_i)). Remember that all elements * in sch have by definition S_i <= V so if sch is not empty, * V is surely the max and we must not update it. Conversely, * if sch is empty we only need to look at neh. * We don't need to move the queues, as it will be done at the * next enqueue */ if (sch->elements == 0 && neh->elements > 0) { si->V = MAX64(si->V, HEAP_TOP(neh)->key); } while (neh->elements > 0 && DN_KEY_LEQ(HEAP_TOP(neh)->key, si->V)) { q = HEAP_TOP(neh)->object; alg_fq = (struct wf2qp_queue *)q; heap_extract(neh, NULL); heap_insert(sch, alg_fq->F, q); } if (m) /* pkt found in previous iteration */ break; /* ok we have at least one eligible pkt */ q = HEAP_TOP(sch)->object; alg_fq = (struct wf2qp_queue *)q; m = dn_dequeue(q); heap_extract(sch, NULL); /* Remove queue from heap. */ si->V += (uint64_t)(m->m_pkthdr.len) * si->inv_wsum; alg_fq->S = alg_fq->F; /* Update start time. */ if (q->mq.head == 0) { /* not backlogged any more. */ heap_insert(&si->idle_heap, alg_fq->F, q); } else { /* Still backlogged. */ /* Update F, store in neh or sch */ uint64_t len = q->mq.head->m_pkthdr.len; alg_fq->F += len * alg_fq->inv_w; if (DN_KEY_LEQ(alg_fq->S, si->V)) { heap_insert(sch, alg_fq->F, q); } else { heap_insert(neh, alg_fq->S, q); } } } return m; } static int wf2qp_new_sched(struct dn_sch_inst *_si) { struct wf2qp_si *si = (struct wf2qp_si *)(_si + 1); int ofs = offsetof(struct wf2qp_queue, heap_pos); /* all heaps support extract from middle */ if (heap_init(&si->idle_heap, 16, ofs) || heap_init(&si->sch_heap, 16, ofs) || heap_init(&si->ne_heap, 16, ofs)) { heap_free(&si->ne_heap); heap_free(&si->sch_heap); heap_free(&si->idle_heap); return ENOMEM; } return 0; } static int wf2qp_free_sched(struct dn_sch_inst *_si) { struct wf2qp_si *si = (struct wf2qp_si *)(_si + 1); heap_free(&si->sch_heap); heap_free(&si->ne_heap); heap_free(&si->idle_heap); return 0; } static int wf2qp_new_fsk(struct dn_fsk *fs) { ipdn_bound_var(&fs->fs.par[0], 1, 1, 100, "WF2Q+ weight"); return 0; } static int wf2qp_new_queue(struct dn_queue *_q) { struct wf2qp_queue *q = (struct wf2qp_queue *)_q; _q->ni.oid.subtype = DN_SCHED_WF2QP; q->F = 0; /* not strictly necessary */ q->S = q->F + 1; /* mark timestamp as invalid. */ q->inv_w = ONE_FP / _q->fs->fs.par[0]; if (_q->mq.head != NULL) { wf2qp_enqueue(_q->_si, _q, _q->mq.head); } return 0; } /* * Called when the infrastructure removes a queue (e.g. flowset * is reconfigured). Nothing to do if we did not 'own' the queue, * otherwise remove it from the right heap and adjust the sum * of weights. */ static int wf2qp_free_queue(struct dn_queue *q) { struct wf2qp_queue *alg_fq = (struct wf2qp_queue *)q; struct wf2qp_si *si = (struct wf2qp_si *)(q->_si + 1); if (alg_fq->S >= alg_fq->F + 1) return 0; /* nothing to do, not in any heap */ si->wsum -= q->fs->fs.par[0]; if (si->wsum > 0) si->inv_wsum = ONE_FP/si->wsum; /* extract from the heap. XXX TODO we may need to adjust V * to make sure the invariants hold. */ if (q->mq.head == NULL) { heap_extract(&si->idle_heap, q); } else if (DN_KEY_LT(si->V, alg_fq->S)) { heap_extract(&si->ne_heap, q); } else { heap_extract(&si->sch_heap, q); } return 0; } /* * WF2Q+ scheduler descriptor * contains the type of the scheduler, the name, the size of the * structures and function pointers. */ static struct dn_alg wf2qp_desc = { _SI( .type = ) DN_SCHED_WF2QP, _SI( .name = ) "WF2Q+", _SI( .flags = ) DN_MULTIQUEUE, /* we need extra space in the si and the queue */ _SI( .schk_datalen = ) 0, _SI( .si_datalen = ) sizeof(struct wf2qp_si), _SI( .q_datalen = ) sizeof(struct wf2qp_queue) - sizeof(struct dn_queue), _SI( .enqueue = ) wf2qp_enqueue, _SI( .dequeue = ) wf2qp_dequeue, _SI( .config = ) NULL, _SI( .destroy = ) NULL, _SI( .new_sched = ) wf2qp_new_sched, _SI( .free_sched = ) wf2qp_free_sched, _SI( .new_fsk = ) wf2qp_new_fsk, _SI( .free_fsk = ) NULL, _SI( .new_queue = ) wf2qp_new_queue, _SI( .free_queue = ) wf2qp_free_queue, +#ifdef NEW_AQM + _SI( .getconfig = ) NULL, +#endif + }; DECLARE_DNSCHED_MODULE(dn_wf2qp, &wf2qp_desc); Index: head/sys/netpfil/ipfw/ip_dn_glue.c =================================================================== --- head/sys/netpfil/ipfw/ip_dn_glue.c (revision 300778) +++ head/sys/netpfil/ipfw/ip_dn_glue.c (revision 300779) @@ -1,846 +1,849 @@ /*- * Copyright (c) 2010 Riccardo Panicucci, Universita` di Pisa * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ * * Binary compatibility support for /sbin/ipfw RELENG_7 and RELENG_8 */ #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ #include #include /* ip_output(), IP_FORWARDING */ #include #include #include #include #include +#ifdef NEW_AQM +#include +#endif #include /* FREEBSD7.2 ip_dummynet.h r191715*/ struct dn_heap_entry7 { int64_t key; /* sorting key. Topmost element is smallest one */ void *object; /* object pointer */ }; struct dn_heap7 { int size; int elements; int offset; /* XXX if > 0 this is the offset of direct ptr to obj */ struct dn_heap_entry7 *p; /* really an array of "size" entries */ }; /* Common to 7.2 and 8 */ struct dn_flow_set { SLIST_ENTRY(dn_flow_set) next; /* linked list in a hash slot */ u_short fs_nr ; /* flow_set number */ u_short flags_fs; #define DNOLD_HAVE_FLOW_MASK 0x0001 #define DNOLD_IS_RED 0x0002 #define DNOLD_IS_GENTLE_RED 0x0004 #define DNOLD_QSIZE_IS_BYTES 0x0008 /* queue size is measured in bytes */ #define DNOLD_NOERROR 0x0010 /* do not report ENOBUFS on drops */ #define DNOLD_HAS_PROFILE 0x0020 /* the pipe has a delay profile. */ #define DNOLD_IS_PIPE 0x4000 #define DNOLD_IS_QUEUE 0x8000 struct dn_pipe7 *pipe ; /* pointer to parent pipe */ u_short parent_nr ; /* parent pipe#, 0 if local to a pipe */ int weight ; /* WFQ queue weight */ int qsize ; /* queue size in slots or bytes */ int plr ; /* pkt loss rate (2^31-1 means 100%) */ struct ipfw_flow_id flow_mask ; /* hash table of queues onto this flow_set */ int rq_size ; /* number of slots */ int rq_elements ; /* active elements */ struct dn_flow_queue7 **rq; /* array of rq_size entries */ u_int32_t last_expired ; /* do not expire too frequently */ int backlogged ; /* #active queues for this flowset */ /* RED parameters */ #define SCALE_RED 16 #define SCALE(x) ( (x) << SCALE_RED ) #define SCALE_VAL(x) ( (x) >> SCALE_RED ) #define SCALE_MUL(x,y) ( ( (x) * (y) ) >> SCALE_RED ) int w_q ; /* queue weight (scaled) */ int max_th ; /* maximum threshold for queue (scaled) */ int min_th ; /* minimum threshold for queue (scaled) */ int max_p ; /* maximum value for p_b (scaled) */ u_int c_1 ; /* max_p/(max_th-min_th) (scaled) */ u_int c_2 ; /* max_p*min_th/(max_th-min_th) (scaled) */ u_int c_3 ; /* for GRED, (1-max_p)/max_th (scaled) */ u_int c_4 ; /* for GRED, 1 - 2*max_p (scaled) */ u_int * w_q_lookup ; /* lookup table for computing (1-w_q)^t */ u_int lookup_depth ; /* depth of lookup table */ int lookup_step ; /* granularity inside the lookup table */ int lookup_weight ; /* equal to (1-w_q)^t / (1-w_q)^(t+1) */ int avg_pkt_size ; /* medium packet size */ int max_pkt_size ; /* max packet size */ }; SLIST_HEAD(dn_flow_set_head, dn_flow_set); #define DN_IS_PIPE 0x4000 #define DN_IS_QUEUE 0x8000 struct dn_flow_queue7 { struct dn_flow_queue7 *next ; struct ipfw_flow_id id ; struct mbuf *head, *tail ; /* queue of packets */ u_int len ; u_int len_bytes ; u_long numbytes; u_int64_t tot_pkts ; /* statistics counters */ u_int64_t tot_bytes ; u_int32_t drops ; int hash_slot ; /* debugging/diagnostic */ /* RED parameters */ int avg ; /* average queue length est. (scaled) */ int count ; /* arrivals since last RED drop */ int random ; /* random value (scaled) */ u_int32_t q_time; /* start of queue idle time */ /* WF2Q+ support */ struct dn_flow_set *fs ; /* parent flow set */ int heap_pos ; /* position (index) of struct in heap */ int64_t sched_time ; /* current time when queue enters ready_heap */ int64_t S,F ; /* start time, finish time */ }; struct dn_pipe7 { /* a pipe */ SLIST_ENTRY(dn_pipe7) next; /* linked list in a hash slot */ int pipe_nr ; /* number */ int bandwidth; /* really, bytes/tick. */ int delay ; /* really, ticks */ struct mbuf *head, *tail ; /* packets in delay line */ /* WF2Q+ */ struct dn_heap7 scheduler_heap ; /* top extract - key Finish time*/ struct dn_heap7 not_eligible_heap; /* top extract- key Start time */ struct dn_heap7 idle_heap ; /* random extract - key Start=Finish time */ int64_t V ; /* virtual time */ int sum; /* sum of weights of all active sessions */ int numbytes; int64_t sched_time ; /* time pipe was scheduled in ready_heap */ /* * When the tx clock come from an interface (if_name[0] != '\0'), its name * is stored below, whereas the ifp is filled when the rule is configured. */ char if_name[IFNAMSIZ]; struct ifnet *ifp ; int ready ; /* set if ifp != NULL and we got a signal from it */ struct dn_flow_set fs ; /* used with fixed-rate flows */ }; SLIST_HEAD(dn_pipe_head7, dn_pipe7); /* FREEBSD8 ip_dummynet.h r196045 */ struct dn_flow_queue8 { struct dn_flow_queue8 *next ; struct ipfw_flow_id id ; struct mbuf *head, *tail ; /* queue of packets */ u_int len ; u_int len_bytes ; uint64_t numbytes ; /* credit for transmission (dynamic queues) */ int64_t extra_bits; /* extra bits simulating unavailable channel */ u_int64_t tot_pkts ; /* statistics counters */ u_int64_t tot_bytes ; u_int32_t drops ; int hash_slot ; /* debugging/diagnostic */ /* RED parameters */ int avg ; /* average queue length est. (scaled) */ int count ; /* arrivals since last RED drop */ int random ; /* random value (scaled) */ int64_t idle_time; /* start of queue idle time */ /* WF2Q+ support */ struct dn_flow_set *fs ; /* parent flow set */ int heap_pos ; /* position (index) of struct in heap */ int64_t sched_time ; /* current time when queue enters ready_heap */ int64_t S,F ; /* start time, finish time */ }; struct dn_pipe8 { /* a pipe */ SLIST_ENTRY(dn_pipe8) next; /* linked list in a hash slot */ int pipe_nr ; /* number */ int bandwidth; /* really, bytes/tick. */ int delay ; /* really, ticks */ struct mbuf *head, *tail ; /* packets in delay line */ /* WF2Q+ */ struct dn_heap7 scheduler_heap ; /* top extract - key Finish time*/ struct dn_heap7 not_eligible_heap; /* top extract- key Start time */ struct dn_heap7 idle_heap ; /* random extract - key Start=Finish time */ int64_t V ; /* virtual time */ int sum; /* sum of weights of all active sessions */ /* Same as in dn_flow_queue, numbytes can become large */ int64_t numbytes; /* bits I can transmit (more or less). */ uint64_t burst; /* burst size, scaled: bits * hz */ int64_t sched_time ; /* time pipe was scheduled in ready_heap */ int64_t idle_time; /* start of pipe idle time */ char if_name[IFNAMSIZ]; struct ifnet *ifp ; int ready ; /* set if ifp != NULL and we got a signal from it */ struct dn_flow_set fs ; /* used with fixed-rate flows */ /* fields to simulate a delay profile */ #define ED_MAX_NAME_LEN 32 char name[ED_MAX_NAME_LEN]; int loss_level; int samples_no; int *samples; }; #define ED_MAX_SAMPLES_NO 1024 struct dn_pipe_max8 { struct dn_pipe8 pipe; int samples[ED_MAX_SAMPLES_NO]; }; SLIST_HEAD(dn_pipe_head8, dn_pipe8); /* * Changes from 7.2 to 8: * dn_pipe: * numbytes from int to int64_t * add burst (int64_t) * add idle_time (int64_t) * add profile * add struct dn_pipe_max * add flag DN_HAS_PROFILE * * dn_flow_queue * numbytes from u_long to int64_t * add extra_bits (int64_t) * q_time from u_int32_t to int64_t and name idle_time * * dn_flow_set unchanged * */ /* NOTE:XXX copied from dummynet.c */ #define O_NEXT(p, len) ((void *)((char *)p + len)) static void oid_fill(struct dn_id *oid, int len, int type, uintptr_t id) { oid->len = len; oid->type = type; oid->subtype = 0; oid->id = id; } /* make room in the buffer and move the pointer forward */ static void * o_next(struct dn_id **o, int len, int type) { struct dn_id *ret = *o; oid_fill(ret, len, type, 0); *o = O_NEXT(*o, len); return ret; } static size_t pipesize7 = sizeof(struct dn_pipe7); static size_t pipesize8 = sizeof(struct dn_pipe8); static size_t pipesizemax8 = sizeof(struct dn_pipe_max8); /* Indicate 'ipfw' version * 1: from FreeBSD 7.2 * 0: from FreeBSD 8 * -1: unknown (for now is unused) * * It is update when a IP_DUMMYNET_DEL or IP_DUMMYNET_CONFIGURE request arrives * NOTE: if a IP_DUMMYNET_GET arrives and the 'ipfw' version is unknown, * it is suppose to be the FreeBSD 8 version. */ static int is7 = 0; static int convertflags2new(int src) { int dst = 0; if (src & DNOLD_HAVE_FLOW_MASK) dst |= DN_HAVE_MASK; if (src & DNOLD_QSIZE_IS_BYTES) dst |= DN_QSIZE_BYTES; if (src & DNOLD_NOERROR) dst |= DN_NOERROR; if (src & DNOLD_IS_RED) dst |= DN_IS_RED; if (src & DNOLD_IS_GENTLE_RED) dst |= DN_IS_GENTLE_RED; if (src & DNOLD_HAS_PROFILE) dst |= DN_HAS_PROFILE; return dst; } static int convertflags2old(int src) { int dst = 0; if (src & DN_HAVE_MASK) dst |= DNOLD_HAVE_FLOW_MASK; if (src & DN_IS_RED) dst |= DNOLD_IS_RED; if (src & DN_IS_GENTLE_RED) dst |= DNOLD_IS_GENTLE_RED; if (src & DN_NOERROR) dst |= DNOLD_NOERROR; if (src & DN_HAS_PROFILE) dst |= DNOLD_HAS_PROFILE; if (src & DN_QSIZE_BYTES) dst |= DNOLD_QSIZE_IS_BYTES; return dst; } static int dn_compat_del(void *v) { struct dn_pipe7 *p = (struct dn_pipe7 *) v; struct dn_pipe8 *p8 = (struct dn_pipe8 *) v; struct { struct dn_id oid; uintptr_t a[1]; /* add more if we want a list */ } cmd; /* XXX DN_API_VERSION ??? */ oid_fill((void *)&cmd, sizeof(cmd), DN_CMD_DELETE, DN_API_VERSION); if (is7) { if (p->pipe_nr == 0 && p->fs.fs_nr == 0) return EINVAL; if (p->pipe_nr != 0 && p->fs.fs_nr != 0) return EINVAL; } else { if (p8->pipe_nr == 0 && p8->fs.fs_nr == 0) return EINVAL; if (p8->pipe_nr != 0 && p8->fs.fs_nr != 0) return EINVAL; } if (p->pipe_nr != 0) { /* pipe x delete */ cmd.a[0] = p->pipe_nr; cmd.oid.subtype = DN_LINK; } else { /* queue x delete */ cmd.oid.subtype = DN_FS; cmd.a[0] = (is7) ? p->fs.fs_nr : p8->fs.fs_nr; } return do_config(&cmd, cmd.oid.len); } static int dn_compat_config_queue(struct dn_fs *fs, void* v) { struct dn_pipe7 *p7 = (struct dn_pipe7 *)v; struct dn_pipe8 *p8 = (struct dn_pipe8 *)v; struct dn_flow_set *f; if (is7) f = &p7->fs; else f = &p8->fs; fs->fs_nr = f->fs_nr; fs->sched_nr = f->parent_nr; fs->flow_mask = f->flow_mask; fs->buckets = f->rq_size; fs->qsize = f->qsize; fs->plr = f->plr; fs->par[0] = f->weight; fs->flags = convertflags2new(f->flags_fs); if (fs->flags & DN_IS_GENTLE_RED || fs->flags & DN_IS_RED) { fs->w_q = f->w_q; fs->max_th = f->max_th; fs->min_th = f->min_th; fs->max_p = f->max_p; } return 0; } static int dn_compat_config_pipe(struct dn_sch *sch, struct dn_link *p, struct dn_fs *fs, void* v) { struct dn_pipe7 *p7 = (struct dn_pipe7 *)v; struct dn_pipe8 *p8 = (struct dn_pipe8 *)v; int i = p7->pipe_nr; sch->sched_nr = i; sch->oid.subtype = 0; p->link_nr = i; fs->fs_nr = i + 2*DN_MAX_ID; fs->sched_nr = i + DN_MAX_ID; /* Common to 7 and 8 */ p->bandwidth = p7->bandwidth; p->delay = p7->delay; if (!is7) { /* FreeBSD 8 has burst */ p->burst = p8->burst; } /* fill the fifo flowset */ dn_compat_config_queue(fs, v); fs->fs_nr = i + 2*DN_MAX_ID; fs->sched_nr = i + DN_MAX_ID; /* Move scheduler related parameter from fs to sch */ sch->buckets = fs->buckets; /*XXX*/ fs->buckets = 0; if (fs->flags & DN_HAVE_MASK) { sch->flags |= DN_HAVE_MASK; fs->flags &= ~DN_HAVE_MASK; sch->sched_mask = fs->flow_mask; bzero(&fs->flow_mask, sizeof(struct ipfw_flow_id)); } return 0; } static int dn_compat_config_profile(struct dn_profile *pf, struct dn_link *p, void *v) { struct dn_pipe8 *p8 = (struct dn_pipe8 *)v; p8->samples = &(((struct dn_pipe_max8 *)p8)->samples[0]); pf->link_nr = p->link_nr; pf->loss_level = p8->loss_level; // pf->bandwidth = p->bandwidth; //XXX bandwidth redundant? pf->samples_no = p8->samples_no; strncpy(pf->name, p8->name,sizeof(pf->name)); bcopy(p8->samples, pf->samples, sizeof(pf->samples)); return 0; } /* * If p->pipe_nr != 0 the command is 'pipe x config', so need to create * the three main struct, else only a flowset is created */ static int dn_compat_configure(void *v) { struct dn_id *buf = NULL, *base; struct dn_sch *sch = NULL; struct dn_link *p = NULL; struct dn_fs *fs = NULL; struct dn_profile *pf = NULL; int lmax; int error; struct dn_pipe7 *p7 = (struct dn_pipe7 *)v; struct dn_pipe8 *p8 = (struct dn_pipe8 *)v; int i; /* number of object to configure */ lmax = sizeof(struct dn_id); /* command header */ lmax += sizeof(struct dn_sch) + sizeof(struct dn_link) + sizeof(struct dn_fs) + sizeof(struct dn_profile); base = buf = malloc(lmax, M_DUMMYNET, M_WAITOK|M_ZERO); o_next(&buf, sizeof(struct dn_id), DN_CMD_CONFIG); base->id = DN_API_VERSION; /* pipe_nr is the same in p7 and p8 */ i = p7->pipe_nr; if (i != 0) { /* pipe config */ sch = o_next(&buf, sizeof(*sch), DN_SCH); p = o_next(&buf, sizeof(*p), DN_LINK); fs = o_next(&buf, sizeof(*fs), DN_FS); error = dn_compat_config_pipe(sch, p, fs, v); if (error) { free(buf, M_DUMMYNET); return error; } if (!is7 && p8->samples_no > 0) { /* Add profiles*/ pf = o_next(&buf, sizeof(*pf), DN_PROFILE); error = dn_compat_config_profile(pf, p, v); if (error) { free(buf, M_DUMMYNET); return error; } } } else { /* queue config */ fs = o_next(&buf, sizeof(*fs), DN_FS); error = dn_compat_config_queue(fs, v); if (error) { free(buf, M_DUMMYNET); return error; } } error = do_config(base, (char *)buf - (char *)base); if (buf) free(buf, M_DUMMYNET); return error; } int dn_compat_calc_size(void) { int need = 0; /* XXX use FreeBSD 8 struct size */ /* NOTE: * - half scheduler: schk_count/2 * - all flowset: fsk_count * - all flowset queues: queue_count * - all pipe queue: si_count */ need += dn_cfg.schk_count * sizeof(struct dn_pipe8) / 2; need += dn_cfg.fsk_count * sizeof(struct dn_flow_set); need += dn_cfg.si_count * sizeof(struct dn_flow_queue8); need += dn_cfg.queue_count * sizeof(struct dn_flow_queue8); return need; } int dn_c_copy_q (void *_ni, void *arg) { struct copy_args *a = arg; struct dn_flow_queue7 *fq7 = (struct dn_flow_queue7 *)*a->start; struct dn_flow_queue8 *fq8 = (struct dn_flow_queue8 *)*a->start; struct dn_flow *ni = (struct dn_flow *)_ni; int size = 0; /* XXX hash slot not set */ /* No difference between 7.2/8 */ fq7->len = ni->length; fq7->len_bytes = ni->len_bytes; fq7->id = ni->fid; if (is7) { size = sizeof(struct dn_flow_queue7); fq7->tot_pkts = ni->tot_pkts; fq7->tot_bytes = ni->tot_bytes; fq7->drops = ni->drops; } else { size = sizeof(struct dn_flow_queue8); fq8->tot_pkts = ni->tot_pkts; fq8->tot_bytes = ni->tot_bytes; fq8->drops = ni->drops; } *a->start += size; return 0; } int dn_c_copy_pipe(struct dn_schk *s, struct copy_args *a, int nq) { struct dn_link *l = &s->link; struct dn_fsk *f = s->fs; struct dn_pipe7 *pipe7 = (struct dn_pipe7 *)*a->start; struct dn_pipe8 *pipe8 = (struct dn_pipe8 *)*a->start; struct dn_flow_set *fs; int size = 0; if (is7) { fs = &pipe7->fs; size = sizeof(struct dn_pipe7); } else { fs = &pipe8->fs; size = sizeof(struct dn_pipe8); } /* These 4 field are the same in pipe7 and pipe8 */ pipe7->next.sle_next = (struct dn_pipe7 *)DN_IS_PIPE; pipe7->bandwidth = l->bandwidth; pipe7->delay = l->delay * 1000 / hz; pipe7->pipe_nr = l->link_nr - DN_MAX_ID; if (!is7) { if (s->profile) { struct dn_profile *pf = s->profile; strncpy(pipe8->name, pf->name, sizeof(pf->name)); pipe8->loss_level = pf->loss_level; pipe8->samples_no = pf->samples_no; } pipe8->burst = div64(l->burst , 8 * hz); } fs->flow_mask = s->sch.sched_mask; fs->rq_size = s->sch.buckets ? s->sch.buckets : 1; fs->parent_nr = l->link_nr - DN_MAX_ID; fs->qsize = f->fs.qsize; fs->plr = f->fs.plr; fs->w_q = f->fs.w_q; fs->max_th = f->max_th; fs->min_th = f->min_th; fs->max_p = f->fs.max_p; fs->rq_elements = nq; fs->flags_fs = convertflags2old(f->fs.flags); *a->start += size; return 0; } int dn_compat_copy_pipe(struct copy_args *a, void *_o) { int have = a->end - *a->start; int need = 0; int pipe_size = sizeof(struct dn_pipe8); int queue_size = sizeof(struct dn_flow_queue8); int n_queue = 0; /* number of queues */ struct dn_schk *s = (struct dn_schk *)_o; /* calculate needed space: * - struct dn_pipe * - if there are instances, dn_queue * n_instances */ n_queue = (s->sch.flags & DN_HAVE_MASK ? dn_ht_entries(s->siht) : (s->siht ? 1 : 0)); need = pipe_size + queue_size * n_queue; if (have < need) { D("have %d < need %d", have, need); return 1; } /* copy pipe */ dn_c_copy_pipe(s, a, n_queue); /* copy queues */ if (s->sch.flags & DN_HAVE_MASK) dn_ht_scan(s->siht, dn_c_copy_q, a); else if (s->siht) dn_c_copy_q(s->siht, a); return 0; } int dn_c_copy_fs(struct dn_fsk *f, struct copy_args *a, int nq) { struct dn_flow_set *fs = (struct dn_flow_set *)*a->start; fs->next.sle_next = (struct dn_flow_set *)DN_IS_QUEUE; fs->fs_nr = f->fs.fs_nr; fs->qsize = f->fs.qsize; fs->plr = f->fs.plr; fs->w_q = f->fs.w_q; fs->max_th = f->max_th; fs->min_th = f->min_th; fs->max_p = f->fs.max_p; fs->flow_mask = f->fs.flow_mask; fs->rq_elements = nq; fs->rq_size = (f->fs.buckets ? f->fs.buckets : 1); fs->parent_nr = f->fs.sched_nr; fs->weight = f->fs.par[0]; fs->flags_fs = convertflags2old(f->fs.flags); *a->start += sizeof(struct dn_flow_set); return 0; } int dn_compat_copy_queue(struct copy_args *a, void *_o) { int have = a->end - *a->start; int need = 0; int fs_size = sizeof(struct dn_flow_set); int queue_size = sizeof(struct dn_flow_queue8); struct dn_fsk *fs = (struct dn_fsk *)_o; int n_queue = 0; /* number of queues */ n_queue = (fs->fs.flags & DN_HAVE_MASK ? dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0)); need = fs_size + queue_size * n_queue; if (have < need) { D("have < need"); return 1; } /* copy flowset */ dn_c_copy_fs(fs, a, n_queue); /* copy queues */ if (fs->fs.flags & DN_HAVE_MASK) dn_ht_scan(fs->qht, dn_c_copy_q, a); else if (fs->qht) dn_c_copy_q(fs->qht, a); return 0; } int copy_data_helper_compat(void *_o, void *_arg) { struct copy_args *a = _arg; if (a->type == DN_COMPAT_PIPE) { struct dn_schk *s = _o; if (s->sch.oid.subtype != 1 || s->sch.sched_nr <= DN_MAX_ID) { return 0; /* not old type */ } /* copy pipe parameters, and if instance exists, copy * other parameters and eventually queues. */ if(dn_compat_copy_pipe(a, _o)) return DNHT_SCAN_END; } else if (a->type == DN_COMPAT_QUEUE) { struct dn_fsk *fs = _o; if (fs->fs.fs_nr >= DN_MAX_ID) return 0; if (dn_compat_copy_queue(a, _o)) return DNHT_SCAN_END; } return 0; } /* Main function to manage old requests */ int ip_dummynet_compat(struct sockopt *sopt) { int error=0; void *v = NULL; struct dn_id oid; /* Length of data, used to found ipfw version... */ int len = sopt->sopt_valsize; /* len can be 0 if command was dummynet_flush */ if (len == pipesize7) { D("setting compatibility with FreeBSD 7.2"); is7 = 1; } else if (len == pipesize8 || len == pipesizemax8) { D("setting compatibility with FreeBSD 8"); is7 = 0; } switch (sopt->sopt_name) { default: printf("dummynet: -- unknown option %d", sopt->sopt_name); error = EINVAL; break; case IP_DUMMYNET_FLUSH: oid_fill(&oid, sizeof(oid), DN_CMD_FLUSH, DN_API_VERSION); do_config(&oid, oid.len); break; case IP_DUMMYNET_DEL: v = malloc(len, M_TEMP, M_WAITOK); error = sooptcopyin(sopt, v, len, len); if (error) break; error = dn_compat_del(v); free(v, M_TEMP); break; case IP_DUMMYNET_CONFIGURE: v = malloc(len, M_TEMP, M_WAITOK); error = sooptcopyin(sopt, v, len, len); if (error) break; error = dn_compat_configure(v); free(v, M_TEMP); break; case IP_DUMMYNET_GET: { void *buf; int ret; int original_size = sopt->sopt_valsize; int size; ret = dummynet_get(sopt, &buf); if (ret) return 0;//XXX ? size = sopt->sopt_valsize; sopt->sopt_valsize = original_size; D("size=%d, buf=%p", size, buf); ret = sooptcopyout(sopt, buf, size); if (ret) printf(" %s ERROR sooptcopyout\n", __FUNCTION__); if (buf) free(buf, M_DUMMYNET); } } return error; } Index: head/sys/netpfil/ipfw/ip_dn_io.c =================================================================== --- head/sys/netpfil/ipfw/ip_dn_io.c (revision 300778) +++ head/sys/netpfil/ipfw/ip_dn_io.c (revision 300779) @@ -1,951 +1,984 @@ /*- * Copyright (c) 2010 Luigi Rizzo, Riccardo Panicucci, Universita` di Pisa * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Dummynet portions related to packet handling. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ #include #include #include #include /* ip_len, ip_off */ #include /* ip_output(), IP_FORWARDING */ #include #include #include /* various ether_* routines */ #include /* for ip6_input, ip6_output prototypes */ #include #include #include #include +#ifdef NEW_AQM +#include +#endif #include /* * We keep a private variable for the simulation time, but we could * probably use an existing one ("softticks" in sys/kern/kern_timeout.c) * instead of dn_cfg.curr_time */ struct dn_parms dn_cfg; //VNET_DEFINE(struct dn_parms, _base_dn_cfg); static long tick_last; /* Last tick duration (usec). */ static long tick_delta; /* Last vs standard tick diff (usec). */ static long tick_delta_sum; /* Accumulated tick difference (usec).*/ static long tick_adjustment; /* Tick adjustments done. */ static long tick_lost; /* Lost(coalesced) ticks number. */ /* Adjusted vs non-adjusted curr_time difference (ticks). */ static long tick_diff; static unsigned long io_pkt; static unsigned long io_pkt_fast; -static unsigned long io_pkt_drop; +#ifdef NEW_AQM +unsigned long io_pkt_drop; +#else +static unsigned long io_pkt_drop; +#endif /* * We use a heap to store entities for which we have pending timer events. * The heap is checked at every tick and all entities with expired events * are extracted. */ MALLOC_DEFINE(M_DUMMYNET, "dummynet", "dummynet heap"); extern void (*bridge_dn_p)(struct mbuf *, struct ifnet *); #ifdef SYSCTL_NODE /* * Because of the way the SYSBEGIN/SYSEND macros work on other * platforms, there should not be functions between them. * So keep the handlers outside the block. */ static int sysctl_hash_size(SYSCTL_HANDLER_ARGS) { int error, value; value = dn_cfg.hash_size; error = sysctl_handle_int(oidp, &value, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (value < 16 || value > 65536) return (EINVAL); dn_cfg.hash_size = value; return (0); } static int sysctl_limits(SYSCTL_HANDLER_ARGS) { int error; long value; if (arg2 != 0) value = dn_cfg.slot_limit; else value = dn_cfg.byte_limit; error = sysctl_handle_long(oidp, &value, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (arg2 != 0) { if (value < 1) return (EINVAL); dn_cfg.slot_limit = value; } else { if (value < 1500) return (EINVAL); dn_cfg.byte_limit = value; } return (0); } SYSBEGIN(f4) SYSCTL_DECL(_net_inet); SYSCTL_DECL(_net_inet_ip); +#ifdef NEW_AQM +SYSCTL_NODE(_net_inet_ip, OID_AUTO, dummynet, CTLFLAG_RW, 0, "Dummynet"); +#else static SYSCTL_NODE(_net_inet_ip, OID_AUTO, dummynet, CTLFLAG_RW, 0, "Dummynet"); +#endif /* wrapper to pass dn_cfg fields to SYSCTL_* */ //#define DC(x) (&(VNET_NAME(_base_dn_cfg).x)) #define DC(x) (&(dn_cfg.x)) /* parameters */ SYSCTL_PROC(_net_inet_ip_dummynet, OID_AUTO, hash_size, CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_hash_size, "I", "Default hash table size"); SYSCTL_PROC(_net_inet_ip_dummynet, OID_AUTO, pipe_slot_limit, CTLTYPE_LONG | CTLFLAG_RW, 0, 1, sysctl_limits, "L", "Upper limit in slots for pipe queue."); SYSCTL_PROC(_net_inet_ip_dummynet, OID_AUTO, pipe_byte_limit, CTLTYPE_LONG | CTLFLAG_RW, 0, 0, sysctl_limits, "L", "Upper limit in bytes for pipe queue."); SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, io_fast, CTLFLAG_RW, DC(io_fast), 0, "Enable fast dummynet io."); SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, debug, CTLFLAG_RW, DC(debug), 0, "Dummynet debug level"); /* RED parameters */ SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, red_lookup_depth, CTLFLAG_RD, DC(red_lookup_depth), 0, "Depth of RED lookup table"); SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, red_avg_pkt_size, CTLFLAG_RD, DC(red_avg_pkt_size), 0, "RED Medium packet size"); SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, red_max_pkt_size, CTLFLAG_RD, DC(red_max_pkt_size), 0, "RED Max packet size"); /* time adjustment */ SYSCTL_LONG(_net_inet_ip_dummynet, OID_AUTO, tick_delta, CTLFLAG_RD, &tick_delta, 0, "Last vs standard tick difference (usec)."); SYSCTL_LONG(_net_inet_ip_dummynet, OID_AUTO, tick_delta_sum, CTLFLAG_RD, &tick_delta_sum, 0, "Accumulated tick difference (usec)."); SYSCTL_LONG(_net_inet_ip_dummynet, OID_AUTO, tick_adjustment, CTLFLAG_RD, &tick_adjustment, 0, "Tick adjustments done."); SYSCTL_LONG(_net_inet_ip_dummynet, OID_AUTO, tick_diff, CTLFLAG_RD, &tick_diff, 0, "Adjusted vs non-adjusted curr_time difference (ticks)."); SYSCTL_LONG(_net_inet_ip_dummynet, OID_AUTO, tick_lost, CTLFLAG_RD, &tick_lost, 0, "Number of ticks coalesced by dummynet taskqueue."); /* Drain parameters */ SYSCTL_UINT(_net_inet_ip_dummynet, OID_AUTO, expire, CTLFLAG_RW, DC(expire), 0, "Expire empty queues/pipes"); SYSCTL_UINT(_net_inet_ip_dummynet, OID_AUTO, expire_cycle, CTLFLAG_RD, DC(expire_cycle), 0, "Expire cycle for queues/pipes"); /* statistics */ SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, schk_count, CTLFLAG_RD, DC(schk_count), 0, "Number of schedulers"); SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, si_count, CTLFLAG_RD, DC(si_count), 0, "Number of scheduler instances"); SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, fsk_count, CTLFLAG_RD, DC(fsk_count), 0, "Number of flowsets"); SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, queue_count, CTLFLAG_RD, DC(queue_count), 0, "Number of queues"); SYSCTL_ULONG(_net_inet_ip_dummynet, OID_AUTO, io_pkt, CTLFLAG_RD, &io_pkt, 0, "Number of packets passed to dummynet."); SYSCTL_ULONG(_net_inet_ip_dummynet, OID_AUTO, io_pkt_fast, CTLFLAG_RD, &io_pkt_fast, 0, "Number of packets bypassed dummynet scheduler."); SYSCTL_ULONG(_net_inet_ip_dummynet, OID_AUTO, io_pkt_drop, CTLFLAG_RD, &io_pkt_drop, 0, "Number of packets dropped by dummynet."); #undef DC SYSEND #endif static void dummynet_send(struct mbuf *); /* * Packets processed by dummynet have an mbuf tag associated with * them that carries their dummynet state. * Outside dummynet, only the 'rule' field is relevant, and it must * be at the beginning of the structure. */ struct dn_pkt_tag { struct ipfw_rule_ref rule; /* matching rule */ /* second part, dummynet specific */ int dn_dir; /* action when packet comes out.*/ /* see ip_fw_private.h */ uint64_t output_time; /* when the pkt is due for delivery*/ struct ifnet *ifp; /* interface, for ip_output */ struct _ip6dn_args ip6opt; /* XXX ipv6 options */ }; /* * Return the mbuf tag holding the dummynet state (it should * be the first one on the list). */ static struct dn_pkt_tag * dn_tag_get(struct mbuf *m) { struct m_tag *mtag = m_tag_first(m); +#ifdef NEW_AQM + /* XXX: to skip ts m_tag. For Debugging only*/ + if (mtag != NULL && mtag->m_tag_id == DN_AQM_MTAG_TS) { + m_tag_delete(m,mtag); + mtag = m_tag_first(m); + D("skip TS tag"); + } +#endif KASSERT(mtag != NULL && mtag->m_tag_cookie == MTAG_ABI_COMPAT && mtag->m_tag_id == PACKET_TAG_DUMMYNET, ("packet on dummynet queue w/o dummynet tag!")); return (struct dn_pkt_tag *)(mtag+1); } +#ifndef NEW_AQM static inline void mq_append(struct mq *q, struct mbuf *m) { #ifdef USERSPACE // buffers from netmap need to be copied // XXX note that the routine is not expected to fail ND("append %p to %p", m, q); if (m->m_flags & M_STACK) { struct mbuf *m_new; void *p; int l, ofs; ofs = m->m_data - m->__m_extbuf; // XXX allocate MGETHDR(m_new, M_NOWAIT, MT_DATA); ND("*** WARNING, volatile buf %p ext %p %d dofs %d m_new %p", m, m->__m_extbuf, m->__m_extlen, ofs, m_new); p = m_new->__m_extbuf; /* new pointer */ l = m_new->__m_extlen; /* new len */ if (l <= m->__m_extlen) { panic("extlen too large"); } *m_new = *m; // copy m_new->m_flags &= ~M_STACK; m_new->__m_extbuf = p; // point to new buffer _pkt_copy(m->__m_extbuf, p, m->__m_extlen); m_new->m_data = p + ofs; m = m_new; } #endif /* USERSPACE */ if (q->head == NULL) q->head = m; else q->tail->m_nextpkt = m; q->count++; q->tail = m; m->m_nextpkt = NULL; } +#endif /* * Dispose a list of packet. Use a functions so if we need to do * more work, this is a central point to do it. */ void dn_free_pkts(struct mbuf *mnext) { struct mbuf *m; while ((m = mnext) != NULL) { mnext = m->m_nextpkt; FREE_PKT(m); } } static int red_drops (struct dn_queue *q, int len) { /* * RED algorithm * * RED calculates the average queue size (avg) using a low-pass filter * with an exponential weighted (w_q) moving average: * avg <- (1-w_q) * avg + w_q * q_size * where q_size is the queue length (measured in bytes or * packets). * * If q_size == 0, we compute the idle time for the link, and set * avg = (1 - w_q)^(idle/s) * where s is the time needed for transmitting a medium-sized packet. * * Now, if avg < min_th the packet is enqueued. * If avg > max_th the packet is dropped. Otherwise, the packet is * dropped with probability P function of avg. */ struct dn_fsk *fs = q->fs; int64_t p_b = 0; /* Queue in bytes or packets? */ uint32_t q_size = (fs->fs.flags & DN_QSIZE_BYTES) ? q->ni.len_bytes : q->ni.length; /* Average queue size estimation. */ if (q_size != 0) { /* Queue is not empty, avg <- avg + (q_size - avg) * w_q */ int diff = SCALE(q_size) - q->avg; int64_t v = SCALE_MUL((int64_t)diff, (int64_t)fs->w_q); q->avg += (int)v; } else { /* * Queue is empty, find for how long the queue has been * empty and use a lookup table for computing * (1 - * w_q)^(idle_time/s) where s is the time to send a * (small) packet. * XXX check wraps... */ if (q->avg) { u_int t = div64((dn_cfg.curr_time - q->q_time), fs->lookup_step); q->avg = (t < fs->lookup_depth) ? SCALE_MUL(q->avg, fs->w_q_lookup[t]) : 0; } } /* Should i drop? */ if (q->avg < fs->min_th) { q->count = -1; return (0); /* accept packet */ } if (q->avg >= fs->max_th) { /* average queue >= max threshold */ if (fs->fs.flags & DN_IS_ECN) return (1); if (fs->fs.flags & DN_IS_GENTLE_RED) { /* * According to Gentle-RED, if avg is greater than * max_th the packet is dropped with a probability * p_b = c_3 * avg - c_4 * where c_3 = (1 - max_p) / max_th * c_4 = 1 - 2 * max_p */ p_b = SCALE_MUL((int64_t)fs->c_3, (int64_t)q->avg) - fs->c_4; } else { q->count = -1; return (1); } } else if (q->avg > fs->min_th) { if (fs->fs.flags & DN_IS_ECN) return (1); /* * We compute p_b using the linear dropping function * p_b = c_1 * avg - c_2 * where c_1 = max_p / (max_th - min_th) * c_2 = max_p * min_th / (max_th - min_th) */ p_b = SCALE_MUL((int64_t)fs->c_1, (int64_t)q->avg) - fs->c_2; } if (fs->fs.flags & DN_QSIZE_BYTES) p_b = div64((p_b * len) , fs->max_pkt_size); if (++q->count == 0) q->random = random() & 0xffff; else { /* * q->count counts packets arrived since last drop, so a greater * value of q->count means a greater packet drop probability. */ if (SCALE_MUL(p_b, SCALE((int64_t)q->count)) > q->random) { q->count = 0; /* After a drop we calculate a new random value. */ q->random = random() & 0xffff; return (1); /* drop */ } } /* End of RED algorithm. */ return (0); /* accept */ } /* * ECN/ECT Processing (partially adopted from altq) */ -static int +#ifndef NEW_AQM +static +#endif +int ecn_mark(struct mbuf* m) { struct ip *ip; ip = mtod(m, struct ip *); switch (ip->ip_v) { case IPVERSION: { uint16_t old; if ((ip->ip_tos & IPTOS_ECN_MASK) == IPTOS_ECN_NOTECT) return (0); /* not-ECT */ if ((ip->ip_tos & IPTOS_ECN_MASK) == IPTOS_ECN_CE) return (1); /* already marked */ /* * ecn-capable but not marked, * mark CE and update checksum */ old = *(uint16_t *)ip; ip->ip_tos |= IPTOS_ECN_CE; ip->ip_sum = cksum_adjust(ip->ip_sum, old, *(uint16_t *)ip); return (1); } #ifdef INET6 case (IPV6_VERSION >> 4): { struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); u_int32_t flowlabel; flowlabel = ntohl(ip6->ip6_flow); if ((flowlabel >> 28) != 6) return (0); /* version mismatch! */ if ((flowlabel & (IPTOS_ECN_MASK << 20)) == (IPTOS_ECN_NOTECT << 20)) return (0); /* not-ECT */ if ((flowlabel & (IPTOS_ECN_MASK << 20)) == (IPTOS_ECN_CE << 20)) return (1); /* already marked */ /* * ecn-capable but not marked, mark CE */ flowlabel |= (IPTOS_ECN_CE << 20); ip6->ip6_flow = htonl(flowlabel); return (1); } #endif } return (0); } /* * Enqueue a packet in q, subject to space and queue management policy * (whose parameters are in q->fs). * Update stats for the queue and the scheduler. * Return 0 on success, 1 on drop. The packet is consumed anyways. */ int dn_enqueue(struct dn_queue *q, struct mbuf* m, int drop) { struct dn_fs *f; struct dn_flow *ni; /* stats for scheduler instance */ uint64_t len; if (q->fs == NULL || q->_si == NULL) { printf("%s fs %p si %p, dropping\n", __FUNCTION__, q->fs, q->_si); FREE_PKT(m); return 1; } f = &(q->fs->fs); ni = &q->_si->ni; len = m->m_pkthdr.len; /* Update statistics, then check reasons to drop pkt. */ q->ni.tot_bytes += len; q->ni.tot_pkts++; ni->tot_bytes += len; ni->tot_pkts++; if (drop) goto drop; if (f->plr && random() < f->plr) goto drop; +#ifdef NEW_AQM + /* Call AQM enqueue function */ + if (q->fs->aqmfp) + return q->fs->aqmfp->enqueue(q ,m); +#endif if (f->flags & DN_IS_RED && red_drops(q, m->m_pkthdr.len)) { if (!(f->flags & DN_IS_ECN) || !ecn_mark(m)) goto drop; } if (f->flags & DN_QSIZE_BYTES) { if (q->ni.len_bytes > f->qsize) goto drop; } else if (q->ni.length >= f->qsize) { goto drop; } mq_append(&q->mq, m); q->ni.length++; q->ni.len_bytes += len; ni->length++; ni->len_bytes += len; return (0); drop: io_pkt_drop++; q->ni.drops++; ni->drops++; FREE_PKT(m); return (1); } /* * Fetch packets from the delay line which are due now. If there are * leftover packets, reinsert the delay line in the heap. * Runs under scheduler lock. */ static void transmit_event(struct mq *q, struct delay_line *dline, uint64_t now) { struct mbuf *m; struct dn_pkt_tag *pkt = NULL; dline->oid.subtype = 0; /* not in heap */ while ((m = dline->mq.head) != NULL) { pkt = dn_tag_get(m); if (!DN_KEY_LEQ(pkt->output_time, now)) break; dline->mq.head = m->m_nextpkt; dline->mq.count--; mq_append(q, m); } if (m != NULL) { dline->oid.subtype = 1; /* in heap */ heap_insert(&dn_cfg.evheap, pkt->output_time, dline); } } /* * Convert the additional MAC overheads/delays into an equivalent * number of bits for the given data rate. The samples are * in milliseconds so we need to divide by 1000. */ static uint64_t extra_bits(struct mbuf *m, struct dn_schk *s) { int index; uint64_t bits; struct dn_profile *pf = s->profile; if (!pf || pf->samples_no == 0) return 0; index = random() % pf->samples_no; bits = div64((uint64_t)pf->samples[index] * s->link.bandwidth, 1000); if (index >= pf->loss_level) { struct dn_pkt_tag *dt = dn_tag_get(m); if (dt) dt->dn_dir = DIR_DROP; } return bits; } /* * Send traffic from a scheduler instance due by 'now'. * Return a pointer to the head of the queue. */ static struct mbuf * serve_sched(struct mq *q, struct dn_sch_inst *si, uint64_t now) { struct mq def_q; struct dn_schk *s = si->sched; struct mbuf *m = NULL; int delay_line_idle = (si->dline.mq.head == NULL); int done, bw; if (q == NULL) { q = &def_q; q->head = NULL; } bw = s->link.bandwidth; si->kflags &= ~DN_ACTIVE; if (bw > 0) si->credit += (now - si->sched_time) * bw; else si->credit = 0; si->sched_time = now; done = 0; while (si->credit >= 0 && (m = s->fp->dequeue(si)) != NULL) { uint64_t len_scaled; done++; len_scaled = (bw == 0) ? 0 : hz * (m->m_pkthdr.len * 8 + extra_bits(m, s)); si->credit -= len_scaled; /* Move packet in the delay line */ dn_tag_get(m)->output_time = dn_cfg.curr_time + s->link.delay ; mq_append(&si->dline.mq, m); } /* * If credit >= 0 the instance is idle, mark time. * Otherwise put back in the heap, and adjust the output * time of the last inserted packet, m, which was too early. */ if (si->credit >= 0) { si->idle_time = now; } else { uint64_t t; KASSERT (bw > 0, ("bw=0 and credit<0 ?")); t = div64(bw - 1 - si->credit, bw); if (m) dn_tag_get(m)->output_time += t; si->kflags |= DN_ACTIVE; heap_insert(&dn_cfg.evheap, now + t, si); } if (delay_line_idle && done) transmit_event(q, &si->dline, now); return q->head; } /* * The timer handler for dummynet. Time is computed in ticks, but * but the code is tolerant to the actual rate at which this is called. * Once complete, the function reschedules itself for the next tick. */ void dummynet_task(void *context, int pending) { struct timeval t; struct mq q = { NULL, NULL }; /* queue to accumulate results */ CURVNET_SET((struct vnet *)context); DN_BH_WLOCK(); /* Update number of lost(coalesced) ticks. */ tick_lost += pending - 1; getmicrouptime(&t); /* Last tick duration (usec). */ tick_last = (t.tv_sec - dn_cfg.prev_t.tv_sec) * 1000000 + (t.tv_usec - dn_cfg.prev_t.tv_usec); /* Last tick vs standard tick difference (usec). */ tick_delta = (tick_last * hz - 1000000) / hz; /* Accumulated tick difference (usec). */ tick_delta_sum += tick_delta; dn_cfg.prev_t = t; /* * Adjust curr_time if the accumulated tick difference is * greater than the 'standard' tick. Since curr_time should * be monotonically increasing, we do positive adjustments * as required, and throttle curr_time in case of negative * adjustment. */ dn_cfg.curr_time++; if (tick_delta_sum - tick >= 0) { int diff = tick_delta_sum / tick; dn_cfg.curr_time += diff; tick_diff += diff; tick_delta_sum %= tick; tick_adjustment++; } else if (tick_delta_sum + tick <= 0) { dn_cfg.curr_time--; tick_diff--; tick_delta_sum += tick; tick_adjustment++; } /* serve pending events, accumulate in q */ for (;;) { struct dn_id *p; /* generic parameter to handler */ if (dn_cfg.evheap.elements == 0 || DN_KEY_LT(dn_cfg.curr_time, HEAP_TOP(&dn_cfg.evheap)->key)) break; p = HEAP_TOP(&dn_cfg.evheap)->object; heap_extract(&dn_cfg.evheap, NULL); if (p->type == DN_SCH_I) { serve_sched(&q, (struct dn_sch_inst *)p, dn_cfg.curr_time); } else { /* extracted a delay line */ transmit_event(&q, (struct delay_line *)p, dn_cfg.curr_time); } } if (dn_cfg.expire && ++dn_cfg.expire_cycle >= dn_cfg.expire) { dn_cfg.expire_cycle = 0; dn_drain_scheduler(); dn_drain_queue(); } dn_reschedule(); DN_BH_WUNLOCK(); if (q.head != NULL) dummynet_send(q.head); CURVNET_RESTORE(); } /* * forward a chain of packets to the proper destination. * This runs outside the dummynet lock. */ static void dummynet_send(struct mbuf *m) { struct mbuf *n; for (; m != NULL; m = n) { struct ifnet *ifp = NULL; /* gcc 3.4.6 complains */ struct m_tag *tag; int dst; n = m->m_nextpkt; m->m_nextpkt = NULL; tag = m_tag_first(m); if (tag == NULL) { /* should not happen */ dst = DIR_DROP; } else { struct dn_pkt_tag *pkt = dn_tag_get(m); /* extract the dummynet info, rename the tag * to carry reinject info. */ if (pkt->dn_dir == (DIR_OUT | PROTO_LAYER2) && pkt->ifp == NULL) { dst = DIR_DROP; } else { dst = pkt->dn_dir; ifp = pkt->ifp; tag->m_tag_cookie = MTAG_IPFW_RULE; tag->m_tag_id = 0; } } switch (dst) { case DIR_OUT: ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL); break ; case DIR_IN : netisr_dispatch(NETISR_IP, m); break; #ifdef INET6 case DIR_IN | PROTO_IPV6: netisr_dispatch(NETISR_IPV6, m); break; case DIR_OUT | PROTO_IPV6: ip6_output(m, NULL, NULL, IPV6_FORWARDING, NULL, NULL, NULL); break; #endif case DIR_FWD | PROTO_IFB: /* DN_TO_IFB_FWD: */ if (bridge_dn_p != NULL) ((*bridge_dn_p)(m, ifp)); else printf("dummynet: if_bridge not loaded\n"); break; case DIR_IN | PROTO_LAYER2: /* DN_TO_ETH_DEMUX: */ /* * The Ethernet code assumes the Ethernet header is * contiguous in the first mbuf header. * Insure this is true. */ if (m->m_len < ETHER_HDR_LEN && (m = m_pullup(m, ETHER_HDR_LEN)) == NULL) { printf("dummynet/ether: pullup failed, " "dropping packet\n"); break; } ether_demux(m->m_pkthdr.rcvif, m); break; case DIR_OUT | PROTO_LAYER2: /* N_TO_ETH_OUT: */ ether_output_frame(ifp, m); break; case DIR_DROP: /* drop the packet after some time */ FREE_PKT(m); break; default: printf("dummynet: bad switch %d!\n", dst); FREE_PKT(m); break; } } } static inline int tag_mbuf(struct mbuf *m, int dir, struct ip_fw_args *fwa) { struct dn_pkt_tag *dt; struct m_tag *mtag; mtag = m_tag_get(PACKET_TAG_DUMMYNET, sizeof(*dt), M_NOWAIT | M_ZERO); if (mtag == NULL) return 1; /* Cannot allocate packet header. */ m_tag_prepend(m, mtag); /* Attach to mbuf chain. */ dt = (struct dn_pkt_tag *)(mtag + 1); dt->rule = fwa->rule; dt->rule.info &= IPFW_ONEPASS; /* only keep this info */ dt->dn_dir = dir; dt->ifp = fwa->oif; /* dt->output tame is updated as we move through */ dt->output_time = dn_cfg.curr_time; return 0; } /* * dummynet hook for packets. * We use the argument to locate the flowset fs and the sched_set sch * associated to it. The we apply flow_mask and sched_mask to * determine the queue and scheduler instances. * * dir where shall we send the packet after dummynet. * *m0 the mbuf with the packet * ifp the 'ifp' parameter from the caller. * NULL in ip_input, destination interface in ip_output, */ int dummynet_io(struct mbuf **m0, int dir, struct ip_fw_args *fwa) { struct mbuf *m = *m0; struct dn_fsk *fs = NULL; struct dn_sch_inst *si; struct dn_queue *q = NULL; /* default */ int fs_id = (fwa->rule.info & IPFW_INFO_MASK) + ((fwa->rule.info & IPFW_IS_PIPE) ? 2*DN_MAX_ID : 0); DN_BH_WLOCK(); io_pkt++; /* we could actually tag outside the lock, but who cares... */ if (tag_mbuf(m, dir, fwa)) goto dropit; if (dn_cfg.busy) { /* if the upper half is busy doing something expensive, * lets queue the packet and move forward */ mq_append(&dn_cfg.pending, m); m = *m0 = NULL; /* consumed */ goto done; /* already active, nothing to do */ } /* XXX locate_flowset could be optimised with a direct ref. */ fs = dn_ht_find(dn_cfg.fshash, fs_id, 0, NULL); if (fs == NULL) goto dropit; /* This queue/pipe does not exist! */ if (fs->sched == NULL) /* should not happen */ goto dropit; /* find scheduler instance, possibly applying sched_mask */ si = ipdn_si_find(fs->sched, &(fwa->f_id)); if (si == NULL) goto dropit; /* * If the scheduler supports multiple queues, find the right one * (otherwise it will be ignored by enqueue). */ if (fs->sched->fp->flags & DN_MULTIQUEUE) { q = ipdn_q_find(fs, si, &(fwa->f_id)); if (q == NULL) goto dropit; } if (fs->sched->fp->enqueue(si, q, m)) { /* packet was dropped by enqueue() */ m = *m0 = NULL; + + /* dn_enqueue already increases io_pkt_drop */ + io_pkt_drop--; + goto dropit; } if (si->kflags & DN_ACTIVE) { m = *m0 = NULL; /* consumed */ goto done; /* already active, nothing to do */ } /* compute the initial allowance */ if (si->idle_time < dn_cfg.curr_time) { /* Do this only on the first packet on an idle pipe */ struct dn_link *p = &fs->sched->link; si->sched_time = dn_cfg.curr_time; si->credit = dn_cfg.io_fast ? p->bandwidth : 0; if (p->burst) { uint64_t burst = (dn_cfg.curr_time - si->idle_time) * p->bandwidth; if (burst > p->burst) burst = p->burst; si->credit += burst; } } /* pass through scheduler and delay line */ m = serve_sched(NULL, si, dn_cfg.curr_time); /* optimization -- pass it back to ipfw for immediate send */ /* XXX Don't call dummynet_send() if scheduler return the packet * just enqueued. This avoid a lock order reversal. * */ if (/*dn_cfg.io_fast &&*/ m == *m0 && (dir & PROTO_LAYER2) == 0 ) { /* fast io, rename the tag * to carry reinject info. */ struct m_tag *tag = m_tag_first(m); tag->m_tag_cookie = MTAG_IPFW_RULE; tag->m_tag_id = 0; io_pkt_fast++; if (m->m_nextpkt != NULL) { printf("dummynet: fast io: pkt chain detected!\n"); m->m_nextpkt = NULL; } m = NULL; } else { *m0 = NULL; } done: DN_BH_WUNLOCK(); if (m) dummynet_send(m); return 0; dropit: io_pkt_drop++; DN_BH_WUNLOCK(); if (m) FREE_PKT(m); *m0 = NULL; return (fs && (fs->fs.flags & DN_NOERROR)) ? 0 : ENOBUFS; } Index: head/sys/netpfil/ipfw/ip_dn_private.h =================================================================== --- head/sys/netpfil/ipfw/ip_dn_private.h (revision 300778) +++ head/sys/netpfil/ipfw/ip_dn_private.h (revision 300779) @@ -1,404 +1,463 @@ /*- * Copyright (c) 2010 Luigi Rizzo, Riccardo Panicucci, Universita` di Pisa * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * internal dummynet APIs. * * $FreeBSD$ */ #ifndef _IP_DN_PRIVATE_H #define _IP_DN_PRIVATE_H /* debugging support * use ND() to remove debugging, D() to print a line, * DX(level, ...) to print above a certain level * If you redefine D() you are expected to redefine all. */ #ifndef D #define ND(fmt, ...) do {} while (0) #define D1(fmt, ...) do {} while (0) #define D(fmt, ...) printf("%-10s " fmt "\n", \ __FUNCTION__, ## __VA_ARGS__) #define DX(lev, fmt, ...) do { \ if (dn_cfg.debug > lev) D(fmt, ## __VA_ARGS__); } while (0) #endif MALLOC_DECLARE(M_DUMMYNET); #ifndef __linux__ #define div64(a, b) ((int64_t)(a) / (int64_t)(b)) #endif #define DN_LOCK_INIT() do { \ mtx_init(&dn_cfg.uh_mtx, "dn_uh", NULL, MTX_DEF); \ mtx_init(&dn_cfg.bh_mtx, "dn_bh", NULL, MTX_DEF); \ } while (0) #define DN_LOCK_DESTROY() do { \ mtx_destroy(&dn_cfg.uh_mtx); \ mtx_destroy(&dn_cfg.bh_mtx); \ } while (0) #if 0 /* not used yet */ #define DN_UH_RLOCK() mtx_lock(&dn_cfg.uh_mtx) #define DN_UH_RUNLOCK() mtx_unlock(&dn_cfg.uh_mtx) #define DN_UH_WLOCK() mtx_lock(&dn_cfg.uh_mtx) #define DN_UH_WUNLOCK() mtx_unlock(&dn_cfg.uh_mtx) #define DN_UH_LOCK_ASSERT() mtx_assert(&dn_cfg.uh_mtx, MA_OWNED) #endif #define DN_BH_RLOCK() mtx_lock(&dn_cfg.uh_mtx) #define DN_BH_RUNLOCK() mtx_unlock(&dn_cfg.uh_mtx) #define DN_BH_WLOCK() mtx_lock(&dn_cfg.uh_mtx) #define DN_BH_WUNLOCK() mtx_unlock(&dn_cfg.uh_mtx) #define DN_BH_LOCK_ASSERT() mtx_assert(&dn_cfg.uh_mtx, MA_OWNED) SLIST_HEAD(dn_schk_head, dn_schk); SLIST_HEAD(dn_sch_inst_head, dn_sch_inst); SLIST_HEAD(dn_fsk_head, dn_fsk); SLIST_HEAD(dn_queue_head, dn_queue); SLIST_HEAD(dn_alg_head, dn_alg); +#ifdef NEW_AQM +SLIST_HEAD(dn_aqm_head, dn_aqm); /* for new AQMs */ +#endif + struct mq { /* a basic queue of packets*/ struct mbuf *head, *tail; int count; }; static inline void set_oid(struct dn_id *o, int type, int len) { o->type = type; o->len = len; o->subtype = 0; } /* * configuration and global data for a dummynet instance * * When a configuration is modified from userland, 'id' is incremented * so we can use the value to check for stale pointers. */ struct dn_parms { uint32_t id; /* configuration version */ /* defaults (sysctl-accessible) */ int red_lookup_depth; int red_avg_pkt_size; int red_max_pkt_size; int hash_size; int max_hash_size; long byte_limit; /* max queue sizes */ long slot_limit; int io_fast; int debug; /* timekeeping */ struct timeval prev_t; /* last time dummynet_tick ran */ struct dn_heap evheap; /* scheduled events */ /* counters of objects -- used for reporting space */ int schk_count; int si_count; int fsk_count; int queue_count; /* ticks and other stuff */ uint64_t curr_time; /* flowsets and schedulers are in hash tables, with 'hash_size' * buckets. fshash is looked up at every packet arrival * so better be generous if we expect many entries. */ struct dn_ht *fshash; struct dn_ht *schedhash; /* list of flowsets without a scheduler -- use sch_chain */ struct dn_fsk_head fsu; /* list of unlinked flowsets */ struct dn_alg_head schedlist; /* list of algorithms */ +#ifdef NEW_AQM + struct dn_aqm_head aqmlist; /* list of AQMs */ +#endif /* Store the fs/sch to scan when draining. The value is the * bucket number of the hash table. Expire can be disabled * with net.inet.ip.dummynet.expire=0, or it happens every * expire ticks. **/ int drain_fs; int drain_sch; uint32_t expire; uint32_t expire_cycle; /* tick count */ int init_done; /* if the upper half is busy doing something long, * can set the busy flag and we will enqueue packets in * a queue for later processing. */ int busy; struct mq pending; #ifdef _KERNEL /* * This file is normally used in the kernel, unless we do * some userland tests, in which case we do not need a mtx. * uh_mtx arbitrates between system calls and also * protects fshash, schedhash and fsunlinked. * These structures are readonly for the lower half. * bh_mtx protects all other structures which may be * modified upon packet arrivals */ #if defined( __linux__ ) || defined( _WIN32 ) spinlock_t uh_mtx; spinlock_t bh_mtx; #else struct mtx uh_mtx; struct mtx bh_mtx; #endif #endif /* _KERNEL */ }; /* * Delay line, contains all packets on output from a link. * Every scheduler instance has one. */ struct delay_line { struct dn_id oid; struct dn_sch_inst *si; struct mq mq; }; /* * The kernel side of a flowset. It is linked in a hash table * of flowsets, and in a list of children of their parent scheduler. * qht is either the queue or (if HAVE_MASK) a hash table queues. * Note that the mask to use is the (flow_mask|sched_mask), which * changes as we attach/detach schedulers. So we store it here. * * XXX If we want to add scheduler-specific parameters, we need to * put them in external storage because the scheduler may not be * available when the fsk is created. */ struct dn_fsk { /* kernel side of a flowset */ struct dn_fs fs; SLIST_ENTRY(dn_fsk) fsk_next; /* hash chain for fshash */ struct ipfw_flow_id fsk_mask; /* qht is a hash table of queues, or just a single queue * a bit in fs.flags tells us which one */ struct dn_ht *qht; struct dn_schk *sched; /* Sched we are linked to */ SLIST_ENTRY(dn_fsk) sch_chain; /* list of fsk attached to sched */ /* bucket index used by drain routine to drain queues for this * flowset */ int drain_bucket; /* Parameter realted to RED / GRED */ /* original values are in dn_fs*/ int w_q ; /* queue weight (scaled) */ int max_th ; /* maximum threshold for queue (scaled) */ int min_th ; /* minimum threshold for queue (scaled) */ int max_p ; /* maximum value for p_b (scaled) */ u_int c_1 ; /* max_p/(max_th-min_th) (scaled) */ u_int c_2 ; /* max_p*min_th/(max_th-min_th) (scaled) */ u_int c_3 ; /* for GRED, (1-max_p)/max_th (scaled) */ u_int c_4 ; /* for GRED, 1 - 2*max_p (scaled) */ u_int * w_q_lookup ; /* lookup table for computing (1-w_q)^t */ u_int lookup_depth ; /* depth of lookup table */ int lookup_step ; /* granularity inside the lookup table */ int lookup_weight ; /* equal to (1-w_q)^t / (1-w_q)^(t+1) */ int avg_pkt_size ; /* medium packet size */ int max_pkt_size ; /* max packet size */ +#ifdef NEW_AQM + struct dn_aqm *aqmfp; /* Pointer to AQM functions */ + void *aqmcfg; /* configuration parameters for AQM */ +#endif }; /* * A queue is created as a child of a flowset unless it belongs to * a !MULTIQUEUE scheduler. It is normally in a hash table in the * flowset. fs always points to the parent flowset. * si normally points to the sch_inst, unless the flowset has been * detached from the scheduler -- in this case si == NULL and we * should not enqueue. */ struct dn_queue { struct dn_flow ni; /* oid, flow_id, stats */ struct mq mq; /* packets queue */ struct dn_sch_inst *_si; /* owner scheduler instance */ SLIST_ENTRY(dn_queue) q_next; /* hash chain list for qht */ struct dn_fsk *fs; /* parent flowset. */ /* RED parameters */ int avg; /* average queue length est. (scaled) */ int count; /* arrivals since last RED drop */ int random; /* random value (scaled) */ uint64_t q_time; /* start of queue idle time */ +#ifdef NEW_AQM + void *aqm_status; /* per-queue status variables*/ +#endif }; /* * The kernel side of a scheduler. Contains the userland config, * a link, pointer to extra config arguments from command line, * kernel flags, and a pointer to the scheduler methods. * It is stored in a hash table, and holds a list of all * flowsets and scheduler instances. * XXX sch must be at the beginning, see schk_hash(). */ struct dn_schk { struct dn_sch sch; struct dn_alg *fp; /* Pointer to scheduler functions */ struct dn_link link; /* The link, embedded */ struct dn_profile *profile; /* delay profile, if any */ struct dn_id *cfg; /* extra config arguments */ SLIST_ENTRY(dn_schk) schk_next; /* hash chain for schedhash */ struct dn_fsk_head fsk_list; /* all fsk linked to me */ struct dn_fsk *fs; /* Flowset for !MULTIQUEUE */ /* bucket index used by the drain routine to drain the scheduler * instance for this flowset. */ int drain_bucket; /* Hash table of all instances (through sch.sched_mask) * or single instance if no mask. Always valid. */ struct dn_ht *siht; }; /* * Scheduler instance. * Contains variables and all queues relative to a this instance. * This struct is created a runtime. */ struct dn_sch_inst { struct dn_flow ni; /* oid, flowid and stats */ SLIST_ENTRY(dn_sch_inst) si_next; /* hash chain for siht */ struct delay_line dline; struct dn_schk *sched; /* the template */ int kflags; /* DN_ACTIVE */ int64_t credit; /* bits I can transmit (more or less). */ uint64_t sched_time; /* time link was scheduled in ready_heap */ uint64_t idle_time; /* start of scheduler instance idle time */ /* q_count is the number of queues that this instance is using. * The counter is incremented or decremented when * a reference from the queue is created or deleted. * It is used to make sure that a scheduler instance can be safely * deleted by the drain routine. See notes below. */ int q_count; }; /* * NOTE about object drain. * The system will automatically (XXX check when) drain queues and * scheduler instances when they are idle. * A queue is idle when it has no packets; an instance is idle when * it is not in the evheap heap, and the corresponding delay line is empty. * A queue can be safely deleted when it is idle because of the scheduler * function xxx_free_queue() will remove any references to it. * An instance can be only deleted when no queues reference it. To be sure * of that, a counter (q_count) stores the number of queues that are pointing * to the instance. * * XXX * Order of scan: * - take all flowset in a bucket for the flowset hash table * - take all queues in a bucket for the flowset * - increment the queue bucket * - scan next flowset bucket * Nothing is done if a bucket contains no entries. * * The same schema is used for sceduler instances */ /* kernel-side flags. Linux has DN_DELETE in fcntl.h */ enum { /* 1 and 2 are reserved for the SCAN flags */ DN_DESTROY = 0x0004, /* destroy */ DN_DELETE_FS = 0x0008, /* destroy flowset */ DN_DETACH = 0x0010, DN_ACTIVE = 0x0020, /* object is in evheap */ DN_F_DLINE = 0x0040, /* object is a delay line */ DN_DEL_SAFE = 0x0080, /* delete a queue only if no longer needed * by scheduler */ DN_QHT_IS_Q = 0x0100, /* in flowset, qht is a single queue */ }; extern struct dn_parms dn_cfg; //VNET_DECLARE(struct dn_parms, _base_dn_cfg); //#define dn_cfg VNET(_base_dn_cfg) int dummynet_io(struct mbuf **, int , struct ip_fw_args *); void dummynet_task(void *context, int pending); void dn_reschedule(void); struct dn_queue *ipdn_q_find(struct dn_fsk *, struct dn_sch_inst *, struct ipfw_flow_id *); struct dn_sch_inst *ipdn_si_find(struct dn_schk *, struct ipfw_flow_id *); /* * copy_range is a template for requests for ranges of pipes/queues/scheds. * The number of ranges is variable and can be derived by o.len. * As a default, we use a small number of entries so that the struct * fits easily on the stack and is sufficient for most common requests. */ #define DEFAULT_RANGES 5 struct copy_range { struct dn_id o; uint32_t r[ 2 * DEFAULT_RANGES ]; }; struct copy_args { char **start; char *end; int flags; int type; struct copy_range *extra; /* extra filtering */ }; struct sockopt; int ip_dummynet_compat(struct sockopt *sopt); int dummynet_get(struct sockopt *sopt, void **compat); int dn_c_copy_q (void *_ni, void *arg); int dn_c_copy_pipe(struct dn_schk *s, struct copy_args *a, int nq); int dn_c_copy_fs(struct dn_fsk *f, struct copy_args *a, int nq); int dn_compat_copy_queue(struct copy_args *a, void *_o); int dn_compat_copy_pipe(struct copy_args *a, void *_o); int copy_data_helper_compat(void *_o, void *_arg); int dn_compat_calc_size(void); int do_config(void *p, int l); /* function to drain idle object */ void dn_drain_scheduler(void); void dn_drain_queue(void); + +#ifdef NEW_AQM +int ecn_mark(struct mbuf* m); + +/* moved from ip_dn_io.c to here to be available for AQMs modules*/ +static inline void +mq_append(struct mq *q, struct mbuf *m) +{ +#ifdef USERSPACE + // buffers from netmap need to be copied + // XXX note that the routine is not expected to fail + ND("append %p to %p", m, q); + if (m->m_flags & M_STACK) { + struct mbuf *m_new; + void *p; + int l, ofs; + + ofs = m->m_data - m->__m_extbuf; + // XXX allocate + MGETHDR(m_new, M_NOWAIT, MT_DATA); + ND("*** WARNING, volatile buf %p ext %p %d dofs %d m_new %p", + m, m->__m_extbuf, m->__m_extlen, ofs, m_new); + p = m_new->__m_extbuf; /* new pointer */ + l = m_new->__m_extlen; /* new len */ + if (l <= m->__m_extlen) { + panic("extlen too large"); + } + + *m_new = *m; // copy + m_new->m_flags &= ~M_STACK; + m_new->__m_extbuf = p; // point to new buffer + _pkt_copy(m->__m_extbuf, p, m->__m_extlen); + m_new->m_data = p + ofs; + m = m_new; + } +#endif /* USERSPACE */ + if (q->head == NULL) + q->head = m; + else + q->tail->m_nextpkt = m; + q->count++; + q->tail = m; + m->m_nextpkt = NULL; +} +#endif /* NEW_AQM */ #endif /* _IP_DN_PRIVATE_H */ Index: head/sys/netpfil/ipfw/ip_dummynet.c =================================================================== --- head/sys/netpfil/ipfw/ip_dummynet.c (revision 300778) +++ head/sys/netpfil/ipfw/ip_dummynet.c (revision 300779) @@ -1,2321 +1,2746 @@ /*- + * Codel/FQ_Codel and PIE/FQ-PIE Code: + * Copyright (C) 2016 Centre for Advanced Internet Architectures, + * Swinburne University of Technology, Melbourne, Australia. + * Portions of this code were made possible in part by a gift from + * The Comcast Innovation Fund. + * Implemented by Rasool Al-Saadi + * * Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa * Portions Copyright (c) 2000 Akamba Corp. * All rights reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Configuration and internal object management for dummynet. */ #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ #include #include /* ip_output(), IP_FORWARDING */ #include #include #include #include #include +#ifdef NEW_AQM +#include +#endif #include /* which objects to copy */ #define DN_C_LINK 0x01 #define DN_C_SCH 0x02 #define DN_C_FLOW 0x04 #define DN_C_FS 0x08 #define DN_C_QUEUE 0x10 /* we use this argument in case of a schk_new */ struct schk_new_arg { struct dn_alg *fp; struct dn_sch *sch; }; /*---- callout hooks. ----*/ static struct callout dn_timeout; static int dn_gone; static struct task dn_task; static struct taskqueue *dn_tq = NULL; static void dummynet(void *arg) { (void)arg; /* UNUSED */ taskqueue_enqueue(dn_tq, &dn_task); } void dn_reschedule(void) { if (dn_gone != 0) return; callout_reset_sbt(&dn_timeout, tick_sbt, 0, dummynet, NULL, C_HARDCLOCK | C_DIRECT_EXEC); } /*----- end of callout hooks -----*/ +#ifdef NEW_AQM +/* Return AQM descriptor for given type or name. */ +static struct dn_aqm * +find_aqm_type(int type, char *name) +{ + struct dn_aqm *d; + + SLIST_FOREACH(d, &dn_cfg.aqmlist, next) { + if (d->type == type || (name && !strcasecmp(d->name, name))) + return d; + } + return NULL; /* not found */ +} +#endif + /* Return a scheduler descriptor given the type or name. */ static struct dn_alg * find_sched_type(int type, char *name) { struct dn_alg *d; SLIST_FOREACH(d, &dn_cfg.schedlist, next) { if (d->type == type || (name && !strcasecmp(d->name, name))) return d; } return NULL; /* not found */ } int ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg) { int oldv = *v; const char *op = NULL; if (dflt < lo) dflt = lo; if (dflt > hi) dflt = hi; if (oldv < lo) { *v = dflt; op = "Bump"; } else if (oldv > hi) { *v = hi; op = "Clamp"; } else return *v; if (op && msg) printf("%s %s to %d (was %d)\n", op, msg, *v, oldv); return *v; } /*---- flow_id mask, hash and compare functions ---*/ /* * The flow_id includes the 5-tuple, the queue/pipe number * which we store in the extra area in host order, * and for ipv6 also the flow_id6. * XXX see if we want the tos byte (can store in 'flags') */ static struct ipfw_flow_id * flow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id) { int is_v6 = IS_IP6_FLOW_ID(id); id->dst_port &= mask->dst_port; id->src_port &= mask->src_port; id->proto &= mask->proto; id->extra &= mask->extra; if (is_v6) { APPLY_MASK(&id->dst_ip6, &mask->dst_ip6); APPLY_MASK(&id->src_ip6, &mask->src_ip6); id->flow_id6 &= mask->flow_id6; } else { id->dst_ip &= mask->dst_ip; id->src_ip &= mask->src_ip; } return id; } /* computes an OR of two masks, result in dst and also returned */ static struct ipfw_flow_id * flow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst) { int is_v6 = IS_IP6_FLOW_ID(dst); dst->dst_port |= src->dst_port; dst->src_port |= src->src_port; dst->proto |= src->proto; dst->extra |= src->extra; if (is_v6) { #define OR_MASK(_d, _s) \ (_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \ (_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \ (_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \ (_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3]; OR_MASK(&dst->dst_ip6, &src->dst_ip6); OR_MASK(&dst->src_ip6, &src->src_ip6); #undef OR_MASK dst->flow_id6 |= src->flow_id6; } else { dst->dst_ip |= src->dst_ip; dst->src_ip |= src->src_ip; } return dst; } static int nonzero_mask(struct ipfw_flow_id *m) { if (m->dst_port || m->src_port || m->proto || m->extra) return 1; if (IS_IP6_FLOW_ID(m)) { return m->dst_ip6.__u6_addr.__u6_addr32[0] || m->dst_ip6.__u6_addr.__u6_addr32[1] || m->dst_ip6.__u6_addr.__u6_addr32[2] || m->dst_ip6.__u6_addr.__u6_addr32[3] || m->src_ip6.__u6_addr.__u6_addr32[0] || m->src_ip6.__u6_addr.__u6_addr32[1] || m->src_ip6.__u6_addr.__u6_addr32[2] || m->src_ip6.__u6_addr.__u6_addr32[3] || m->flow_id6; } else { return m->dst_ip || m->src_ip; } } /* XXX we may want a better hash function */ static uint32_t flow_id_hash(struct ipfw_flow_id *id) { uint32_t i; if (IS_IP6_FLOW_ID(id)) { uint32_t *d = (uint32_t *)&id->dst_ip6; uint32_t *s = (uint32_t *)&id->src_ip6; i = (d[0] ) ^ (d[1]) ^ (d[2] ) ^ (d[3]) ^ (d[0] >> 15) ^ (d[1] >> 15) ^ (d[2] >> 15) ^ (d[3] >> 15) ^ (s[0] << 1) ^ (s[1] << 1) ^ (s[2] << 1) ^ (s[3] << 1) ^ (s[0] << 16) ^ (s[1] << 16) ^ (s[2] << 16) ^ (s[3] << 16) ^ (id->dst_port << 1) ^ (id->src_port) ^ (id->extra) ^ (id->proto ) ^ (id->flow_id6); } else { i = (id->dst_ip) ^ (id->dst_ip >> 15) ^ (id->src_ip << 1) ^ (id->src_ip >> 16) ^ (id->extra) ^ (id->dst_port << 1) ^ (id->src_port) ^ (id->proto); } return i; } /* Like bcmp, returns 0 if ids match, 1 otherwise. */ static int flow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2) { int is_v6 = IS_IP6_FLOW_ID(id1); if (!is_v6) { if (IS_IP6_FLOW_ID(id2)) return 1; /* different address families */ return (id1->dst_ip == id2->dst_ip && id1->src_ip == id2->src_ip && id1->dst_port == id2->dst_port && id1->src_port == id2->src_port && id1->proto == id2->proto && id1->extra == id2->extra) ? 0 : 1; } /* the ipv6 case */ return ( !bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) && !bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) && id1->dst_port == id2->dst_port && id1->src_port == id2->src_port && id1->proto == id2->proto && id1->extra == id2->extra && id1->flow_id6 == id2->flow_id6) ? 0 : 1; } /*--------- end of flow-id mask, hash and compare ---------*/ /*--- support functions for the qht hashtable ---- * Entries are hashed by flow-id */ static uint32_t q_hash(uintptr_t key, int flags, void *arg) { /* compute the hash slot from the flow id */ struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? &((struct dn_queue *)key)->ni.fid : (struct ipfw_flow_id *)key; return flow_id_hash(id); } static int q_match(void *obj, uintptr_t key, int flags, void *arg) { struct dn_queue *o = (struct dn_queue *)obj; struct ipfw_flow_id *id2; if (flags & DNHT_KEY_IS_OBJ) { /* compare pointers */ id2 = &((struct dn_queue *)key)->ni.fid; } else { id2 = (struct ipfw_flow_id *)key; } return (0 == flow_id_cmp(&o->ni.fid, id2)); } /* * create a new queue instance for the given 'key'. */ static void * q_new(uintptr_t key, int flags, void *arg) { struct dn_queue *q, *template = arg; struct dn_fsk *fs = template->fs; int size = sizeof(*q) + fs->sched->fp->q_datalen; q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO); if (q == NULL) { D("no memory for new queue"); return NULL; } set_oid(&q->ni.oid, DN_QUEUE, size); if (fs->fs.flags & DN_QHT_HASH) q->ni.fid = *(struct ipfw_flow_id *)key; q->fs = fs; q->_si = template->_si; q->_si->q_count++; if (fs->sched->fp->new_queue) fs->sched->fp->new_queue(q); + +#ifdef NEW_AQM + /* call AQM init function after creating a queue*/ + if (fs->aqmfp && fs->aqmfp->init) + if(fs->aqmfp->init(q)) + D("unable to init AQM for fs %d", fs->fs.fs_nr); +#endif dn_cfg.queue_count++; + return q; } /* * Notify schedulers that a queue is going away. * If (flags & DN_DESTROY), also free the packets. * The version for callbacks is called q_delete_cb(). */ static void dn_delete_queue(struct dn_queue *q, int flags) { struct dn_fsk *fs = q->fs; +#ifdef NEW_AQM + /* clean up AQM status for queue 'q' + * cleanup here is called just with MULTIQUEUE + */ + if (fs && fs->aqmfp && fs->aqmfp->cleanup) + fs->aqmfp->cleanup(q); +#endif // D("fs %p si %p\n", fs, q->_si); /* notify the parent scheduler that the queue is going away */ if (fs && fs->sched->fp->free_queue) fs->sched->fp->free_queue(q); q->_si->q_count--; q->_si = NULL; if (flags & DN_DESTROY) { if (q->mq.head) dn_free_pkts(q->mq.head); bzero(q, sizeof(*q)); // safety free(q, M_DUMMYNET); dn_cfg.queue_count--; } } static int q_delete_cb(void *q, void *arg) { int flags = (int)(uintptr_t)arg; dn_delete_queue(q, flags); return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0; } /* * calls dn_delete_queue/q_delete_cb on all queues, * which notifies the parent scheduler and possibly drains packets. * flags & DN_DESTROY: drains queues and destroy qht; */ static void qht_delete(struct dn_fsk *fs, int flags) { ND("fs %d start flags %d qht %p", fs->fs.fs_nr, flags, fs->qht); if (!fs->qht) return; if (fs->fs.flags & DN_QHT_HASH) { dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags); if (flags & DN_DESTROY) { dn_ht_free(fs->qht, 0); fs->qht = NULL; } } else { dn_delete_queue((struct dn_queue *)(fs->qht), flags); if (flags & DN_DESTROY) fs->qht = NULL; } } /* * Find and possibly create the queue for a MULTIQUEUE scheduler. * We never call it for !MULTIQUEUE (the queue is in the sch_inst). */ struct dn_queue * ipdn_q_find(struct dn_fsk *fs, struct dn_sch_inst *si, struct ipfw_flow_id *id) { struct dn_queue template; template._si = si; template.fs = fs; if (fs->fs.flags & DN_QHT_HASH) { struct ipfw_flow_id masked_id; if (fs->qht == NULL) { fs->qht = dn_ht_init(NULL, fs->fs.buckets, offsetof(struct dn_queue, q_next), q_hash, q_match, q_new); if (fs->qht == NULL) return NULL; } masked_id = *id; flow_id_mask(&fs->fsk_mask, &masked_id); return dn_ht_find(fs->qht, (uintptr_t)&masked_id, DNHT_INSERT, &template); } else { if (fs->qht == NULL) fs->qht = q_new(0, 0, &template); return (struct dn_queue *)fs->qht; } } /*--- end of queue hash table ---*/ /*--- support functions for the sch_inst hashtable ---- * * These are hashed by flow-id */ static uint32_t si_hash(uintptr_t key, int flags, void *arg) { /* compute the hash slot from the flow id */ struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? &((struct dn_sch_inst *)key)->ni.fid : (struct ipfw_flow_id *)key; return flow_id_hash(id); } static int si_match(void *obj, uintptr_t key, int flags, void *arg) { struct dn_sch_inst *o = obj; struct ipfw_flow_id *id2; id2 = (flags & DNHT_KEY_IS_OBJ) ? &((struct dn_sch_inst *)key)->ni.fid : (struct ipfw_flow_id *)key; return flow_id_cmp(&o->ni.fid, id2) == 0; } /* * create a new instance for the given 'key' * Allocate memory for instance, delay line and scheduler private data. */ static void * si_new(uintptr_t key, int flags, void *arg) { struct dn_schk *s = arg; struct dn_sch_inst *si; int l = sizeof(*si) + s->fp->si_datalen; si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); if (si == NULL) goto error; /* Set length only for the part passed up to userland. */ set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow)); set_oid(&(si->dline.oid), DN_DELAY_LINE, sizeof(struct delay_line)); /* mark si and dline as outside the event queue */ si->ni.oid.id = si->dline.oid.id = -1; si->sched = s; si->dline.si = si; if (s->fp->new_sched && s->fp->new_sched(si)) { D("new_sched error"); goto error; } if (s->sch.flags & DN_HAVE_MASK) si->ni.fid = *(struct ipfw_flow_id *)key; +#ifdef NEW_AQM + /* init AQM status for !DN_MULTIQUEUE sched*/ + if (!(s->fp->flags & DN_MULTIQUEUE)) + if (s->fs->aqmfp && s->fs->aqmfp->init) + if(s->fs->aqmfp->init((struct dn_queue *)(si + 1))) { + D("unable to init AQM for fs %d", s->fs->fs.fs_nr); + goto error; + } +#endif + dn_cfg.si_count++; return si; error: if (si) { bzero(si, sizeof(*si)); // safety free(si, M_DUMMYNET); } return NULL; } /* * Callback from siht to delete all scheduler instances. Remove * si and delay line from the system heap, destroy all queues. * We assume that all flowset have been notified and do not * point to us anymore. */ static int si_destroy(void *_si, void *arg) { struct dn_sch_inst *si = _si; struct dn_schk *s = si->sched; struct delay_line *dl = &si->dline; if (dl->oid.subtype) /* remove delay line from event heap */ heap_extract(&dn_cfg.evheap, dl); dn_free_pkts(dl->mq.head); /* drain delay line */ if (si->kflags & DN_ACTIVE) /* remove si from event heap */ heap_extract(&dn_cfg.evheap, si); + +#ifdef NEW_AQM + /* clean up AQM status for !DN_MULTIQUEUE sched + * Note that all queues belong to fs were cleaned up in fsk_detach. + * When drain_scheduler is called s->fs and q->fs are pointing + * to a correct fs, so we can use fs in this case. + */ + if (!(s->fp->flags & DN_MULTIQUEUE)) { + struct dn_queue *q = (struct dn_queue *)(si + 1); + if (q->aqm_status && q->fs->aqmfp) + if (q->fs->aqmfp->cleanup) + q->fs->aqmfp->cleanup(q); + } +#endif if (s->fp->free_sched) s->fp->free_sched(si); bzero(si, sizeof(*si)); /* safety */ free(si, M_DUMMYNET); dn_cfg.si_count--; return DNHT_SCAN_DEL; } /* * Find the scheduler instance for this packet. If we need to apply * a mask, do on a local copy of the flow_id to preserve the original. * Assume siht is always initialized if we have a mask. */ struct dn_sch_inst * ipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id) { if (s->sch.flags & DN_HAVE_MASK) { struct ipfw_flow_id id_t = *id; flow_id_mask(&s->sch.sched_mask, &id_t); return dn_ht_find(s->siht, (uintptr_t)&id_t, DNHT_INSERT, s); } if (!s->siht) s->siht = si_new(0, 0, s); return (struct dn_sch_inst *)s->siht; } /* callback to flush credit for the scheduler instance */ static int si_reset_credit(void *_si, void *arg) { struct dn_sch_inst *si = _si; struct dn_link *p = &si->sched->link; si->credit = p->burst + (dn_cfg.io_fast ? p->bandwidth : 0); return 0; } static void schk_reset_credit(struct dn_schk *s) { if (s->sch.flags & DN_HAVE_MASK) dn_ht_scan(s->siht, si_reset_credit, NULL); else if (s->siht) si_reset_credit(s->siht, NULL); } /*---- end of sch_inst hashtable ---------------------*/ /*------------------------------------------------------- * flowset hash (fshash) support. Entries are hashed by fs_nr. * New allocations are put in the fsunlinked list, from which * they are removed when they point to a specific scheduler. */ static uint32_t fsk_hash(uintptr_t key, int flags, void *arg) { uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : ((struct dn_fsk *)key)->fs.fs_nr; return ( (i>>8)^(i>>4)^i ); } static int fsk_match(void *obj, uintptr_t key, int flags, void *arg) { struct dn_fsk *fs = obj; int i = !(flags & DNHT_KEY_IS_OBJ) ? key : ((struct dn_fsk *)key)->fs.fs_nr; return (fs->fs.fs_nr == i); } static void * fsk_new(uintptr_t key, int flags, void *arg) { struct dn_fsk *fs; fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO); if (fs) { set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs)); dn_cfg.fsk_count++; fs->drain_bucket = 0; SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); } return fs; } +#ifdef NEW_AQM +/* callback function for cleaning up AQM queue status belongs to a flowset + * connected to scheduler instance '_si' (for !DN_MULTIQUEUE only). + */ +static int +si_cleanup_q(void *_si, void *arg) +{ + struct dn_sch_inst *si = _si; + + if (!(si->sched->fp->flags & DN_MULTIQUEUE)) { + if (si->sched->fs->aqmfp && si->sched->fs->aqmfp->cleanup) + si->sched->fs->aqmfp->cleanup((struct dn_queue *) (si+1)); + } + return 0; +} + +/* callback to clean up queue AQM status.*/ +static int +q_cleanup_q(void *_q, void *arg) +{ + struct dn_queue *q = _q; + q->fs->aqmfp->cleanup(q); + return 0; +} + +/* Clean up all AQM queues status belongs to flowset 'fs' and then + * deconfig AQM for flowset 'fs' + */ +static void +aqm_cleanup_deconfig_fs(struct dn_fsk *fs) +{ + struct dn_sch_inst *si; + + /* clean up AQM status for all queues for !DN_MULTIQUEUE sched*/ + if (fs->fs.fs_nr > DN_MAX_ID) { + if (fs->sched && !(fs->sched->fp->flags & DN_MULTIQUEUE)) { + if (fs->sched->sch.flags & DN_HAVE_MASK) + dn_ht_scan(fs->sched->siht, si_cleanup_q, NULL); + else { + /* single si i.e. no sched mask */ + si = (struct dn_sch_inst *) fs->sched->siht; + if (si && fs->aqmfp && fs->aqmfp->cleanup) + fs->aqmfp->cleanup((struct dn_queue *) (si+1)); + } + } + } + + /* clean up AQM status for all queues for DN_MULTIQUEUE sched*/ + if (fs->sched && fs->sched->fp->flags & DN_MULTIQUEUE && fs->qht) { + if (fs->fs.flags & DN_QHT_HASH) + dn_ht_scan(fs->qht, q_cleanup_q, NULL); + else + fs->aqmfp->cleanup((struct dn_queue *)(fs->qht)); + } + + /* deconfig AQM */ + if(fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) + fs->aqmfp->deconfig(fs); +} +#endif + /* * detach flowset from its current scheduler. Flags as follows: * DN_DETACH removes from the fsk_list * DN_DESTROY deletes individual queues * DN_DELETE_FS destroys the flowset (otherwise goes in unlinked). */ static void fsk_detach(struct dn_fsk *fs, int flags) { if (flags & DN_DELETE_FS) flags |= DN_DESTROY; ND("fs %d from sched %d flags %s %s %s", fs->fs.fs_nr, fs->fs.sched_nr, (flags & DN_DELETE_FS) ? "DEL_FS":"", (flags & DN_DESTROY) ? "DEL":"", (flags & DN_DETACH) ? "DET":""); if (flags & DN_DETACH) { /* detach from the list */ struct dn_fsk_head *h; h = fs->sched ? &fs->sched->fsk_list : &dn_cfg.fsu; SLIST_REMOVE(h, fs, dn_fsk, sch_chain); } /* Free the RED parameters, they will be recomputed on * subsequent attach if needed. */ if (fs->w_q_lookup) free(fs->w_q_lookup, M_DUMMYNET); fs->w_q_lookup = NULL; qht_delete(fs, flags); +#ifdef NEW_AQM + aqm_cleanup_deconfig_fs(fs); +#endif + if (fs->sched && fs->sched->fp->free_fsk) fs->sched->fp->free_fsk(fs); fs->sched = NULL; if (flags & DN_DELETE_FS) { bzero(fs, sizeof(*fs)); /* safety */ free(fs, M_DUMMYNET); dn_cfg.fsk_count--; } else { SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); } } /* * Detach or destroy all flowsets in a list. * flags specifies what to do: * DN_DESTROY: flush all queues * DN_DELETE_FS: DN_DESTROY + destroy flowset * DN_DELETE_FS implies DN_DESTROY */ static void fsk_detach_list(struct dn_fsk_head *h, int flags) { struct dn_fsk *fs; int n = 0; /* only for stats */ ND("head %p flags %x", h, flags); while ((fs = SLIST_FIRST(h))) { SLIST_REMOVE_HEAD(h, sch_chain); n++; fsk_detach(fs, flags); } ND("done %d flowsets", n); } /* * called on 'queue X delete' -- removes the flowset from fshash, * deletes all queues for the flowset, and removes the flowset. */ static int delete_fs(int i, int locked) { struct dn_fsk *fs; int err = 0; if (!locked) DN_BH_WLOCK(); fs = dn_ht_find(dn_cfg.fshash, i, DNHT_REMOVE, NULL); ND("fs %d found %p", i, fs); if (fs) { fsk_detach(fs, DN_DETACH | DN_DELETE_FS); err = 0; } else err = EINVAL; if (!locked) DN_BH_WUNLOCK(); return err; } /*----- end of flowset hashtable support -------------*/ /*------------------------------------------------------------ * Scheduler hash. When searching by index we pass sched_nr, * otherwise we pass struct dn_sch * which is the first field in * struct dn_schk so we can cast between the two. We use this trick * because in the create phase (but it should be fixed). */ static uint32_t schk_hash(uintptr_t key, int flags, void *_arg) { uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : ((struct dn_schk *)key)->sch.sched_nr; return ( (i>>8)^(i>>4)^i ); } static int schk_match(void *obj, uintptr_t key, int flags, void *_arg) { struct dn_schk *s = (struct dn_schk *)obj; int i = !(flags & DNHT_KEY_IS_OBJ) ? key : ((struct dn_schk *)key)->sch.sched_nr; return (s->sch.sched_nr == i); } /* * Create the entry and intialize with the sched hash if needed. * Leave s->fp unset so we can tell whether a dn_ht_find() returns * a new object or a previously existing one. */ static void * schk_new(uintptr_t key, int flags, void *arg) { struct schk_new_arg *a = arg; struct dn_schk *s; int l = sizeof(*s) +a->fp->schk_datalen; s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); if (s == NULL) return NULL; set_oid(&s->link.oid, DN_LINK, sizeof(s->link)); s->sch = *a->sch; // copy initial values s->link.link_nr = s->sch.sched_nr; SLIST_INIT(&s->fsk_list); /* initialize the hash table or create the single instance */ s->fp = a->fp; /* si_new needs this */ s->drain_bucket = 0; if (s->sch.flags & DN_HAVE_MASK) { s->siht = dn_ht_init(NULL, s->sch.buckets, offsetof(struct dn_sch_inst, si_next), si_hash, si_match, si_new); if (s->siht == NULL) { free(s, M_DUMMYNET); return NULL; } } s->fp = NULL; /* mark as a new scheduler */ dn_cfg.schk_count++; return s; } /* * Callback for sched delete. Notify all attached flowsets to * detach from the scheduler, destroy the internal flowset, and * all instances. The scheduler goes away too. * arg is 0 (only detach flowsets and destroy instances) * DN_DESTROY (detach & delete queues, delete schk) * or DN_DELETE_FS (delete queues and flowsets, delete schk) */ static int schk_delete_cb(void *obj, void *arg) { struct dn_schk *s = obj; #if 0 int a = (int)arg; ND("sched %d arg %s%s", s->sch.sched_nr, a&DN_DESTROY ? "DEL ":"", a&DN_DELETE_FS ? "DEL_FS":""); #endif fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0); /* no more flowset pointing to us now */ if (s->sch.flags & DN_HAVE_MASK) { dn_ht_scan(s->siht, si_destroy, NULL); dn_ht_free(s->siht, 0); } else if (s->siht) si_destroy(s->siht, NULL); if (s->profile) { free(s->profile, M_DUMMYNET); s->profile = NULL; } s->siht = NULL; if (s->fp->destroy) s->fp->destroy(s); bzero(s, sizeof(*s)); // safety free(obj, M_DUMMYNET); dn_cfg.schk_count--; return DNHT_SCAN_DEL; } /* * called on a 'sched X delete' command. Deletes a single scheduler. * This is done by removing from the schedhash, unlinking all * flowsets and deleting their traffic. */ static int delete_schk(int i) { struct dn_schk *s; s = dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); ND("%d %p", i, s); if (!s) return EINVAL; delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */ /* then detach flowsets, delete traffic */ schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY); return 0; } /*--- end of schk hashtable support ---*/ static int copy_obj(char **start, char *end, void *_o, const char *msg, int i) { struct dn_id *o = _o; int have = end - *start; if (have < o->len || o->len == 0 || o->type == 0) { D("(WARN) type %d %s %d have %d need %d", o->type, msg, i, have, o->len); return 1; } ND("type %d %s %d len %d", o->type, msg, i, o->len); bcopy(_o, *start, o->len); if (o->type == DN_LINK) { /* Adjust burst parameter for link */ struct dn_link *l = (struct dn_link *)*start; l->burst = div64(l->burst, 8 * hz); l->delay = l->delay * 1000 / hz; } else if (o->type == DN_SCH) { /* Set id->id to the number of instances */ struct dn_schk *s = _o; struct dn_id *id = (struct dn_id *)(*start); id->id = (s->sch.flags & DN_HAVE_MASK) ? dn_ht_entries(s->siht) : (s->siht ? 1 : 0); } *start += o->len; return 0; } /* Specific function to copy a queue. * Copies only the user-visible part of a queue (which is in * a struct dn_flow), and sets len accordingly. */ static int copy_obj_q(char **start, char *end, void *_o, const char *msg, int i) { struct dn_id *o = _o; int have = end - *start; int len = sizeof(struct dn_flow); /* see above comment */ if (have < len || o->len == 0 || o->type != DN_QUEUE) { D("ERROR type %d %s %d have %d need %d", o->type, msg, i, have, len); return 1; } ND("type %d %s %d len %d", o->type, msg, i, len); bcopy(_o, *start, len); ((struct dn_id*)(*start))->len = len; *start += len; return 0; } static int copy_q_cb(void *obj, void *arg) { struct dn_queue *q = obj; struct copy_args *a = arg; struct dn_flow *ni = (struct dn_flow *)(*a->start); if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1)) return DNHT_SCAN_END; ni->oid.type = DN_FLOW; /* override the DN_QUEUE */ ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL); return 0; } static int copy_q(struct copy_args *a, struct dn_fsk *fs, int flags) { if (!fs->qht) return 0; if (fs->fs.flags & DN_QHT_HASH) dn_ht_scan(fs->qht, copy_q_cb, a); else copy_q_cb(fs->qht, a); return 0; } /* * This routine only copies the initial part of a profile ? XXX */ static int copy_profile(struct copy_args *a, struct dn_profile *p) { int have = a->end - *a->start; /* XXX here we check for max length */ int profile_len = sizeof(struct dn_profile) - ED_MAX_SAMPLES_NO*sizeof(int); if (p == NULL) return 0; if (have < profile_len) { D("error have %d need %d", have, profile_len); return 1; } bcopy(p, *a->start, profile_len); ((struct dn_id *)(*a->start))->len = profile_len; *a->start += profile_len; return 0; } static int copy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags) { struct dn_fs *ufs = (struct dn_fs *)(*a->start); if (!fs) return 0; ND("flowset %d", fs->fs.fs_nr); if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr)) return DNHT_SCAN_END; ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ? dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0); if (flags) { /* copy queues */ copy_q(a, fs, 0); } return 0; } static int copy_si_cb(void *obj, void *arg) { struct dn_sch_inst *si = obj; struct copy_args *a = arg; struct dn_flow *ni = (struct dn_flow *)(*a->start); if (copy_obj(a->start, a->end, &si->ni, "inst", si->sched->sch.sched_nr)) return DNHT_SCAN_END; ni->oid.type = DN_FLOW; /* override the DN_SCH_I */ ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL); return 0; } static int copy_si(struct copy_args *a, struct dn_schk *s, int flags) { if (s->sch.flags & DN_HAVE_MASK) dn_ht_scan(s->siht, copy_si_cb, a); else if (s->siht) copy_si_cb(s->siht, a); return 0; } /* * compute a list of children of a scheduler and copy up */ static int copy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags) { struct dn_fsk *fs; struct dn_id *o; uint32_t *p; int n = 0, space = sizeof(*o); SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { if (fs->fs.fs_nr < DN_MAX_ID) n++; } space += n * sizeof(uint32_t); DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n); if (a->end - *(a->start) < space) return DNHT_SCAN_END; o = (struct dn_id *)(*(a->start)); o->len = space; *a->start += o->len; o->type = DN_TEXT; p = (uint32_t *)(o+1); SLIST_FOREACH(fs, &s->fsk_list, sch_chain) if (fs->fs.fs_nr < DN_MAX_ID) *p++ = fs->fs.fs_nr; return 0; } static int copy_data_helper(void *_o, void *_arg) { struct copy_args *a = _arg; uint32_t *r = a->extra->r; /* start of first range */ uint32_t *lim; /* first invalid pointer */ int n; lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len); if (a->type == DN_LINK || a->type == DN_SCH) { /* pipe|sched show, we receive a dn_schk */ struct dn_schk *s = _o; n = s->sch.sched_nr; if (a->type == DN_SCH && n >= DN_MAX_ID) return 0; /* not a scheduler */ if (a->type == DN_LINK && n <= DN_MAX_ID) return 0; /* not a pipe */ /* see if the object is within one of our ranges */ for (;r < lim; r += 2) { if (n < r[0] || n > r[1]) continue; /* Found a valid entry, copy and we are done */ if (a->flags & DN_C_LINK) { if (copy_obj(a->start, a->end, &s->link, "link", n)) return DNHT_SCAN_END; if (copy_profile(a, s->profile)) return DNHT_SCAN_END; if (copy_flowset(a, s->fs, 0)) return DNHT_SCAN_END; } if (a->flags & DN_C_SCH) { if (copy_obj(a->start, a->end, &s->sch, "sched", n)) return DNHT_SCAN_END; /* list all attached flowsets */ if (copy_fsk_list(a, s, 0)) return DNHT_SCAN_END; } if (a->flags & DN_C_FLOW) copy_si(a, s, 0); break; } } else if (a->type == DN_FS) { /* queue show, skip internal flowsets */ struct dn_fsk *fs = _o; n = fs->fs.fs_nr; if (n >= DN_MAX_ID) return 0; /* see if the object is within one of our ranges */ for (;r < lim; r += 2) { if (n < r[0] || n > r[1]) continue; if (copy_flowset(a, fs, 0)) return DNHT_SCAN_END; copy_q(a, fs, 0); break; /* we are done */ } } return 0; } static inline struct dn_schk * locate_scheduler(int i) { return dn_ht_find(dn_cfg.schedhash, i, 0, NULL); } /* * red parameters are in fixed point arithmetic. */ static int config_red(struct dn_fsk *fs) { int64_t s, idle, weight, w0; int t, i; fs->w_q = fs->fs.w_q; fs->max_p = fs->fs.max_p; ND("called"); /* Doing stuff that was in userland */ i = fs->sched->link.bandwidth; s = (i <= 0) ? 0 : hz * dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i; idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */ fs->lookup_step = div64(idle , dn_cfg.red_lookup_depth); /* fs->lookup_step not scaled, */ if (!fs->lookup_step) fs->lookup_step = 1; w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled for (t = fs->lookup_step; t > 1; --t) weight = SCALE_MUL(weight, w0); fs->lookup_weight = (int)(weight); // scaled /* Now doing stuff that was in kerneland */ fs->min_th = SCALE(fs->fs.min_th); fs->max_th = SCALE(fs->fs.max_th); if (fs->fs.max_th == fs->fs.min_th) fs->c_1 = fs->max_p; else fs->c_1 = SCALE((int64_t)(fs->max_p)) / (fs->fs.max_th - fs->fs.min_th); fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th)); if (fs->fs.flags & DN_IS_GENTLE_RED) { fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th; fs->c_4 = SCALE(1) - 2 * fs->max_p; } /* If the lookup table already exist, free and create it again. */ if (fs->w_q_lookup) { free(fs->w_q_lookup, M_DUMMYNET); fs->w_q_lookup = NULL; } if (dn_cfg.red_lookup_depth == 0) { printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth" "must be > 0\n"); fs->fs.flags &= ~DN_IS_RED; fs->fs.flags &= ~DN_IS_GENTLE_RED; return (EINVAL); } fs->lookup_depth = dn_cfg.red_lookup_depth; fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int), M_DUMMYNET, M_NOWAIT); if (fs->w_q_lookup == NULL) { printf("dummynet: sorry, cannot allocate red lookup table\n"); fs->fs.flags &= ~DN_IS_RED; fs->fs.flags &= ~DN_IS_GENTLE_RED; return(ENOSPC); } /* Fill the lookup table with (1 - w_q)^x */ fs->w_q_lookup[0] = SCALE(1) - fs->w_q; for (i = 1; i < fs->lookup_depth; i++) fs->w_q_lookup[i] = SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight); if (dn_cfg.red_avg_pkt_size < 1) dn_cfg.red_avg_pkt_size = 512; fs->avg_pkt_size = dn_cfg.red_avg_pkt_size; if (dn_cfg.red_max_pkt_size < 1) dn_cfg.red_max_pkt_size = 1500; fs->max_pkt_size = dn_cfg.red_max_pkt_size; ND("exit"); return 0; } /* Scan all flowset attached to this scheduler and update red */ static void update_red(struct dn_schk *s) { struct dn_fsk *fs; SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { if (fs && (fs->fs.flags & DN_IS_RED)) config_red(fs); } } /* attach flowset to scheduler s, possibly requeue */ static void fsk_attach(struct dn_fsk *fs, struct dn_schk *s) { ND("remove fs %d from fsunlinked, link to sched %d", fs->fs.fs_nr, s->sch.sched_nr); SLIST_REMOVE(&dn_cfg.fsu, fs, dn_fsk, sch_chain); fs->sched = s; SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain); if (s->fp->new_fsk) s->fp->new_fsk(fs); /* XXX compute fsk_mask */ fs->fsk_mask = fs->fs.flow_mask; if (fs->sched->sch.flags & DN_HAVE_MASK) flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask); if (fs->qht) { /* * we must drain qht according to the old * type, and reinsert according to the new one. * The requeue is complex -- in general we need to * reclassify every single packet. * For the time being, let's hope qht is never set * when we reach this point. */ D("XXX TODO requeue from fs %d to sch %d", fs->fs.fs_nr, s->sch.sched_nr); fs->qht = NULL; } /* set the new type for qht */ if (nonzero_mask(&fs->fsk_mask)) fs->fs.flags |= DN_QHT_HASH; else fs->fs.flags &= ~DN_QHT_HASH; /* XXX config_red() can fail... */ if (fs->fs.flags & DN_IS_RED) config_red(fs); } /* update all flowsets which may refer to this scheduler */ static void update_fs(struct dn_schk *s) { struct dn_fsk *fs, *tmp; SLIST_FOREACH_SAFE(fs, &dn_cfg.fsu, sch_chain, tmp) { if (s->sch.sched_nr != fs->fs.sched_nr) { D("fs %d for sch %d not %d still unlinked", fs->fs.fs_nr, fs->fs.sched_nr, s->sch.sched_nr); continue; } fsk_attach(fs, s); } } +#ifdef NEW_AQM +/* Retrieve AQM configurations to ipfw userland + */ +static int +get_aqm_parms(struct sockopt *sopt) +{ + struct dn_extra_parms *ep; + struct dn_fsk *fs; + size_t sopt_valsize; + int l, err = 0; + + sopt_valsize = sopt->sopt_valsize; + l = sizeof(*ep); + if (sopt->sopt_valsize < l) { + D("bad len sopt->sopt_valsize %d len %d", + (int) sopt->sopt_valsize , l); + err = EINVAL; + return err; + } + ep = malloc(l, M_DUMMYNET, M_WAITOK); + if(!ep) { + err = ENOMEM ; + return err; + } + do { + err = sooptcopyin(sopt, ep, l, l); + if(err) + break; + sopt->sopt_valsize = sopt_valsize; + if (ep->oid.len < l) { + err = EINVAL; + break; + } + + fs = dn_ht_find(dn_cfg.fshash, ep->nr, 0, NULL); + if (!fs) { + D("fs %d not found", ep->nr); + err = EINVAL; + break; + } + + if (fs->aqmfp && fs->aqmfp->getconfig) { + if(fs->aqmfp->getconfig(fs, ep)) { + D("Error while trying to get AQM params"); + err = EINVAL; + break; + } + ep->oid.len = l; + err = sooptcopyout(sopt, ep, l); + } + }while(0); + + free(ep, M_DUMMYNET); + return err; +} + +/* Retrieve AQM configurations to ipfw userland + */ +static int +get_sched_parms(struct sockopt *sopt) +{ + struct dn_extra_parms *ep; + struct dn_schk *schk; + size_t sopt_valsize; + int l, err = 0; + + sopt_valsize = sopt->sopt_valsize; + l = sizeof(*ep); + if (sopt->sopt_valsize < l) { + D("bad len sopt->sopt_valsize %d len %d", + (int) sopt->sopt_valsize , l); + err = EINVAL; + return err; + } + ep = malloc(l, M_DUMMYNET, M_WAITOK); + if(!ep) { + err = ENOMEM ; + return err; + } + do { + err = sooptcopyin(sopt, ep, l, l); + if(err) + break; + sopt->sopt_valsize = sopt_valsize; + if (ep->oid.len < l) { + err = EINVAL; + break; + } + + schk = locate_scheduler(ep->nr); + if (!schk) { + D("sched %d not found", ep->nr); + err = EINVAL; + break; + } + + if (schk->fp && schk->fp->getconfig) { + if(schk->fp->getconfig(schk, ep)) { + D("Error while trying to get sched params"); + err = EINVAL; + break; + } + ep->oid.len = l; + err = sooptcopyout(sopt, ep, l); + } + }while(0); + free(ep, M_DUMMYNET); + + return err; +} + +/* Configure AQM for flowset 'fs'. + * extra parameters are passed from userland. + */ +static int +config_aqm(struct dn_fsk *fs, struct dn_extra_parms *ep, int busy) +{ + int err = 0; + + do { + /* no configurations */ + if (!ep) { + err = 0; + break; + } + + /* no AQM for this flowset*/ + if (!strcmp(ep->name,"")) { + err = 0; + break; + } + if (ep->oid.len < sizeof(*ep)) { + D("short aqm len %d", ep->oid.len); + err = EINVAL; + break; + } + + if (busy) { + D("Unable to configure flowset, flowset busy!"); + err = EINVAL; + break; + } + + /* deconfigure old aqm if exist */ + if (fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) { + aqm_cleanup_deconfig_fs(fs); + } + + if (!(fs->aqmfp = find_aqm_type(0, ep->name))) { + D("AQM functions not found for type %s!", ep->name); + fs->fs.flags &= ~DN_IS_AQM; + err = EINVAL; + break; + } else + fs->fs.flags |= DN_IS_AQM; + + if (ep->oid.subtype != DN_AQM_PARAMS) { + D("Wrong subtype"); + err = EINVAL; + break; + } + + if (fs->aqmfp->config) { + err = fs->aqmfp->config(fs, ep, ep->oid.len); + if (err) { + D("Unable to configure AQM for FS %d", fs->fs.fs_nr ); + fs->fs.flags &= ~DN_IS_AQM; + fs->aqmfp = NULL; + break; + } + } + } while(0); + + return err; +} +#endif + /* * Configuration -- to preserve backward compatibility we use * the following scheme (N is 65536) * NUMBER SCHED LINK FLOWSET * 1 .. N-1 (1)WFQ (2)WFQ (3)queue * N+1 .. 2N-1 (4)FIFO (5)FIFO (6)FIFO for sched 1..N-1 * 2N+1 .. 3N-1 -- -- (7)FIFO for sched N+1..2N-1 * * "pipe i config" configures #1, #2 and #3 * "sched i config" configures #1 and possibly #6 * "queue i config" configures #3 * #1 is configured with 'pipe i config' or 'sched i config' * #2 is configured with 'pipe i config', and created if not * existing with 'sched i config' * #3 is configured with 'queue i config' * #4 is automatically configured after #1, can only be FIFO * #5 is automatically configured after #2 * #6 is automatically created when #1 is !MULTIQUEUE, * and can be updated. * #7 is automatically configured after #2 */ /* * configure a link (and its FIFO instance) */ static int config_link(struct dn_link *p, struct dn_id *arg) { int i; if (p->oid.len != sizeof(*p)) { D("invalid pipe len %d", p->oid.len); return EINVAL; } i = p->link_nr; if (i <= 0 || i >= DN_MAX_ID) return EINVAL; /* * The config program passes parameters as follows: * bw = bits/second (0 means no limits), * delay = ms, must be translated into ticks. * qsize = slots/bytes * burst ??? */ p->delay = (p->delay * hz) / 1000; /* Scale burst size: bytes -> bits * hz */ p->burst *= 8 * hz; DN_BH_WLOCK(); /* do it twice, base link and FIFO link */ for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { struct dn_schk *s = locate_scheduler(i); if (s == NULL) { DN_BH_WUNLOCK(); D("sched %d not found", i); return EINVAL; } /* remove profile if exists */ if (s->profile) { free(s->profile, M_DUMMYNET); s->profile = NULL; } /* copy all parameters */ s->link.oid = p->oid; s->link.link_nr = i; s->link.delay = p->delay; if (s->link.bandwidth != p->bandwidth) { /* XXX bandwidth changes, need to update red params */ s->link.bandwidth = p->bandwidth; update_red(s); } s->link.burst = p->burst; schk_reset_credit(s); } dn_cfg.id++; DN_BH_WUNLOCK(); return 0; } /* * configure a flowset. Can be called from inside with locked=1, */ static struct dn_fsk * config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) { int i; struct dn_fsk *fs; if (nfs->oid.len != sizeof(*nfs)) { D("invalid flowset len %d", nfs->oid.len); return NULL; } i = nfs->fs_nr; if (i <= 0 || i >= 3*DN_MAX_ID) return NULL; ND("flowset %d", i); /* XXX other sanity checks */ if (nfs->flags & DN_QSIZE_BYTES) { ipdn_bound_var(&nfs->qsize, 16384, 1500, dn_cfg.byte_limit, NULL); // "queue byte size"); } else { ipdn_bound_var(&nfs->qsize, 50, 1, dn_cfg.slot_limit, NULL); // "queue slot size"); } if (nfs->flags & DN_HAVE_MASK) { /* make sure we have some buckets */ ipdn_bound_var((int *)&nfs->buckets, dn_cfg.hash_size, 1, dn_cfg.max_hash_size, "flowset buckets"); } else { nfs->buckets = 1; /* we only need 1 */ } if (!locked) DN_BH_WLOCK(); do { /* exit with break when done */ struct dn_schk *s; int flags = nfs->sched_nr ? DNHT_INSERT : 0; int j; int oldc = dn_cfg.fsk_count; fs = dn_ht_find(dn_cfg.fshash, i, flags, NULL); if (fs == NULL) { D("missing sched for flowset %d", i); break; } /* grab some defaults from the existing one */ if (nfs->sched_nr == 0) /* reuse */ nfs->sched_nr = fs->fs.sched_nr; for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) { if (nfs->par[j] == -1) /* reuse */ nfs->par[j] = fs->fs.par[j]; } if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) { ND("flowset %d unchanged", i); +#ifdef NEW_AQM + /* reconfigure AQM as the parameters can be changed. + * we consider the flowsetis busy if it has scheduler instance(s) + */ + s = locate_scheduler(nfs->sched_nr); + config_aqm(fs, (struct dn_extra_parms *) arg, + s != NULL && s->siht != NULL); +#endif break; /* no change, nothing to do */ } if (oldc != dn_cfg.fsk_count) /* new item */ dn_cfg.id++; s = locate_scheduler(nfs->sched_nr); /* detach from old scheduler if needed, preserving * queues if we need to reattach. Then update the * configuration, and possibly attach to the new sched. */ DX(2, "fs %d changed sched %d@%p to %d@%p", fs->fs.fs_nr, fs->fs.sched_nr, fs->sched, nfs->sched_nr, s); if (fs->sched) { int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY); flags |= DN_DESTROY; /* XXX temporary */ fsk_detach(fs, flags); } fs->fs = *nfs; /* copy configuration */ +#ifdef NEW_AQM + fs->aqmfp = NULL; + config_aqm(fs, (struct dn_extra_parms *) arg, s != NULL && s->siht != NULL); +#endif if (s != NULL) fsk_attach(fs, s); } while (0); if (!locked) DN_BH_WUNLOCK(); return fs; } /* * config/reconfig a scheduler and its FIFO variant. * For !MULTIQUEUE schedulers, also set up the flowset. * * On reconfigurations (detected because s->fp is set), * detach existing flowsets preserving traffic, preserve link, * and delete the old scheduler creating a new one. */ static int config_sched(struct dn_sch *_nsch, struct dn_id *arg) { struct dn_schk *s; struct schk_new_arg a; /* argument for schk_new */ int i; struct dn_link p; /* copy of oldlink */ struct dn_profile *pf = NULL; /* copy of old link profile */ /* Used to preserv mask parameter */ struct ipfw_flow_id new_mask; int new_buckets = 0; int new_flags = 0; int pipe_cmd; int err = ENOMEM; a.sch = _nsch; if (a.sch->oid.len != sizeof(*a.sch)) { D("bad sched len %d", a.sch->oid.len); return EINVAL; } i = a.sch->sched_nr; if (i <= 0 || i >= DN_MAX_ID) return EINVAL; /* make sure we have some buckets */ if (a.sch->flags & DN_HAVE_MASK) ipdn_bound_var((int *)&a.sch->buckets, dn_cfg.hash_size, 1, dn_cfg.max_hash_size, "sched buckets"); /* XXX other sanity checks */ bzero(&p, sizeof(p)); pipe_cmd = a.sch->flags & DN_PIPE_CMD; a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set? if (pipe_cmd) { /* Copy mask parameter */ new_mask = a.sch->sched_mask; new_buckets = a.sch->buckets; new_flags = a.sch->flags; } DN_BH_WLOCK(); again: /* run twice, for wfq and fifo */ /* * lookup the type. If not supplied, use the previous one * or default to WF2Q+. Otherwise, return an error. */ dn_cfg.id++; a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name); if (a.fp != NULL) { /* found. Lookup or create entry */ s = dn_ht_find(dn_cfg.schedhash, i, DNHT_INSERT, &a); } else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) { /* No type. search existing s* or retry with WF2Q+ */ s = dn_ht_find(dn_cfg.schedhash, i, 0, &a); if (s != NULL) { a.fp = s->fp; /* Scheduler exists, skip to FIFO scheduler * if command was pipe config... */ if (pipe_cmd) goto next; } else { /* New scheduler, create a wf2q+ with no mask * if command was pipe config... */ if (pipe_cmd) { /* clear mask parameter */ bzero(&a.sch->sched_mask, sizeof(new_mask)); a.sch->buckets = 0; a.sch->flags &= ~DN_HAVE_MASK; } a.sch->oid.subtype = DN_SCHED_WF2QP; goto again; } } else { D("invalid scheduler type %d %s", a.sch->oid.subtype, a.sch->name); err = EINVAL; goto error; } /* normalize name and subtype */ a.sch->oid.subtype = a.fp->type; bzero(a.sch->name, sizeof(a.sch->name)); strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name)); if (s == NULL) { D("cannot allocate scheduler %d", i); goto error; } /* restore existing link if any */ if (p.link_nr) { s->link = p; if (!pf || pf->link_nr != p.link_nr) { /* no saved value */ s->profile = NULL; /* XXX maybe not needed */ } else { s->profile = malloc(sizeof(struct dn_profile), M_DUMMYNET, M_NOWAIT | M_ZERO); if (s->profile == NULL) { D("cannot allocate profile"); goto error; //XXX } bcopy(pf, s->profile, sizeof(*pf)); } } p.link_nr = 0; if (s->fp == NULL) { DX(2, "sched %d new type %s", i, a.fp->name); } else if (s->fp != a.fp || bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) { /* already existing. */ DX(2, "sched %d type changed from %s to %s", i, s->fp->name, a.fp->name); DX(4, " type/sub %d/%d -> %d/%d", s->sch.oid.type, s->sch.oid.subtype, a.sch->oid.type, a.sch->oid.subtype); if (s->link.link_nr == 0) D("XXX WARNING link 0 for sched %d", i); p = s->link; /* preserve link */ if (s->profile) {/* preserve profile */ if (!pf) pf = malloc(sizeof(*pf), M_DUMMYNET, M_NOWAIT | M_ZERO); if (pf) /* XXX should issue a warning otherwise */ bcopy(s->profile, pf, sizeof(*pf)); } /* remove from the hash */ dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); /* Detach flowsets, preserve queues. */ // schk_delete_cb(s, NULL); // XXX temporarily, kill queues schk_delete_cb(s, (void *)DN_DESTROY); goto again; } else { DX(4, "sched %d unchanged type %s", i, a.fp->name); } /* complete initialization */ s->sch = *a.sch; s->fp = a.fp; s->cfg = arg; // XXX schk_reset_credit(s); /* create the internal flowset if needed, * trying to reuse existing ones if available */ if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) { s->fs = dn_ht_find(dn_cfg.fshash, i, 0, NULL); if (!s->fs) { struct dn_fs fs; bzero(&fs, sizeof(fs)); set_oid(&fs.oid, DN_FS, sizeof(fs)); fs.fs_nr = i + DN_MAX_ID; fs.sched_nr = i; s->fs = config_fs(&fs, NULL, 1 /* locked */); } if (!s->fs) { schk_delete_cb(s, (void *)DN_DESTROY); D("error creating internal fs for %d", i); goto error; } } /* call init function after the flowset is created */ if (s->fp->config) s->fp->config(s); update_fs(s); next: if (i < DN_MAX_ID) { /* now configure the FIFO instance */ i += DN_MAX_ID; if (pipe_cmd) { /* Restore mask parameter for FIFO */ a.sch->sched_mask = new_mask; a.sch->buckets = new_buckets; a.sch->flags = new_flags; } else { /* sched config shouldn't modify the FIFO scheduler */ if (dn_ht_find(dn_cfg.schedhash, i, 0, &a) != NULL) { /* FIFO already exist, don't touch it */ err = 0; /* and this is not an error */ goto error; } } a.sch->sched_nr = i; a.sch->oid.subtype = DN_SCHED_FIFO; bzero(a.sch->name, sizeof(a.sch->name)); goto again; } err = 0; error: DN_BH_WUNLOCK(); if (pf) free(pf, M_DUMMYNET); return err; } /* * attach a profile to a link */ static int config_profile(struct dn_profile *pf, struct dn_id *arg) { struct dn_schk *s; int i, olen, err = 0; if (pf->oid.len < sizeof(*pf)) { D("short profile len %d", pf->oid.len); return EINVAL; } i = pf->link_nr; if (i <= 0 || i >= DN_MAX_ID) return EINVAL; /* XXX other sanity checks */ DN_BH_WLOCK(); for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { s = locate_scheduler(i); if (s == NULL) { err = EINVAL; break; } dn_cfg.id++; /* * If we had a profile and the new one does not fit, * or it is deleted, then we need to free memory. */ if (s->profile && (pf->samples_no == 0 || s->profile->oid.len < pf->oid.len)) { free(s->profile, M_DUMMYNET); s->profile = NULL; } if (pf->samples_no == 0) continue; /* * new profile, possibly allocate memory * and copy data. */ if (s->profile == NULL) s->profile = malloc(pf->oid.len, M_DUMMYNET, M_NOWAIT | M_ZERO); if (s->profile == NULL) { D("no memory for profile %d", i); err = ENOMEM; break; } /* preserve larger length XXX double check */ olen = s->profile->oid.len; if (olen < pf->oid.len) olen = pf->oid.len; bcopy(pf, s->profile, pf->oid.len); s->profile->oid.len = olen; } DN_BH_WUNLOCK(); return err; } /* * Delete all objects: */ static void dummynet_flush(void) { /* delete all schedulers and related links/queues/flowsets */ dn_ht_scan(dn_cfg.schedhash, schk_delete_cb, (void *)(uintptr_t)DN_DELETE_FS); /* delete all remaining (unlinked) flowsets */ DX(4, "still %d unlinked fs", dn_cfg.fsk_count); dn_ht_free(dn_cfg.fshash, DNHT_REMOVE); fsk_detach_list(&dn_cfg.fsu, DN_DELETE_FS); /* Reinitialize system heap... */ heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); } /* * Main handler for configuration. We are guaranteed to be called * with an oid which is at least a dn_id. * - the first object is the command (config, delete, flush, ...) * - config_link must be issued after the corresponding config_sched * - parameters (DN_TXT) for an object must precede the object * processed on a config_sched. */ int do_config(void *p, int l) { struct dn_id *next, *o; int err = 0, err2 = 0; struct dn_id *arg = NULL; uintptr_t *a; o = p; if (o->id != DN_API_VERSION) { D("invalid api version got %d need %d", o->id, DN_API_VERSION); return EINVAL; } for (; l >= sizeof(*o); o = next) { struct dn_id *prev = arg; if (o->len < sizeof(*o) || l < o->len) { D("bad len o->len %d len %d", o->len, l); err = EINVAL; break; } l -= o->len; next = (struct dn_id *)((char *)o + o->len); err = 0; switch (o->type) { default: D("cmd %d not implemented", o->type); break; #ifdef EMULATE_SYSCTL /* sysctl emulation. * if we recognize the command, jump to the correct * handler and return */ case DN_SYSCTL_SET: err = kesysctl_emu_set(p, l); return err; #endif case DN_CMD_CONFIG: /* simply a header */ break; case DN_CMD_DELETE: /* the argument is in the first uintptr_t after o */ a = (uintptr_t *)(o+1); if (o->len < sizeof(*o) + sizeof(*a)) { err = EINVAL; break; } switch (o->subtype) { case DN_LINK: /* delete base and derived schedulers */ DN_BH_WLOCK(); err = delete_schk(*a); err2 = delete_schk(*a + DN_MAX_ID); DN_BH_WUNLOCK(); if (!err) err = err2; break; default: D("invalid delete type %d", o->subtype); err = EINVAL; break; case DN_FS: err = (*a <1 || *a >= DN_MAX_ID) ? EINVAL : delete_fs(*a, 0) ; break; } break; case DN_CMD_FLUSH: DN_BH_WLOCK(); dummynet_flush(); DN_BH_WUNLOCK(); break; case DN_TEXT: /* store argument the next block */ prev = NULL; arg = o; break; case DN_LINK: err = config_link((struct dn_link *)o, arg); break; case DN_PROFILE: err = config_profile((struct dn_profile *)o, arg); break; case DN_SCH: err = config_sched((struct dn_sch *)o, arg); break; case DN_FS: err = (NULL==config_fs((struct dn_fs *)o, arg, 0)); break; } if (prev) arg = NULL; if (err != 0) break; } return err; } static int compute_space(struct dn_id *cmd, struct copy_args *a) { int x = 0, need = 0; int profile_size = sizeof(struct dn_profile) - ED_MAX_SAMPLES_NO*sizeof(int); /* NOTE about compute space: * NP = dn_cfg.schk_count * NSI = dn_cfg.si_count * NF = dn_cfg.fsk_count * NQ = dn_cfg.queue_count * - ipfw pipe show * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler * link, scheduler template, flowset * integrated in scheduler and header * for flowset list * (NSI)*(dn_flow) all scheduler instance (includes * the queue instance) * - ipfw sched show * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler * link, scheduler template, flowset * integrated in scheduler and header * for flowset list * (NSI * dn_flow) all scheduler instances * (NF * sizeof(uint_32)) space for flowset list linked to scheduler * (NQ * dn_queue) all queue [XXXfor now not listed] * - ipfw queue show * (NF * dn_fs) all flowset * (NQ * dn_queue) all queues */ switch (cmd->subtype) { default: return -1; /* XXX where do LINK and SCH differ ? */ /* 'ipfw sched show' could list all queues associated to * a scheduler. This feature for now is disabled */ case DN_LINK: /* pipe show */ x = DN_C_LINK | DN_C_SCH | DN_C_FLOW; need += dn_cfg.schk_count * (sizeof(struct dn_fs) + profile_size) / 2; need += dn_cfg.fsk_count * sizeof(uint32_t); break; case DN_SCH: /* sched show */ need += dn_cfg.schk_count * (sizeof(struct dn_fs) + profile_size) / 2; need += dn_cfg.fsk_count * sizeof(uint32_t); x = DN_C_SCH | DN_C_LINK | DN_C_FLOW; break; case DN_FS: /* queue show */ x = DN_C_FS | DN_C_QUEUE; break; case DN_GET_COMPAT: /* compatibility mode */ need = dn_compat_calc_size(); break; } a->flags = x; if (x & DN_C_SCH) { need += dn_cfg.schk_count * sizeof(struct dn_sch) / 2; /* NOT also, each fs might be attached to a sched */ need += dn_cfg.schk_count * sizeof(struct dn_id) / 2; } if (x & DN_C_FS) need += dn_cfg.fsk_count * sizeof(struct dn_fs); if (x & DN_C_LINK) { need += dn_cfg.schk_count * sizeof(struct dn_link) / 2; } /* * When exporting a queue to userland, only pass up the * struct dn_flow, which is the only visible part. */ if (x & DN_C_QUEUE) need += dn_cfg.queue_count * sizeof(struct dn_flow); if (x & DN_C_FLOW) need += dn_cfg.si_count * (sizeof(struct dn_flow)); return need; } /* * If compat != NULL dummynet_get is called in compatibility mode. * *compat will be the pointer to the buffer to pass to ipfw */ int dummynet_get(struct sockopt *sopt, void **compat) { int have, i, need, error; char *start = NULL, *buf; size_t sopt_valsize; struct dn_id *cmd; struct copy_args a; struct copy_range r; int l = sizeof(struct dn_id); bzero(&a, sizeof(a)); bzero(&r, sizeof(r)); /* save and restore original sopt_valsize around copyin */ sopt_valsize = sopt->sopt_valsize; cmd = &r.o; if (!compat) { /* copy at least an oid, and possibly a full object */ error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd)); sopt->sopt_valsize = sopt_valsize; if (error) goto done; l = cmd->len; #ifdef EMULATE_SYSCTL /* sysctl emulation. */ if (cmd->type == DN_SYSCTL_GET) return kesysctl_emu_get(sopt); #endif if (l > sizeof(r)) { /* request larger than default, allocate buffer */ cmd = malloc(l, M_DUMMYNET, M_WAITOK); error = sooptcopyin(sopt, cmd, l, l); sopt->sopt_valsize = sopt_valsize; if (error) goto done; } } else { /* compatibility */ error = 0; cmd->type = DN_CMD_GET; cmd->len = sizeof(struct dn_id); cmd->subtype = DN_GET_COMPAT; // cmd->id = sopt_valsize; D("compatibility mode"); } + +#ifdef NEW_AQM + /* get AQM params */ + if(cmd->subtype == DN_AQM_PARAMS) { + error = get_aqm_parms(sopt); + goto done; + /* get Scheduler params */ + } else if (cmd->subtype == DN_SCH_PARAMS) { + error = get_sched_parms(sopt); + goto done; + } +#endif + a.extra = (struct copy_range *)cmd; if (cmd->len == sizeof(*cmd)) { /* no range, create a default */ uint32_t *rp = (uint32_t *)(cmd + 1); cmd->len += 2* sizeof(uint32_t); rp[0] = 1; rp[1] = DN_MAX_ID - 1; if (cmd->subtype == DN_LINK) { rp[0] += DN_MAX_ID; rp[1] += DN_MAX_ID; } } /* Count space (under lock) and allocate (outside lock). * Exit with lock held if we manage to get enough buffer. * Try a few times then give up. */ for (have = 0, i = 0; i < 10; i++) { DN_BH_WLOCK(); need = compute_space(cmd, &a); /* if there is a range, ignore value from compute_space() */ if (l > sizeof(*cmd)) need = sopt_valsize - sizeof(*cmd); if (need < 0) { DN_BH_WUNLOCK(); error = EINVAL; goto done; } need += sizeof(*cmd); cmd->id = need; if (have >= need) break; DN_BH_WUNLOCK(); if (start) free(start, M_DUMMYNET); start = NULL; if (need > sopt_valsize) break; have = need; start = malloc(have, M_DUMMYNET, M_WAITOK | M_ZERO); } if (start == NULL) { if (compat) { *compat = NULL; error = 1; // XXX } else { error = sooptcopyout(sopt, cmd, sizeof(*cmd)); } goto done; } ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, " "%d:%d si %d, %d:%d queues %d", dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH, dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK, dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS, dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I, dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE); sopt->sopt_valsize = sopt_valsize; a.type = cmd->subtype; if (compat == NULL) { bcopy(cmd, start, sizeof(*cmd)); ((struct dn_id*)(start))->len = sizeof(struct dn_id); buf = start + sizeof(*cmd); } else buf = start; a.start = &buf; a.end = start + have; /* start copying other objects */ if (compat) { a.type = DN_COMPAT_PIPE; dn_ht_scan(dn_cfg.schedhash, copy_data_helper_compat, &a); a.type = DN_COMPAT_QUEUE; dn_ht_scan(dn_cfg.fshash, copy_data_helper_compat, &a); } else if (a.type == DN_FS) { dn_ht_scan(dn_cfg.fshash, copy_data_helper, &a); } else { dn_ht_scan(dn_cfg.schedhash, copy_data_helper, &a); } DN_BH_WUNLOCK(); if (compat) { *compat = start; sopt->sopt_valsize = buf - start; /* free() is done by ip_dummynet_compat() */ start = NULL; //XXX hack } else { error = sooptcopyout(sopt, start, buf - start); } done: if (cmd && cmd != &r.o) free(cmd, M_DUMMYNET); if (start) free(start, M_DUMMYNET); return error; } /* Callback called on scheduler instance to delete it if idle */ static int drain_scheduler_cb(void *_si, void *arg) { struct dn_sch_inst *si = _si; if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL) return 0; if (si->sched->fp->flags & DN_MULTIQUEUE) { if (si->q_count == 0) return si_destroy(si, NULL); else return 0; } else { /* !DN_MULTIQUEUE */ if ((si+1)->ni.length == 0) return si_destroy(si, NULL); else return 0; } return 0; /* unreachable */ } /* Callback called on scheduler to check if it has instances */ static int drain_scheduler_sch_cb(void *_s, void *arg) { struct dn_schk *s = _s; if (s->sch.flags & DN_HAVE_MASK) { dn_ht_scan_bucket(s->siht, &s->drain_bucket, drain_scheduler_cb, NULL); s->drain_bucket++; } else { if (s->siht) { if (drain_scheduler_cb(s->siht, NULL) == DNHT_SCAN_DEL) s->siht = NULL; } } return 0; } /* Called every tick, try to delete a 'bucket' of scheduler */ void dn_drain_scheduler(void) { dn_ht_scan_bucket(dn_cfg.schedhash, &dn_cfg.drain_sch, drain_scheduler_sch_cb, NULL); dn_cfg.drain_sch++; } /* Callback called on queue to delete if it is idle */ static int drain_queue_cb(void *_q, void *arg) { struct dn_queue *q = _q; if (q->ni.length == 0) { dn_delete_queue(q, DN_DESTROY); return DNHT_SCAN_DEL; /* queue is deleted */ } return 0; /* queue isn't deleted */ } /* Callback called on flowset used to check if it has queues */ static int drain_queue_fs_cb(void *_fs, void *arg) { struct dn_fsk *fs = _fs; if (fs->fs.flags & DN_QHT_HASH) { /* Flowset has a hash table for queues */ dn_ht_scan_bucket(fs->qht, &fs->drain_bucket, drain_queue_cb, NULL); fs->drain_bucket++; } else { /* No hash table for this flowset, null the pointer * if the queue is deleted */ if (fs->qht) { if (drain_queue_cb(fs->qht, NULL) == DNHT_SCAN_DEL) fs->qht = NULL; } } return 0; } /* Called every tick, try to delete a 'bucket' of queue */ void dn_drain_queue(void) { /* scan a bucket of flowset */ dn_ht_scan_bucket(dn_cfg.fshash, &dn_cfg.drain_fs, drain_queue_fs_cb, NULL); dn_cfg.drain_fs++; } /* * Handler for the various dummynet socket options */ static int ip_dn_ctl(struct sockopt *sopt) { void *p = NULL; int error, l; error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET); if (error) return (error); /* Disallow sets in really-really secure mode. */ if (sopt->sopt_dir == SOPT_SET) { error = securelevel_ge(sopt->sopt_td->td_ucred, 3); if (error) return (error); } switch (sopt->sopt_name) { default : D("dummynet: unknown option %d", sopt->sopt_name); error = EINVAL; break; case IP_DUMMYNET_FLUSH: case IP_DUMMYNET_CONFIGURE: case IP_DUMMYNET_DEL: /* remove a pipe or queue */ case IP_DUMMYNET_GET: D("dummynet: compat option %d", sopt->sopt_name); error = ip_dummynet_compat(sopt); break; case IP_DUMMYNET3 : if (sopt->sopt_dir == SOPT_GET) { error = dummynet_get(sopt, NULL); break; } l = sopt->sopt_valsize; if (l < sizeof(struct dn_id) || l > 12000) { D("argument len %d invalid", l); break; } p = malloc(l, M_TEMP, M_WAITOK); // XXX can it fail ? error = sooptcopyin(sopt, p, l, l); if (error) break ; error = do_config(p, l); break; } if (p != NULL) free(p, M_TEMP); return error ; } static void ip_dn_init(void) { if (dn_cfg.init_done) return; printf("DUMMYNET %p with IPv6 initialized (100409)\n", curvnet); dn_cfg.init_done = 1; /* Set defaults here. MSVC does not accept initializers, * and this is also useful for vimages */ /* queue limits */ dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */ dn_cfg.byte_limit = 1024 * 1024; dn_cfg.expire = 1; /* RED parameters */ dn_cfg.red_lookup_depth = 256; /* default lookup table depth */ dn_cfg.red_avg_pkt_size = 512; /* default medium packet size */ dn_cfg.red_max_pkt_size = 1500; /* default max packet size */ /* hash tables */ dn_cfg.max_hash_size = 65536; /* max in the hash tables */ dn_cfg.hash_size = 64; /* default hash size */ /* create hash tables for schedulers and flowsets. * In both we search by key and by pointer. */ dn_cfg.schedhash = dn_ht_init(NULL, dn_cfg.hash_size, offsetof(struct dn_schk, schk_next), schk_hash, schk_match, schk_new); dn_cfg.fshash = dn_ht_init(NULL, dn_cfg.hash_size, offsetof(struct dn_fsk, fsk_next), fsk_hash, fsk_match, fsk_new); /* bucket index to drain object */ dn_cfg.drain_fs = 0; dn_cfg.drain_sch = 0; heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); SLIST_INIT(&dn_cfg.fsu); SLIST_INIT(&dn_cfg.schedlist); DN_LOCK_INIT(); TASK_INIT(&dn_task, 0, dummynet_task, curvnet); dn_tq = taskqueue_create_fast("dummynet", M_WAITOK, taskqueue_thread_enqueue, &dn_tq); taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet"); callout_init(&dn_timeout, 1); dn_reschedule(); /* Initialize curr_time adjustment mechanics. */ getmicrouptime(&dn_cfg.prev_t); } static void ip_dn_destroy(int last) { DN_BH_WLOCK(); /* ensure no more callouts are started */ dn_gone = 1; /* check for last */ if (last) { ND("removing last instance\n"); ip_dn_ctl_ptr = NULL; ip_dn_io_ptr = NULL; } dummynet_flush(); DN_BH_WUNLOCK(); callout_drain(&dn_timeout); taskqueue_drain(dn_tq, &dn_task); taskqueue_free(dn_tq); dn_ht_free(dn_cfg.schedhash, 0); dn_ht_free(dn_cfg.fshash, 0); heap_free(&dn_cfg.evheap); DN_LOCK_DESTROY(); } static int dummynet_modevent(module_t mod, int type, void *data) { if (type == MOD_LOAD) { if (ip_dn_io_ptr) { printf("DUMMYNET already loaded\n"); return EEXIST ; } ip_dn_init(); ip_dn_ctl_ptr = ip_dn_ctl; ip_dn_io_ptr = dummynet_io; return 0; } else if (type == MOD_UNLOAD) { ip_dn_destroy(1 /* last */); return 0; } else return EOPNOTSUPP; } /* modevent helpers for the modules */ static int load_dn_sched(struct dn_alg *d) { struct dn_alg *s; if (d == NULL) return 1; /* error */ ip_dn_init(); /* just in case, we need the lock */ /* Check that mandatory funcs exists */ if (d->enqueue == NULL || d->dequeue == NULL) { D("missing enqueue or dequeue for %s", d->name); return 1; } /* Search if scheduler already exists */ DN_BH_WLOCK(); SLIST_FOREACH(s, &dn_cfg.schedlist, next) { if (strcmp(s->name, d->name) == 0) { D("%s already loaded", d->name); break; /* scheduler already exists */ } } if (s == NULL) SLIST_INSERT_HEAD(&dn_cfg.schedlist, d, next); DN_BH_WUNLOCK(); D("dn_sched %s %sloaded", d->name, s ? "not ":""); return s ? 1 : 0; } static int unload_dn_sched(struct dn_alg *s) { struct dn_alg *tmp, *r; int err = EINVAL; ND("called for %s", s->name); DN_BH_WLOCK(); SLIST_FOREACH_SAFE(r, &dn_cfg.schedlist, next, tmp) { if (strcmp(s->name, r->name) != 0) continue; ND("ref_count = %d", r->ref_count); err = (r->ref_count != 0) ? EBUSY : 0; if (err == 0) SLIST_REMOVE(&dn_cfg.schedlist, r, dn_alg, next); break; } DN_BH_WUNLOCK(); D("dn_sched %s %sunloaded", s->name, err ? "not ":""); return err; } int dn_sched_modevent(module_t mod, int cmd, void *arg) { struct dn_alg *sch = arg; if (cmd == MOD_LOAD) return load_dn_sched(sch); else if (cmd == MOD_UNLOAD) return unload_dn_sched(sch); else return EINVAL; } static moduledata_t dummynet_mod = { "dummynet", dummynet_modevent, NULL }; #define DN_SI_SUB SI_SUB_PROTO_IFATTACHDOMAIN #define DN_MODEV_ORD (SI_ORDER_ANY - 128) /* after ipfw */ DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD); MODULE_DEPEND(dummynet, ipfw, 3, 3, 3); MODULE_VERSION(dummynet, 3); /* * Starting up. Done in order after dummynet_modevent() has been called. * VNET_SYSINIT is also called for each existing vnet and each new vnet. */ //VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_init, NULL); /* * Shutdown handlers up shop. These are done in REVERSE ORDER, but still * after dummynet_modevent() has been called. Not called on reboot. * VNET_SYSUNINIT is also called for each exiting vnet as it exits. * or when the module is unloaded. */ //VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_destroy, NULL); +#ifdef NEW_AQM + +/* modevent helpers for the AQM modules */ +static int +load_dn_aqm(struct dn_aqm *d) +{ + struct dn_aqm *aqm=NULL; + + if (d == NULL) + return 1; /* error */ + ip_dn_init(); /* just in case, we need the lock */ + + /* Check that mandatory funcs exists */ + if (d->enqueue == NULL || d->dequeue == NULL) { + D("missing enqueue or dequeue for %s", d->name); + return 1; + } + + /* Search if AQM already exists */ + DN_BH_WLOCK(); + SLIST_FOREACH(aqm, &dn_cfg.aqmlist, next) { + if (strcmp(aqm->name, d->name) == 0) { + D("%s already loaded", d->name); + break; /* AQM already exists */ + } + } + if (aqm == NULL) + SLIST_INSERT_HEAD(&dn_cfg.aqmlist, d, next); + DN_BH_WUNLOCK(); + D("dn_aqm %s %sloaded", d->name, aqm ? "not ":""); + return aqm ? 1 : 0; +} + + +/* Callback to clean up AQM status for queues connected to a flowset + * and then deconfigure the flowset. + * This function is called before an AQM module is unloaded + */ +static int +fs_cleanup(void *_fs, void *arg) +{ + struct dn_fsk *fs = _fs; + uint32_t type = *(uint32_t *)arg; + + if (fs->aqmfp && fs->aqmfp->type == type) + aqm_cleanup_deconfig_fs(fs); + + return 0; +} + +static int +unload_dn_aqm(struct dn_aqm *aqm) +{ + struct dn_aqm *tmp, *r; + int err = EINVAL; + err = 0; + ND("called for %s", aqm->name); + + DN_BH_WLOCK(); + + /* clean up AQM status and deconfig flowset */ + dn_ht_scan(dn_cfg.fshash, fs_cleanup, &aqm->type); + + SLIST_FOREACH_SAFE(r, &dn_cfg.aqmlist, next, tmp) { + if (strcmp(aqm->name, r->name) != 0) + continue; + ND("ref_count = %d", r->ref_count); + err = (r->ref_count != 0 || r->cfg_ref_count != 0) ? EBUSY : 0; + if (err == 0) + SLIST_REMOVE(&dn_cfg.aqmlist, r, dn_aqm, next); + break; + } + DN_BH_WUNLOCK(); + D("%s %sunloaded", aqm->name, err ? "not ":""); + if (err) + D("ref_count=%d, cfg_ref_count=%d", r->ref_count, r->cfg_ref_count); + return err; +} + +int +dn_aqm_modevent(module_t mod, int cmd, void *arg) +{ + struct dn_aqm *aqm = arg; + + if (cmd == MOD_LOAD) + return load_dn_aqm(aqm); + else if (cmd == MOD_UNLOAD) + return unload_dn_aqm(aqm); + else + return EINVAL; +} +#endif + /* end of file */ +