Page MenuHomeFreeBSD
Differential D46183 Diff 141565 sys/netpfil/ipfw/nat64/nat64lsn.c

Changeset View

Standalone View

View Options

sys/netpfil/ipfw/nat64/nat64lsn.c

/*- /*-
* SPDX-License-Identifier: BSD-2-Clause * SPDX-License-Identifier: BSD-2-Clause
* *
* Copyright (c) 2015-2019 Yandex LLC * Copyright (c) 2015-2020 Yandex LLC
* Copyright (c) 2015 Alexander V. Chernikov <melifaro@FreeBSD.org> * Copyright (c) 2015 Alexander V. Chernikov <melifaro@FreeBSD.org>
* Copyright (c) 2016-2019 Andrey V. Elsukov <ae@FreeBSD.org> * Copyright (c) 2016-2020 Andrey V. Elsukov <ae@FreeBSD.org>
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions * modification, are permitted provided that the following conditions
* are met: * are met:
* *
* 1. Redistributions of source code must retain the above copyright * 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer. * notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright * 2. Redistributions in binary form must reproduce the above copyright
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines
static uma_zone_t nat64lsn_pg_zone; static uma_zone_t nat64lsn_pg_zone;
static uma_zone_t nat64lsn_aliaslink_zone; static uma_zone_t nat64lsn_aliaslink_zone;
static uma_zone_t nat64lsn_state_zone; static uma_zone_t nat64lsn_state_zone;
static uma_zone_t nat64lsn_job_zone; static uma_zone_t nat64lsn_job_zone;
static void nat64lsn_periodic(void *data); static void nat64lsn_periodic(void *data);
#define PERIODIC_DELAY 4 #define PERIODIC_DELAY 4
#define NAT64_LOOKUP(chain, cmd) \ #define NAT64_LOOKUP(chain, cmd) \
(struct nat64lsn_cfg *)SRV_OBJECT((chain), (cmd)->arg1) (struct nat64lsn_instance *)SRV_OBJECT((chain), insntod(cmd, kidx)->kidx)
/* /*
* Delayed job queue, used to create new hosts * Delayed job queue, used to create new hosts
* and new portgroups * and new portgroups
*/ */
enum nat64lsn_jtype { enum nat64lsn_jtype {
JTYPE_NEWHOST = 1, JTYPE_NEWHOST = 1,
JTYPE_NEWPORTGROUP, JTYPE_NEWPORTGROUP,
JTYPE_DESTROY, JTYPE_DESTROY,
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines
} }
static void static void
nat64lsn_log(struct pfloghdr *plog, struct mbuf *m, sa_family_t family, nat64lsn_log(struct pfloghdr *plog, struct mbuf *m, sa_family_t family,
struct nat64lsn_state *state) struct nat64lsn_state *state)
{ {
memset(plog, 0, sizeof(*plog)); memset(plog, 0, sizeof(*plog));
plog->length = PFLOG_HDRLEN; plog->length = PFLOG_REAL_HDRLEN;
plog->af = family; plog->af = family;
plog->action = PF_NAT; plog->action = PF_NAT;
plog->dir = PF_IN; plog->dir = PF_IN;
plog->rulenr = htonl(state->ip_src); plog->rulenr = htonl(state->ip_src);
plog->subrulenr = htonl((uint32_t)(state->aport << 16) | plog->subrulenr = htonl((uint32_t)(state->aport << 16) |
(state->proto << 8) | (state->ip_dst & 0xff)); (state->proto << 8) | (state->ip_dst & 0xff));
plog->ruleset[0] = '\0'; plog->ruleset[0] = '\0';
strlcpy(plog->ifname, "NAT64LSN", sizeof(plog->ifname)); strlcpy(plog->ifname, "NAT64LSN", sizeof(plog->ifname));
Show All 17 Linesnat64lsn_get_aliaslink(struct nat64lsn_cfg *cfg __unused,
/* /*
* We can implement some different algorithms how * We can implement some different algorithms how
* select an alias address. * select an alias address.
* XXX: for now we use first available. * XXX: for now we use first available.
*/ */
return (CK_SLIST_FIRST(&host->aliases)); return (CK_SLIST_FIRST(&host->aliases));
} }
static struct nat64lsn_alias*
nat64lsn_get_alias(struct nat64lsn_cfg *cfg,
const struct ipfw_flow_id *f_id __unused)
{
static uint32_t idx = 0;
/*
* We can choose alias by number of allocated PGs,
* not used yet by other hosts, or some static configured
* by user.
* XXX: for now we choose it using round robin.
*/
return (&ALIAS_BYHASH(cfg, idx++));
}
#define STATE_HVAL(c, d) HVAL((d), 2, (c)->hash_seed) #define STATE_HVAL(c, d) HVAL((d), 2, (c)->hash_seed)
#define STATE_HASH(h, v) \ #define STATE_HASH(h, v) \
((h)->states_hash[(v) & ((h)->states_hashsize - 1)]) ((h)->states_hash[(v) & ((h)->states_hashsize - 1)])
#define STATES_CHUNK(p, v) \ #define STATES_CHUNK(p, v) \
((p)->chunks_count == 1 ? (p)->states : \ ((p)->chunks_count == 1 ? (p)->states : \
((p)->states_chunk[CHUNK_BY_FADDR(p, v)])) ((p)->states_chunk[CHUNK_BY_FADDR(p, v)]))
#ifdef __LP64__ #ifdef __LP64__
Show All 27 Lines#define FREEMASK_BTS(pg, faddr, bit) \
ck_pr_bts_32(FREEMASK_CHUNK((pg), (faddr)) + (bit) / 32, (bit) % 32) ck_pr_bts_32(FREEMASK_CHUNK((pg), (faddr)) + (bit) / 32, (bit) % 32)
#define FREEMASK_ISSET(pg, faddr, bit) \ #define FREEMASK_ISSET(pg, faddr, bit) \
ISSET32(*(FREEMASK_CHUNK((pg), (faddr)) + (bit) / 32), (bit) % 32) ISSET32(*(FREEMASK_CHUNK((pg), (faddr)) + (bit) / 32), (bit) % 32)
#define FREEMASK_COPY(pg, n, out) \ #define FREEMASK_COPY(pg, n, out) \
(out) = ck_pr_load_32(FREEMASK_CHUNK((pg), (n))) | \ (out) = ck_pr_load_32(FREEMASK_CHUNK((pg), (n))) | \
((uint64_t)ck_pr_load_32(FREEMASK_CHUNK((pg), (n)) + 1) << 32) ((uint64_t)ck_pr_load_32(FREEMASK_CHUNK((pg), (n)) + 1) << 32)
#endif /* !__LP64__ */ #endif /* !__LP64__ */
#define NAT64LSN_TRY_PGCNT 32
#define NAT64LSN_TRY_PGCNT 36
static struct nat64lsn_pg* static struct nat64lsn_pg*
nat64lsn_get_pg(uint32_t *chunkmask, uint32_t *pgmask, nat64lsn_get_pg(uint32_t *chunkmask, uint32_t *pgmask,
struct nat64lsn_pgchunk **chunks, struct nat64lsn_pg **pgptr, struct nat64lsn_pgchunk **chunks, uint32_t *pgidx, in_addr_t faddr)
uint32_t *pgidx, in_addr_t faddr)
{ {
struct nat64lsn_pg *pg, *oldpg; struct nat64lsn_pg *pg;
uint32_t idx, oldidx; uint32_t idx, oldidx;
int cnt; int cnt;
cnt = 0; /* First try last used PG. */
/* First try last used PG */
oldpg = pg = ck_pr_load_ptr(pgptr);
idx = oldidx = ck_pr_load_32(pgidx); idx = oldidx = ck_pr_load_32(pgidx);
/* If pgidx is out of range, reset it to the first pgchunk */ MPASS(idx < 1024);
if (!ISSET32(*chunkmask, idx / 32)) cnt = 0;
idx = 0;
do { do {
ck_pr_fence_load(); ck_pr_fence_load();
if (pg != NULL && FREEMASK_BITCOUNT(pg, faddr) > 0) { if (idx > 1023 || !ISSET32(*chunkmask, idx / 32)) {
/* If it is first try, reset idx to first PG */
idx = 0;
/* Stop if idx is out of range */
if (cnt > 0)
break;
}
if (ISSET32(pgmask[idx / 32], idx % 32)) {
pg = ck_pr_load_ptr(
&chunks[idx / 32]->pgptr[idx % 32]);
ck_pr_fence_load();
/* /*
* If last used PG has not free states, * Make sure that pg did not become DEAD.
* try to update pointer.
* NOTE: it can be already updated by jobs handler,
* thus we use CAS operation.
*/ */
if ((pg->flags & NAT64LSN_DEADPG) == 0 &&
FREEMASK_BITCOUNT(pg, faddr) > 0) {
if (cnt > 0) if (cnt > 0)
ck_pr_cas_ptr(pgptr, oldpg, pg); ck_pr_cas_32(pgidx, oldidx, idx);
return (pg); return (pg);
} }
/* Stop if idx is out of range */ }
if (!ISSET32(*chunkmask, idx / 32))
break;
if (ISSET32(pgmask[idx / 32], idx % 32))
pg = ck_pr_load_ptr(
&chunks[idx / 32]->pgptr[idx % 32]);
else
pg = NULL;
idx++; idx++;
} while (++cnt < NAT64LSN_TRY_PGCNT); } while (++cnt < NAT64LSN_TRY_PGCNT);
if (oldidx != idx)
/* If pgidx is out of range, reset it to the first pgchunk */
if (!ISSET32(*chunkmask, idx / 32))
idx = 0;
ck_pr_cas_32(pgidx, oldidx, idx); ck_pr_cas_32(pgidx, oldidx, idx);
return (NULL); return (NULL);
} }
static struct nat64lsn_state* static struct nat64lsn_state*
nat64lsn_get_state6to4(struct nat64lsn_cfg *cfg, struct nat64lsn_host *host, nat64lsn_get_state6to4(struct nat64lsn_cfg *cfg, struct nat64lsn_host *host,
const struct ipfw_flow_id *f_id, uint32_t hval, in_addr_t faddr, const struct ipfw_flow_id *f_id, uint32_t hval, in_addr_t faddr,
uint16_t port, uint8_t proto) uint16_t port, uint8_t proto)
{ {
Show All 12 Linesnat64lsn_get_state6to4(struct nat64lsn_cfg *cfg, struct nat64lsn_host *host,
} }
link = nat64lsn_get_aliaslink(cfg, host, f_id); link = nat64lsn_get_aliaslink(cfg, host, f_id);
if (link == NULL) if (link == NULL)
return (NULL); return (NULL);
switch (proto) { switch (proto) {
case IPPROTO_TCP: case IPPROTO_TCP:
pg = nat64lsn_get_pg( pg = nat64lsn_get_pg(&link->alias->tcp_chunkmask,
&link->alias->tcp_chunkmask, link->alias->tcp_pgmask, link->alias->tcp_pgmask, link->alias->tcp,
link->alias->tcp, &link->alias->tcp_pg,
&link->alias->tcp_pgidx, faddr); &link->alias->tcp_pgidx, faddr);
break; break;
case IPPROTO_UDP: case IPPROTO_UDP:
pg = nat64lsn_get_pg( pg = nat64lsn_get_pg(&link->alias->udp_chunkmask,
&link->alias->udp_chunkmask, link->alias->udp_pgmask, link->alias->udp_pgmask, link->alias->udp,
link->alias->udp, &link->alias->udp_pg,
&link->alias->udp_pgidx, faddr); &link->alias->udp_pgidx, faddr);
break; break;
case IPPROTO_ICMP: case IPPROTO_ICMP:
pg = nat64lsn_get_pg( pg = nat64lsn_get_pg(&link->alias->icmp_chunkmask,
&link->alias->icmp_chunkmask, link->alias->icmp_pgmask, link->alias->icmp_pgmask, link->alias->icmp,
link->alias->icmp, &link->alias->icmp_pg,
&link->alias->icmp_pgidx, faddr); &link->alias->icmp_pgidx, faddr);
break; break;
default: default:
panic("%s: wrong proto %d", __func__, proto); panic("%s: wrong proto %d", __func__, proto);
} }
if (pg == NULL) if (pg == NULL || (pg->flags & NAT64LSN_DEADPG) != 0)
return (NULL); return (NULL);
/* Check that PG has some free states */ /* Check that PG has some free states */
state = NULL; state = NULL;
i = FREEMASK_BITCOUNT(pg, faddr); i = FREEMASK_BITCOUNT(pg, faddr);
while (i-- > 0) { while (i-- > 0) {
offset = FREEMASK_FFSLL(pg, faddr); offset = FREEMASK_FFSLL(pg, faddr);
if (offset == 0) { if (offset == 0) {
Show All 18 Lines if (FREEMASK_BTR(pg, faddr, offset - 1)) {
state->ip_dst = faddr; state->ip_dst = faddr;
state->ip_src = link->alias->addr; state->ip_src = link->alias->addr;
state->hval = hval; state->hval = hval;
state->host = host; state->host = host;
SET_AGE(state->timestamp); SET_AGE(state->timestamp);
/* Insert new state into host's hash table */ /* Insert new state into host's hash table */
HOST_LOCK(host); HOST_LOCK(host);
SET_AGE(host->timestamp);
CK_SLIST_INSERT_HEAD(&STATE_HASH(host, hval), CK_SLIST_INSERT_HEAD(&STATE_HASH(host, hval),
state, entries); state, entries);
host->states_count++; host->states_count++;
/*
* XXX: In case if host is going to be expired,
* reset NAT64LSN_DEADHOST flag.
*/
host->flags &= ~NAT64LSN_DEADHOST;
HOST_UNLOCK(host); HOST_UNLOCK(host);
NAT64STAT_INC(&cfg->base.stats, screated); NAT64STAT_INC(&cfg->base.stats, screated);
/* Mark the state as ready for translate4 */ /* Mark the state as ready for translate4 */
ck_pr_fence_store(); ck_pr_fence_store();
ck_pr_bts_32(&state->flags, NAT64_BIT_READY_IPV4); ck_pr_bts_32(&state->flags, NAT64_BIT_READY_IPV4);
break; break;
} }
} }
▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Linesnat64lsn_reassemble4(struct nat64lsn_cfg *cfg, struct mbuf *m,
m = ip_reass(m); m = ip_reass(m);
if (m == NULL) if (m == NULL)
return (NULL); return (NULL);
/* IP header must be contigious after ip_reass() */ /* IP header must be contigious after ip_reass() */
ip = mtod(m, struct ip *); ip = mtod(m, struct ip *);
len = ip->ip_hl << 2; len = ip->ip_hl << 2;
switch (ip->ip_p) { switch (ip->ip_p) {
case IPPROTO_ICMP: case IPPROTO_ICMP:
len += ICMP_MINLEN; /* Enough to get icmp_id */ len += ICMP_MINLEN;
break; break;
case IPPROTO_TCP: case IPPROTO_TCP:
len += sizeof(struct tcphdr); len += sizeof(struct tcphdr);
break; break;
case IPPROTO_UDP: case IPPROTO_UDP:
len += sizeof(struct udphdr); len += sizeof(struct udphdr);
break; break;
default: default:
▲ Show 20 Lines • Show All 160 Lines • ▼ Show 20 Linesnat64lsn_check_state(struct nat64lsn_cfg *cfg, struct nat64lsn_state *state)
case IPPROTO_ICMP: case IPPROTO_ICMP:
if (age > cfg->st_icmp_ttl) if (age > cfg->st_icmp_ttl)
return (1); return (1);
break; break;
} }
return (0); return (0);
} }
#define PGCOUNT_ADD(alias, proto, value) \
switch (proto) { \
case IPPROTO_TCP: (alias)->tcp_pgcount += (value); break; \
case IPPROTO_UDP: (alias)->udp_pgcount += (value); break; \
case IPPROTO_ICMP: (alias)->icmp_pgcount += (value); break; \
}
#define PGCOUNT_INC(alias, proto) PGCOUNT_ADD(alias, proto, 1)
#define PGCOUNT_DEC(alias, proto) PGCOUNT_ADD(alias, proto, -1)
static inline void
nat64lsn_state_cleanup(struct nat64lsn_state *state)
{
/*
* Reset READY flag and wait until it become
* safe for translate4.
*/
ck_pr_btr_32(&state->flags, NAT64_BIT_READY_IPV4);
/*
* And set STALE flag for deferred deletion in the
* next pass of nat64lsn_maintain_pg().
*/
ck_pr_bts_32(&state->flags, NAT64_BIT_STALE);
ck_pr_fence_store();
}
static int static int
nat64lsn_maintain_pg(struct nat64lsn_cfg *cfg, struct nat64lsn_pg *pg) nat64lsn_maintain_pg(struct nat64lsn_cfg *cfg, struct nat64lsn_pg *pg)
{ {
struct nat64lsn_state *state; struct nat64lsn_state *state;
struct nat64lsn_host *host; struct nat64lsn_host *host;
uint64_t freemask; uint64_t freemask;
int c, i, update_age; int c, i, update_age;
Show All 25 Lines for (i = 0; i < 64; i++) {
continue; continue;
} }
MPASS(state->flags & NAT64_FLAG_READY); MPASS(state->flags & NAT64_FLAG_READY);
host = state->host; host = state->host;
HOST_LOCK(host); HOST_LOCK(host);
CK_SLIST_REMOVE(&STATE_HASH(host, state->hval), CK_SLIST_REMOVE(&STATE_HASH(host, state->hval),
state, nat64lsn_state, entries); state, nat64lsn_state, entries);
host->states_count--;
HOST_UNLOCK(host);
/* Reset READY flag */
ck_pr_btr_32(&state->flags, NAT64_BIT_READY_IPV4);
/* And set STALE flag */
ck_pr_bts_32(&state->flags, NAT64_BIT_STALE);
ck_pr_fence_store();
/* /*
* Now translate6 will not use this state, wait * Now translate6 will not use this state.
* until it become safe for translate4, then mark
* state as free.
*/ */
host->states_count--;
HOST_UNLOCK(host);
nat64lsn_state_cleanup(state);
} }
} }
/* /*
* We have some alive states, update timestamp. * We have some alive states, update timestamp.
*/ */
if (update_age) if (update_age)
SET_AGE(pg->timestamp); SET_AGE(pg->timestamp);
if (GET_AGE(pg->timestamp) < cfg->pg_delete_delay) if (GET_AGE(pg->timestamp) < cfg->pg_delete_delay)
return (0); return (0);
return (1); return (1);
} }
static void static void
nat64lsn_expire_portgroups(struct nat64lsn_cfg *cfg, nat64lsn_expire_portgroups(struct nat64lsn_cfg *cfg,
struct nat64lsn_pg_slist *portgroups) struct nat64lsn_pg_slist *portgroups)
{ {
struct nat64lsn_alias *alias; struct nat64lsn_alias *alias;
struct nat64lsn_pg *pg, *tpg, *firstpg, **pgptr; struct nat64lsn_pg *pg, *tpg;
uint32_t *pgmask, *pgidx; uint32_t *pgmask, *pgidx;
int i, idx; int i, idx;
for (i = 0; i < 1 << (32 - cfg->plen4); i++) { for (i = 0; i < 1 << (32 - cfg->plen4); i++) {
alias = &cfg->aliases[i]; alias = &cfg->aliases[i];
CK_SLIST_FOREACH_SAFE(pg, &alias->portgroups, entries, tpg) { CK_SLIST_FOREACH_SAFE(pg, &alias->portgroups, entries, tpg) {
if (nat64lsn_maintain_pg(cfg, pg) == 0) if (nat64lsn_maintain_pg(cfg, pg) == 0)
continue; continue;
/* Always keep first PG */ /* Always keep first PG */
if (pg->base_port == NAT64_MIN_PORT) if (pg->base_port == NAT64_MIN_PORT)
continue; continue;
/* /*
* PG is expired, unlink it and schedule for * PG expires in two passes:
* deferred destroying. * 1. Reset bit in pgmask, mark it as DEAD.
* 2. Unlink it and schedule for deferred destroying.
*/ */
idx = (pg->base_port - NAT64_MIN_PORT) / 64; idx = (pg->base_port - NAT64_MIN_PORT) / 64;
switch (pg->proto) { switch (pg->proto) {
case IPPROTO_TCP: case IPPROTO_TCP:
pgmask = alias->tcp_pgmask; pgmask = alias->tcp_pgmask;
pgptr = &alias->tcp_pg;
pgidx = &alias->tcp_pgidx; pgidx = &alias->tcp_pgidx;
firstpg = alias->tcp[0]->pgptr[0];
break; break;
case IPPROTO_UDP: case IPPROTO_UDP:
pgmask = alias->udp_pgmask; pgmask = alias->udp_pgmask;
pgptr = &alias->udp_pg;
pgidx = &alias->udp_pgidx; pgidx = &alias->udp_pgidx;
firstpg = alias->udp[0]->pgptr[0];
break; break;
case IPPROTO_ICMP: case IPPROTO_ICMP:
pgmask = alias->icmp_pgmask; pgmask = alias->icmp_pgmask;
pgptr = &alias->icmp_pg;
pgidx = &alias->icmp_pgidx; pgidx = &alias->icmp_pgidx;
firstpg = alias->icmp[0]->pgptr[0];
break; break;
} }
/* Reset the corresponding bit in pgmask array. */ if (pg->flags & NAT64LSN_DEADPG) {
ck_pr_btr_32(&pgmask[idx / 32], idx % 32);
ck_pr_fence_store();
/* If last used PG points to this PG, reset it. */
ck_pr_cas_ptr(pgptr, pg, firstpg);
ck_pr_cas_32(pgidx, idx, 0);
/* Unlink PG from alias's chain */ /* Unlink PG from alias's chain */
ALIAS_LOCK(alias); ALIAS_LOCK(alias);
CK_SLIST_REMOVE(&alias->portgroups, pg, CK_SLIST_REMOVE(&alias->portgroups, pg,
nat64lsn_pg, entries); nat64lsn_pg, entries);
alias->portgroups_count--; PGCOUNT_DEC(alias, pg->proto);
ALIAS_UNLOCK(alias); ALIAS_UNLOCK(alias);
/* And link to job's chain for deferred destroying */ /*
* Link it to job's chain for deferred
* destroying.
*/
NAT64STAT_INC(&cfg->base.stats, spgdeleted); NAT64STAT_INC(&cfg->base.stats, spgdeleted);
CK_SLIST_INSERT_HEAD(portgroups, pg, entries); CK_SLIST_INSERT_HEAD(portgroups, pg, entries);
continue;
} }
/* Reset the corresponding bit in pgmask array. */
ck_pr_btr_32(&pgmask[idx / 32], idx % 32);
pg->flags |= NAT64LSN_DEADPG;
ck_pr_fence_store();
/* If last used PG points to this PG, reset it. */
ck_pr_cas_32(pgidx, idx, 0);
} }
} }
}
static void static void
nat64lsn_expire_hosts(struct nat64lsn_cfg *cfg, nat64lsn_expire_hosts(struct nat64lsn_cfg *cfg,
struct nat64lsn_hosts_slist *hosts) struct nat64lsn_hosts_slist *hosts)
{ {
struct nat64lsn_host *host, *tmp; struct nat64lsn_host *host, *tmp;
int i; int i;
for (i = 0; i < cfg->hosts_hashsize; i++) { for (i = 0; i < cfg->hosts_hashsize; i++) {
CK_SLIST_FOREACH_SAFE(host, &cfg->hosts_hash[i], CK_SLIST_FOREACH_SAFE(host, &cfg->hosts_hash[i],
entries, tmp) { entries, tmp) {
/* Is host was marked in previous call? */ /* Is host was marked in previous call? */
if (host->flags & NAT64LSN_DEADHOST) { if (host->flags & NAT64LSN_DEADHOST) {
if (host->states_count > 0) { if (host->states_count > 0 ||
GET_AGE(host->timestamp) <
cfg->host_delete_delay) {
host->flags &= ~NAT64LSN_DEADHOST; host->flags &= ~NAT64LSN_DEADHOST;
continue; continue;
} }
/* /*
* Unlink host from hash table and schedule * Unlink host from hash table and schedule
* it for deferred destroying. * it for deferred destroying.
*/ */
CFG_LOCK(cfg); CFG_LOCK(cfg);
CK_SLIST_REMOVE(&cfg->hosts_hash[i], host, CK_SLIST_REMOVE(&cfg->hosts_hash[i], host,
nat64lsn_host, entries); nat64lsn_host, entries);
cfg->hosts_count--; cfg->hosts_count--;
CFG_UNLOCK(cfg); CFG_UNLOCK(cfg);
CK_SLIST_INSERT_HEAD(hosts, host, entries); CK_SLIST_INSERT_HEAD(hosts, host, entries);
continue; continue;
} }
if (GET_AGE(host->timestamp) < cfg->host_delete_delay) if (host->states_count > 0 ||
GET_AGE(host->timestamp) < cfg->host_delete_delay)
continue; continue;
if (host->states_count > 0)
continue;
/* Mark host as going to be expired in next pass */ /* Mark host as going to be expired in next pass */
host->flags |= NAT64LSN_DEADHOST; host->flags |= NAT64LSN_DEADHOST;
ck_pr_fence_store(); ck_pr_fence_store();
} }
} }
} }
static struct nat64lsn_pgchunk* static struct nat64lsn_pgchunk*
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines CK_SLIST_FOREACH(h, &cfg->hosts_hash[i], entries) {
ck_pr_bts_32(&h->flags, NAT64LSN_GROWHASH); ck_pr_bts_32(&h->flags, NAT64LSN_GROWHASH);
} }
} }
} }
#endif #endif
/* /*
* This procedure is used to perform various maintenance * This procedure is used to perform various maintance
* on dynamic hash list. Currently it is called every 4 seconds. * on dynamic hash list. Currently it is called every 4 seconds.
*/ */
static void static void
nat64lsn_periodic(void *data) nat64lsn_periodic(void *data)
{ {
struct nat64lsn_job_item *ji; struct nat64lsn_job_item *ji;
struct nat64lsn_cfg *cfg; struct nat64lsn_cfg *cfg;
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Linesnat64lsn_alloc_host(struct nat64lsn_cfg *cfg, struct nat64lsn_job_item *ji)
/* Initialize */ /* Initialize */
HOST_LOCK_INIT(host); HOST_LOCK_INIT(host);
SET_AGE(host->timestamp); SET_AGE(host->timestamp);
host->addr = ji->f_id.src_ip6; host->addr = ji->f_id.src_ip6;
host->hval = ji->src6_hval; host->hval = ji->src6_hval;
host->flags = 0; host->flags = 0;
host->states_count = 0; host->states_count = 0;
host->states_hashsize = NAT64LSN_HSIZE;
CK_SLIST_INIT(&host->aliases); CK_SLIST_INIT(&host->aliases);
for (i = 0; i < host->states_hashsize; i++) for (i = 0; i < host->states_hashsize; i++)
CK_SLIST_INIT(&host->states_hash[i]); CK_SLIST_INIT(&host->states_hash[i]);
/* Determine alias from flow hash. */ link->alias = nat64lsn_get_alias(cfg, &ji->f_id);
hval = ALIASLINK_HVAL(cfg, &ji->f_id);
link->alias = &ALIAS_BYHASH(cfg, hval);
CK_SLIST_INSERT_HEAD(&host->aliases, link, host_entries); CK_SLIST_INSERT_HEAD(&host->aliases, link, host_entries);
ALIAS_LOCK(link->alias); ALIAS_LOCK(link->alias);
CK_SLIST_INSERT_HEAD(&link->alias->hosts, link, alias_entries); CK_SLIST_INSERT_HEAD(&link->alias->hosts, link, alias_entries);
link->alias->hosts_count++; link->alias->hosts_count++;
ALIAS_UNLOCK(link->alias); ALIAS_UNLOCK(link->alias);
CFG_LOCK(cfg); CFG_LOCK(cfg);
Show All 35 Lines if (~data[i] == 0)
continue; continue;
return (i * 32 + ffs(~data[i]) - 1); return (i * 32 + ffs(~data[i]) - 1);
} }
return (-1); return (-1);
} }
static int static int
nat64lsn_alloc_proto_pg(struct nat64lsn_cfg *cfg, nat64lsn_alloc_proto_pg(struct nat64lsn_cfg *cfg,
struct nat64lsn_alias *alias, uint32_t *chunkmask, struct nat64lsn_alias *alias, uint32_t *chunkmask, uint32_t *pgmask,
uint32_t *pgmask, struct nat64lsn_pgchunk **chunks, struct nat64lsn_pgchunk **chunks, uint32_t *pgidx, uint8_t proto)
struct nat64lsn_pg **pgptr, uint8_t proto)
{ {
struct nat64lsn_pg *pg; struct nat64lsn_pg *pg;
int i, pg_idx, chunk_idx; int i, pg_idx, chunk_idx;
/* Find place in pgchunk where PG can be added */ /* Find place in pgchunk where PG can be added */
pg_idx = nat64lsn_find_pg_place(pgmask); pg_idx = nat64lsn_find_pg_place(pgmask);
if (pg_idx < 0) /* no more PGs */ if (pg_idx < 0) /* no more PGs */
return (PG_ERROR(1)); return (PG_ERROR(1));
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines if (pg->states == NULL) {
uma_zfree(nat64lsn_pg_zone, pg); uma_zfree(nat64lsn_pg_zone, pg);
return (PG_ERROR(6)); return (PG_ERROR(6));
} }
memset(&pg->freemask64, 0xff, sizeof(uint64_t)); memset(&pg->freemask64, 0xff, sizeof(uint64_t));
} }
/* Initialize PG and hook it to pgchunk */ /* Initialize PG and hook it to pgchunk */
SET_AGE(pg->timestamp); SET_AGE(pg->timestamp);
pg->flags = 0;
pg->proto = proto; pg->proto = proto;
pg->base_port = NAT64_MIN_PORT + 64 * pg_idx; pg->base_port = NAT64_MIN_PORT + 64 * pg_idx;
ck_pr_store_ptr(&chunks[chunk_idx]->pgptr[pg_idx % 32], pg); ck_pr_store_ptr(&chunks[chunk_idx]->pgptr[pg_idx % 32], pg);
ck_pr_fence_store(); ck_pr_fence_store();
ck_pr_bts_32(&pgmask[pg_idx / 32], pg_idx % 32);
ck_pr_store_ptr(pgptr, pg);
/* Set bit in pgmask and set index of last used PG */
ck_pr_bts_32(&pgmask[chunk_idx], pg_idx % 32);
ck_pr_store_32(pgidx, pg_idx);
ALIAS_LOCK(alias); ALIAS_LOCK(alias);
CK_SLIST_INSERT_HEAD(&alias->portgroups, pg, entries); CK_SLIST_INSERT_HEAD(&alias->portgroups, pg, entries);
SET_AGE(alias->timestamp); SET_AGE(alias->timestamp);
alias->portgroups_count++; PGCOUNT_INC(alias, proto);
ALIAS_UNLOCK(alias); ALIAS_UNLOCK(alias);
NAT64STAT_INC(&cfg->base.stats, spgcreated); NAT64STAT_INC(&cfg->base.stats, spgcreated);
return (PG_ERROR(0)); return (PG_ERROR(0));
states_failed: states_failed:
for (i = 0; i < pg->chunks_count; i++) for (i = 0; i < pg->chunks_count; i++)
uma_zfree(nat64lsn_state_zone, pg->states_chunk[i]); uma_zfree(nat64lsn_state_zone, pg->states_chunk[i]);
free(pg->freemask_chunk, M_NAT64LSN); free(pg->freemask_chunk, M_NAT64LSN);
Show All 20 Linesnat64lsn_alloc_pg(struct nat64lsn_cfg *cfg, struct nat64lsn_job_item *ji)
ret = 0; ret = 0;
alias = link->alias; alias = link->alias;
/* Find place in pgchunk where PG can be added */ /* Find place in pgchunk where PG can be added */
switch (ji->proto) { switch (ji->proto) {
case IPPROTO_TCP: case IPPROTO_TCP:
ret = nat64lsn_alloc_proto_pg(cfg, alias, ret = nat64lsn_alloc_proto_pg(cfg, alias,
&alias->tcp_chunkmask, alias->tcp_pgmask, &alias->tcp_chunkmask, alias->tcp_pgmask,
alias->tcp, &alias->tcp_pg, ji->proto); alias->tcp, &alias->tcp_pgidx, ji->proto);
break; break;
case IPPROTO_UDP: case IPPROTO_UDP:
ret = nat64lsn_alloc_proto_pg(cfg, alias, ret = nat64lsn_alloc_proto_pg(cfg, alias,
&alias->udp_chunkmask, alias->udp_pgmask, &alias->udp_chunkmask, alias->udp_pgmask,
alias->udp, &alias->udp_pg, ji->proto); alias->udp, &alias->udp_pgidx, ji->proto);
break; break;
case IPPROTO_ICMP: case IPPROTO_ICMP:
ret = nat64lsn_alloc_proto_pg(cfg, alias, ret = nat64lsn_alloc_proto_pg(cfg, alias,
&alias->icmp_chunkmask, alias->icmp_pgmask, &alias->icmp_chunkmask, alias->icmp_pgmask,
alias->icmp, &alias->icmp_pg, ji->proto); alias->icmp, &alias->icmp_pgidx, ji->proto);
break; break;
default: default:
panic("%s: wrong proto %d", __func__, ji->proto); panic("%s: wrong proto %d", __func__, ji->proto);
} }
if (ret == PG_ERROR(1)) { if (ret == PG_ERROR(1)) {
/* /*
* PG_ERROR(1) means that alias lacks free PGs * PG_ERROR(1) means that alias lacks free PGs
* XXX: try next alias. * XXX: try next alias.
▲ Show 20 Lines • Show All 125 Lines • ▼ Show 20 Linesnat64lsn_enqueue_job(struct nat64lsn_cfg *cfg, struct nat64lsn_job_item *ji)
NAT64STAT_INC(&cfg->base.stats, jrequests); NAT64STAT_INC(&cfg->base.stats, jrequests);
cfg->jlen++; cfg->jlen++;
if (callout_pending(&cfg->jcallout) == 0) if (callout_pending(&cfg->jcallout) == 0)
callout_reset(&cfg->jcallout, 1, nat64lsn_do_request, cfg); callout_reset(&cfg->jcallout, 1, nat64lsn_do_request, cfg);
JQUEUE_UNLOCK(); JQUEUE_UNLOCK();
} }
/*
* This function is used to clean up the result of less likely possible
* race condition, when host object was deleted, but some translation
* state was created before it is destroyed.
*
* Since the state expiration removes state from host's hash table,
* we need to be sure, that there will not any states, that are linked
* with this host entry.
*/
static void static void
nat64lsn_host_cleanup(struct nat64lsn_host *host)
{
struct nat64lsn_state *state, *ts;
int i;
printf("NAT64LSN: %s: race condition has been detected for host %p\n",
__func__, host);
for (i = 0; i < host->states_hashsize; i++) {
CK_SLIST_FOREACH_SAFE(state, &host->states_hash[i],
entries, ts) {
/*
* We can remove the state without lock,
* because this host entry is unlinked and will
* be destroyed.
*/
CK_SLIST_REMOVE(&host->states_hash[i], state,
nat64lsn_state, entries);
host->states_count--;
nat64lsn_state_cleanup(state);
}
}
MPASS(host->states_count == 0);
}
/*
* This function is used to clean up the result of less likely possible
* race condition, when portgroup was deleted, but some translation state
* was created before it is destroyed.
*
* Since states entries are accessible via host's hash table, we need
* to be sure, that there will not any states from this PG, that are
* linked with any host entries.
*/
static void
nat64lsn_pg_cleanup(struct nat64lsn_pg *pg)
{
struct nat64lsn_state *state;
uint64_t usedmask;
int c, i;
printf("NAT64LSN: %s: race condition has been detected for pg %p\n",
__func__, pg);
for (c = 0; c < pg->chunks_count; c++) {
/*
* Use inverted freemask to find what state was created.
*/
usedmask = ~(*FREEMASK_CHUNK(pg, c));
if (usedmask == 0)
continue;
for (i = 0; i < 64; i++) {
if (!ISSET64(usedmask, i))
continue;
state = &STATES_CHUNK(pg, c)->state[i];
/*
* If we have STALE bit, this means that state
* is already unlinked from host's hash table.
* Thus we can just reset the bit in mask and
* schedule destroying in the next epoch call.
*/
if (ISSET32(state->flags, NAT64_BIT_STALE)) {
FREEMASK_BTS(pg, c, i);
continue;
}
/*
* There is small window, when we have bit
* grabbed from freemask, but state is not yet
* linked into host's hash table.
* Check for READY flag, it is set just after
* linking. If it is not set, defer cleanup
* for next call.
*/
if (ISSET32(state->flags, NAT64_BIT_READY_IPV4)) {
struct nat64lsn_host *host;
host = state->host;
HOST_LOCK(host);
CK_SLIST_REMOVE(&STATE_HASH(host,
state->hval), state, nat64lsn_state,
entries);
host->states_count--;
HOST_UNLOCK(host);
nat64lsn_state_cleanup(state);
}
}
}
}
static void
nat64lsn_job_destroy(epoch_context_t ctx) nat64lsn_job_destroy(epoch_context_t ctx)
{ {
struct nat64lsn_hosts_slist hosts;
struct nat64lsn_pg_slist portgroups;
struct nat64lsn_job_item *ji; struct nat64lsn_job_item *ji;
struct nat64lsn_host *host; struct nat64lsn_host *host;
struct nat64lsn_pg *pg; struct nat64lsn_pg *pg;
int i; int i;
CK_SLIST_INIT(&hosts);
CK_SLIST_INIT(&portgroups);
ji = __containerof(ctx, struct nat64lsn_job_item, epoch_ctx); ji = __containerof(ctx, struct nat64lsn_job_item, epoch_ctx);
MPASS(ji->jtype == JTYPE_DESTROY); MPASS(ji->jtype == JTYPE_DESTROY);
while (!CK_SLIST_EMPTY(&ji->hosts)) { while (!CK_SLIST_EMPTY(&ji->hosts)) {
host = CK_SLIST_FIRST(&ji->hosts); host = CK_SLIST_FIRST(&ji->hosts);
CK_SLIST_REMOVE_HEAD(&ji->hosts, entries); CK_SLIST_REMOVE_HEAD(&ji->hosts, entries);
if (host->states_count > 0) { if (host->states_count > 0) {
/* /*
* XXX: The state has been created * The state has been created during host deletion.
* during host deletion.
*/ */
printf("NAT64LSN: %s: destroying host with %d " printf("NAT64LSN: %s: destroying host with %d "
"states\n", __func__, host->states_count); "states\n", __func__, host->states_count);
/*
* We need to cleanup these states to avoid
* possible access to already deleted host in
* the state expiration code.
*/
nat64lsn_host_cleanup(host);
CK_SLIST_INSERT_HEAD(&hosts, host, entries);
/*
* Keep host entry for next deferred destroying.
* In the next epoch its states will be not
* accessible.
*/
continue;
} }
nat64lsn_destroy_host(host); nat64lsn_destroy_host(host);
} }
while (!CK_SLIST_EMPTY(&ji->portgroups)) { while (!CK_SLIST_EMPTY(&ji->portgroups)) {
pg = CK_SLIST_FIRST(&ji->portgroups); pg = CK_SLIST_FIRST(&ji->portgroups);
CK_SLIST_REMOVE_HEAD(&ji->portgroups, entries); CK_SLIST_REMOVE_HEAD(&ji->portgroups, entries);
for (i = 0; i < pg->chunks_count; i++) { for (i = 0; i < pg->chunks_count; i++) {
if (FREEMASK_BITCOUNT(pg, i) != 64) { if (FREEMASK_BITCOUNT(pg, i) != 64) {
/* /*
* XXX: The state has been created during * A state has been created during
* PG deletion. * PG deletion.
*/ */
printf("NAT64LSN: %s: destroying PG %p " printf("NAT64LSN: %s: destroying PG %p "
"with non-empty chunk %d\n", __func__, "with non-empty chunk %d\n", __func__,
pg, i); pg, i);
nat64lsn_pg_cleanup(pg);
CK_SLIST_INSERT_HEAD(&portgroups,
pg, entries);
i = -1;
break;
} }
} }
if (i != -1)
nat64lsn_destroy_pg(pg); nat64lsn_destroy_pg(pg);
} }
if (CK_SLIST_EMPTY(&hosts) &&
CK_SLIST_EMPTY(&portgroups)) {
uma_zfree(nat64lsn_pgchunk_zone, ji->pgchunk); uma_zfree(nat64lsn_pgchunk_zone, ji->pgchunk);
uma_zfree(nat64lsn_job_zone, ji); uma_zfree(nat64lsn_job_zone, ji);
return;
} }
/* Schedule job item again */
CK_SLIST_MOVE(&ji->hosts, &hosts, entries);
CK_SLIST_MOVE(&ji->portgroups, &portgroups, entries);
NAT64LSN_EPOCH_CALL(&ji->epoch_ctx, nat64lsn_job_destroy);
}
static int static int
nat64lsn_request_host(struct nat64lsn_cfg *cfg, nat64lsn_request_host(struct nat64lsn_cfg *cfg,
const struct ipfw_flow_id *f_id, struct mbuf **mp, uint32_t hval, const struct ipfw_flow_id *f_id, struct mbuf **mp, uint32_t hval,
in_addr_t faddr, uint16_t port, uint8_t proto) in_addr_t faddr, uint16_t port, uint8_t proto)
{ {
struct nat64lsn_job_item *ji; struct nat64lsn_job_item *ji;
ji = nat64lsn_create_job(cfg, JTYPE_NEWHOST); ji = nat64lsn_create_job(cfg, JTYPE_NEWHOST);
▲ Show 20 Lines • Show All 148 Lines • ▼ Show 20 Lines
/* /*
* Main dataplane entry point. * Main dataplane entry point.
*/ */
int int
ipfw_nat64lsn(struct ip_fw_chain *ch, struct ip_fw_args *args, ipfw_nat64lsn(struct ip_fw_chain *ch, struct ip_fw_args *args,
ipfw_insn *cmd, int *done) ipfw_insn *cmd, int *done)
{ {
struct nat64lsn_cfg *cfg; struct nat64lsn_instance *i;
ipfw_insn *icmd; ipfw_insn *icmd;
int ret; int ret;
IPFW_RLOCK_ASSERT(ch); IPFW_RLOCK_ASSERT(ch);
*done = 0; /* continue the search in case of failure */ *done = 0; /* continue the search in case of failure */
icmd = cmd + 1; icmd = cmd + F_LEN(cmd);
if (cmd->opcode != O_EXTERNAL_ACTION || if (cmd->opcode != O_EXTERNAL_ACTION ||
cmd->arg1 != V_nat64lsn_eid || insntod(cmd, kidx)->kidx != V_nat64lsn_eid ||
icmd->opcode != O_EXTERNAL_INSTANCE || icmd->opcode != O_EXTERNAL_INSTANCE ||
(cfg = NAT64_LOOKUP(ch, icmd)) == NULL) (i = NAT64_LOOKUP(ch, icmd)) == NULL)
return (IP_FW_DENY); return (IP_FW_DENY);
*done = 1; /* terminate the search */ *done = 1; /* terminate the search */
switch (args->f_id.addr_type) { switch (args->f_id.addr_type) {
case 4: case 4:
ret = nat64lsn_translate4(cfg, &args->f_id, &args->m); ret = nat64lsn_translate4(i->cfg, &args->f_id, &args->m);
break; break;
case 6: case 6:
/* /*
* Check that destination IPv6 address matches our prefix6. * Check that destination IPv6 address matches our prefix6.
*/ */
if ((cfg->base.flags & NAT64LSN_ANYPREFIX) == 0 && if ((i->cfg->base.flags & NAT64LSN_ANYPREFIX) == 0 &&
memcmp(&args->f_id.dst_ip6, &cfg->base.plat_prefix, memcmp(&args->f_id.dst_ip6, &i->cfg->base.plat_prefix,
cfg->base.plat_plen / 8) != 0) { i->cfg->base.plat_plen / 8) != 0) {
ret = cfg->nomatch_verdict; ret = i->cfg->nomatch_verdict;
break; break;
} }
ret = nat64lsn_translate6(cfg, &args->f_id, &args->m); ret = nat64lsn_translate6(i->cfg, &args->f_id, &args->m);
break; break;
default: default:
ret = cfg->nomatch_verdict; ret = i->cfg->nomatch_verdict;
} }
if (ret != IP_FW_PASS && args->m != NULL) { if (ret != IP_FW_PASS && args->m != NULL) {
m_freem(args->m); m_freem(args->m);
args->m = NULL; args->m = NULL;
} }
return (ret); return (ret);
} }
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Linesnat64lsn_start_instance(struct nat64lsn_cfg *cfg)
CALLOUT_LOCK(cfg); CALLOUT_LOCK(cfg);
callout_reset(&cfg->periodic, hz * PERIODIC_DELAY, callout_reset(&cfg->periodic, hz * PERIODIC_DELAY,
nat64lsn_periodic, cfg); nat64lsn_periodic, cfg);
CALLOUT_UNLOCK(cfg); CALLOUT_UNLOCK(cfg);
} }
struct nat64lsn_cfg * struct nat64lsn_cfg *
nat64lsn_init_instance(struct ip_fw_chain *ch, in_addr_t prefix, int plen) nat64lsn_init_config(struct ip_fw_chain *ch, in_addr_t prefix, int plen)
{ {
struct nat64lsn_cfg *cfg; struct nat64lsn_cfg *cfg;
struct nat64lsn_alias *alias; struct nat64lsn_alias *alias;
int i, naddr; int i, naddr;
cfg = malloc(sizeof(struct nat64lsn_cfg), M_NAT64LSN, cfg = malloc(sizeof(struct nat64lsn_cfg), M_NAT64LSN,
M_WAITOK | M_ZERO); M_WAITOK | M_ZERO);
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines while (!CK_SLIST_EMPTY(&host->aliases)) {
uma_zfree(nat64lsn_aliaslink_zone, link); uma_zfree(nat64lsn_aliaslink_zone, link);
} }
HOST_LOCK_DESTROY(host); HOST_LOCK_DESTROY(host);
free(host->states_hash, M_NAT64LSN); free(host->states_hash, M_NAT64LSN);
uma_zfree(nat64lsn_host_zone, host); uma_zfree(nat64lsn_host_zone, host);
} }
void void
nat64lsn_destroy_instance(struct nat64lsn_cfg *cfg) nat64lsn_destroy_config(struct nat64lsn_cfg *cfg)
{ {
struct nat64lsn_host *host; struct nat64lsn_host *host;
int i; int i;
CALLOUT_LOCK(cfg); CALLOUT_LOCK(cfg);
callout_drain(&cfg->periodic); callout_drain(&cfg->periodic);
CALLOUT_UNLOCK(cfg); CALLOUT_UNLOCK(cfg);
callout_drain(&cfg->jcallout); callout_drain(&cfg->jcallout);
Show All 11 Linesnat64lsn_destroy_config(struct nat64lsn_cfg *cfg)
CALLOUT_LOCK_DESTROY(cfg); CALLOUT_LOCK_DESTROY(cfg);
CFG_LOCK_DESTROY(cfg); CFG_LOCK_DESTROY(cfg);
COUNTER_ARRAY_FREE(cfg->base.stats.cnt, NAT64STATS); COUNTER_ARRAY_FREE(cfg->base.stats.cnt, NAT64STATS);
free(cfg->hosts_hash, M_NAT64LSN); free(cfg->hosts_hash, M_NAT64LSN);
free(cfg->aliases, M_NAT64LSN); free(cfg->aliases, M_NAT64LSN);
free(cfg, M_NAT64LSN); free(cfg, M_NAT64LSN);
} }