Changeset View
Changeset View
Standalone View
Standalone View
head/sys/dev/netmap/netmap_pipe.c
/*- | /*- | ||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD | * SPDX-License-Identifier: BSD-2-Clause-FreeBSD | ||||
* | * | ||||
* Copyright (C) 2014-2016 Giuseppe Lettieri | * Copyright (C) 2014-2018 Giuseppe Lettieri | ||||
* All rights reserved. | * All rights reserved. | ||||
* | * | ||||
* Redistribution and use in source and binary forms, with or without | * Redistribution and use in source and binary forms, with or without | ||||
* modification, are permitted provided that the following conditions | * modification, are permitted provided that the following conditions | ||||
* are met: | * are met: | ||||
* 1. Redistributions of source code must retain the above copyright | * 1. Redistributions of source code must retain the above copyright | ||||
* notice, this list of conditions and the following disclaimer. | * notice, this list of conditions and the following disclaimer. | ||||
* 2. Redistributions in binary form must reproduce the above copyright | * 2. Redistributions in binary form must reproduce the above copyright | ||||
▲ Show 20 Lines • Show All 167 Lines • ▼ Show 20 Lines | netmap_pipe_remove(struct netmap_adapter *parent, struct netmap_pipe_adapter *na) | ||||
} | } | ||||
parent->na_pipes[n] = NULL; | parent->na_pipes[n] = NULL; | ||||
} | } | ||||
int | int | ||||
netmap_pipe_txsync(struct netmap_kring *txkring, int flags) | netmap_pipe_txsync(struct netmap_kring *txkring, int flags) | ||||
{ | { | ||||
struct netmap_kring *rxkring = txkring->pipe; | struct netmap_kring *rxkring = txkring->pipe; | ||||
u_int k, lim = txkring->nkr_num_slots - 1; | u_int k, lim = txkring->nkr_num_slots - 1, nk; | ||||
int m; /* slots to transfer */ | int m; /* slots to transfer */ | ||||
int complete; /* did we see a complete packet ? */ | |||||
struct netmap_ring *txring = txkring->ring, *rxring = rxkring->ring; | struct netmap_ring *txring = txkring->ring, *rxring = rxkring->ring; | ||||
ND("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name); | ND("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name); | ||||
ND(20, "TX before: hwcur %d hwtail %d cur %d head %d tail %d", | ND(20, "TX before: hwcur %d hwtail %d cur %d head %d tail %d", | ||||
txkring->nr_hwcur, txkring->nr_hwtail, | txkring->nr_hwcur, txkring->nr_hwtail, | ||||
txkring->rcur, txkring->rhead, txkring->rtail); | txkring->rcur, txkring->rhead, txkring->rtail); | ||||
/* update the hwtail */ | |||||
txkring->nr_hwtail = txkring->pipe_tail; | |||||
m = txkring->rhead - txkring->nr_hwcur; /* new slots */ | m = txkring->rhead - txkring->nr_hwcur; /* new slots */ | ||||
if (m < 0) | if (m < 0) | ||||
m += txkring->nkr_num_slots; | m += txkring->nkr_num_slots; | ||||
if (m == 0) { | if (m == 0) { | ||||
/* nothing to send */ | /* nothing to send */ | ||||
return 0; | return 0; | ||||
} | } | ||||
for (k = txkring->nr_hwcur; m; m--, k = nm_next(k, lim)) { | for (k = txkring->nr_hwcur, nk = lim + 1, complete = 0; m; | ||||
m--, k = nm_next(k, lim), nk = (complete ? k : nk)) { | |||||
struct netmap_slot *rs = &rxring->slot[k]; | struct netmap_slot *rs = &rxring->slot[k]; | ||||
struct netmap_slot *ts = &txring->slot[k]; | struct netmap_slot *ts = &txring->slot[k]; | ||||
rs->len = ts->len; | *rs = *ts; | ||||
rs->ptr = ts->ptr; | |||||
if (ts->flags & NS_BUF_CHANGED) { | if (ts->flags & NS_BUF_CHANGED) { | ||||
rs->buf_idx = ts->buf_idx; | |||||
rs->flags |= NS_BUF_CHANGED; | |||||
ts->flags &= ~NS_BUF_CHANGED; | ts->flags &= ~NS_BUF_CHANGED; | ||||
} | } | ||||
complete = !(ts->flags & NS_MOREFRAG); | |||||
} | } | ||||
mb(); /* make sure the slots are updated before publishing them */ | |||||
rxkring->nr_hwtail = k; | |||||
txkring->nr_hwcur = k; | txkring->nr_hwcur = k; | ||||
ND(20, "TX after : hwcur %d hwtail %d cur %d head %d tail %d k %d", | ND(20, "TX after : hwcur %d hwtail %d cur %d head %d tail %d k %d", | ||||
txkring->nr_hwcur, txkring->nr_hwtail, | txkring->nr_hwcur, txkring->nr_hwtail, | ||||
txkring->rcur, txkring->rhead, txkring->rtail, k); | txkring->rcur, txkring->rhead, txkring->rtail, k); | ||||
if (likely(nk <= lim)) { | |||||
mb(); /* make sure the slots are updated before publishing them */ | |||||
rxkring->pipe_tail = nk; /* only publish complete packets */ | |||||
rxkring->nm_notify(rxkring, 0); | rxkring->nm_notify(rxkring, 0); | ||||
} | |||||
return 0; | return 0; | ||||
} | } | ||||
int | int | ||||
netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags) | netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags) | ||||
{ | { | ||||
struct netmap_kring *txkring = rxkring->pipe; | struct netmap_kring *txkring = rxkring->pipe; | ||||
u_int k, lim = rxkring->nkr_num_slots - 1; | u_int k, lim = rxkring->nkr_num_slots - 1; | ||||
int m; /* slots to release */ | int m; /* slots to release */ | ||||
struct netmap_ring *txring = txkring->ring, *rxring = rxkring->ring; | struct netmap_ring *txring = txkring->ring, *rxring = rxkring->ring; | ||||
ND("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name); | ND("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name); | ||||
ND(20, "RX before: hwcur %d hwtail %d cur %d head %d tail %d", | ND(20, "RX before: hwcur %d hwtail %d cur %d head %d tail %d", | ||||
rxkring->nr_hwcur, rxkring->nr_hwtail, | rxkring->nr_hwcur, rxkring->nr_hwtail, | ||||
rxkring->rcur, rxkring->rhead, rxkring->rtail); | rxkring->rcur, rxkring->rhead, rxkring->rtail); | ||||
/* update the hwtail */ | |||||
rxkring->nr_hwtail = rxkring->pipe_tail; | |||||
m = rxkring->rhead - rxkring->nr_hwcur; /* released slots */ | m = rxkring->rhead - rxkring->nr_hwcur; /* released slots */ | ||||
if (m < 0) | if (m < 0) | ||||
m += rxkring->nkr_num_slots; | m += rxkring->nkr_num_slots; | ||||
if (m == 0) { | if (m == 0) { | ||||
/* nothing to release */ | /* nothing to release */ | ||||
return 0; | return 0; | ||||
} | } | ||||
for (k = rxkring->nr_hwcur; m; m--, k = nm_next(k, lim)) { | for (k = rxkring->nr_hwcur; m; m--, k = nm_next(k, lim)) { | ||||
struct netmap_slot *rs = &rxring->slot[k]; | struct netmap_slot *rs = &rxring->slot[k]; | ||||
struct netmap_slot *ts = &txring->slot[k]; | struct netmap_slot *ts = &txring->slot[k]; | ||||
if (rs->flags & NS_BUF_CHANGED) { | if (rs->flags & NS_BUF_CHANGED) { | ||||
/* copy the slot and report the buffer change */ | /* copy the slot and report the buffer change */ | ||||
*ts = *rs; | *ts = *rs; | ||||
rs->flags &= ~NS_BUF_CHANGED; | rs->flags &= ~NS_BUF_CHANGED; | ||||
} | } | ||||
} | } | ||||
mb(); /* make sure the slots are updated before publishing them */ | mb(); /* make sure the slots are updated before publishing them */ | ||||
txkring->nr_hwtail = nm_prev(k, lim); | txkring->pipe_tail = nm_prev(k, lim); | ||||
rxkring->nr_hwcur = k; | rxkring->nr_hwcur = k; | ||||
ND(20, "RX after : hwcur %d hwtail %d cur %d head %d tail %d k %d", | ND(20, "RX after : hwcur %d hwtail %d cur %d head %d tail %d k %d", | ||||
rxkring->nr_hwcur, rxkring->nr_hwtail, | rxkring->nr_hwcur, rxkring->nr_hwtail, | ||||
rxkring->rcur, rxkring->rhead, rxkring->rtail, k); | rxkring->rcur, rxkring->rhead, rxkring->rtail, k); | ||||
txkring->nm_notify(txkring, 0); | txkring->nm_notify(txkring, 0); | ||||
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | if (pna->peer_ref) { | ||||
if (error) | if (error) | ||||
goto err; | goto err; | ||||
/* create the krings of the other end */ | /* create the krings of the other end */ | ||||
error = netmap_krings_create(ona, 0); | error = netmap_krings_create(ona, 0); | ||||
if (error) | if (error) | ||||
goto del_krings1; | goto del_krings1; | ||||
/* cross link the krings */ | /* cross link the krings and initialize the pipe_tails */ | ||||
for_rx_tx(t) { | for_rx_tx(t) { | ||||
enum txrx r = nm_txrx_swap(t); /* swap NR_TX <-> NR_RX */ | enum txrx r = nm_txrx_swap(t); /* swap NR_TX <-> NR_RX */ | ||||
for (i = 0; i < nma_get_nrings(na, t); i++) { | for (i = 0; i < nma_get_nrings(na, t); i++) { | ||||
NMR(na, t)[i]->pipe = NMR(ona, r)[i]; | struct netmap_kring *k1 = NMR(na, t)[i], | ||||
NMR(ona, r)[i]->pipe = NMR(na, t)[i]; | *k2 = NMR(ona, r)[i]; | ||||
k1->pipe = k2; | |||||
k2->pipe = k1; | |||||
/* mark all peer-adapter rings as fake */ | /* mark all peer-adapter rings as fake */ | ||||
NMR(ona, r)[i]->nr_kflags |= NKR_FAKERING; | k2->nr_kflags |= NKR_FAKERING; | ||||
/* init tails */ | |||||
k1->pipe_tail = k1->nr_hwtail; | |||||
k2->pipe_tail = k2->nr_hwtail; | |||||
} | } | ||||
} | } | ||||
} | } | ||||
return 0; | return 0; | ||||
del_krings1: | del_krings1: | ||||
netmap_krings_delete(na); | netmap_krings_delete(na); | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | if (onoff) { | ||||
/* In case of no error we put our rings in netmap mode */ | /* In case of no error we put our rings in netmap mode */ | ||||
for_rx_tx(t) { | for_rx_tx(t) { | ||||
for (i = 0; i < nma_get_nrings(na, t) + 1; i++) { | for (i = 0; i < nma_get_nrings(na, t) + 1; i++) { | ||||
struct netmap_kring *kring = NMR(na, t)[i]; | struct netmap_kring *kring = NMR(na, t)[i]; | ||||
if (nm_kring_pending_on(kring)) { | if (nm_kring_pending_on(kring)) { | ||||
struct netmap_kring *sring, *dring; | struct netmap_kring *sring, *dring; | ||||
kring->nr_mode = NKR_NETMAP_ON; | |||||
if ((kring->nr_kflags & NKR_FAKERING) && | |||||
(kring->pipe->nr_kflags & NKR_FAKERING)) { | |||||
/* this is a re-open of a pipe | |||||
* end-point kept alive by the other end. | |||||
* We need to leave everything as it is | |||||
*/ | |||||
continue; | |||||
} | |||||
/* copy the buffers from the non-fake ring */ | /* copy the buffers from the non-fake ring */ | ||||
if (kring->nr_kflags & NKR_FAKERING) { | if (kring->nr_kflags & NKR_FAKERING) { | ||||
sring = kring->pipe; | sring = kring->pipe; | ||||
dring = kring; | dring = kring; | ||||
} else { | } else { | ||||
sring = kring; | sring = kring; | ||||
dring = kring->pipe; | dring = kring->pipe; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | for (i = 0; i < nma_get_nrings(sna, t) + 1; i++) { | ||||
uint32_t j, lim = kring->nkr_num_slots - 1; | uint32_t j, lim = kring->nkr_num_slots - 1; | ||||
ND("%s ring %p hwtail %u hwcur %u", | ND("%s ring %p hwtail %u hwcur %u", | ||||
kring->name, ring, kring->nr_hwtail, kring->nr_hwcur); | kring->name, ring, kring->nr_hwtail, kring->nr_hwcur); | ||||
if (ring == NULL) | if (ring == NULL) | ||||
continue; | continue; | ||||
if (kring->nr_hwtail == kring->nr_hwcur) | if (kring->tx == NR_RX) | ||||
ring->slot[kring->nr_hwtail].buf_idx = 0; | ring->slot[kring->pipe_tail].buf_idx = 0; | ||||
for (j = nm_next(kring->nr_hwtail, lim); | for (j = nm_next(kring->pipe_tail, lim); | ||||
j != kring->nr_hwcur; | j != kring->nr_hwcur; | ||||
j = nm_next(j, lim)) | j = nm_next(j, lim)) | ||||
{ | { | ||||
ND("%s[%d] %u", kring->name, j, ring->slot[j].buf_idx); | ND("%s[%d] %u", kring->name, j, ring->slot[j].buf_idx); | ||||
ring->slot[j].buf_idx = 0; | ring->slot[j].buf_idx = 0; | ||||
} | } | ||||
kring->nr_kflags &= ~(NKR_FAKERING | NKR_NEEDRING); | kring->nr_kflags &= ~(NKR_FAKERING | NKR_NEEDRING); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 249 Lines • Show Last 20 Lines |