Page MenuHomeFreeBSD
Files F87411083

No OneTemporary
Actions

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
Index: head/sys/amd64/amd64/mp_watchdog.c
===================================================================
--- head/sys/amd64/amd64/mp_watchdog.c (revision 283290)
+++ head/sys/amd64/amd64/mp_watchdog.c (revision 283291)
@@ -1,210 +1,210 @@
/*-
* Copyright (c) 2004 Robert N. M. Watson
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include "opt_mp_watchdog.h"
#include "opt_sched.h"
#ifdef SCHED_ULE
#error MP_WATCHDOG cannot currently be used with SCHED_ULE
#endif
#include <sys/param.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/pcpu.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <machine/smp.h>
#include <x86/apicreg.h>
#include <x86/apicvar.h>
#include <machine/mp_watchdog.h>
/*
* mp_watchdog hijacks the idle thread on a specified CPU, prevents new work
* from being scheduled there, and uses it as a "watchdog" to detect kernel
* failure on other CPUs. This is made reasonable by inclusion of logical
* processors in Xeon hardware. The watchdog is configured by setting the
* debug.watchdog sysctl/tunable to the CPU of interest. A callout will then
* begin executing reseting a timer that is gradually lowered by the watching
* thread. If the timer reaches 0, the watchdog fires by ether dropping
* directly to the debugger, or by sending an NMI IPI to the boot processor.
* This is a somewhat less efficient substitute for dedicated watchdog
* hardware, but can be quite an effective tool for debugging hangs.
*
* XXXRW: This should really use the watchdog(9)/watchdog(4) framework, but
* doesn't yet.
*/
static int watchdog_cpu = -1;
static int watchdog_dontfire = 1;
static int watchdog_timer = -1;
static int watchdog_nmi = 1;
SYSCTL_INT(_debug, OID_AUTO, watchdog_nmi, CTLFLAG_RWTUN, &watchdog_nmi, 0,
"IPI the boot processor with an NMI to enter the debugger");
static struct callout watchdog_callout;
static void watchdog_change(int wdcpu);
/*
* Number of seconds before the watchdog will fire if the callout fails to
* reset the timer.
*/
#define WATCHDOG_THRESHOLD 10
static void
watchdog_init(void *arg)
{
- callout_init(&watchdog_callout, CALLOUT_MPSAFE);
+ callout_init(&watchdog_callout, 1);
if (watchdog_cpu != -1)
watchdog_change(watchdog_cpu);
}
/*
* This callout resets a timer until the watchdog kicks in. It acquires some
* critical locks to make sure things haven't gotten wedged with hose locks
* held.
*/
static void
watchdog_function(void *arg)
{
/*
* Since the timer ran, we must not be wedged. Acquire some critical
* locks to make sure. Then reset the timer.
*/
mtx_lock(&Giant);
watchdog_timer = WATCHDOG_THRESHOLD;
mtx_unlock(&Giant);
callout_reset(&watchdog_callout, 1 * hz, watchdog_function, NULL);
}
SYSINIT(watchdog_init, SI_SUB_DRIVERS, SI_ORDER_ANY, watchdog_init, NULL);
static void
watchdog_change(int wdcpu)
{
if (wdcpu == -1 || wdcpu == 0xffffffff) {
/*
* Disable the watchdog.
*/
watchdog_cpu = -1;
watchdog_dontfire = 1;
callout_stop(&watchdog_callout);
printf("watchdog stopped\n");
} else {
watchdog_timer = WATCHDOG_THRESHOLD;
watchdog_dontfire = 0;
watchdog_cpu = wdcpu;
callout_reset(&watchdog_callout, 1 * hz, watchdog_function,
NULL);
}
}
/*
* This sysctl sets which CPU is the watchdog CPU. Set to -1 or 0xffffffff
* to disable the watchdog.
*/
static int
sysctl_watchdog(SYSCTL_HANDLER_ARGS)
{
int error, temp;
temp = watchdog_cpu;
error = sysctl_handle_int(oidp, &temp, 0, req);
if (error)
return (error);
if (req->newptr != NULL)
watchdog_change(temp);
return (0);
}
SYSCTL_PROC(_debug, OID_AUTO, watchdog, CTLTYPE_INT|CTLFLAG_RW, 0, 0,
sysctl_watchdog, "I", "");
/*
* Drop into the debugger by sending an IPI NMI to the boot processor.
*/
static void
watchdog_ipi_nmi(void)
{
/*
* Deliver NMI to the boot processor. Why not?
*/
lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_NMI,
boot_cpu_id);
lapic_ipi_wait(-1);
}
/*
* ap_watchdog() is called by the SMP idle loop code. It works on the same
* premise that the disabling of logical processors does: that if the cpu is
* idle, then it can ignore the world from then on, as nothing will be
* scheduled on it. Leaving aside multi-runqueue schedulers (SCHED_ULE) and
* explicit process migration (sched_bind()), this is not an unreasonable
* assumption.
*/
void
ap_watchdog(u_int cpuid)
{
char old_pcomm[MAXCOMLEN + 1];
struct proc *p;
if (watchdog_cpu != cpuid)
return;
printf("watchdog started on cpu %d\n", cpuid);
p = curproc;
bcopy(p->p_comm, old_pcomm, MAXCOMLEN + 1);
snprintf(p->p_comm, MAXCOMLEN + 1, "mp_watchdog cpu %d", cpuid);
while (1) {
DELAY(1000000); /* One second. */
if (watchdog_cpu != cpuid)
break;
atomic_subtract_int(&watchdog_timer, 1);
if (watchdog_timer < 4)
printf("Watchdog timer: %d\n", watchdog_timer);
if (watchdog_timer == 0 && watchdog_dontfire == 0) {
printf("Watchdog firing!\n");
watchdog_dontfire = 1;
if (watchdog_nmi)
watchdog_ipi_nmi();
else
kdb_enter(KDB_WHY_WATCHDOG, "mp_watchdog");
}
}
bcopy(old_pcomm, p->p_comm, MAXCOMLEN + 1);
printf("watchdog stopped on cpu %d\n", cpuid);
}
Index: head/sys/amd64/vmm/io/vatpit.c
===================================================================
--- head/sys/amd64/vmm/io/vatpit.c (revision 283290)
+++ head/sys/amd64/vmm/io/vatpit.c (revision 283291)
@@ -1,457 +1,457 @@
/*-
* Copyright (c) 2014 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
* Copyright (c) 2011 NetApp, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/types.h>
#include <sys/queue.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/systm.h>
#include <machine/vmm.h>
#include "vmm_ktr.h"
#include "vatpic.h"
#include "vioapic.h"
#include "vatpit.h"
static MALLOC_DEFINE(M_VATPIT, "atpit", "bhyve virtual atpit (8254)");
#define VATPIT_LOCK(vatpit) mtx_lock_spin(&((vatpit)->mtx))
#define VATPIT_UNLOCK(vatpit) mtx_unlock_spin(&((vatpit)->mtx))
#define VATPIT_LOCKED(vatpit) mtx_owned(&((vatpit)->mtx))
#define TIMER_SEL_MASK 0xc0
#define TIMER_RW_MASK 0x30
#define TIMER_MODE_MASK 0x0f
#define TIMER_SEL_READBACK 0xc0
#define TIMER_STS_OUT 0x80
#define TIMER_STS_NULLCNT 0x40
#define TIMER_RB_LCTR 0x20
#define TIMER_RB_LSTATUS 0x10
#define TIMER_RB_CTR_2 0x08
#define TIMER_RB_CTR_1 0x04
#define TIMER_RB_CTR_0 0x02
#define TMR2_OUT_STS 0x20
#define PIT_8254_FREQ 1193182
#define TIMER_DIV(freq, hz) (((freq) + (hz) / 2) / (hz))
struct vatpit_callout_arg {
struct vatpit *vatpit;
int channel_num;
};
struct channel {
int mode;
uint16_t initial; /* initial counter value */
sbintime_t now_sbt; /* uptime when counter was loaded */
uint8_t cr[2];
uint8_t ol[2];
bool slatched; /* status latched */
uint8_t status;
int crbyte;
int olbyte;
int frbyte;
struct callout callout;
sbintime_t callout_sbt; /* target time */
struct vatpit_callout_arg callout_arg;
};
struct vatpit {
struct vm *vm;
struct mtx mtx;
sbintime_t freq_sbt;
struct channel channel[3];
};
static void pit_timer_start_cntr0(struct vatpit *vatpit);
static int
vatpit_get_out(struct vatpit *vatpit, int channel)
{
struct channel *c;
sbintime_t delta_ticks;
int out;
c = &vatpit->channel[channel];
switch (c->mode) {
case TIMER_INTTC:
delta_ticks = (sbinuptime() - c->now_sbt) / vatpit->freq_sbt;
out = ((c->initial - delta_ticks) <= 0);
break;
default:
out = 0;
break;
}
return (out);
}
static void
vatpit_callout_handler(void *a)
{
struct vatpit_callout_arg *arg = a;
struct vatpit *vatpit;
struct callout *callout;
struct channel *c;
vatpit = arg->vatpit;
c = &vatpit->channel[arg->channel_num];
callout = &c->callout;
VM_CTR1(vatpit->vm, "atpit t%d fired", arg->channel_num);
VATPIT_LOCK(vatpit);
if (callout_pending(callout)) /* callout was reset */
goto done;
if (!callout_active(callout)) /* callout was stopped */
goto done;
callout_deactivate(callout);
if (c->mode == TIMER_RATEGEN) {
pit_timer_start_cntr0(vatpit);
}
vatpic_pulse_irq(vatpit->vm, 0);
vioapic_pulse_irq(vatpit->vm, 2);
done:
VATPIT_UNLOCK(vatpit);
return;
}
static void
pit_timer_start_cntr0(struct vatpit *vatpit)
{
struct channel *c;
sbintime_t now, delta, precision;
c = &vatpit->channel[0];
if (c->initial != 0) {
delta = c->initial * vatpit->freq_sbt;
precision = delta >> tc_precexp;
c->callout_sbt = c->callout_sbt + delta;
/*
* Reset 'callout_sbt' if the time that the callout
* was supposed to fire is more than 'c->initial'
* ticks in the past.
*/
now = sbinuptime();
if (c->callout_sbt < now)
c->callout_sbt = now + delta;
callout_reset_sbt(&c->callout, c->callout_sbt,
precision, vatpit_callout_handler, &c->callout_arg,
C_ABSOLUTE);
}
}
static uint16_t
pit_update_counter(struct vatpit *vatpit, struct channel *c, bool latch)
{
uint16_t lval;
sbintime_t delta_ticks;
/* cannot latch a new value until the old one has been consumed */
if (latch && c->olbyte != 0)
return (0);
if (c->initial == 0) {
/*
* This is possibly an o/s bug - reading the value of
* the timer without having set up the initial value.
*
* The original user-space version of this code set
* the timer to 100hz in this condition; do the same
* here.
*/
c->initial = TIMER_DIV(PIT_8254_FREQ, 100);
c->now_sbt = sbinuptime();
c->status &= ~TIMER_STS_NULLCNT;
}
delta_ticks = (sbinuptime() - c->now_sbt) / vatpit->freq_sbt;
lval = c->initial - delta_ticks % c->initial;
if (latch) {
c->olbyte = 2;
c->ol[1] = lval; /* LSB */
c->ol[0] = lval >> 8; /* MSB */
}
return (lval);
}
static int
pit_readback1(struct vatpit *vatpit, int channel, uint8_t cmd)
{
struct channel *c;
c = &vatpit->channel[channel];
/*
* Latch the count/status of the timer if not already latched.
* N.B. that the count/status latch-select bits are active-low.
*/
if (!(cmd & TIMER_RB_LCTR) && !c->olbyte) {
(void) pit_update_counter(vatpit, c, true);
}
if (!(cmd & TIMER_RB_LSTATUS) && !c->slatched) {
c->slatched = true;
/*
* For mode 0, see if the elapsed time is greater
* than the initial value - this results in the
* output pin being set to 1 in the status byte.
*/
if (c->mode == TIMER_INTTC && vatpit_get_out(vatpit, channel))
c->status |= TIMER_STS_OUT;
else
c->status &= ~TIMER_STS_OUT;
}
return (0);
}
static int
pit_readback(struct vatpit *vatpit, uint8_t cmd)
{
int error;
/*
* The readback command can apply to all timers.
*/
error = 0;
if (cmd & TIMER_RB_CTR_0)
error = pit_readback1(vatpit, 0, cmd);
if (!error && cmd & TIMER_RB_CTR_1)
error = pit_readback1(vatpit, 1, cmd);
if (!error && cmd & TIMER_RB_CTR_2)
error = pit_readback1(vatpit, 2, cmd);
return (error);
}
static int
vatpit_update_mode(struct vatpit *vatpit, uint8_t val)
{
struct channel *c;
int sel, rw, mode;
sel = val & TIMER_SEL_MASK;
rw = val & TIMER_RW_MASK;
mode = val & TIMER_MODE_MASK;
if (sel == TIMER_SEL_READBACK)
return (pit_readback(vatpit, val));
if (rw != TIMER_LATCH && rw != TIMER_16BIT)
return (-1);
if (rw != TIMER_LATCH) {
/*
* Counter mode is not affected when issuing a
* latch command.
*/
if (mode != TIMER_INTTC &&
mode != TIMER_RATEGEN &&
mode != TIMER_SQWAVE &&
mode != TIMER_SWSTROBE)
return (-1);
}
c = &vatpit->channel[sel >> 6];
if (rw == TIMER_LATCH)
pit_update_counter(vatpit, c, true);
else {
c->mode = mode;
c->olbyte = 0; /* reset latch after reprogramming */
c->status |= TIMER_STS_NULLCNT;
}
return (0);
}
int
vatpit_handler(struct vm *vm, int vcpuid, bool in, int port, int bytes,
uint32_t *eax)
{
struct vatpit *vatpit;
struct channel *c;
uint8_t val;
int error;
vatpit = vm_atpit(vm);
if (bytes != 1)
return (-1);
val = *eax;
if (port == TIMER_MODE) {
if (in) {
VM_CTR0(vatpit->vm, "vatpit attempt to read mode");
return (-1);
}
VATPIT_LOCK(vatpit);
error = vatpit_update_mode(vatpit, val);
VATPIT_UNLOCK(vatpit);
return (error);
}
/* counter ports */
KASSERT(port >= TIMER_CNTR0 && port <= TIMER_CNTR2,
("invalid port 0x%x", port));
c = &vatpit->channel[port - TIMER_CNTR0];
VATPIT_LOCK(vatpit);
if (in && c->slatched) {
/*
* Return the status byte if latched
*/
*eax = c->status;
c->slatched = false;
c->status = 0;
} else if (in) {
/*
* The spec says that once the output latch is completely
* read it should revert to "following" the counter. Use
* the free running counter for this case (i.e. Linux
* TSC calibration). Assuming the access mode is 16-bit,
* toggle the MSB/LSB bit on each read.
*/
if (c->olbyte == 0) {
uint16_t tmp;
tmp = pit_update_counter(vatpit, c, false);
if (c->frbyte)
tmp >>= 8;
tmp &= 0xff;
*eax = tmp;
c->frbyte ^= 1;
} else
*eax = c->ol[--c->olbyte];
} else {
c->cr[c->crbyte++] = *eax;
if (c->crbyte == 2) {
c->status &= ~TIMER_STS_NULLCNT;
c->frbyte = 0;
c->crbyte = 0;
c->initial = c->cr[0] | (uint16_t)c->cr[1] << 8;
c->now_sbt = sbinuptime();
/* Start an interval timer for channel 0 */
if (port == TIMER_CNTR0) {
c->callout_sbt = c->now_sbt;
pit_timer_start_cntr0(vatpit);
}
if (c->initial == 0)
c->initial = 0xffff;
}
}
VATPIT_UNLOCK(vatpit);
return (0);
}
int
vatpit_nmisc_handler(struct vm *vm, int vcpuid, bool in, int port, int bytes,
uint32_t *eax)
{
struct vatpit *vatpit;
vatpit = vm_atpit(vm);
if (in) {
VATPIT_LOCK(vatpit);
if (vatpit_get_out(vatpit, 2))
*eax = TMR2_OUT_STS;
else
*eax = 0;
VATPIT_UNLOCK(vatpit);
}
return (0);
}
struct vatpit *
vatpit_init(struct vm *vm)
{
struct vatpit *vatpit;
struct bintime bt;
struct vatpit_callout_arg *arg;
int i;
vatpit = malloc(sizeof(struct vatpit), M_VATPIT, M_WAITOK | M_ZERO);
vatpit->vm = vm;
mtx_init(&vatpit->mtx, "vatpit lock", NULL, MTX_SPIN);
FREQ2BT(PIT_8254_FREQ, &bt);
vatpit->freq_sbt = bttosbt(bt);
for (i = 0; i < 3; i++) {
- callout_init(&vatpit->channel[i].callout, true);
+ callout_init(&vatpit->channel[i].callout, 1);
arg = &vatpit->channel[i].callout_arg;
arg->vatpit = vatpit;
arg->channel_num = i;
}
return (vatpit);
}
void
vatpit_cleanup(struct vatpit *vatpit)
{
int i;
for (i = 0; i < 3; i++)
callout_drain(&vatpit->channel[i].callout);
free(vatpit, M_VATPIT);
}
Index: head/sys/arm/amlogic/aml8726/aml8726_rng.c
===================================================================
--- head/sys/arm/amlogic/aml8726/aml8726_rng.c (revision 283290)
+++ head/sys/arm/amlogic/aml8726/aml8726_rng.c (revision 283291)
@@ -1,155 +1,155 @@
/*-
* Copyright 2014 John Wehle <john@feith.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Amlogic aml8726 random number generator driver.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/conf.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/resource.h>
#include <sys/rman.h>
#include <sys/random.h>
#include <machine/bus.h>
#include <dev/fdt/fdt_common.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
struct aml8726_rng_softc {
device_t dev;
struct resource *res[1];
struct callout co;
int ticks;
};
static struct resource_spec aml8726_rng_spec[] = {
{ SYS_RES_MEMORY, 0, RF_ACTIVE },
{ -1, 0 }
};
#define AML_RNG_0_REG 0
#define AML_RNG_1_REG 4
#define CSR_READ_4(sc, reg) bus_read_4((sc)->res[0], reg)
static void
aml8726_rng_harvest(void *arg)
{
struct aml8726_rng_softc *sc = arg;
uint32_t rn[2];
rn[0] = CSR_READ_4(sc, AML_RNG_0_REG);
rn[1] = CSR_READ_4(sc, AML_RNG_1_REG);
random_harvest(rn, sizeof(rn), sizeof(rn) * NBBY / 2,
RANDOM_PURE_AML8726);
callout_reset(&sc->co, sc->ticks, aml8726_rng_harvest, sc);
}
static int
aml8726_rng_probe(device_t dev)
{
if (!ofw_bus_status_okay(dev))
return (ENXIO);
if (!ofw_bus_is_compatible(dev, "amlogic,aml8726-rng"))
return (ENXIO);
device_set_desc(dev, "Amlogic aml8726 RNG");
return (BUS_PROBE_DEFAULT);
}
static int
aml8726_rng_attach(device_t dev)
{
struct aml8726_rng_softc *sc = device_get_softc(dev);
sc->dev = dev;
if (bus_alloc_resources(dev, aml8726_rng_spec, sc->res)) {
device_printf(dev, "can not allocate resources for device\n");
return (ENXIO);
}
/* Install a periodic collector for the RNG */
if (hz > 100)
sc->ticks = hz / 100;
else
sc->ticks = 1;
- callout_init(&sc->co, CALLOUT_MPSAFE);
+ callout_init(&sc->co, 1);
callout_reset(&sc->co, sc->ticks, aml8726_rng_harvest, sc);
return (0);
}
static int
aml8726_rng_detach(device_t dev)
{
struct aml8726_rng_softc *sc = device_get_softc(dev);
callout_drain(&sc->co);
bus_release_resources(dev, aml8726_rng_spec, sc->res);
return (0);
}
static device_method_t aml8726_rng_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, aml8726_rng_probe),
DEVMETHOD(device_attach, aml8726_rng_attach),
DEVMETHOD(device_detach, aml8726_rng_detach),
DEVMETHOD_END
};
static driver_t aml8726_rng_driver = {
"rng",
aml8726_rng_methods,
sizeof(struct aml8726_rng_softc),
};
static devclass_t aml8726_rng_devclass;
DRIVER_MODULE(aml8726_rng, simplebus, aml8726_rng_driver,
aml8726_rng_devclass, 0, 0);
MODULE_DEPEND(aml8726_rng, random, 1, 1, 1);
Index: head/sys/arm/freescale/imx/imx_sdhci.c
===================================================================
--- head/sys/arm/freescale/imx/imx_sdhci.c (revision 283290)
+++ head/sys/arm/freescale/imx/imx_sdhci.c (revision 283291)
@@ -1,830 +1,830 @@
/*-
* Copyright (c) 2013 Ian Lepore <ian@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* SDHCI driver glue for Freescale i.MX SoC family.
*
* This supports both eSDHC (earlier SoCs) and uSDHC (more recent SoCs).
*/
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/bus.h>
#include <sys/callout.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/resource.h>
#include <sys/rman.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/time.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <machine/intr.h>
#include <arm/freescale/imx/imx_ccmvar.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#include <dev/mmc/bridge.h>
#include <dev/mmc/mmcreg.h>
#include <dev/mmc/mmcbrvar.h>
#include <dev/sdhci/sdhci.h>
#include "sdhci_if.h"
struct imx_sdhci_softc {
device_t dev;
struct resource * mem_res;
struct resource * irq_res;
void * intr_cookie;
struct sdhci_slot slot;
struct callout r1bfix_callout;
sbintime_t r1bfix_timeout_at;
uint32_t baseclk_hz;
uint32_t sdclockreg_freq_bits;
uint32_t cmd_and_mode;
uint32_t r1bfix_intmask;
uint8_t r1bfix_type;
uint8_t hwtype;
boolean_t force_card_present;
};
#define R1BFIX_NONE 0 /* No fix needed at next interrupt. */
#define R1BFIX_NODATA 1 /* Synthesize DATA_END for R1B w/o data. */
#define R1BFIX_AC12 2 /* Wait for busy after auto command 12. */
#define HWTYPE_NONE 0 /* Hardware not recognized/supported. */
#define HWTYPE_ESDHC 1 /* imx5x and earlier. */
#define HWTYPE_USDHC 2 /* imx6. */
#define SDHC_WTMK_LVL 0x44 /* Watermark Level register. */
#define USDHC_MIX_CONTROL 0x48 /* Mix(ed) Control register. */
#define SDHC_VEND_SPEC 0xC0 /* Vendor-specific register. */
#define SDHC_VEND_FRC_SDCLK_ON (1 << 8)
#define SDHC_VEND_IPGEN (1 << 11)
#define SDHC_VEND_HCKEN (1 << 12)
#define SDHC_VEND_PEREN (1 << 13)
#define SDHC_PRES_STATE 0x24
#define SDHC_PRES_CIHB (1 << 0)
#define SDHC_PRES_CDIHB (1 << 1)
#define SDHC_PRES_DLA (1 << 2)
#define SDHC_PRES_SDSTB (1 << 3)
#define SDHC_PRES_IPGOFF (1 << 4)
#define SDHC_PRES_HCKOFF (1 << 5)
#define SDHC_PRES_PEROFF (1 << 6)
#define SDHC_PRES_SDOFF (1 << 7)
#define SDHC_PRES_WTA (1 << 8)
#define SDHC_PRES_RTA (1 << 9)
#define SDHC_PRES_BWEN (1 << 10)
#define SDHC_PRES_BREN (1 << 11)
#define SDHC_PRES_RTR (1 << 12)
#define SDHC_PRES_CINST (1 << 16)
#define SDHC_PRES_CDPL (1 << 18)
#define SDHC_PRES_WPSPL (1 << 19)
#define SDHC_PRES_CLSL (1 << 23)
#define SDHC_PRES_DLSL_SHIFT 24
#define SDHC_PRES_DLSL_MASK (0xffU << SDHC_PRES_DLSL_SHIFT)
#define SDHC_PROT_CTRL 0x28
#define SDHC_PROT_LED (1 << 0)
#define SDHC_PROT_WIDTH_1BIT (0 << 1)
#define SDHC_PROT_WIDTH_4BIT (1 << 1)
#define SDHC_PROT_WIDTH_8BIT (2 << 1)
#define SDHC_PROT_WIDTH_MASK (3 << 1)
#define SDHC_PROT_D3CD (1 << 3)
#define SDHC_PROT_EMODE_BIG (0 << 4)
#define SDHC_PROT_EMODE_HALF (1 << 4)
#define SDHC_PROT_EMODE_LITTLE (2 << 4)
#define SDHC_PROT_EMODE_MASK (3 << 4)
#define SDHC_PROT_SDMA (0 << 8)
#define SDHC_PROT_ADMA1 (1 << 8)
#define SDHC_PROT_ADMA2 (2 << 8)
#define SDHC_PROT_ADMA264 (3 << 8)
#define SDHC_PROT_DMA_MASK (3 << 8)
#define SDHC_PROT_CDTL (1 << 6)
#define SDHC_PROT_CDSS (1 << 7)
#define SDHC_INT_STATUS 0x30
#define SDHC_CLK_IPGEN (1 << 0)
#define SDHC_CLK_HCKEN (1 << 1)
#define SDHC_CLK_PEREN (1 << 2)
#define SDHC_CLK_DIVISOR_MASK 0x000000f0
#define SDHC_CLK_DIVISOR_SHIFT 4
#define SDHC_CLK_PRESCALE_MASK 0x0000ff00
#define SDHC_CLK_PRESCALE_SHIFT 8
static struct ofw_compat_data compat_data[] = {
{"fsl,imx6q-usdhc", HWTYPE_USDHC},
{"fsl,imx6sl-usdhc", HWTYPE_USDHC},
{"fsl,imx53-esdhc", HWTYPE_ESDHC},
{"fsl,imx51-esdhc", HWTYPE_ESDHC},
{NULL, HWTYPE_NONE},
};
static void imx_sdhc_set_clock(struct imx_sdhci_softc *sc, int enable);
static void imx_sdhci_r1bfix_func(void *arg);
static inline uint32_t
RD4(struct imx_sdhci_softc *sc, bus_size_t off)
{
return (bus_read_4(sc->mem_res, off));
}
static inline void
WR4(struct imx_sdhci_softc *sc, bus_size_t off, uint32_t val)
{
bus_write_4(sc->mem_res, off, val);
}
static uint8_t
imx_sdhci_read_1(device_t dev, struct sdhci_slot *slot, bus_size_t off)
{
struct imx_sdhci_softc *sc = device_get_softc(dev);
uint32_t val32, wrk32;
/*
* Most of the things in the standard host control register are in the
* hardware's wider protocol control register, but some of the bits are
* moved around.
*/
if (off == SDHCI_HOST_CONTROL) {
wrk32 = RD4(sc, SDHC_PROT_CTRL);
val32 = wrk32 & (SDHCI_CTRL_LED | SDHCI_CTRL_CARD_DET |
SDHCI_CTRL_FORCE_CARD);
switch (wrk32 & SDHC_PROT_WIDTH_MASK) {
case SDHC_PROT_WIDTH_1BIT:
/* Value is already 0. */
break;
case SDHC_PROT_WIDTH_4BIT:
val32 |= SDHCI_CTRL_4BITBUS;
break;
case SDHC_PROT_WIDTH_8BIT:
val32 |= SDHCI_CTRL_8BITBUS;
break;
}
switch (wrk32 & SDHC_PROT_DMA_MASK) {
case SDHC_PROT_SDMA:
/* Value is already 0. */
break;
case SDHC_PROT_ADMA1:
/* This value is deprecated, should never appear. */
break;
case SDHC_PROT_ADMA2:
val32 |= SDHCI_CTRL_ADMA2;
break;
case SDHC_PROT_ADMA264:
val32 |= SDHCI_CTRL_ADMA264;
break;
}
return val32;
}
/*
* XXX can't find the bus power on/off knob. For now we have to say the
* power is always on and always set to the same voltage.
*/
if (off == SDHCI_POWER_CONTROL) {
return (SDHCI_POWER_ON | SDHCI_POWER_300);
}
return ((RD4(sc, off & ~3) >> (off & 3) * 8) & 0xff);
}
static uint16_t
imx_sdhci_read_2(device_t dev, struct sdhci_slot *slot, bus_size_t off)
{
struct imx_sdhci_softc *sc = device_get_softc(dev);
uint32_t val32, wrk32;
if (sc->hwtype == HWTYPE_USDHC) {
/*
* The USDHC hardware has nothing in the version register, but
* it's v3 compatible with all our translation code.
*/
if (off == SDHCI_HOST_VERSION) {
return (SDHCI_SPEC_300 << SDHCI_SPEC_VER_SHIFT);
}
/*
* The USDHC hardware moved the transfer mode bits to the mixed
* control register, fetch them from there.
*/
if (off == SDHCI_TRANSFER_MODE)
return (RD4(sc, USDHC_MIX_CONTROL) & 0x37);
} else if (sc->hwtype == HWTYPE_ESDHC) {
/*
* The ESDHC hardware has the typical 32-bit combined "command
* and mode" register that we have to cache so that command
* isn't written until after mode. On a read, just retrieve the
* cached values last written.
*/
if (off == SDHCI_TRANSFER_MODE) {
return (sc->cmd_and_mode >> 16);
} else if (off == SDHCI_COMMAND_FLAGS) {
return (sc->cmd_and_mode & 0x0000ffff);
}
}
/*
* This hardware only manages one slot. Synthesize a slot interrupt
* status register... if there are any enabled interrupts active they
* must be coming from our one and only slot.
*/
if (off == SDHCI_SLOT_INT_STATUS) {
val32 = RD4(sc, SDHCI_INT_STATUS);
val32 &= RD4(sc, SDHCI_SIGNAL_ENABLE);
return (val32 ? 1 : 0);
}
/*
* The clock enable bit is in the vendor register and the clock-stable
* bit is in the present state register. Transcribe them as if they
* were in the clock control register where they should be.
* XXX Is it important that we distinguish between "internal" and "card"
* clocks? Probably not; transcribe the card clock status to both bits.
*/
if (off == SDHCI_CLOCK_CONTROL) {
val32 = 0;
wrk32 = RD4(sc, SDHC_VEND_SPEC);
if (wrk32 & SDHC_VEND_FRC_SDCLK_ON)
val32 |= SDHCI_CLOCK_INT_EN | SDHCI_CLOCK_CARD_EN;
wrk32 = RD4(sc, SDHC_PRES_STATE);
if (wrk32 & SDHC_PRES_SDSTB)
val32 |= SDHCI_CLOCK_INT_STABLE;
val32 |= sc->sdclockreg_freq_bits;
return (val32);
}
return ((RD4(sc, off & ~3) >> (off & 3) * 8) & 0xffff);
}
static uint32_t
imx_sdhci_read_4(device_t dev, struct sdhci_slot *slot, bus_size_t off)
{
struct imx_sdhci_softc *sc = device_get_softc(dev);
uint32_t val32, wrk32;
val32 = RD4(sc, off);
/*
* The hardware leaves the base clock frequency out of the capabilities
* register; fill it in. The timeout clock is the same as the active
* output sdclock; we indicate that with a quirk setting so don't
* populate the timeout frequency bits.
*
* XXX Turn off (for now) features the hardware can do but this driver
* doesn't yet handle (1.8v, suspend/resume, etc).
*/
if (off == SDHCI_CAPABILITIES) {
val32 &= ~SDHCI_CAN_VDD_180;
val32 &= ~SDHCI_CAN_DO_SUSPEND;
val32 |= SDHCI_CAN_DO_8BITBUS;
val32 |= (sc->baseclk_hz / 1000000) << SDHCI_CLOCK_BASE_SHIFT;
return (val32);
}
/*
* The hardware moves bits around in the present state register to make
* room for all 8 data line state bits. To translate, mask out all the
* bits which are not in the same position in both registers (this also
* masks out some freescale-specific bits in locations defined as
* reserved by sdhci), then shift the data line and retune request bits
* down to their standard locations.
*/
if (off == SDHCI_PRESENT_STATE) {
wrk32 = val32;
val32 &= 0x000F0F07;
val32 |= (wrk32 >> 4) & SDHCI_STATE_DAT_MASK;
val32 |= (wrk32 >> 9) & SDHCI_RETUNE_REQUEST;
if (sc->force_card_present)
val32 |= SDHCI_CARD_PRESENT;
return (val32);
}
/*
* imx_sdhci_intr() can synthesize a DATA_END interrupt following a
* command with an R1B response, mix it into the hardware status.
*/
if (off == SDHCI_INT_STATUS) {
return (val32 | sc->r1bfix_intmask);
}
return val32;
}
static void
imx_sdhci_read_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off,
uint32_t *data, bus_size_t count)
{
struct imx_sdhci_softc *sc = device_get_softc(dev);
bus_read_multi_4(sc->mem_res, off, data, count);
}
static void
imx_sdhci_write_1(device_t dev, struct sdhci_slot *slot, bus_size_t off, uint8_t val)
{
struct imx_sdhci_softc *sc = device_get_softc(dev);
uint32_t val32;
/*
* Most of the things in the standard host control register are in the
* hardware's wider protocol control register, but some of the bits are
* moved around.
*/
if (off == SDHCI_HOST_CONTROL) {
val32 = RD4(sc, SDHC_PROT_CTRL);
val32 &= ~(SDHC_PROT_LED | SDHC_PROT_DMA_MASK |
SDHC_PROT_WIDTH_MASK | SDHC_PROT_CDTL | SDHC_PROT_CDSS);
val32 |= (val & SDHCI_CTRL_LED);
if (val & SDHCI_CTRL_8BITBUS)
val32 |= SDHC_PROT_WIDTH_8BIT;
else
val32 |= (val & SDHCI_CTRL_4BITBUS);
val32 |= (val & (SDHCI_CTRL_SDMA | SDHCI_CTRL_ADMA2)) << 4;
val32 |= (val & (SDHCI_CTRL_CARD_DET | SDHCI_CTRL_FORCE_CARD));
WR4(sc, SDHC_PROT_CTRL, val32);
return;
}
/* XXX I can't find the bus power on/off knob; do nothing. */
if (off == SDHCI_POWER_CONTROL) {
return;
}
val32 = RD4(sc, off & ~3);
val32 &= ~(0xff << (off & 3) * 8);
val32 |= (val << (off & 3) * 8);
WR4(sc, off & ~3, val32);
}
static void
imx_sdhci_write_2(device_t dev, struct sdhci_slot *slot, bus_size_t off, uint16_t val)
{
struct imx_sdhci_softc *sc = device_get_softc(dev);
uint32_t val32;
/* The USDHC hardware moved the transfer mode bits to mixed control. */
if (sc->hwtype == HWTYPE_USDHC) {
if (off == SDHCI_TRANSFER_MODE) {
val32 = RD4(sc, USDHC_MIX_CONTROL);
val32 &= ~0x3f;
val32 |= val & 0x37;
// XXX acmd23 not supported here (or by sdhci driver)
WR4(sc, USDHC_MIX_CONTROL, val32);
return;
}
}
/*
* The clock control stuff is complex enough to have its own routine
* that can both change speeds and en/disable the clock output. Also,
* save the register bits in SDHCI format so that we can play them back
* in the read2 routine without complex decoding.
*/
if (off == SDHCI_CLOCK_CONTROL) {
sc->sdclockreg_freq_bits = val & 0xffc0;
if (val & SDHCI_CLOCK_CARD_EN) {
imx_sdhc_set_clock(sc, true);
} else {
imx_sdhc_set_clock(sc, false);
}
}
/*
* Figure out whether we need to check the DAT0 line for busy status at
* interrupt time. The controller should be doing this, but for some
* reason it doesn't. There are two cases:
* - R1B response with no data transfer should generate a DATA_END (aka
* TRANSFER_COMPLETE) interrupt after waiting for busy, but if
* there's no data transfer there's no DATA_END interrupt. This is
* documented; they seem to think it's a feature.
* - R1B response after Auto-CMD12 appears to not work, even though
* there's a control bit for it (bit 3) in the vendor register.
* When we're starting a command that needs a manual DAT0 line check at
* interrupt time, we leave ourselves a note in r1bfix_type so that we
* can do the extra work in imx_sdhci_intr().
*/
if (off == SDHCI_COMMAND_FLAGS) {
if (val & SDHCI_CMD_DATA) {
const uint32_t MBAUTOCMD = SDHCI_TRNS_ACMD12 | SDHCI_TRNS_MULTI;
val32 = RD4(sc, USDHC_MIX_CONTROL);
if ((val32 & MBAUTOCMD) == MBAUTOCMD)
sc->r1bfix_type = R1BFIX_AC12;
} else {
if ((val & SDHCI_CMD_RESP_MASK) == SDHCI_CMD_RESP_SHORT_BUSY) {
WR4(sc, SDHCI_INT_ENABLE, slot->intmask | SDHCI_INT_RESPONSE);
WR4(sc, SDHCI_SIGNAL_ENABLE, slot->intmask | SDHCI_INT_RESPONSE);
sc->r1bfix_type = R1BFIX_NODATA;
}
}
}
val32 = RD4(sc, off & ~3);
val32 &= ~(0xffff << (off & 3) * 8);
val32 |= ((val & 0xffff) << (off & 3) * 8);
WR4(sc, off & ~3, val32);
}
static void
imx_sdhci_write_4(device_t dev, struct sdhci_slot *slot, bus_size_t off, uint32_t val)
{
struct imx_sdhci_softc *sc = device_get_softc(dev);
/* Clear synthesized interrupts, then pass the value to the hardware. */
if (off == SDHCI_INT_STATUS) {
sc->r1bfix_intmask &= ~val;
}
WR4(sc, off, val);
}
static void
imx_sdhci_write_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off,
uint32_t *data, bus_size_t count)
{
struct imx_sdhci_softc *sc = device_get_softc(dev);
bus_write_multi_4(sc->mem_res, off, data, count);
}
static void
imx_sdhc_set_clock(struct imx_sdhci_softc *sc, int enable)
{
uint32_t divisor, enable_bits, enable_reg, freq, prescale, val32;
if (sc->hwtype == HWTYPE_ESDHC) {
divisor = (sc->sdclockreg_freq_bits >> SDHCI_DIVIDER_SHIFT) &
SDHCI_DIVIDER_MASK;
enable_reg = SDHCI_CLOCK_CONTROL;
enable_bits = SDHC_CLK_IPGEN | SDHC_CLK_HCKEN |
SDHC_CLK_PEREN;
} else {
divisor = (sc->sdclockreg_freq_bits >> SDHCI_DIVIDER_SHIFT) &
SDHCI_DIVIDER_MASK;
divisor |= ((sc->sdclockreg_freq_bits >>
SDHCI_DIVIDER_HI_SHIFT) &
SDHCI_DIVIDER_HI_MASK) << SDHCI_DIVIDER_MASK_LEN;
enable_reg = SDHCI_CLOCK_CONTROL;
enable_bits = SDHC_VEND_IPGEN | SDHC_VEND_HCKEN |
SDHC_VEND_PEREN;
}
WR4(sc, SDHC_VEND_SPEC,
RD4(sc, SDHC_VEND_SPEC) & ~SDHC_VEND_FRC_SDCLK_ON);
WR4(sc, enable_reg, RD4(sc, enable_reg) & ~enable_bits);
if (!enable)
return;
if (divisor == 0)
freq = sc->baseclk_hz;
else
freq = sc->baseclk_hz / (2 * divisor);
for (prescale = 2; prescale < freq / prescale / 16;)
prescale <<= 1;
for (divisor = 1; freq < freq / prescale / divisor;)
++divisor;
prescale >>= 1;
divisor -= 1;
val32 = RD4(sc, SDHCI_CLOCK_CONTROL);
val32 &= ~SDHC_CLK_DIVISOR_MASK;
val32 |= divisor << SDHC_CLK_DIVISOR_SHIFT;
val32 &= ~SDHC_CLK_PRESCALE_MASK;
val32 |= prescale << SDHC_CLK_PRESCALE_SHIFT;
WR4(sc, SDHCI_CLOCK_CONTROL, val32);
WR4(sc, enable_reg, RD4(sc, enable_reg) | enable_bits);
WR4(sc, SDHC_VEND_SPEC,
RD4(sc, SDHC_VEND_SPEC) | SDHC_VEND_FRC_SDCLK_ON);
}
static boolean_t
imx_sdhci_r1bfix_is_wait_done(struct imx_sdhci_softc *sc)
{
uint32_t inhibit;
mtx_assert(&sc->slot.mtx, MA_OWNED);
/*
* Check the DAT0 line status using both the DLA (data line active) and
* CDIHB (data inhibit) bits in the present state register. In theory
* just DLA should do the trick, but in practice it takes both. If the
* DAT0 line is still being held and we're not yet beyond the timeout
* point, just schedule another callout to check again later.
*/
inhibit = RD4(sc, SDHC_PRES_STATE) & (SDHC_PRES_DLA | SDHC_PRES_CDIHB);
if (inhibit && getsbinuptime() < sc->r1bfix_timeout_at) {
callout_reset_sbt(&sc->r1bfix_callout, SBT_1MS, 0,
imx_sdhci_r1bfix_func, sc, 0);
return (false);
}
/*
* If we reach this point with the inhibit bits still set, we've got a
* timeout, synthesize a DATA_TIMEOUT interrupt. Otherwise the DAT0
* line has been released, and we synthesize a DATA_END, and if the type
* of fix needed was on a command-without-data we also now add in the
* original INT_RESPONSE that we suppressed earlier.
*/
if (inhibit)
sc->r1bfix_intmask |= SDHCI_INT_DATA_TIMEOUT;
else {
sc->r1bfix_intmask |= SDHCI_INT_DATA_END;
if (sc->r1bfix_type == R1BFIX_NODATA)
sc->r1bfix_intmask |= SDHCI_INT_RESPONSE;
}
sc->r1bfix_type = R1BFIX_NONE;
return (true);
}
static void
imx_sdhci_r1bfix_func(void * arg)
{
struct imx_sdhci_softc *sc = arg;
boolean_t r1bwait_done;
mtx_lock(&sc->slot.mtx);
r1bwait_done = imx_sdhci_r1bfix_is_wait_done(sc);
mtx_unlock(&sc->slot.mtx);
if (r1bwait_done)
sdhci_generic_intr(&sc->slot);
}
static void
imx_sdhci_intr(void *arg)
{
struct imx_sdhci_softc *sc = arg;
uint32_t intmask;
mtx_lock(&sc->slot.mtx);
/*
* Manually check the DAT0 line for R1B response types that the
* controller fails to handle properly. The controller asserts the done
* interrupt while the card is still asserting busy with the DAT0 line.
*
* We check DAT0 immediately because most of the time, especially on a
* read, the card will actually be done by time we get here. If it's
* not, then the wait_done routine will schedule a callout to re-check
* periodically until it is done. In that case we clear the interrupt
* out of the hardware now so that we can present it later when the DAT0
* line is released.
*
* If we need to wait for the the DAT0 line to be released, we set up a
* timeout point 250ms in the future. This number comes from the SD
* spec, which allows a command to take that long. In the real world,
* cards tend to take 10-20ms for a long-running command such as a write
* or erase that spans two pages.
*/
switch (sc->r1bfix_type) {
case R1BFIX_NODATA:
intmask = RD4(sc, SDHC_INT_STATUS) & SDHCI_INT_RESPONSE;
break;
case R1BFIX_AC12:
intmask = RD4(sc, SDHC_INT_STATUS) & SDHCI_INT_DATA_END;
break;
default:
intmask = 0;
break;
}
if (intmask) {
sc->r1bfix_timeout_at = getsbinuptime() + 250 * SBT_1MS;
if (!imx_sdhci_r1bfix_is_wait_done(sc)) {
WR4(sc, SDHC_INT_STATUS, intmask);
bus_barrier(sc->mem_res, SDHC_INT_STATUS, 4,
BUS_SPACE_BARRIER_WRITE);
}
}
mtx_unlock(&sc->slot.mtx);
sdhci_generic_intr(&sc->slot);
}
static int
imx_sdhci_get_ro(device_t bus, device_t child)
{
return (false);
}
static int
imx_sdhci_detach(device_t dev)
{
return (EBUSY);
}
static int
imx_sdhci_attach(device_t dev)
{
struct imx_sdhci_softc *sc = device_get_softc(dev);
int rid, err;
phandle_t node;
sc->dev = dev;
sc->hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
if (sc->hwtype == HWTYPE_NONE)
panic("Impossible: not compatible in imx_sdhci_attach()");
rid = 0;
sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (!sc->mem_res) {
device_printf(dev, "cannot allocate memory window\n");
err = ENXIO;
goto fail;
}
rid = 0;
sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_ACTIVE);
if (!sc->irq_res) {
device_printf(dev, "cannot allocate interrupt\n");
err = ENXIO;
goto fail;
}
if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
NULL, imx_sdhci_intr, sc, &sc->intr_cookie)) {
device_printf(dev, "cannot setup interrupt handler\n");
err = ENXIO;
goto fail;
}
sc->slot.quirks |= SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK;
/*
* DMA is not really broken, I just haven't implemented it yet.
*/
sc->slot.quirks |= SDHCI_QUIRK_BROKEN_DMA;
/*
* Set the buffer watermark level to 128 words (512 bytes) for both read
* and write. The hardware has a restriction that when the read or
* write ready status is asserted, that means you can read exactly the
* number of words set in the watermark register before you have to
* re-check the status and potentially wait for more data. The main
* sdhci driver provides no hook for doing status checking on less than
* a full block boundary, so we set the watermark level to be a full
* block. Reads and writes where the block size is less than the
* watermark size will work correctly too, no need to change the
* watermark for different size blocks. However, 128 is the maximum
* allowed for the watermark, so PIO is limitted to 512 byte blocks
* (which works fine for SD cards, may be a problem for SDIO some day).
*
* XXX need named constants for this stuff.
*/
WR4(sc, SDHC_WTMK_LVL, 0x08800880);
sc->baseclk_hz = imx_ccm_sdhci_hz();
/*
* If the slot is flagged with the non-removable property, set our flag
* to always force the SDHCI_CARD_PRESENT bit on.
*
* XXX Workaround for gpio-based card detect...
*
* We don't have gpio support yet. If there's a cd-gpios property just
* force the SDHCI_CARD_PRESENT bit on for now. If there isn't really a
* card there it will fail to probe at the mmc layer and nothing bad
* happens except instantiating an mmcN device for an empty slot.
*/
node = ofw_bus_get_node(dev);
if (OF_hasprop(node, "non-removable"))
sc->force_card_present = true;
else if (OF_hasprop(node, "cd-gpios")) {
/* XXX put real gpio hookup here. */
sc->force_card_present = true;
}
- callout_init(&sc->r1bfix_callout, true);
+ callout_init(&sc->r1bfix_callout, 1);
sdhci_init_slot(dev, &sc->slot, 0);
bus_generic_probe(dev);
bus_generic_attach(dev);
sdhci_start_slot(&sc->slot);
return (0);
fail:
if (sc->intr_cookie)
bus_teardown_intr(dev, sc->irq_res, sc->intr_cookie);
if (sc->irq_res)
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->irq_res);
if (sc->mem_res)
bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->mem_res);
return (err);
}
static int
imx_sdhci_probe(device_t dev)
{
if (!ofw_bus_status_okay(dev))
return (ENXIO);
switch (ofw_bus_search_compatible(dev, compat_data)->ocd_data) {
case HWTYPE_ESDHC:
device_set_desc(dev, "Freescale eSDHC controller");
return (BUS_PROBE_DEFAULT);
case HWTYPE_USDHC:
device_set_desc(dev, "Freescale uSDHC controller");
return (BUS_PROBE_DEFAULT);
default:
break;
}
return (ENXIO);
}
static device_method_t imx_sdhci_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, imx_sdhci_probe),
DEVMETHOD(device_attach, imx_sdhci_attach),
DEVMETHOD(device_detach, imx_sdhci_detach),
/* Bus interface */
DEVMETHOD(bus_read_ivar, sdhci_generic_read_ivar),
DEVMETHOD(bus_write_ivar, sdhci_generic_write_ivar),
DEVMETHOD(bus_print_child, bus_generic_print_child),
/* MMC bridge interface */
DEVMETHOD(mmcbr_update_ios, sdhci_generic_update_ios),
DEVMETHOD(mmcbr_request, sdhci_generic_request),
DEVMETHOD(mmcbr_get_ro, imx_sdhci_get_ro),
DEVMETHOD(mmcbr_acquire_host, sdhci_generic_acquire_host),
DEVMETHOD(mmcbr_release_host, sdhci_generic_release_host),
/* SDHCI registers accessors */
DEVMETHOD(sdhci_read_1, imx_sdhci_read_1),
DEVMETHOD(sdhci_read_2, imx_sdhci_read_2),
DEVMETHOD(sdhci_read_4, imx_sdhci_read_4),
DEVMETHOD(sdhci_read_multi_4, imx_sdhci_read_multi_4),
DEVMETHOD(sdhci_write_1, imx_sdhci_write_1),
DEVMETHOD(sdhci_write_2, imx_sdhci_write_2),
DEVMETHOD(sdhci_write_4, imx_sdhci_write_4),
DEVMETHOD(sdhci_write_multi_4, imx_sdhci_write_multi_4),
{ 0, 0 }
};
static devclass_t imx_sdhci_devclass;
static driver_t imx_sdhci_driver = {
"sdhci_imx",
imx_sdhci_methods,
sizeof(struct imx_sdhci_softc),
};
DRIVER_MODULE(sdhci_imx, simplebus, imx_sdhci_driver, imx_sdhci_devclass, 0, 0);
MODULE_DEPEND(sdhci_imx, sdhci, 1, 1, 1);
Index: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
===================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c (revision 283290)
+++ head/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c (revision 283291)
@@ -1,17999 +1,17999 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*
* $FreeBSD$
*/
/*
* Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, Joyent, Inc. All rights reserved.
* Copyright (c) 2012, 2014 by Delphix. All rights reserved.
*/
/*
* DTrace - Dynamic Tracing for Solaris
*
* This is the implementation of the Solaris Dynamic Tracing framework
* (DTrace). The user-visible interface to DTrace is described at length in
* the "Solaris Dynamic Tracing Guide". The interfaces between the libdtrace
* library, the in-kernel DTrace framework, and the DTrace providers are
* described in the block comments in the <sys/dtrace.h> header file. The
* internal architecture of DTrace is described in the block comments in the
* <sys/dtrace_impl.h> header file. The comments contained within the DTrace
* implementation very much assume mastery of all of these sources; if one has
* an unanswered question about the implementation, one should consult them
* first.
*
* The functions here are ordered roughly as follows:
*
* - Probe context functions
* - Probe hashing functions
* - Non-probe context utility functions
* - Matching functions
* - Provider-to-Framework API functions
* - Probe management functions
* - DIF object functions
* - Format functions
* - Predicate functions
* - ECB functions
* - Buffer functions
* - Enabling functions
* - DOF functions
* - Anonymous enabling functions
* - Consumer state functions
* - Helper functions
* - Hook functions
* - Driver cookbook functions
*
* Each group of functions begins with a block comment labelled the "DTrace
* [Group] Functions", allowing one to find each block by searching forward
* on capital-f functions.
*/
#include <sys/errno.h>
#ifndef illumos
#include <sys/time.h>
#endif
#include <sys/stat.h>
#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/systm.h>
#ifdef illumos
#include <sys/ddi.h>
#include <sys/sunddi.h>
#endif
#include <sys/cpuvar.h>
#include <sys/kmem.h>
#ifdef illumos
#include <sys/strsubr.h>
#endif
#include <sys/sysmacros.h>
#include <sys/dtrace_impl.h>
#include <sys/atomic.h>
#include <sys/cmn_err.h>
#ifdef illumos
#include <sys/mutex_impl.h>
#include <sys/rwlock_impl.h>
#endif
#include <sys/ctf_api.h>
#ifdef illumos
#include <sys/panic.h>
#include <sys/priv_impl.h>
#endif
#include <sys/policy.h>
#ifdef illumos
#include <sys/cred_impl.h>
#include <sys/procfs_isa.h>
#endif
#include <sys/taskq.h>
#ifdef illumos
#include <sys/mkdev.h>
#include <sys/kdi.h>
#endif
#include <sys/zone.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "strtolctype.h"
/* FreeBSD includes: */
#ifndef illumos
#include <sys/callout.h>
#include <sys/ctype.h>
#include <sys/eventhandler.h>
#include <sys/limits.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/rwlock.h>
#include <sys/sx.h>
#include <sys/dtrace_bsd.h>
#include <netinet/in.h>
#include "dtrace_cddl.h"
#include "dtrace_debug.c"
#endif
/*
* DTrace Tunable Variables
*
* The following variables may be tuned by adding a line to /etc/system that
* includes both the name of the DTrace module ("dtrace") and the name of the
* variable. For example:
*
* set dtrace:dtrace_destructive_disallow = 1
*
* In general, the only variables that one should be tuning this way are those
* that affect system-wide DTrace behavior, and for which the default behavior
* is undesirable. Most of these variables are tunable on a per-consumer
* basis using DTrace options, and need not be tuned on a system-wide basis.
* When tuning these variables, avoid pathological values; while some attempt
* is made to verify the integrity of these variables, they are not considered
* part of the supported interface to DTrace, and they are therefore not
* checked comprehensively. Further, these variables should not be tuned
* dynamically via "mdb -kw" or other means; they should only be tuned via
* /etc/system.
*/
int dtrace_destructive_disallow = 0;
dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024);
size_t dtrace_difo_maxsize = (256 * 1024);
dtrace_optval_t dtrace_dof_maxsize = (8 * 1024 * 1024);
size_t dtrace_global_maxsize = (16 * 1024);
size_t dtrace_actions_max = (16 * 1024);
size_t dtrace_retain_max = 1024;
dtrace_optval_t dtrace_helper_actions_max = 128;
dtrace_optval_t dtrace_helper_providers_max = 32;
dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024);
size_t dtrace_strsize_default = 256;
dtrace_optval_t dtrace_cleanrate_default = 9900990; /* 101 hz */
dtrace_optval_t dtrace_cleanrate_min = 200000; /* 5000 hz */
dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC; /* 1/minute */
dtrace_optval_t dtrace_aggrate_default = NANOSEC; /* 1 hz */
dtrace_optval_t dtrace_statusrate_default = NANOSEC; /* 1 hz */
dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC; /* 6/minute */
dtrace_optval_t dtrace_switchrate_default = NANOSEC; /* 1 hz */
dtrace_optval_t dtrace_nspec_default = 1;
dtrace_optval_t dtrace_specsize_default = 32 * 1024;
dtrace_optval_t dtrace_stackframes_default = 20;
dtrace_optval_t dtrace_ustackframes_default = 20;
dtrace_optval_t dtrace_jstackframes_default = 50;
dtrace_optval_t dtrace_jstackstrsize_default = 512;
int dtrace_msgdsize_max = 128;
hrtime_t dtrace_chill_max = MSEC2NSEC(500); /* 500 ms */
hrtime_t dtrace_chill_interval = NANOSEC; /* 1000 ms */
int dtrace_devdepth_max = 32;
int dtrace_err_verbose;
hrtime_t dtrace_deadman_interval = NANOSEC;
hrtime_t dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
hrtime_t dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
hrtime_t dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC;
#ifndef illumos
int dtrace_memstr_max = 4096;
#endif
/*
* DTrace External Variables
*
* As dtrace(7D) is a kernel module, any DTrace variables are obviously
* available to DTrace consumers via the backtick (`) syntax. One of these,
* dtrace_zero, is made deliberately so: it is provided as a source of
* well-known, zero-filled memory. While this variable is not documented,
* it is used by some translators as an implementation detail.
*/
const char dtrace_zero[256] = { 0 }; /* zero-filled memory */
/*
* DTrace Internal Variables
*/
#ifdef illumos
static dev_info_t *dtrace_devi; /* device info */
#endif
#ifdef illumos
static vmem_t *dtrace_arena; /* probe ID arena */
static vmem_t *dtrace_minor; /* minor number arena */
#else
static taskq_t *dtrace_taskq; /* task queue */
static struct unrhdr *dtrace_arena; /* Probe ID number. */
#endif
static dtrace_probe_t **dtrace_probes; /* array of all probes */
static int dtrace_nprobes; /* number of probes */
static dtrace_provider_t *dtrace_provider; /* provider list */
static dtrace_meta_t *dtrace_meta_pid; /* user-land meta provider */
static int dtrace_opens; /* number of opens */
static int dtrace_helpers; /* number of helpers */
static int dtrace_getf; /* number of unpriv getf()s */
#ifdef illumos
static void *dtrace_softstate; /* softstate pointer */
#endif
static dtrace_hash_t *dtrace_bymod; /* probes hashed by module */
static dtrace_hash_t *dtrace_byfunc; /* probes hashed by function */
static dtrace_hash_t *dtrace_byname; /* probes hashed by name */
static dtrace_toxrange_t *dtrace_toxrange; /* toxic range array */
static int dtrace_toxranges; /* number of toxic ranges */
static int dtrace_toxranges_max; /* size of toxic range array */
static dtrace_anon_t dtrace_anon; /* anonymous enabling */
static kmem_cache_t *dtrace_state_cache; /* cache for dynamic state */
static uint64_t dtrace_vtime_references; /* number of vtimestamp refs */
static kthread_t *dtrace_panicked; /* panicking thread */
static dtrace_ecb_t *dtrace_ecb_create_cache; /* cached created ECB */
static dtrace_genid_t dtrace_probegen; /* current probe generation */
static dtrace_helpers_t *dtrace_deferred_pid; /* deferred helper list */
static dtrace_enabling_t *dtrace_retained; /* list of retained enablings */
static dtrace_genid_t dtrace_retained_gen; /* current retained enab gen */
static dtrace_dynvar_t dtrace_dynhash_sink; /* end of dynamic hash chains */
static int dtrace_dynvar_failclean; /* dynvars failed to clean */
#ifndef illumos
static struct mtx dtrace_unr_mtx;
MTX_SYSINIT(dtrace_unr_mtx, &dtrace_unr_mtx, "Unique resource identifier", MTX_DEF);
int dtrace_in_probe; /* non-zero if executing a probe */
#if defined(__i386__) || defined(__amd64__) || defined(__mips__) || defined(__powerpc__)
uintptr_t dtrace_in_probe_addr; /* Address of invop when already in probe */
#endif
static eventhandler_tag dtrace_kld_load_tag;
static eventhandler_tag dtrace_kld_unload_try_tag;
#endif
/*
* DTrace Locking
* DTrace is protected by three (relatively coarse-grained) locks:
*
* (1) dtrace_lock is required to manipulate essentially any DTrace state,
* including enabling state, probes, ECBs, consumer state, helper state,
* etc. Importantly, dtrace_lock is _not_ required when in probe context;
* probe context is lock-free -- synchronization is handled via the
* dtrace_sync() cross call mechanism.
*
* (2) dtrace_provider_lock is required when manipulating provider state, or
* when provider state must be held constant.
*
* (3) dtrace_meta_lock is required when manipulating meta provider state, or
* when meta provider state must be held constant.
*
* The lock ordering between these three locks is dtrace_meta_lock before
* dtrace_provider_lock before dtrace_lock. (In particular, there are
* several places where dtrace_provider_lock is held by the framework as it
* calls into the providers -- which then call back into the framework,
* grabbing dtrace_lock.)
*
* There are two other locks in the mix: mod_lock and cpu_lock. With respect
* to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
* role as a coarse-grained lock; it is acquired before both of these locks.
* With respect to dtrace_meta_lock, its behavior is stranger: cpu_lock must
* be acquired _between_ dtrace_meta_lock and any other DTrace locks.
* mod_lock is similar with respect to dtrace_provider_lock in that it must be
* acquired _between_ dtrace_provider_lock and dtrace_lock.
*/
static kmutex_t dtrace_lock; /* probe state lock */
static kmutex_t dtrace_provider_lock; /* provider state lock */
static kmutex_t dtrace_meta_lock; /* meta-provider state lock */
#ifndef illumos
/* XXX FreeBSD hacks. */
#define cr_suid cr_svuid
#define cr_sgid cr_svgid
#define ipaddr_t in_addr_t
#define mod_modname pathname
#define vuprintf vprintf
#define ttoproc(_a) ((_a)->td_proc)
#define crgetzoneid(_a) 0
#define NCPU MAXCPU
#define SNOCD 0
#define CPU_ON_INTR(_a) 0
#define PRIV_EFFECTIVE (1 << 0)
#define PRIV_DTRACE_KERNEL (1 << 1)
#define PRIV_DTRACE_PROC (1 << 2)
#define PRIV_DTRACE_USER (1 << 3)
#define PRIV_PROC_OWNER (1 << 4)
#define PRIV_PROC_ZONE (1 << 5)
#define PRIV_ALL ~0
SYSCTL_DECL(_debug_dtrace);
SYSCTL_DECL(_kern_dtrace);
#endif
#ifdef illumos
#define curcpu CPU->cpu_id
#endif
/*
* DTrace Provider Variables
*
* These are the variables relating to DTrace as a provider (that is, the
* provider of the BEGIN, END, and ERROR probes).
*/
static dtrace_pattr_t dtrace_provider_attr = {
{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
};
static void
dtrace_nullop(void)
{}
static dtrace_pops_t dtrace_provider_ops = {
(void (*)(void *, dtrace_probedesc_t *))dtrace_nullop,
(void (*)(void *, modctl_t *))dtrace_nullop,
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
NULL,
NULL,
NULL,
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop
};
static dtrace_id_t dtrace_probeid_begin; /* special BEGIN probe */
static dtrace_id_t dtrace_probeid_end; /* special END probe */
dtrace_id_t dtrace_probeid_error; /* special ERROR probe */
/*
* DTrace Helper Tracing Variables
*
* These variables should be set dynamically to enable helper tracing. The
* only variables that should be set are dtrace_helptrace_enable (which should
* be set to a non-zero value to allocate helper tracing buffers on the next
* open of /dev/dtrace) and dtrace_helptrace_disable (which should be set to a
* non-zero value to deallocate helper tracing buffers on the next close of
* /dev/dtrace). When (and only when) helper tracing is disabled, the
* buffer size may also be set via dtrace_helptrace_bufsize.
*/
int dtrace_helptrace_enable = 0;
int dtrace_helptrace_disable = 0;
int dtrace_helptrace_bufsize = 16 * 1024 * 1024;
uint32_t dtrace_helptrace_nlocals;
static dtrace_helptrace_t *dtrace_helptrace_buffer;
static uint32_t dtrace_helptrace_next = 0;
static int dtrace_helptrace_wrapped = 0;
/*
* DTrace Error Hashing
*
* On DEBUG kernels, DTrace will track the errors that has seen in a hash
* table. This is very useful for checking coverage of tests that are
* expected to induce DIF or DOF processing errors, and may be useful for
* debugging problems in the DIF code generator or in DOF generation . The
* error hash may be examined with the ::dtrace_errhash MDB dcmd.
*/
#ifdef DEBUG
static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ];
static const char *dtrace_errlast;
static kthread_t *dtrace_errthread;
static kmutex_t dtrace_errlock;
#endif
/*
* DTrace Macros and Constants
*
* These are various macros that are useful in various spots in the
* implementation, along with a few random constants that have no meaning
* outside of the implementation. There is no real structure to this cpp
* mishmash -- but is there ever?
*/
#define DTRACE_HASHSTR(hash, probe) \
dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
#define DTRACE_HASHNEXT(hash, probe) \
(dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
#define DTRACE_HASHPREV(hash, probe) \
(dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
#define DTRACE_HASHEQ(hash, lhs, rhs) \
(strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
*((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
#define DTRACE_AGGHASHSIZE_SLEW 17
#define DTRACE_V4MAPPED_OFFSET (sizeof (uint32_t) * 3)
/*
* The key for a thread-local variable consists of the lower 61 bits of the
* t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
* We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
* equal to a variable identifier. This is necessary (but not sufficient) to
* assure that global associative arrays never collide with thread-local
* variables. To guarantee that they cannot collide, we must also define the
* order for keying dynamic variables. That order is:
*
* [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
*
* Because the variable-key and the tls-key are in orthogonal spaces, there is
* no way for a global variable key signature to match a thread-local key
* signature.
*/
#ifdef illumos
#define DTRACE_TLS_THRKEY(where) { \
uint_t intr = 0; \
uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
for (; actv; actv >>= 1) \
intr++; \
ASSERT(intr < (1 << 3)); \
(where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
(((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
}
#else
#define DTRACE_TLS_THRKEY(where) { \
solaris_cpu_t *_c = &solaris_cpu[curcpu]; \
uint_t intr = 0; \
uint_t actv = _c->cpu_intr_actv; \
for (; actv; actv >>= 1) \
intr++; \
ASSERT(intr < (1 << 3)); \
(where) = ((curthread->td_tid + DIF_VARIABLE_MAX) & \
(((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
}
#endif
#define DT_BSWAP_8(x) ((x) & 0xff)
#define DT_BSWAP_16(x) ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))
#define DT_BSWAP_32(x) ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))
#define DT_BSWAP_64(x) ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))
#define DT_MASK_LO 0x00000000FFFFFFFFULL
#define DTRACE_STORE(type, tomax, offset, what) \
*((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
#ifndef __x86
#define DTRACE_ALIGNCHECK(addr, size, flags) \
if (addr & (size - 1)) { \
*flags |= CPU_DTRACE_BADALIGN; \
cpu_core[curcpu].cpuc_dtrace_illval = addr; \
return (0); \
}
#else
#define DTRACE_ALIGNCHECK(addr, size, flags)
#endif
/*
* Test whether a range of memory starting at testaddr of size testsz falls
* within the range of memory described by addr, sz. We take care to avoid
* problems with overflow and underflow of the unsigned quantities, and
* disallow all negative sizes. Ranges of size 0 are allowed.
*/
#define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \
((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \
(testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \
(testaddr) + (testsz) >= (testaddr))
/*
* Test whether alloc_sz bytes will fit in the scratch region. We isolate
* alloc_sz on the righthand side of the comparison in order to avoid overflow
* or underflow in the comparison with it. This is simpler than the INRANGE
* check above, because we know that the dtms_scratch_ptr is valid in the
* range. Allocations of size zero are allowed.
*/
#define DTRACE_INSCRATCH(mstate, alloc_sz) \
((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \
(mstate)->dtms_scratch_ptr >= (alloc_sz))
#define DTRACE_LOADFUNC(bits) \
/*CSTYLED*/ \
uint##bits##_t \
dtrace_load##bits(uintptr_t addr) \
{ \
size_t size = bits / NBBY; \
/*CSTYLED*/ \
uint##bits##_t rval; \
int i; \
volatile uint16_t *flags = (volatile uint16_t *) \
&cpu_core[curcpu].cpuc_dtrace_flags; \
\
DTRACE_ALIGNCHECK(addr, size, flags); \
\
for (i = 0; i < dtrace_toxranges; i++) { \
if (addr >= dtrace_toxrange[i].dtt_limit) \
continue; \
\
if (addr + size <= dtrace_toxrange[i].dtt_base) \
continue; \
\
/* \
* This address falls within a toxic region; return 0. \
*/ \
*flags |= CPU_DTRACE_BADADDR; \
cpu_core[curcpu].cpuc_dtrace_illval = addr; \
return (0); \
} \
\
*flags |= CPU_DTRACE_NOFAULT; \
/*CSTYLED*/ \
rval = *((volatile uint##bits##_t *)addr); \
*flags &= ~CPU_DTRACE_NOFAULT; \
\
return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0); \
}
#ifdef _LP64
#define dtrace_loadptr dtrace_load64
#else
#define dtrace_loadptr dtrace_load32
#endif
#define DTRACE_DYNHASH_FREE 0
#define DTRACE_DYNHASH_SINK 1
#define DTRACE_DYNHASH_VALID 2
#define DTRACE_MATCH_NEXT 0
#define DTRACE_MATCH_DONE 1
#define DTRACE_ANCHORED(probe) ((probe)->dtpr_func[0] != '\0')
#define DTRACE_STATE_ALIGN 64
#define DTRACE_FLAGS2FLT(flags) \
(((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR : \
((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP : \
((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO : \
((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV : \
((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV : \
((flags) & CPU_DTRACE_TUPOFLOW) ? DTRACEFLT_TUPOFLOW : \
((flags) & CPU_DTRACE_BADALIGN) ? DTRACEFLT_BADALIGN : \
((flags) & CPU_DTRACE_NOSCRATCH) ? DTRACEFLT_NOSCRATCH : \
((flags) & CPU_DTRACE_BADSTACK) ? DTRACEFLT_BADSTACK : \
DTRACEFLT_UNKNOWN)
#define DTRACEACT_ISSTRING(act) \
((act)->dta_kind == DTRACEACT_DIFEXPR && \
(act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
/* Function prototype definitions: */
static size_t dtrace_strlen(const char *, size_t);
static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
static void dtrace_enabling_provide(dtrace_provider_t *);
static int dtrace_enabling_match(dtrace_enabling_t *, int *);
static void dtrace_enabling_matchall(void);
static void dtrace_enabling_reap(void);
static dtrace_state_t *dtrace_anon_grab(void);
static uint64_t dtrace_helper(int, dtrace_mstate_t *,
dtrace_state_t *, uint64_t, uint64_t);
static dtrace_helpers_t *dtrace_helpers_create(proc_t *);
static void dtrace_buffer_drop(dtrace_buffer_t *);
static int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when);
static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
dtrace_state_t *, dtrace_mstate_t *);
static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
dtrace_optval_t);
static int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);
uint16_t dtrace_load16(uintptr_t);
uint32_t dtrace_load32(uintptr_t);
uint64_t dtrace_load64(uintptr_t);
uint8_t dtrace_load8(uintptr_t);
void dtrace_dynvar_clean(dtrace_dstate_t *);
dtrace_dynvar_t *dtrace_dynvar(dtrace_dstate_t *, uint_t, dtrace_key_t *,
size_t, dtrace_dynvar_op_t, dtrace_mstate_t *, dtrace_vstate_t *);
uintptr_t dtrace_dif_varstr(uintptr_t, dtrace_state_t *, dtrace_mstate_t *);
static int dtrace_priv_proc(dtrace_state_t *);
static void dtrace_getf_barrier(void);
/*
* DTrace Probe Context Functions
*
* These functions are called from probe context. Because probe context is
* any context in which C may be called, arbitrarily locks may be held,
* interrupts may be disabled, we may be in arbitrary dispatched state, etc.
* As a result, functions called from probe context may only call other DTrace
* support functions -- they may not interact at all with the system at large.
* (Note that the ASSERT macro is made probe-context safe by redefining it in
* terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
* loads are to be performed from probe context, they _must_ be in terms of
* the safe dtrace_load*() variants.
*
* Some functions in this block are not actually called from probe context;
* for these functions, there will be a comment above the function reading
* "Note: not called from probe context."
*/
void
dtrace_panic(const char *format, ...)
{
va_list alist;
va_start(alist, format);
#ifdef __FreeBSD__
vpanic(format, alist);
#else
dtrace_vpanic(format, alist);
#endif
va_end(alist);
}
int
dtrace_assfail(const char *a, const char *f, int l)
{
dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
/*
* We just need something here that even the most clever compiler
* cannot optimize away.
*/
return (a[(uintptr_t)f]);
}
/*
* Atomically increment a specified error counter from probe context.
*/
static void
dtrace_error(uint32_t *counter)
{
/*
* Most counters stored to in probe context are per-CPU counters.
* However, there are some error conditions that are sufficiently
* arcane that they don't merit per-CPU storage. If these counters
* are incremented concurrently on different CPUs, scalability will be
* adversely affected -- but we don't expect them to be white-hot in a
* correctly constructed enabling...
*/
uint32_t oval, nval;
do {
oval = *counter;
if ((nval = oval + 1) == 0) {
/*
* If the counter would wrap, set it to 1 -- assuring
* that the counter is never zero when we have seen
* errors. (The counter must be 32-bits because we
* aren't guaranteed a 64-bit compare&swap operation.)
* To save this code both the infamy of being fingered
* by a priggish news story and the indignity of being
* the target of a neo-puritan witch trial, we're
* carefully avoiding any colorful description of the
* likelihood of this condition -- but suffice it to
* say that it is only slightly more likely than the
* overflow of predicate cache IDs, as discussed in
* dtrace_predicate_create().
*/
nval = 1;
}
} while (dtrace_cas32(counter, oval, nval) != oval);
}
/*
* Use the DTRACE_LOADFUNC macro to define functions for each of loading a
* uint8_t, a uint16_t, a uint32_t and a uint64_t.
*/
DTRACE_LOADFUNC(8)
DTRACE_LOADFUNC(16)
DTRACE_LOADFUNC(32)
DTRACE_LOADFUNC(64)
static int
dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
{
if (dest < mstate->dtms_scratch_base)
return (0);
if (dest + size < dest)
return (0);
if (dest + size > mstate->dtms_scratch_ptr)
return (0);
return (1);
}
static int
dtrace_canstore_statvar(uint64_t addr, size_t sz,
dtrace_statvar_t **svars, int nsvars)
{
int i;
for (i = 0; i < nsvars; i++) {
dtrace_statvar_t *svar = svars[i];
if (svar == NULL || svar->dtsv_size == 0)
continue;
if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))
return (1);
}
return (0);
}
/*
* Check to see if the address is within a memory region to which a store may
* be issued. This includes the DTrace scratch areas, and any DTrace variable
* region. The caller of dtrace_canstore() is responsible for performing any
* alignment checks that are needed before stores are actually executed.
*/
static int
dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
dtrace_vstate_t *vstate)
{
/*
* First, check to see if the address is in scratch space...
*/
if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base,
mstate->dtms_scratch_size))
return (1);
/*
* Now check to see if it's a dynamic variable. This check will pick
* up both thread-local variables and any global dynamically-allocated
* variables.
*/
if (DTRACE_INRANGE(addr, sz, vstate->dtvs_dynvars.dtds_base,
vstate->dtvs_dynvars.dtds_size)) {
dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
uintptr_t base = (uintptr_t)dstate->dtds_base +
(dstate->dtds_hashsize * sizeof (dtrace_dynhash_t));
uintptr_t chunkoffs;
/*
* Before we assume that we can store here, we need to make
* sure that it isn't in our metadata -- storing to our
* dynamic variable metadata would corrupt our state. For
* the range to not include any dynamic variable metadata,
* it must:
*
* (1) Start above the hash table that is at the base of
* the dynamic variable space
*
* (2) Have a starting chunk offset that is beyond the
* dtrace_dynvar_t that is at the base of every chunk
*
* (3) Not span a chunk boundary
*
*/
if (addr < base)
return (0);
chunkoffs = (addr - base) % dstate->dtds_chunksize;
if (chunkoffs < sizeof (dtrace_dynvar_t))
return (0);
if (chunkoffs + sz > dstate->dtds_chunksize)
return (0);
return (1);
}
/*
* Finally, check the static local and global variables. These checks
* take the longest, so we perform them last.
*/
if (dtrace_canstore_statvar(addr, sz,
vstate->dtvs_locals, vstate->dtvs_nlocals))
return (1);
if (dtrace_canstore_statvar(addr, sz,
vstate->dtvs_globals, vstate->dtvs_nglobals))
return (1);
return (0);
}
/*
* Convenience routine to check to see if the address is within a memory
* region in which a load may be issued given the user's privilege level;
* if not, it sets the appropriate error flags and loads 'addr' into the
* illegal value slot.
*
* DTrace subroutines (DIF_SUBR_*) should use this helper to implement
* appropriate memory access protection.
*/
static int
dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
dtrace_vstate_t *vstate)
{
volatile uintptr_t *illval = &cpu_core[curcpu].cpuc_dtrace_illval;
file_t *fp;
/*
* If we hold the privilege to read from kernel memory, then
* everything is readable.
*/
if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
return (1);
/*
* You can obviously read that which you can store.
*/
if (dtrace_canstore(addr, sz, mstate, vstate))
return (1);
/*
* We're allowed to read from our own string table.
*/
if (DTRACE_INRANGE(addr, sz, mstate->dtms_difo->dtdo_strtab,
mstate->dtms_difo->dtdo_strlen))
return (1);
if (vstate->dtvs_state != NULL &&
dtrace_priv_proc(vstate->dtvs_state)) {
proc_t *p;
/*
* When we have privileges to the current process, there are
* several context-related kernel structures that are safe to
* read, even absent the privilege to read from kernel memory.
* These reads are safe because these structures contain only
* state that (1) we're permitted to read, (2) is harmless or
* (3) contains pointers to additional kernel state that we're
* not permitted to read (and as such, do not present an
* opportunity for privilege escalation). Finally (and
* critically), because of the nature of their relation with
* the current thread context, the memory associated with these
* structures cannot change over the duration of probe context,
* and it is therefore impossible for this memory to be
* deallocated and reallocated as something else while it's
* being operated upon.
*/
if (DTRACE_INRANGE(addr, sz, curthread, sizeof (kthread_t)))
return (1);
if ((p = curthread->t_procp) != NULL && DTRACE_INRANGE(addr,
sz, curthread->t_procp, sizeof (proc_t))) {
return (1);
}
if (curthread->t_cred != NULL && DTRACE_INRANGE(addr, sz,
curthread->t_cred, sizeof (cred_t))) {
return (1);
}
#ifdef illumos
if (p != NULL && p->p_pidp != NULL && DTRACE_INRANGE(addr, sz,
&(p->p_pidp->pid_id), sizeof (pid_t))) {
return (1);
}
if (curthread->t_cpu != NULL && DTRACE_INRANGE(addr, sz,
curthread->t_cpu, offsetof(cpu_t, cpu_pause_thread))) {
return (1);
}
#endif
}
if ((fp = mstate->dtms_getf) != NULL) {
uintptr_t psz = sizeof (void *);
vnode_t *vp;
vnodeops_t *op;
/*
* When getf() returns a file_t, the enabling is implicitly
* granted the (transient) right to read the returned file_t
* as well as the v_path and v_op->vnop_name of the underlying
* vnode. These accesses are allowed after a successful
* getf() because the members that they refer to cannot change
* once set -- and the barrier logic in the kernel's closef()
* path assures that the file_t and its referenced vode_t
* cannot themselves be stale (that is, it impossible for
* either dtms_getf itself or its f_vnode member to reference
* freed memory).
*/
if (DTRACE_INRANGE(addr, sz, fp, sizeof (file_t)))
return (1);
if ((vp = fp->f_vnode) != NULL) {
#ifdef illumos
if (DTRACE_INRANGE(addr, sz, &vp->v_path, psz))
return (1);
if (vp->v_path != NULL && DTRACE_INRANGE(addr, sz,
vp->v_path, strlen(vp->v_path) + 1)) {
return (1);
}
#endif
if (DTRACE_INRANGE(addr, sz, &vp->v_op, psz))
return (1);
#ifdef illumos
if ((op = vp->v_op) != NULL &&
DTRACE_INRANGE(addr, sz, &op->vnop_name, psz)) {
return (1);
}
if (op != NULL && op->vnop_name != NULL &&
DTRACE_INRANGE(addr, sz, op->vnop_name,
strlen(op->vnop_name) + 1)) {
return (1);
}
#endif
}
}
DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV);
*illval = addr;
return (0);
}
/*
* Convenience routine to check to see if a given string is within a memory
* region in which a load may be issued given the user's privilege level;
* this exists so that we don't need to issue unnecessary dtrace_strlen()
* calls in the event that the user has all privileges.
*/
static int
dtrace_strcanload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
dtrace_vstate_t *vstate)
{
size_t strsz;
/*
* If we hold the privilege to read from kernel memory, then
* everything is readable.
*/
if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
return (1);
strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, sz);
if (dtrace_canload(addr, strsz, mstate, vstate))
return (1);
return (0);
}
/*
* Convenience routine to check to see if a given variable is within a memory
* region in which a load may be issued given the user's privilege level.
*/
static int
dtrace_vcanload(void *src, dtrace_diftype_t *type, dtrace_mstate_t *mstate,
dtrace_vstate_t *vstate)
{
size_t sz;
ASSERT(type->dtdt_flags & DIF_TF_BYREF);
/*
* If we hold the privilege to read from kernel memory, then
* everything is readable.
*/
if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
return (1);
if (type->dtdt_kind == DIF_TYPE_STRING)
sz = dtrace_strlen(src,
vstate->dtvs_state->dts_options[DTRACEOPT_STRSIZE]) + 1;
else
sz = type->dtdt_size;
return (dtrace_canload((uintptr_t)src, sz, mstate, vstate));
}
/*
* Convert a string to a signed integer using safe loads.
*
* NOTE: This function uses various macros from strtolctype.h to manipulate
* digit values, etc -- these have all been checked to ensure they make
* no additional function calls.
*/
static int64_t
dtrace_strtoll(char *input, int base, size_t limit)
{
uintptr_t pos = (uintptr_t)input;
int64_t val = 0;
int x;
boolean_t neg = B_FALSE;
char c, cc, ccc;
uintptr_t end = pos + limit;
/*
* Consume any whitespace preceding digits.
*/
while ((c = dtrace_load8(pos)) == ' ' || c == '\t')
pos++;
/*
* Handle an explicit sign if one is present.
*/
if (c == '-' || c == '+') {
if (c == '-')
neg = B_TRUE;
c = dtrace_load8(++pos);
}
/*
* Check for an explicit hexadecimal prefix ("0x" or "0X") and skip it
* if present.
*/
if (base == 16 && c == '0' && ((cc = dtrace_load8(pos + 1)) == 'x' ||
cc == 'X') && isxdigit(ccc = dtrace_load8(pos + 2))) {
pos += 2;
c = ccc;
}
/*
* Read in contiguous digits until the first non-digit character.
*/
for (; pos < end && c != '\0' && lisalnum(c) && (x = DIGIT(c)) < base;
c = dtrace_load8(++pos))
val = val * base + x;
return (neg ? -val : val);
}
/*
* Compare two strings using safe loads.
*/
static int
dtrace_strncmp(char *s1, char *s2, size_t limit)
{
uint8_t c1, c2;
volatile uint16_t *flags;
if (s1 == s2 || limit == 0)
return (0);
flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;
do {
if (s1 == NULL) {
c1 = '\0';
} else {
c1 = dtrace_load8((uintptr_t)s1++);
}
if (s2 == NULL) {
c2 = '\0';
} else {
c2 = dtrace_load8((uintptr_t)s2++);
}
if (c1 != c2)
return (c1 - c2);
} while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));
return (0);
}
/*
* Compute strlen(s) for a string using safe memory accesses. The additional
* len parameter is used to specify a maximum length to ensure completion.
*/
static size_t
dtrace_strlen(const char *s, size_t lim)
{
uint_t len;
for (len = 0; len != lim; len++) {
if (dtrace_load8((uintptr_t)s++) == '\0')
break;
}
return (len);
}
/*
* Check if an address falls within a toxic region.
*/
static int
dtrace_istoxic(uintptr_t kaddr, size_t size)
{
uintptr_t taddr, tsize;
int i;
for (i = 0; i < dtrace_toxranges; i++) {
taddr = dtrace_toxrange[i].dtt_base;
tsize = dtrace_toxrange[i].dtt_limit - taddr;
if (kaddr - taddr < tsize) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
cpu_core[curcpu].cpuc_dtrace_illval = kaddr;
return (1);
}
if (taddr - kaddr < size) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
cpu_core[curcpu].cpuc_dtrace_illval = taddr;
return (1);
}
}
return (0);
}
/*
* Copy src to dst using safe memory accesses. The src is assumed to be unsafe
* memory specified by the DIF program. The dst is assumed to be safe memory
* that we can store to directly because it is managed by DTrace. As with
* standard bcopy, overlapping copies are handled properly.
*/
static void
dtrace_bcopy(const void *src, void *dst, size_t len)
{
if (len != 0) {
uint8_t *s1 = dst;
const uint8_t *s2 = src;
if (s1 <= s2) {
do {
*s1++ = dtrace_load8((uintptr_t)s2++);
} while (--len != 0);
} else {
s2 += len;
s1 += len;
do {
*--s1 = dtrace_load8((uintptr_t)--s2);
} while (--len != 0);
}
}
}
/*
* Copy src to dst using safe memory accesses, up to either the specified
* length, or the point that a nul byte is encountered. The src is assumed to
* be unsafe memory specified by the DIF program. The dst is assumed to be
* safe memory that we can store to directly because it is managed by DTrace.
* Unlike dtrace_bcopy(), overlapping regions are not handled.
*/
static void
dtrace_strcpy(const void *src, void *dst, size_t len)
{
if (len != 0) {
uint8_t *s1 = dst, c;
const uint8_t *s2 = src;
do {
*s1++ = c = dtrace_load8((uintptr_t)s2++);
} while (--len != 0 && c != '\0');
}
}
/*
* Copy src to dst, deriving the size and type from the specified (BYREF)
* variable type. The src is assumed to be unsafe memory specified by the DIF
* program. The dst is assumed to be DTrace variable memory that is of the
* specified type; we assume that we can store to directly.
*/
static void
dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type)
{
ASSERT(type->dtdt_flags & DIF_TF_BYREF);
if (type->dtdt_kind == DIF_TYPE_STRING) {
dtrace_strcpy(src, dst, type->dtdt_size);
} else {
dtrace_bcopy(src, dst, type->dtdt_size);
}
}
/*
* Compare s1 to s2 using safe memory accesses. The s1 data is assumed to be
* unsafe memory specified by the DIF program. The s2 data is assumed to be
* safe memory that we can access directly because it is managed by DTrace.
*/
static int
dtrace_bcmp(const void *s1, const void *s2, size_t len)
{
volatile uint16_t *flags;
flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;
if (s1 == s2)
return (0);
if (s1 == NULL || s2 == NULL)
return (1);
if (s1 != s2 && len != 0) {
const uint8_t *ps1 = s1;
const uint8_t *ps2 = s2;
do {
if (dtrace_load8((uintptr_t)ps1++) != *ps2++)
return (1);
} while (--len != 0 && !(*flags & CPU_DTRACE_FAULT));
}
return (0);
}
/*
* Zero the specified region using a simple byte-by-byte loop. Note that this
* is for safe DTrace-managed memory only.
*/
static void
dtrace_bzero(void *dst, size_t len)
{
uchar_t *cp;
for (cp = dst; len != 0; len--)
*cp++ = 0;
}
static void
dtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
{
uint64_t result[2];
result[0] = addend1[0] + addend2[0];
result[1] = addend1[1] + addend2[1] +
(result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
sum[0] = result[0];
sum[1] = result[1];
}
/*
* Shift the 128-bit value in a by b. If b is positive, shift left.
* If b is negative, shift right.
*/
static void
dtrace_shift_128(uint64_t *a, int b)
{
uint64_t mask;
if (b == 0)
return;
if (b < 0) {
b = -b;
if (b >= 64) {
a[0] = a[1] >> (b - 64);
a[1] = 0;
} else {
a[0] >>= b;
mask = 1LL << (64 - b);
mask -= 1;
a[0] |= ((a[1] & mask) << (64 - b));
a[1] >>= b;
}
} else {
if (b >= 64) {
a[1] = a[0] << (b - 64);
a[0] = 0;
} else {
a[1] <<= b;
mask = a[0] >> (64 - b);
a[1] |= mask;
a[0] <<= b;
}
}
}
/*
* The basic idea is to break the 2 64-bit values into 4 32-bit values,
* use native multiplication on those, and then re-combine into the
* resulting 128-bit value.
*
* (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
* hi1 * hi2 << 64 +
* hi1 * lo2 << 32 +
* hi2 * lo1 << 32 +
* lo1 * lo2
*/
static void
dtrace_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
{
uint64_t hi1, hi2, lo1, lo2;
uint64_t tmp[2];
hi1 = factor1 >> 32;
hi2 = factor2 >> 32;
lo1 = factor1 & DT_MASK_LO;
lo2 = factor2 & DT_MASK_LO;
product[0] = lo1 * lo2;
product[1] = hi1 * hi2;
tmp[0] = hi1 * lo2;
tmp[1] = 0;
dtrace_shift_128(tmp, 32);
dtrace_add_128(product, tmp, product);
tmp[0] = hi2 * lo1;
tmp[1] = 0;
dtrace_shift_128(tmp, 32);
dtrace_add_128(product, tmp, product);
}
/*
* This privilege check should be used by actions and subroutines to
* verify that the user credentials of the process that enabled the
* invoking ECB match the target credentials
*/
static int
dtrace_priv_proc_common_user(dtrace_state_t *state)
{
cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
/*
* We should always have a non-NULL state cred here, since if cred
* is null (anonymous tracing), we fast-path bypass this routine.
*/
ASSERT(s_cr != NULL);
if ((cr = CRED()) != NULL &&
s_cr->cr_uid == cr->cr_uid &&
s_cr->cr_uid == cr->cr_ruid &&
s_cr->cr_uid == cr->cr_suid &&
s_cr->cr_gid == cr->cr_gid &&
s_cr->cr_gid == cr->cr_rgid &&
s_cr->cr_gid == cr->cr_sgid)
return (1);
return (0);
}
/*
* This privilege check should be used by actions and subroutines to
* verify that the zone of the process that enabled the invoking ECB
* matches the target credentials
*/
static int
dtrace_priv_proc_common_zone(dtrace_state_t *state)
{
#ifdef illumos
cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
/*
* We should always have a non-NULL state cred here, since if cred
* is null (anonymous tracing), we fast-path bypass this routine.
*/
ASSERT(s_cr != NULL);
if ((cr = CRED()) != NULL && s_cr->cr_zone == cr->cr_zone)
return (1);
return (0);
#else
return (1);
#endif
}
/*
* This privilege check should be used by actions and subroutines to
* verify that the process has not setuid or changed credentials.
*/
static int
dtrace_priv_proc_common_nocd(void)
{
proc_t *proc;
if ((proc = ttoproc(curthread)) != NULL &&
!(proc->p_flag & SNOCD))
return (1);
return (0);
}
static int
dtrace_priv_proc_destructive(dtrace_state_t *state)
{
int action = state->dts_cred.dcr_action;
if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) &&
dtrace_priv_proc_common_zone(state) == 0)
goto bad;
if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) &&
dtrace_priv_proc_common_user(state) == 0)
goto bad;
if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) &&
dtrace_priv_proc_common_nocd() == 0)
goto bad;
return (1);
bad:
cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
return (0);
}
static int
dtrace_priv_proc_control(dtrace_state_t *state)
{
if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL)
return (1);
if (dtrace_priv_proc_common_zone(state) &&
dtrace_priv_proc_common_user(state) &&
dtrace_priv_proc_common_nocd())
return (1);
cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
return (0);
}
static int
dtrace_priv_proc(dtrace_state_t *state)
{
if (state->dts_cred.dcr_action & DTRACE_CRA_PROC)
return (1);
cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
return (0);
}
static int
dtrace_priv_kernel(dtrace_state_t *state)
{
if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL)
return (1);
cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
return (0);
}
static int
dtrace_priv_kernel_destructive(dtrace_state_t *state)
{
if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE)
return (1);
cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
return (0);
}
/*
* Determine if the dte_cond of the specified ECB allows for processing of
* the current probe to continue. Note that this routine may allow continued
* processing, but with access(es) stripped from the mstate's dtms_access
* field.
*/
static int
dtrace_priv_probe(dtrace_state_t *state, dtrace_mstate_t *mstate,
dtrace_ecb_t *ecb)
{
dtrace_probe_t *probe = ecb->dte_probe;
dtrace_provider_t *prov = probe->dtpr_provider;
dtrace_pops_t *pops = &prov->dtpv_pops;
int mode = DTRACE_MODE_NOPRIV_DROP;
ASSERT(ecb->dte_cond);
#ifdef illumos
if (pops->dtps_mode != NULL) {
mode = pops->dtps_mode(prov->dtpv_arg,
probe->dtpr_id, probe->dtpr_arg);
ASSERT((mode & DTRACE_MODE_USER) ||
(mode & DTRACE_MODE_KERNEL));
ASSERT((mode & DTRACE_MODE_NOPRIV_RESTRICT) ||
(mode & DTRACE_MODE_NOPRIV_DROP));
}
/*
* If the dte_cond bits indicate that this consumer is only allowed to
* see user-mode firings of this probe, call the provider's dtps_mode()
* entry point to check that the probe was fired while in a user
* context. If that's not the case, use the policy specified by the
* provider to determine if we drop the probe or merely restrict
* operation.
*/
if (ecb->dte_cond & DTRACE_COND_USERMODE) {
ASSERT(mode != DTRACE_MODE_NOPRIV_DROP);
if (!(mode & DTRACE_MODE_USER)) {
if (mode & DTRACE_MODE_NOPRIV_DROP)
return (0);
mstate->dtms_access &= ~DTRACE_ACCESS_ARGS;
}
}
#endif
/*
* This is more subtle than it looks. We have to be absolutely certain
* that CRED() isn't going to change out from under us so it's only
* legit to examine that structure if we're in constrained situations.
* Currently, the only times we'll this check is if a non-super-user
* has enabled the profile or syscall providers -- providers that
* allow visibility of all processes. For the profile case, the check
* above will ensure that we're examining a user context.
*/
if (ecb->dte_cond & DTRACE_COND_OWNER) {
cred_t *cr;
cred_t *s_cr = state->dts_cred.dcr_cred;
proc_t *proc;
ASSERT(s_cr != NULL);
if ((cr = CRED()) == NULL ||
s_cr->cr_uid != cr->cr_uid ||
s_cr->cr_uid != cr->cr_ruid ||
s_cr->cr_uid != cr->cr_suid ||
s_cr->cr_gid != cr->cr_gid ||
s_cr->cr_gid != cr->cr_rgid ||
s_cr->cr_gid != cr->cr_sgid ||
(proc = ttoproc(curthread)) == NULL ||
(proc->p_flag & SNOCD)) {
if (mode & DTRACE_MODE_NOPRIV_DROP)
return (0);
#ifdef illumos
mstate->dtms_access &= ~DTRACE_ACCESS_PROC;
#endif
}
}
#ifdef illumos
/*
* If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not
* in our zone, check to see if our mode policy is to restrict rather
* than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC
* and DTRACE_ACCESS_ARGS
*/
if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {
cred_t *cr;
cred_t *s_cr = state->dts_cred.dcr_cred;
ASSERT(s_cr != NULL);
if ((cr = CRED()) == NULL ||
s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) {
if (mode & DTRACE_MODE_NOPRIV_DROP)
return (0);
mstate->dtms_access &=
~(DTRACE_ACCESS_PROC | DTRACE_ACCESS_ARGS);
}
}
#endif
return (1);
}
/*
* Note: not called from probe context. This function is called
* asynchronously (and at a regular interval) from outside of probe context to
* clean the dirty dynamic variable lists on all CPUs. Dynamic variable
* cleaning is explained in detail in <sys/dtrace_impl.h>.
*/
void
dtrace_dynvar_clean(dtrace_dstate_t *dstate)
{
dtrace_dynvar_t *dirty;
dtrace_dstate_percpu_t *dcpu;
dtrace_dynvar_t **rinsep;
int i, j, work = 0;
for (i = 0; i < NCPU; i++) {
dcpu = &dstate->dtds_percpu[i];
rinsep = &dcpu->dtdsc_rinsing;
/*
* If the dirty list is NULL, there is no dirty work to do.
*/
if (dcpu->dtdsc_dirty == NULL)
continue;
if (dcpu->dtdsc_rinsing != NULL) {
/*
* If the rinsing list is non-NULL, then it is because
* this CPU was selected to accept another CPU's
* dirty list -- and since that time, dirty buffers
* have accumulated. This is a highly unlikely
* condition, but we choose to ignore the dirty
* buffers -- they'll be picked up a future cleanse.
*/
continue;
}
if (dcpu->dtdsc_clean != NULL) {
/*
* If the clean list is non-NULL, then we're in a
* situation where a CPU has done deallocations (we
* have a non-NULL dirty list) but no allocations (we
* also have a non-NULL clean list). We can't simply
* move the dirty list into the clean list on this
* CPU, yet we also don't want to allow this condition
* to persist, lest a short clean list prevent a
* massive dirty list from being cleaned (which in
* turn could lead to otherwise avoidable dynamic
* drops). To deal with this, we look for some CPU
* with a NULL clean list, NULL dirty list, and NULL
* rinsing list -- and then we borrow this CPU to
* rinse our dirty list.
*/
for (j = 0; j < NCPU; j++) {
dtrace_dstate_percpu_t *rinser;
rinser = &dstate->dtds_percpu[j];
if (rinser->dtdsc_rinsing != NULL)
continue;
if (rinser->dtdsc_dirty != NULL)
continue;
if (rinser->dtdsc_clean != NULL)
continue;
rinsep = &rinser->dtdsc_rinsing;
break;
}
if (j == NCPU) {
/*
* We were unable to find another CPU that
* could accept this dirty list -- we are
* therefore unable to clean it now.
*/
dtrace_dynvar_failclean++;
continue;
}
}
work = 1;
/*
* Atomically move the dirty list aside.
*/
do {
dirty = dcpu->dtdsc_dirty;
/*
* Before we zap the dirty list, set the rinsing list.
* (This allows for a potential assertion in
* dtrace_dynvar(): if a free dynamic variable appears
* on a hash chain, either the dirty list or the
* rinsing list for some CPU must be non-NULL.)
*/
*rinsep = dirty;
dtrace_membar_producer();
} while (dtrace_casptr(&dcpu->dtdsc_dirty,
dirty, NULL) != dirty);
}
if (!work) {
/*
* We have no work to do; we can simply return.
*/
return;
}
dtrace_sync();
for (i = 0; i < NCPU; i++) {
dcpu = &dstate->dtds_percpu[i];
if (dcpu->dtdsc_rinsing == NULL)
continue;
/*
* We are now guaranteed that no hash chain contains a pointer
* into this dirty list; we can make it clean.
*/
ASSERT(dcpu->dtdsc_clean == NULL);
dcpu->dtdsc_clean = dcpu->dtdsc_rinsing;
dcpu->dtdsc_rinsing = NULL;
}
/*
* Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
* sure that all CPUs have seen all of the dtdsc_clean pointers.
* This prevents a race whereby a CPU incorrectly decides that
* the state should be something other than DTRACE_DSTATE_CLEAN
* after dtrace_dynvar_clean() has completed.
*/
dtrace_sync();
dstate->dtds_state = DTRACE_DSTATE_CLEAN;
}
/*
* Depending on the value of the op parameter, this function looks-up,
* allocates or deallocates an arbitrarily-keyed dynamic variable. If an
* allocation is requested, this function will return a pointer to a
* dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no
* variable can be allocated. If NULL is returned, the appropriate counter
* will be incremented.
*/
dtrace_dynvar_t *
dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys,
dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op,
dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
{
uint64_t hashval = DTRACE_DYNHASH_VALID;
dtrace_dynhash_t *hash = dstate->dtds_hash;
dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL;
processorid_t me = curcpu, cpu = me;
dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me];
size_t bucket, ksize;
size_t chunksize = dstate->dtds_chunksize;
uintptr_t kdata, lock, nstate;
uint_t i;
ASSERT(nkeys != 0);
/*
* Hash the key. As with aggregations, we use Jenkins' "One-at-a-time"
* algorithm. For the by-value portions, we perform the algorithm in
* 16-bit chunks (as opposed to 8-bit chunks). This speeds things up a
* bit, and seems to have only a minute effect on distribution. For
* the by-reference data, we perform "One-at-a-time" iterating (safely)
* over each referenced byte. It's painful to do this, but it's much
* better than pathological hash distribution. The efficacy of the
* hashing algorithm (and a comparison with other algorithms) may be
* found by running the ::dtrace_dynstat MDB dcmd.
*/
for (i = 0; i < nkeys; i++) {
if (key[i].dttk_size == 0) {
uint64_t val = key[i].dttk_value;
hashval += (val >> 48) & 0xffff;
hashval += (hashval << 10);
hashval ^= (hashval >> 6);
hashval += (val >> 32) & 0xffff;
hashval += (hashval << 10);
hashval ^= (hashval >> 6);
hashval += (val >> 16) & 0xffff;
hashval += (hashval << 10);
hashval ^= (hashval >> 6);
hashval += val & 0xffff;
hashval += (hashval << 10);
hashval ^= (hashval >> 6);
} else {
/*
* This is incredibly painful, but it beats the hell
* out of the alternative.
*/
uint64_t j, size = key[i].dttk_size;
uintptr_t base = (uintptr_t)key[i].dttk_value;
if (!dtrace_canload(base, size, mstate, vstate))
break;
for (j = 0; j < size; j++) {
hashval += dtrace_load8(base + j);
hashval += (hashval << 10);
hashval ^= (hashval >> 6);
}
}
}
if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
return (NULL);
hashval += (hashval << 3);
hashval ^= (hashval >> 11);
hashval += (hashval << 15);
/*
* There is a remote chance (ideally, 1 in 2^31) that our hashval
* comes out to be one of our two sentinel hash values. If this
* actually happens, we set the hashval to be a value known to be a
* non-sentinel value.
*/
if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK)
hashval = DTRACE_DYNHASH_VALID;
/*
* Yes, it's painful to do a divide here. If the cycle count becomes
* important here, tricks can be pulled to reduce it. (However, it's
* critical that hash collisions be kept to an absolute minimum;
* they're much more painful than a divide.) It's better to have a
* solution that generates few collisions and still keeps things
* relatively simple.
*/
bucket = hashval % dstate->dtds_hashsize;
if (op == DTRACE_DYNVAR_DEALLOC) {
volatile uintptr_t *lockp = &hash[bucket].dtdh_lock;
for (;;) {
while ((lock = *lockp) & 1)
continue;
if (dtrace_casptr((volatile void *)lockp,
(volatile void *)lock, (volatile void *)(lock + 1)) == (void *)lock)
break;
}
dtrace_membar_producer();
}
top:
prev = NULL;
lock = hash[bucket].dtdh_lock;
dtrace_membar_consumer();
start = hash[bucket].dtdh_chain;
ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK ||
start->dtdv_hashval != DTRACE_DYNHASH_FREE ||
op != DTRACE_DYNVAR_DEALLOC));
for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) {
dtrace_tuple_t *dtuple = &dvar->dtdv_tuple;
dtrace_key_t *dkey = &dtuple->dtt_key[0];
if (dvar->dtdv_hashval != hashval) {
if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) {
/*
* We've reached the sink, and therefore the
* end of the hash chain; we can kick out of
* the loop knowing that we have seen a valid
* snapshot of state.
*/
ASSERT(dvar->dtdv_next == NULL);
ASSERT(dvar == &dtrace_dynhash_sink);
break;
}
if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) {
/*
* We've gone off the rails: somewhere along
* the line, one of the members of this hash
* chain was deleted. Note that we could also
* detect this by simply letting this loop run
* to completion, as we would eventually hit
* the end of the dirty list. However, we
* want to avoid running the length of the
* dirty list unnecessarily (it might be quite
* long), so we catch this as early as
* possible by detecting the hash marker. In
* this case, we simply set dvar to NULL and
* break; the conditional after the loop will
* send us back to top.
*/
dvar = NULL;
break;
}
goto next;
}
if (dtuple->dtt_nkeys != nkeys)
goto next;
for (i = 0; i < nkeys; i++, dkey++) {
if (dkey->dttk_size != key[i].dttk_size)
goto next; /* size or type mismatch */
if (dkey->dttk_size != 0) {
if (dtrace_bcmp(
(void *)(uintptr_t)key[i].dttk_value,
(void *)(uintptr_t)dkey->dttk_value,
dkey->dttk_size))
goto next;
} else {
if (dkey->dttk_value != key[i].dttk_value)
goto next;
}
}
if (op != DTRACE_DYNVAR_DEALLOC)
return (dvar);
ASSERT(dvar->dtdv_next == NULL ||
dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE);
if (prev != NULL) {
ASSERT(hash[bucket].dtdh_chain != dvar);
ASSERT(start != dvar);
ASSERT(prev->dtdv_next == dvar);
prev->dtdv_next = dvar->dtdv_next;
} else {
if (dtrace_casptr(&hash[bucket].dtdh_chain,
start, dvar->dtdv_next) != start) {
/*
* We have failed to atomically swing the
* hash table head pointer, presumably because
* of a conflicting allocation on another CPU.
* We need to reread the hash chain and try
* again.
*/
goto top;
}
}
dtrace_membar_producer();
/*
* Now set the hash value to indicate that it's free.
*/
ASSERT(hash[bucket].dtdh_chain != dvar);
dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
dtrace_membar_producer();
/*
* Set the next pointer to point at the dirty list, and
* atomically swing the dirty pointer to the newly freed dvar.
*/
do {
next = dcpu->dtdsc_dirty;
dvar->dtdv_next = next;
} while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next);
/*
* Finally, unlock this hash bucket.
*/
ASSERT(hash[bucket].dtdh_lock == lock);
ASSERT(lock & 1);
hash[bucket].dtdh_lock++;
return (NULL);
next:
prev = dvar;
continue;
}
if (dvar == NULL) {
/*
* If dvar is NULL, it is because we went off the rails:
* one of the elements that we traversed in the hash chain
* was deleted while we were traversing it. In this case,
* we assert that we aren't doing a dealloc (deallocs lock
* the hash bucket to prevent themselves from racing with
* one another), and retry the hash chain traversal.
*/
ASSERT(op != DTRACE_DYNVAR_DEALLOC);
goto top;
}
if (op != DTRACE_DYNVAR_ALLOC) {
/*
* If we are not to allocate a new variable, we want to
* return NULL now. Before we return, check that the value
* of the lock word hasn't changed. If it has, we may have
* seen an inconsistent snapshot.
*/
if (op == DTRACE_DYNVAR_NOALLOC) {
if (hash[bucket].dtdh_lock != lock)
goto top;
} else {
ASSERT(op == DTRACE_DYNVAR_DEALLOC);
ASSERT(hash[bucket].dtdh_lock == lock);
ASSERT(lock & 1);
hash[bucket].dtdh_lock++;
}
return (NULL);
}
/*
* We need to allocate a new dynamic variable. The size we need is the
* size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the
* size of any auxiliary key data (rounded up to 8-byte alignment) plus
* the size of any referred-to data (dsize). We then round the final
* size up to the chunksize for allocation.
*/
for (ksize = 0, i = 0; i < nkeys; i++)
ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
/*
* This should be pretty much impossible, but could happen if, say,
* strange DIF specified the tuple. Ideally, this should be an
* assertion and not an error condition -- but that requires that the
* chunksize calculation in dtrace_difo_chunksize() be absolutely
* bullet-proof. (That is, it must not be able to be fooled by
* malicious DIF.) Given the lack of backwards branches in DIF,
* solving this would presumably not amount to solving the Halting
* Problem -- but it still seems awfully hard.
*/
if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) +
ksize + dsize > chunksize) {
dcpu->dtdsc_drops++;
return (NULL);
}
nstate = DTRACE_DSTATE_EMPTY;
do {
retry:
free = dcpu->dtdsc_free;
if (free == NULL) {
dtrace_dynvar_t *clean = dcpu->dtdsc_clean;
void *rval;
if (clean == NULL) {
/*
* We're out of dynamic variable space on
* this CPU. Unless we have tried all CPUs,
* we'll try to allocate from a different
* CPU.
*/
switch (dstate->dtds_state) {
case DTRACE_DSTATE_CLEAN: {
void *sp = &dstate->dtds_state;
if (++cpu >= NCPU)
cpu = 0;
if (dcpu->dtdsc_dirty != NULL &&
nstate == DTRACE_DSTATE_EMPTY)
nstate = DTRACE_DSTATE_DIRTY;
if (dcpu->dtdsc_rinsing != NULL)
nstate = DTRACE_DSTATE_RINSING;
dcpu = &dstate->dtds_percpu[cpu];
if (cpu != me)
goto retry;
(void) dtrace_cas32(sp,
DTRACE_DSTATE_CLEAN, nstate);
/*
* To increment the correct bean
* counter, take another lap.
*/
goto retry;
}
case DTRACE_DSTATE_DIRTY:
dcpu->dtdsc_dirty_drops++;
break;
case DTRACE_DSTATE_RINSING:
dcpu->dtdsc_rinsing_drops++;
break;
case DTRACE_DSTATE_EMPTY:
dcpu->dtdsc_drops++;
break;
}
DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP);
return (NULL);
}
/*
* The clean list appears to be non-empty. We want to
* move the clean list to the free list; we start by
* moving the clean pointer aside.
*/
if (dtrace_casptr(&dcpu->dtdsc_clean,
clean, NULL) != clean) {
/*
* We are in one of two situations:
*
* (a) The clean list was switched to the
* free list by another CPU.
*
* (b) The clean list was added to by the
* cleansing cyclic.
*
* In either of these situations, we can
* just reattempt the free list allocation.
*/
goto retry;
}
ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE);
/*
* Now we'll move the clean list to our free list.
* It's impossible for this to fail: the only way
* the free list can be updated is through this
* code path, and only one CPU can own the clean list.
* Thus, it would only be possible for this to fail if
* this code were racing with dtrace_dynvar_clean().
* (That is, if dtrace_dynvar_clean() updated the clean
* list, and we ended up racing to update the free
* list.) This race is prevented by the dtrace_sync()
* in dtrace_dynvar_clean() -- which flushes the
* owners of the clean lists out before resetting
* the clean lists.
*/
dcpu = &dstate->dtds_percpu[me];
rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean);
ASSERT(rval == NULL);
goto retry;
}
dvar = free;
new_free = dvar->dtdv_next;
} while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free);
/*
* We have now allocated a new chunk. We copy the tuple keys into the
* tuple array and copy any referenced key data into the data space
* following the tuple array. As we do this, we relocate dttk_value
* in the final tuple to point to the key data address in the chunk.
*/
kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys];
dvar->dtdv_data = (void *)(kdata + ksize);
dvar->dtdv_tuple.dtt_nkeys = nkeys;
for (i = 0; i < nkeys; i++) {
dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i];
size_t kesize = key[i].dttk_size;
if (kesize != 0) {
dtrace_bcopy(
(const void *)(uintptr_t)key[i].dttk_value,
(void *)kdata, kesize);
dkey->dttk_value = kdata;
kdata += P2ROUNDUP(kesize, sizeof (uint64_t));
} else {
dkey->dttk_value = key[i].dttk_value;
}
dkey->dttk_size = kesize;
}
ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE);
dvar->dtdv_hashval = hashval;
dvar->dtdv_next = start;
if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start)
return (dvar);
/*
* The cas has failed. Either another CPU is adding an element to
* this hash chain, or another CPU is deleting an element from this
* hash chain. The simplest way to deal with both of these cases
* (though not necessarily the most efficient) is to free our
* allocated block and tail-call ourselves. Note that the free is
* to the dirty list and _not_ to the free list. This is to prevent
* races with allocators, above.
*/
dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
dtrace_membar_producer();
do {
free = dcpu->dtdsc_dirty;
dvar->dtdv_next = free;
} while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free);
return (dtrace_dynvar(dstate, nkeys, key, dsize, op, mstate, vstate));
}
/*ARGSUSED*/
static void
dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg)
{
if ((int64_t)nval < (int64_t)*oval)
*oval = nval;
}
/*ARGSUSED*/
static void
dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg)
{
if ((int64_t)nval > (int64_t)*oval)
*oval = nval;
}
static void
dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr)
{
int i, zero = DTRACE_QUANTIZE_ZEROBUCKET;
int64_t val = (int64_t)nval;
if (val < 0) {
for (i = 0; i < zero; i++) {
if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) {
quanta[i] += incr;
return;
}
}
} else {
for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) {
if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) {
quanta[i - 1] += incr;
return;
}
}
quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr;
return;
}
ASSERT(0);
}
static void
dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr)
{
uint64_t arg = *lquanta++;
int32_t base = DTRACE_LQUANTIZE_BASE(arg);
uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
int32_t val = (int32_t)nval, level;
ASSERT(step != 0);
ASSERT(levels != 0);
if (val < base) {
/*
* This is an underflow.
*/
lquanta[0] += incr;
return;
}
level = (val - base) / step;
if (level < levels) {
lquanta[level + 1] += incr;
return;
}
/*
* This is an overflow.
*/
lquanta[levels + 1] += incr;
}
static int
dtrace_aggregate_llquantize_bucket(uint16_t factor, uint16_t low,
uint16_t high, uint16_t nsteps, int64_t value)
{
int64_t this = 1, last, next;
int base = 1, order;
ASSERT(factor <= nsteps);
ASSERT(nsteps % factor == 0);
for (order = 0; order < low; order++)
this *= factor;
/*
* If our value is less than our factor taken to the power of the
* low order of magnitude, it goes into the zeroth bucket.
*/
if (value < (last = this))
return (0);
for (this *= factor; order <= high; order++) {
int nbuckets = this > nsteps ? nsteps : this;
if ((next = this * factor) < this) {
/*
* We should not generally get log/linear quantizations
* with a high magnitude that allows 64-bits to
* overflow, but we nonetheless protect against this
* by explicitly checking for overflow, and clamping
* our value accordingly.
*/
value = this - 1;
}
if (value < this) {
/*
* If our value lies within this order of magnitude,
* determine its position by taking the offset within
* the order of magnitude, dividing by the bucket
* width, and adding to our (accumulated) base.
*/
return (base + (value - last) / (this / nbuckets));
}
base += nbuckets - (nbuckets / factor);
last = this;
this = next;
}
/*
* Our value is greater than or equal to our factor taken to the
* power of one plus the high magnitude -- return the top bucket.
*/
return (base);
}
static void
dtrace_aggregate_llquantize(uint64_t *llquanta, uint64_t nval, uint64_t incr)
{
uint64_t arg = *llquanta++;
uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
llquanta[dtrace_aggregate_llquantize_bucket(factor,
low, high, nsteps, nval)] += incr;
}
/*ARGSUSED*/
static void
dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg)
{
data[0]++;
data[1] += nval;
}
/*ARGSUSED*/
static void
dtrace_aggregate_stddev(uint64_t *data, uint64_t nval, uint64_t arg)
{
int64_t snval = (int64_t)nval;
uint64_t tmp[2];
data[0]++;
data[1] += nval;
/*
* What we want to say here is:
*
* data[2] += nval * nval;
*
* But given that nval is 64-bit, we could easily overflow, so
* we do this as 128-bit arithmetic.
*/
if (snval < 0)
snval = -snval;
dtrace_multiply_128((uint64_t)snval, (uint64_t)snval, tmp);
dtrace_add_128(data + 2, tmp, data + 2);
}
/*ARGSUSED*/
static void
dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg)
{
*oval = *oval + 1;
}
/*ARGSUSED*/
static void
dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg)
{
*oval += nval;
}
/*
* Aggregate given the tuple in the principal data buffer, and the aggregating
* action denoted by the specified dtrace_aggregation_t. The aggregation
* buffer is specified as the buf parameter. This routine does not return
* failure; if there is no space in the aggregation buffer, the data will be
* dropped, and a corresponding counter incremented.
*/
static void
dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf,
intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg)
{
dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec;
uint32_t i, ndx, size, fsize;
uint32_t align = sizeof (uint64_t) - 1;
dtrace_aggbuffer_t *agb;
dtrace_aggkey_t *key;
uint32_t hashval = 0, limit, isstr;
caddr_t tomax, data, kdata;
dtrace_actkind_t action;
dtrace_action_t *act;
uintptr_t offs;
if (buf == NULL)
return;
if (!agg->dtag_hasarg) {
/*
* Currently, only quantize() and lquantize() take additional
* arguments, and they have the same semantics: an increment
* value that defaults to 1 when not present. If additional
* aggregating actions take arguments, the setting of the
* default argument value will presumably have to become more
* sophisticated...
*/
arg = 1;
}
action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION;
size = rec->dtrd_offset - agg->dtag_base;
fsize = size + rec->dtrd_size;
ASSERT(dbuf->dtb_tomax != NULL);
data = dbuf->dtb_tomax + offset + agg->dtag_base;
if ((tomax = buf->dtb_tomax) == NULL) {
dtrace_buffer_drop(buf);
return;
}
/*
* The metastructure is always at the bottom of the buffer.
*/
agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size -
sizeof (dtrace_aggbuffer_t));
if (buf->dtb_offset == 0) {
/*
* We just kludge up approximately 1/8th of the size to be
* buckets. If this guess ends up being routinely
* off-the-mark, we may need to dynamically readjust this
* based on past performance.
*/
uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t);
if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) <
(uintptr_t)tomax || hashsize == 0) {
/*
* We've been given a ludicrously small buffer;
* increment our drop count and leave.
*/
dtrace_buffer_drop(buf);
return;
}
/*
* And now, a pathetic attempt to try to get a an odd (or
* perchance, a prime) hash size for better hash distribution.
*/
if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3))
hashsize -= DTRACE_AGGHASHSIZE_SLEW;
agb->dtagb_hashsize = hashsize;
agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb -
agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *));
agb->dtagb_free = (uintptr_t)agb->dtagb_hash;
for (i = 0; i < agb->dtagb_hashsize; i++)
agb->dtagb_hash[i] = NULL;
}
ASSERT(agg->dtag_first != NULL);
ASSERT(agg->dtag_first->dta_intuple);
/*
* Calculate the hash value based on the key. Note that we _don't_
* include the aggid in the hashing (but we will store it as part of
* the key). The hashing algorithm is Bob Jenkins' "One-at-a-time"
* algorithm: a simple, quick algorithm that has no known funnels, and
* gets good distribution in practice. The efficacy of the hashing
* algorithm (and a comparison with other algorithms) may be found by
* running the ::dtrace_aggstat MDB dcmd.
*/
for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
i = act->dta_rec.dtrd_offset - agg->dtag_base;
limit = i + act->dta_rec.dtrd_size;
ASSERT(limit <= size);
isstr = DTRACEACT_ISSTRING(act);
for (; i < limit; i++) {
hashval += data[i];
hashval += (hashval << 10);
hashval ^= (hashval >> 6);
if (isstr && data[i] == '\0')
break;
}
}
hashval += (hashval << 3);
hashval ^= (hashval >> 11);
hashval += (hashval << 15);
/*
* Yes, the divide here is expensive -- but it's generally the least
* of the performance issues given the amount of data that we iterate
* over to compute hash values, compare data, etc.
*/
ndx = hashval % agb->dtagb_hashsize;
for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) {
ASSERT((caddr_t)key >= tomax);
ASSERT((caddr_t)key < tomax + buf->dtb_size);
if (hashval != key->dtak_hashval || key->dtak_size != size)
continue;
kdata = key->dtak_data;
ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size);
for (act = agg->dtag_first; act->dta_intuple;
act = act->dta_next) {
i = act->dta_rec.dtrd_offset - agg->dtag_base;
limit = i + act->dta_rec.dtrd_size;
ASSERT(limit <= size);
isstr = DTRACEACT_ISSTRING(act);
for (; i < limit; i++) {
if (kdata[i] != data[i])
goto next;
if (isstr && data[i] == '\0')
break;
}
}
if (action != key->dtak_action) {
/*
* We are aggregating on the same value in the same
* aggregation with two different aggregating actions.
* (This should have been picked up in the compiler,
* so we may be dealing with errant or devious DIF.)
* This is an error condition; we indicate as much,
* and return.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
return;
}
/*
* This is a hit: we need to apply the aggregator to
* the value at this key.
*/
agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg);
return;
next:
continue;
}
/*
* We didn't find it. We need to allocate some zero-filled space,
* link it into the hash table appropriately, and apply the aggregator
* to the (zero-filled) value.
*/
offs = buf->dtb_offset;
while (offs & (align - 1))
offs += sizeof (uint32_t);
/*
* If we don't have enough room to both allocate a new key _and_
* its associated data, increment the drop count and return.
*/
if ((uintptr_t)tomax + offs + fsize >
agb->dtagb_free - sizeof (dtrace_aggkey_t)) {
dtrace_buffer_drop(buf);
return;
}
/*CONSTCOND*/
ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1)));
key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t));
agb->dtagb_free -= sizeof (dtrace_aggkey_t);
key->dtak_data = kdata = tomax + offs;
buf->dtb_offset = offs + fsize;
/*
* Now copy the data across.
*/
*((dtrace_aggid_t *)kdata) = agg->dtag_id;
for (i = sizeof (dtrace_aggid_t); i < size; i++)
kdata[i] = data[i];
/*
* Because strings are not zeroed out by default, we need to iterate
* looking for actions that store strings, and we need to explicitly
* pad these strings out with zeroes.
*/
for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
int nul;
if (!DTRACEACT_ISSTRING(act))
continue;
i = act->dta_rec.dtrd_offset - agg->dtag_base;
limit = i + act->dta_rec.dtrd_size;
ASSERT(limit <= size);
for (nul = 0; i < limit; i++) {
if (nul) {
kdata[i] = '\0';
continue;
}
if (data[i] != '\0')
continue;
nul = 1;
}
}
for (i = size; i < fsize; i++)
kdata[i] = 0;
key->dtak_hashval = hashval;
key->dtak_size = size;
key->dtak_action = action;
key->dtak_next = agb->dtagb_hash[ndx];
agb->dtagb_hash[ndx] = key;
/*
* Finally, apply the aggregator.
*/
*((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial;
agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg);
}
/*
* Given consumer state, this routine finds a speculation in the INACTIVE
* state and transitions it into the ACTIVE state. If there is no speculation
* in the INACTIVE state, 0 is returned. In this case, no error counter is
* incremented -- it is up to the caller to take appropriate action.
*/
static int
dtrace_speculation(dtrace_state_t *state)
{
int i = 0;
dtrace_speculation_state_t current;
uint32_t *stat = &state->dts_speculations_unavail, count;
while (i < state->dts_nspeculations) {
dtrace_speculation_t *spec = &state->dts_speculations[i];
current = spec->dtsp_state;
if (current != DTRACESPEC_INACTIVE) {
if (current == DTRACESPEC_COMMITTINGMANY ||
current == DTRACESPEC_COMMITTING ||
current == DTRACESPEC_DISCARDING)
stat = &state->dts_speculations_busy;
i++;
continue;
}
if (dtrace_cas32((uint32_t *)&spec->dtsp_state,
current, DTRACESPEC_ACTIVE) == current)
return (i + 1);
}
/*
* We couldn't find a speculation. If we found as much as a single
* busy speculation buffer, we'll attribute this failure as "busy"
* instead of "unavail".
*/
do {
count = *stat;
} while (dtrace_cas32(stat, count, count + 1) != count);
return (0);
}
/*
* This routine commits an active speculation. If the specified speculation
* is not in a valid state to perform a commit(), this routine will silently do
* nothing. The state of the specified speculation is transitioned according
* to the state transition diagram outlined in <sys/dtrace_impl.h>
*/
static void
dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
dtrace_specid_t which)
{
dtrace_speculation_t *spec;
dtrace_buffer_t *src, *dest;
uintptr_t daddr, saddr, dlimit, slimit;
dtrace_speculation_state_t current, new = 0;
intptr_t offs;
uint64_t timestamp;
if (which == 0)
return;
if (which > state->dts_nspeculations) {
cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
return;
}
spec = &state->dts_speculations[which - 1];
src = &spec->dtsp_buffer[cpu];
dest = &state->dts_buffer[cpu];
do {
current = spec->dtsp_state;
if (current == DTRACESPEC_COMMITTINGMANY)
break;
switch (current) {
case DTRACESPEC_INACTIVE:
case DTRACESPEC_DISCARDING:
return;
case DTRACESPEC_COMMITTING:
/*
* This is only possible if we are (a) commit()'ing
* without having done a prior speculate() on this CPU
* and (b) racing with another commit() on a different
* CPU. There's nothing to do -- we just assert that
* our offset is 0.
*/
ASSERT(src->dtb_offset == 0);
return;
case DTRACESPEC_ACTIVE:
new = DTRACESPEC_COMMITTING;
break;
case DTRACESPEC_ACTIVEONE:
/*
* This speculation is active on one CPU. If our
* buffer offset is non-zero, we know that the one CPU
* must be us. Otherwise, we are committing on a
* different CPU from the speculate(), and we must
* rely on being asynchronously cleaned.
*/
if (src->dtb_offset != 0) {
new = DTRACESPEC_COMMITTING;
break;
}
/*FALLTHROUGH*/
case DTRACESPEC_ACTIVEMANY:
new = DTRACESPEC_COMMITTINGMANY;
break;
default:
ASSERT(0);
}
} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
current, new) != current);
/*
* We have set the state to indicate that we are committing this
* speculation. Now reserve the necessary space in the destination
* buffer.
*/
if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset,
sizeof (uint64_t), state, NULL)) < 0) {
dtrace_buffer_drop(dest);
goto out;
}
/*
* We have sufficient space to copy the speculative buffer into the
* primary buffer. First, modify the speculative buffer, filling
* in the timestamp of all entries with the current time. The data
* must have the commit() time rather than the time it was traced,
* so that all entries in the primary buffer are in timestamp order.
*/
timestamp = dtrace_gethrtime();
saddr = (uintptr_t)src->dtb_tomax;
slimit = saddr + src->dtb_offset;
while (saddr < slimit) {
size_t size;
dtrace_rechdr_t *dtrh = (dtrace_rechdr_t *)saddr;
if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
saddr += sizeof (dtrace_epid_t);
continue;
}
ASSERT3U(dtrh->dtrh_epid, <=, state->dts_necbs);
size = state->dts_ecbs[dtrh->dtrh_epid - 1]->dte_size;
ASSERT3U(saddr + size, <=, slimit);
ASSERT3U(size, >=, sizeof (dtrace_rechdr_t));
ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh), ==, UINT64_MAX);
DTRACE_RECORD_STORE_TIMESTAMP(dtrh, timestamp);
saddr += size;
}
/*
* Copy the buffer across. (Note that this is a
* highly subobtimal bcopy(); in the unlikely event that this becomes
* a serious performance issue, a high-performance DTrace-specific
* bcopy() should obviously be invented.)
*/
daddr = (uintptr_t)dest->dtb_tomax + offs;
dlimit = daddr + src->dtb_offset;
saddr = (uintptr_t)src->dtb_tomax;
/*
* First, the aligned portion.
*/
while (dlimit - daddr >= sizeof (uint64_t)) {
*((uint64_t *)daddr) = *((uint64_t *)saddr);
daddr += sizeof (uint64_t);
saddr += sizeof (uint64_t);
}
/*
* Now any left-over bit...
*/
while (dlimit - daddr)
*((uint8_t *)daddr++) = *((uint8_t *)saddr++);
/*
* Finally, commit the reserved space in the destination buffer.
*/
dest->dtb_offset = offs + src->dtb_offset;
out:
/*
* If we're lucky enough to be the only active CPU on this speculation
* buffer, we can just set the state back to DTRACESPEC_INACTIVE.
*/
if (current == DTRACESPEC_ACTIVE ||
(current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {
uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state,
DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE);
ASSERT(rval == DTRACESPEC_COMMITTING);
}
src->dtb_offset = 0;
src->dtb_xamot_drops += src->dtb_drops;
src->dtb_drops = 0;
}
/*
* This routine discards an active speculation. If the specified speculation
* is not in a valid state to perform a discard(), this routine will silently
* do nothing. The state of the specified speculation is transitioned
* according to the state transition diagram outlined in <sys/dtrace_impl.h>
*/
static void
dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
dtrace_specid_t which)
{
dtrace_speculation_t *spec;
dtrace_speculation_state_t current, new = 0;
dtrace_buffer_t *buf;
if (which == 0)
return;
if (which > state->dts_nspeculations) {
cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
return;
}
spec = &state->dts_speculations[which - 1];
buf = &spec->dtsp_buffer[cpu];
do {
current = spec->dtsp_state;
switch (current) {
case DTRACESPEC_INACTIVE:
case DTRACESPEC_COMMITTINGMANY:
case DTRACESPEC_COMMITTING:
case DTRACESPEC_DISCARDING:
return;
case DTRACESPEC_ACTIVE:
case DTRACESPEC_ACTIVEMANY:
new = DTRACESPEC_DISCARDING;
break;
case DTRACESPEC_ACTIVEONE:
if (buf->dtb_offset != 0) {
new = DTRACESPEC_INACTIVE;
} else {
new = DTRACESPEC_DISCARDING;
}
break;
default:
ASSERT(0);
}
} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
current, new) != current);
buf->dtb_offset = 0;
buf->dtb_drops = 0;
}
/*
* Note: not called from probe context. This function is called
* asynchronously from cross call context to clean any speculations that are
* in the COMMITTINGMANY or DISCARDING states. These speculations may not be
* transitioned back to the INACTIVE state until all CPUs have cleaned the
* speculation.
*/
static void
dtrace_speculation_clean_here(dtrace_state_t *state)
{
dtrace_icookie_t cookie;
processorid_t cpu = curcpu;
dtrace_buffer_t *dest = &state->dts_buffer[cpu];
dtrace_specid_t i;
cookie = dtrace_interrupt_disable();
if (dest->dtb_tomax == NULL) {
dtrace_interrupt_enable(cookie);
return;
}
for (i = 0; i < state->dts_nspeculations; i++) {
dtrace_speculation_t *spec = &state->dts_speculations[i];
dtrace_buffer_t *src = &spec->dtsp_buffer[cpu];
if (src->dtb_tomax == NULL)
continue;
if (spec->dtsp_state == DTRACESPEC_DISCARDING) {
src->dtb_offset = 0;
continue;
}
if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
continue;
if (src->dtb_offset == 0)
continue;
dtrace_speculation_commit(state, cpu, i + 1);
}
dtrace_interrupt_enable(cookie);
}
/*
* Note: not called from probe context. This function is called
* asynchronously (and at a regular interval) to clean any speculations that
* are in the COMMITTINGMANY or DISCARDING states. If it discovers that there
* is work to be done, it cross calls all CPUs to perform that work;
* COMMITMANY and DISCARDING speculations may not be transitioned back to the
* INACTIVE state until they have been cleaned by all CPUs.
*/
static void
dtrace_speculation_clean(dtrace_state_t *state)
{
int work = 0, rv;
dtrace_specid_t i;
for (i = 0; i < state->dts_nspeculations; i++) {
dtrace_speculation_t *spec = &state->dts_speculations[i];
ASSERT(!spec->dtsp_cleaning);
if (spec->dtsp_state != DTRACESPEC_DISCARDING &&
spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
continue;
work++;
spec->dtsp_cleaning = 1;
}
if (!work)
return;
dtrace_xcall(DTRACE_CPUALL,
(dtrace_xcall_t)dtrace_speculation_clean_here, state);
/*
* We now know that all CPUs have committed or discarded their
* speculation buffers, as appropriate. We can now set the state
* to inactive.
*/
for (i = 0; i < state->dts_nspeculations; i++) {
dtrace_speculation_t *spec = &state->dts_speculations[i];
dtrace_speculation_state_t current, new;
if (!spec->dtsp_cleaning)
continue;
current = spec->dtsp_state;
ASSERT(current == DTRACESPEC_DISCARDING ||
current == DTRACESPEC_COMMITTINGMANY);
new = DTRACESPEC_INACTIVE;
rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new);
ASSERT(rv == current);
spec->dtsp_cleaning = 0;
}
}
/*
* Called as part of a speculate() to get the speculative buffer associated
* with a given speculation. Returns NULL if the specified speculation is not
* in an ACTIVE state. If the speculation is in the ACTIVEONE state -- and
* the active CPU is not the specified CPU -- the speculation will be
* atomically transitioned into the ACTIVEMANY state.
*/
static dtrace_buffer_t *
dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,
dtrace_specid_t which)
{
dtrace_speculation_t *spec;
dtrace_speculation_state_t current, new = 0;
dtrace_buffer_t *buf;
if (which == 0)
return (NULL);
if (which > state->dts_nspeculations) {
cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
return (NULL);
}
spec = &state->dts_speculations[which - 1];
buf = &spec->dtsp_buffer[cpuid];
do {
current = spec->dtsp_state;
switch (current) {
case DTRACESPEC_INACTIVE:
case DTRACESPEC_COMMITTINGMANY:
case DTRACESPEC_DISCARDING:
return (NULL);
case DTRACESPEC_COMMITTING:
ASSERT(buf->dtb_offset == 0);
return (NULL);
case DTRACESPEC_ACTIVEONE:
/*
* This speculation is currently active on one CPU.
* Check the offset in the buffer; if it's non-zero,
* that CPU must be us (and we leave the state alone).
* If it's zero, assume that we're starting on a new
* CPU -- and change the state to indicate that the
* speculation is active on more than one CPU.
*/
if (buf->dtb_offset != 0)
return (buf);
new = DTRACESPEC_ACTIVEMANY;
break;
case DTRACESPEC_ACTIVEMANY:
return (buf);
case DTRACESPEC_ACTIVE:
new = DTRACESPEC_ACTIVEONE;
break;
default:
ASSERT(0);
}
} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
current, new) != current);
ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY);
return (buf);
}
/*
* Return a string. In the event that the user lacks the privilege to access
* arbitrary kernel memory, we copy the string out to scratch memory so that we
* don't fail access checking.
*
* dtrace_dif_variable() uses this routine as a helper for various
* builtin values such as 'execname' and 'probefunc.'
*/
uintptr_t
dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state,
dtrace_mstate_t *mstate)
{
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
uintptr_t ret;
size_t strsz;
/*
* The easy case: this probe is allowed to read all of memory, so
* we can just return this as a vanilla pointer.
*/
if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
return (addr);
/*
* This is the tougher case: we copy the string in question from
* kernel memory into scratch memory and return it that way: this
* ensures that we won't trip up when access checking tests the
* BYREF return value.
*/
strsz = dtrace_strlen((char *)addr, size) + 1;
if (mstate->dtms_scratch_ptr + strsz >
mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
return (0);
}
dtrace_strcpy((const void *)addr, (void *)mstate->dtms_scratch_ptr,
strsz);
ret = mstate->dtms_scratch_ptr;
mstate->dtms_scratch_ptr += strsz;
return (ret);
}
/*
* Return a string from a memoy address which is known to have one or
* more concatenated, individually zero terminated, sub-strings.
* In the event that the user lacks the privilege to access
* arbitrary kernel memory, we copy the string out to scratch memory so that we
* don't fail access checking.
*
* dtrace_dif_variable() uses this routine as a helper for various
* builtin values such as 'execargs'.
*/
static uintptr_t
dtrace_dif_varstrz(uintptr_t addr, size_t strsz, dtrace_state_t *state,
dtrace_mstate_t *mstate)
{
char *p;
size_t i;
uintptr_t ret;
if (mstate->dtms_scratch_ptr + strsz >
mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
return (0);
}
dtrace_bcopy((const void *)addr, (void *)mstate->dtms_scratch_ptr,
strsz);
/* Replace sub-string termination characters with a space. */
for (p = (char *) mstate->dtms_scratch_ptr, i = 0; i < strsz - 1;
p++, i++)
if (*p == '\0')
*p = ' ';
ret = mstate->dtms_scratch_ptr;
mstate->dtms_scratch_ptr += strsz;
return (ret);
}
/*
* This function implements the DIF emulator's variable lookups. The emulator
* passes a reserved variable identifier and optional built-in array index.
*/
static uint64_t
dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v,
uint64_t ndx)
{
/*
* If we're accessing one of the uncached arguments, we'll turn this
* into a reference in the args array.
*/
if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) {
ndx = v - DIF_VAR_ARG0;
v = DIF_VAR_ARGS;
}
switch (v) {
case DIF_VAR_ARGS:
ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS);
if (ndx >= sizeof (mstate->dtms_arg) /
sizeof (mstate->dtms_arg[0])) {
int aframes = mstate->dtms_probe->dtpr_aframes + 2;
dtrace_provider_t *pv;
uint64_t val;
pv = mstate->dtms_probe->dtpr_provider;
if (pv->dtpv_pops.dtps_getargval != NULL)
val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg,
mstate->dtms_probe->dtpr_id,
mstate->dtms_probe->dtpr_arg, ndx, aframes);
else
val = dtrace_getarg(ndx, aframes);
/*
* This is regrettably required to keep the compiler
* from tail-optimizing the call to dtrace_getarg().
* The condition always evaluates to true, but the
* compiler has no way of figuring that out a priori.
* (None of this would be necessary if the compiler
* could be relied upon to _always_ tail-optimize
* the call to dtrace_getarg() -- but it can't.)
*/
if (mstate->dtms_probe != NULL)
return (val);
ASSERT(0);
}
return (mstate->dtms_arg[ndx]);
#ifdef illumos
case DIF_VAR_UREGS: {
klwp_t *lwp;
if (!dtrace_priv_proc(state))
return (0);
if ((lwp = curthread->t_lwp) == NULL) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
cpu_core[curcpu].cpuc_dtrace_illval = NULL;
return (0);
}
return (dtrace_getreg(lwp->lwp_regs, ndx));
return (0);
}
#else
case DIF_VAR_UREGS: {
struct trapframe *tframe;
if (!dtrace_priv_proc(state))
return (0);
if ((tframe = curthread->td_frame) == NULL) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
cpu_core[curcpu].cpuc_dtrace_illval = 0;
return (0);
}
return (dtrace_getreg(tframe, ndx));
}
#endif
case DIF_VAR_CURTHREAD:
if (!dtrace_priv_proc(state))
return (0);
return ((uint64_t)(uintptr_t)curthread);
case DIF_VAR_TIMESTAMP:
if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
mstate->dtms_timestamp = dtrace_gethrtime();
mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP;
}
return (mstate->dtms_timestamp);
case DIF_VAR_VTIMESTAMP:
ASSERT(dtrace_vtime_references != 0);
return (curthread->t_dtrace_vtime);
case DIF_VAR_WALLTIMESTAMP:
if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) {
mstate->dtms_walltimestamp = dtrace_gethrestime();
mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP;
}
return (mstate->dtms_walltimestamp);
#ifdef illumos
case DIF_VAR_IPL:
if (!dtrace_priv_kernel(state))
return (0);
if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) {
mstate->dtms_ipl = dtrace_getipl();
mstate->dtms_present |= DTRACE_MSTATE_IPL;
}
return (mstate->dtms_ipl);
#endif
case DIF_VAR_EPID:
ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID);
return (mstate->dtms_epid);
case DIF_VAR_ID:
ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
return (mstate->dtms_probe->dtpr_id);
case DIF_VAR_STACKDEPTH:
if (!dtrace_priv_kernel(state))
return (0);
if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) {
int aframes = mstate->dtms_probe->dtpr_aframes + 2;
mstate->dtms_stackdepth = dtrace_getstackdepth(aframes);
mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH;
}
return (mstate->dtms_stackdepth);
case DIF_VAR_USTACKDEPTH:
if (!dtrace_priv_proc(state))
return (0);
if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) {
/*
* See comment in DIF_VAR_PID.
*/
if (DTRACE_ANCHORED(mstate->dtms_probe) &&
CPU_ON_INTR(CPU)) {
mstate->dtms_ustackdepth = 0;
} else {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
mstate->dtms_ustackdepth =
dtrace_getustackdepth();
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
}
mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH;
}
return (mstate->dtms_ustackdepth);
case DIF_VAR_CALLER:
if (!dtrace_priv_kernel(state))
return (0);
if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) {
int aframes = mstate->dtms_probe->dtpr_aframes + 2;
if (!DTRACE_ANCHORED(mstate->dtms_probe)) {
/*
* If this is an unanchored probe, we are
* required to go through the slow path:
* dtrace_caller() only guarantees correct
* results for anchored probes.
*/
pc_t caller[2] = {0, 0};
dtrace_getpcstack(caller, 2, aframes,
(uint32_t *)(uintptr_t)mstate->dtms_arg[0]);
mstate->dtms_caller = caller[1];
} else if ((mstate->dtms_caller =
dtrace_caller(aframes)) == -1) {
/*
* We have failed to do this the quick way;
* we must resort to the slower approach of
* calling dtrace_getpcstack().
*/
pc_t caller = 0;
dtrace_getpcstack(&caller, 1, aframes, NULL);
mstate->dtms_caller = caller;
}
mstate->dtms_present |= DTRACE_MSTATE_CALLER;
}
return (mstate->dtms_caller);
case DIF_VAR_UCALLER:
if (!dtrace_priv_proc(state))
return (0);
if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) {
uint64_t ustack[3];
/*
* dtrace_getupcstack() fills in the first uint64_t
* with the current PID. The second uint64_t will
* be the program counter at user-level. The third
* uint64_t will contain the caller, which is what
* we're after.
*/
ustack[2] = 0;
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
dtrace_getupcstack(ustack, 3);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
mstate->dtms_ucaller = ustack[2];
mstate->dtms_present |= DTRACE_MSTATE_UCALLER;
}
return (mstate->dtms_ucaller);
case DIF_VAR_PROBEPROV:
ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
return (dtrace_dif_varstr(
(uintptr_t)mstate->dtms_probe->dtpr_provider->dtpv_name,
state, mstate));
case DIF_VAR_PROBEMOD:
ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
return (dtrace_dif_varstr(
(uintptr_t)mstate->dtms_probe->dtpr_mod,
state, mstate));
case DIF_VAR_PROBEFUNC:
ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
return (dtrace_dif_varstr(
(uintptr_t)mstate->dtms_probe->dtpr_func,
state, mstate));
case DIF_VAR_PROBENAME:
ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
return (dtrace_dif_varstr(
(uintptr_t)mstate->dtms_probe->dtpr_name,
state, mstate));
case DIF_VAR_PID:
if (!dtrace_priv_proc(state))
return (0);
#ifdef illumos
/*
* Note that we are assuming that an unanchored probe is
* always due to a high-level interrupt. (And we're assuming
* that there is only a single high level interrupt.)
*/
if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
return (pid0.pid_id);
/*
* It is always safe to dereference one's own t_procp pointer:
* it always points to a valid, allocated proc structure.
* Further, it is always safe to dereference the p_pidp member
* of one's own proc structure. (These are truisms becuase
* threads and processes don't clean up their own state --
* they leave that task to whomever reaps them.)
*/
return ((uint64_t)curthread->t_procp->p_pidp->pid_id);
#else
return ((uint64_t)curproc->p_pid);
#endif
case DIF_VAR_PPID:
if (!dtrace_priv_proc(state))
return (0);
#ifdef illumos
/*
* See comment in DIF_VAR_PID.
*/
if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
return (pid0.pid_id);
/*
* It is always safe to dereference one's own t_procp pointer:
* it always points to a valid, allocated proc structure.
* (This is true because threads don't clean up their own
* state -- they leave that task to whomever reaps them.)
*/
return ((uint64_t)curthread->t_procp->p_ppid);
#else
if (curproc->p_pid == proc0.p_pid)
return (curproc->p_pid);
else
return (curproc->p_pptr->p_pid);
#endif
case DIF_VAR_TID:
#ifdef illumos
/*
* See comment in DIF_VAR_PID.
*/
if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
return (0);
#endif
return ((uint64_t)curthread->t_tid);
case DIF_VAR_EXECARGS: {
struct pargs *p_args = curthread->td_proc->p_args;
if (p_args == NULL)
return(0);
return (dtrace_dif_varstrz(
(uintptr_t) p_args->ar_args, p_args->ar_length, state, mstate));
}
case DIF_VAR_EXECNAME:
#ifdef illumos
if (!dtrace_priv_proc(state))
return (0);
/*
* See comment in DIF_VAR_PID.
*/
if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
return ((uint64_t)(uintptr_t)p0.p_user.u_comm);
/*
* It is always safe to dereference one's own t_procp pointer:
* it always points to a valid, allocated proc structure.
* (This is true because threads don't clean up their own
* state -- they leave that task to whomever reaps them.)
*/
return (dtrace_dif_varstr(
(uintptr_t)curthread->t_procp->p_user.u_comm,
state, mstate));
#else
return (dtrace_dif_varstr(
(uintptr_t) curthread->td_proc->p_comm, state, mstate));
#endif
case DIF_VAR_ZONENAME:
#ifdef illumos
if (!dtrace_priv_proc(state))
return (0);
/*
* See comment in DIF_VAR_PID.
*/
if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
return ((uint64_t)(uintptr_t)p0.p_zone->zone_name);
/*
* It is always safe to dereference one's own t_procp pointer:
* it always points to a valid, allocated proc structure.
* (This is true because threads don't clean up their own
* state -- they leave that task to whomever reaps them.)
*/
return (dtrace_dif_varstr(
(uintptr_t)curthread->t_procp->p_zone->zone_name,
state, mstate));
#else
return (0);
#endif
case DIF_VAR_UID:
if (!dtrace_priv_proc(state))
return (0);
#ifdef illumos
/*
* See comment in DIF_VAR_PID.
*/
if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
return ((uint64_t)p0.p_cred->cr_uid);
#endif
/*
* It is always safe to dereference one's own t_procp pointer:
* it always points to a valid, allocated proc structure.
* (This is true because threads don't clean up their own
* state -- they leave that task to whomever reaps them.)
*
* Additionally, it is safe to dereference one's own process
* credential, since this is never NULL after process birth.
*/
return ((uint64_t)curthread->t_procp->p_cred->cr_uid);
case DIF_VAR_GID:
if (!dtrace_priv_proc(state))
return (0);
#ifdef illumos
/*
* See comment in DIF_VAR_PID.
*/
if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
return ((uint64_t)p0.p_cred->cr_gid);
#endif
/*
* It is always safe to dereference one's own t_procp pointer:
* it always points to a valid, allocated proc structure.
* (This is true because threads don't clean up their own
* state -- they leave that task to whomever reaps them.)
*
* Additionally, it is safe to dereference one's own process
* credential, since this is never NULL after process birth.
*/
return ((uint64_t)curthread->t_procp->p_cred->cr_gid);
case DIF_VAR_ERRNO: {
#ifdef illumos
klwp_t *lwp;
if (!dtrace_priv_proc(state))
return (0);
/*
* See comment in DIF_VAR_PID.
*/
if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
return (0);
/*
* It is always safe to dereference one's own t_lwp pointer in
* the event that this pointer is non-NULL. (This is true
* because threads and lwps don't clean up their own state --
* they leave that task to whomever reaps them.)
*/
if ((lwp = curthread->t_lwp) == NULL)
return (0);
return ((uint64_t)lwp->lwp_errno);
#else
return (curthread->td_errno);
#endif
}
#ifndef illumos
case DIF_VAR_CPU: {
return curcpu;
}
#endif
default:
DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
return (0);
}
}
typedef enum dtrace_json_state {
DTRACE_JSON_REST = 1,
DTRACE_JSON_OBJECT,
DTRACE_JSON_STRING,
DTRACE_JSON_STRING_ESCAPE,
DTRACE_JSON_STRING_ESCAPE_UNICODE,
DTRACE_JSON_COLON,
DTRACE_JSON_COMMA,
DTRACE_JSON_VALUE,
DTRACE_JSON_IDENTIFIER,
DTRACE_JSON_NUMBER,
DTRACE_JSON_NUMBER_FRAC,
DTRACE_JSON_NUMBER_EXP,
DTRACE_JSON_COLLECT_OBJECT
} dtrace_json_state_t;
/*
* This function possesses just enough knowledge about JSON to extract a single
* value from a JSON string and store it in the scratch buffer. It is able
* to extract nested object values, and members of arrays by index.
*
* elemlist is a list of JSON keys, stored as packed NUL-terminated strings, to
* be looked up as we descend into the object tree. e.g.
*
* foo[0].bar.baz[32] --> "foo" NUL "0" NUL "bar" NUL "baz" NUL "32" NUL
* with nelems = 5.
*
* The run time of this function must be bounded above by strsize to limit the
* amount of work done in probe context. As such, it is implemented as a
* simple state machine, reading one character at a time using safe loads
* until we find the requested element, hit a parsing error or run off the
* end of the object or string.
*
* As there is no way for a subroutine to return an error without interrupting
* clause execution, we simply return NULL in the event of a missing key or any
* other error condition. Each NULL return in this function is commented with
* the error condition it represents -- parsing or otherwise.
*
* The set of states for the state machine closely matches the JSON
* specification (http://json.org/). Briefly:
*
* DTRACE_JSON_REST:
* Skip whitespace until we find either a top-level Object, moving
* to DTRACE_JSON_OBJECT; or an Array, moving to DTRACE_JSON_VALUE.
*
* DTRACE_JSON_OBJECT:
* Locate the next key String in an Object. Sets a flag to denote
* the next String as a key string and moves to DTRACE_JSON_STRING.
*
* DTRACE_JSON_COLON:
* Skip whitespace until we find the colon that separates key Strings
* from their values. Once found, move to DTRACE_JSON_VALUE.
*
* DTRACE_JSON_VALUE:
* Detects the type of the next value (String, Number, Identifier, Object
* or Array) and routes to the states that process that type. Here we also
* deal with the element selector list if we are requested to traverse down
* into the object tree.
*
* DTRACE_JSON_COMMA:
* Skip whitespace until we find the comma that separates key-value pairs
* in Objects (returning to DTRACE_JSON_OBJECT) or values in Arrays
* (similarly DTRACE_JSON_VALUE). All following literal value processing
* states return to this state at the end of their value, unless otherwise
* noted.
*
* DTRACE_JSON_NUMBER, DTRACE_JSON_NUMBER_FRAC, DTRACE_JSON_NUMBER_EXP:
* Processes a Number literal from the JSON, including any exponent
* component that may be present. Numbers are returned as strings, which
* may be passed to strtoll() if an integer is required.
*
* DTRACE_JSON_IDENTIFIER:
* Processes a "true", "false" or "null" literal in the JSON.
*
* DTRACE_JSON_STRING, DTRACE_JSON_STRING_ESCAPE,
* DTRACE_JSON_STRING_ESCAPE_UNICODE:
* Processes a String literal from the JSON, whether the String denotes
* a key, a value or part of a larger Object. Handles all escape sequences
* present in the specification, including four-digit unicode characters,
* but merely includes the escape sequence without converting it to the
* actual escaped character. If the String is flagged as a key, we
* move to DTRACE_JSON_COLON rather than DTRACE_JSON_COMMA.
*
* DTRACE_JSON_COLLECT_OBJECT:
* This state collects an entire Object (or Array), correctly handling
* embedded strings. If the full element selector list matches this nested
* object, we return the Object in full as a string. If not, we use this
* state to skip to the next value at this level and continue processing.
*
* NOTE: This function uses various macros from strtolctype.h to manipulate
* digit values, etc -- these have all been checked to ensure they make
* no additional function calls.
*/
static char *
dtrace_json(uint64_t size, uintptr_t json, char *elemlist, int nelems,
char *dest)
{
dtrace_json_state_t state = DTRACE_JSON_REST;
int64_t array_elem = INT64_MIN;
int64_t array_pos = 0;
uint8_t escape_unicount = 0;
boolean_t string_is_key = B_FALSE;
boolean_t collect_object = B_FALSE;
boolean_t found_key = B_FALSE;
boolean_t in_array = B_FALSE;
uint32_t braces = 0, brackets = 0;
char *elem = elemlist;
char *dd = dest;
uintptr_t cur;
for (cur = json; cur < json + size; cur++) {
char cc = dtrace_load8(cur);
if (cc == '\0')
return (NULL);
switch (state) {
case DTRACE_JSON_REST:
if (isspace(cc))
break;
if (cc == '{') {
state = DTRACE_JSON_OBJECT;
break;
}
if (cc == '[') {
in_array = B_TRUE;
array_pos = 0;
array_elem = dtrace_strtoll(elem, 10, size);
found_key = array_elem == 0 ? B_TRUE : B_FALSE;
state = DTRACE_JSON_VALUE;
break;
}
/*
* ERROR: expected to find a top-level object or array.
*/
return (NULL);
case DTRACE_JSON_OBJECT:
if (isspace(cc))
break;
if (cc == '"') {
state = DTRACE_JSON_STRING;
string_is_key = B_TRUE;
break;
}
/*
* ERROR: either the object did not start with a key
* string, or we've run off the end of the object
* without finding the requested key.
*/
return (NULL);
case DTRACE_JSON_STRING:
if (cc == '\\') {
*dd++ = '\\';
state = DTRACE_JSON_STRING_ESCAPE;
break;
}
if (cc == '"') {
if (collect_object) {
/*
* We don't reset the dest here, as
* the string is part of a larger
* object being collected.
*/
*dd++ = cc;
collect_object = B_FALSE;
state = DTRACE_JSON_COLLECT_OBJECT;
break;
}
*dd = '\0';
dd = dest; /* reset string buffer */
if (string_is_key) {
if (dtrace_strncmp(dest, elem,
size) == 0)
found_key = B_TRUE;
} else if (found_key) {
if (nelems > 1) {
/*
* We expected an object, not
* this string.
*/
return (NULL);
}
return (dest);
}
state = string_is_key ? DTRACE_JSON_COLON :
DTRACE_JSON_COMMA;
string_is_key = B_FALSE;
break;
}
*dd++ = cc;
break;
case DTRACE_JSON_STRING_ESCAPE:
*dd++ = cc;
if (cc == 'u') {
escape_unicount = 0;
state = DTRACE_JSON_STRING_ESCAPE_UNICODE;
} else {
state = DTRACE_JSON_STRING;
}
break;
case DTRACE_JSON_STRING_ESCAPE_UNICODE:
if (!isxdigit(cc)) {
/*
* ERROR: invalid unicode escape, expected
* four valid hexidecimal digits.
*/
return (NULL);
}
*dd++ = cc;
if (++escape_unicount == 4)
state = DTRACE_JSON_STRING;
break;
case DTRACE_JSON_COLON:
if (isspace(cc))
break;
if (cc == ':') {
state = DTRACE_JSON_VALUE;
break;
}
/*
* ERROR: expected a colon.
*/
return (NULL);
case DTRACE_JSON_COMMA:
if (isspace(cc))
break;
if (cc == ',') {
if (in_array) {
state = DTRACE_JSON_VALUE;
if (++array_pos == array_elem)
found_key = B_TRUE;
} else {
state = DTRACE_JSON_OBJECT;
}
break;
}
/*
* ERROR: either we hit an unexpected character, or
* we reached the end of the object or array without
* finding the requested key.
*/
return (NULL);
case DTRACE_JSON_IDENTIFIER:
if (islower(cc)) {
*dd++ = cc;
break;
}
*dd = '\0';
dd = dest; /* reset string buffer */
if (dtrace_strncmp(dest, "true", 5) == 0 ||
dtrace_strncmp(dest, "false", 6) == 0 ||
dtrace_strncmp(dest, "null", 5) == 0) {
if (found_key) {
if (nelems > 1) {
/*
* ERROR: We expected an object,
* not this identifier.
*/
return (NULL);
}
return (dest);
} else {
cur--;
state = DTRACE_JSON_COMMA;
break;
}
}
/*
* ERROR: we did not recognise the identifier as one
* of those in the JSON specification.
*/
return (NULL);
case DTRACE_JSON_NUMBER:
if (cc == '.') {
*dd++ = cc;
state = DTRACE_JSON_NUMBER_FRAC;
break;
}
if (cc == 'x' || cc == 'X') {
/*
* ERROR: specification explicitly excludes
* hexidecimal or octal numbers.
*/
return (NULL);
}
/* FALLTHRU */
case DTRACE_JSON_NUMBER_FRAC:
if (cc == 'e' || cc == 'E') {
*dd++ = cc;
state = DTRACE_JSON_NUMBER_EXP;
break;
}
if (cc == '+' || cc == '-') {
/*
* ERROR: expect sign as part of exponent only.
*/
return (NULL);
}
/* FALLTHRU */
case DTRACE_JSON_NUMBER_EXP:
if (isdigit(cc) || cc == '+' || cc == '-') {
*dd++ = cc;
break;
}
*dd = '\0';
dd = dest; /* reset string buffer */
if (found_key) {
if (nelems > 1) {
/*
* ERROR: We expected an object, not
* this number.
*/
return (NULL);
}
return (dest);
}
cur--;
state = DTRACE_JSON_COMMA;
break;
case DTRACE_JSON_VALUE:
if (isspace(cc))
break;
if (cc == '{' || cc == '[') {
if (nelems > 1 && found_key) {
in_array = cc == '[' ? B_TRUE : B_FALSE;
/*
* If our element selector directs us
* to descend into this nested object,
* then move to the next selector
* element in the list and restart the
* state machine.
*/
while (*elem != '\0')
elem++;
elem++; /* skip the inter-element NUL */
nelems--;
dd = dest;
if (in_array) {
state = DTRACE_JSON_VALUE;
array_pos = 0;
array_elem = dtrace_strtoll(
elem, 10, size);
found_key = array_elem == 0 ?
B_TRUE : B_FALSE;
} else {
found_key = B_FALSE;
state = DTRACE_JSON_OBJECT;
}
break;
}
/*
* Otherwise, we wish to either skip this
* nested object or return it in full.
*/
if (cc == '[')
brackets = 1;
else
braces = 1;
*dd++ = cc;
state = DTRACE_JSON_COLLECT_OBJECT;
break;
}
if (cc == '"') {
state = DTRACE_JSON_STRING;
break;
}
if (islower(cc)) {
/*
* Here we deal with true, false and null.
*/
*dd++ = cc;
state = DTRACE_JSON_IDENTIFIER;
break;
}
if (cc == '-' || isdigit(cc)) {
*dd++ = cc;
state = DTRACE_JSON_NUMBER;
break;
}
/*
* ERROR: unexpected character at start of value.
*/
return (NULL);
case DTRACE_JSON_COLLECT_OBJECT:
if (cc == '\0')
/*
* ERROR: unexpected end of input.
*/
return (NULL);
*dd++ = cc;
if (cc == '"') {
collect_object = B_TRUE;
state = DTRACE_JSON_STRING;
break;
}
if (cc == ']') {
if (brackets-- == 0) {
/*
* ERROR: unbalanced brackets.
*/
return (NULL);
}
} else if (cc == '}') {
if (braces-- == 0) {
/*
* ERROR: unbalanced braces.
*/
return (NULL);
}
} else if (cc == '{') {
braces++;
} else if (cc == '[') {
brackets++;
}
if (brackets == 0 && braces == 0) {
if (found_key) {
*dd = '\0';
return (dest);
}
dd = dest; /* reset string buffer */
state = DTRACE_JSON_COMMA;
}
break;
}
}
return (NULL);
}
/*
* Emulate the execution of DTrace ID subroutines invoked by the call opcode.
* Notice that we don't bother validating the proper number of arguments or
* their types in the tuple stack. This isn't needed because all argument
* interpretation is safe because of our load safety -- the worst that can
* happen is that a bogus program can obtain bogus results.
*/
static void
dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
dtrace_key_t *tupregs, int nargs,
dtrace_mstate_t *mstate, dtrace_state_t *state)
{
volatile uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags;
volatile uintptr_t *illval = &cpu_core[curcpu].cpuc_dtrace_illval;
dtrace_vstate_t *vstate = &state->dts_vstate;
#ifdef illumos
union {
mutex_impl_t mi;
uint64_t mx;
} m;
union {
krwlock_t ri;
uintptr_t rw;
} r;
#else
struct thread *lowner;
union {
struct lock_object *li;
uintptr_t lx;
} l;
#endif
switch (subr) {
case DIF_SUBR_RAND:
regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875;
break;
#ifdef illumos
case DIF_SUBR_MUTEX_OWNED:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
m.mx = dtrace_load64(tupregs[0].dttk_value);
if (MUTEX_TYPE_ADAPTIVE(&m.mi))
regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER;
else
regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock);
break;
case DIF_SUBR_MUTEX_OWNER:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
m.mx = dtrace_load64(tupregs[0].dttk_value);
if (MUTEX_TYPE_ADAPTIVE(&m.mi) &&
MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER)
regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi);
else
regs[rd] = 0;
break;
case DIF_SUBR_MUTEX_TYPE_ADAPTIVE:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
m.mx = dtrace_load64(tupregs[0].dttk_value);
regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi);
break;
case DIF_SUBR_MUTEX_TYPE_SPIN:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
m.mx = dtrace_load64(tupregs[0].dttk_value);
regs[rd] = MUTEX_TYPE_SPIN(&m.mi);
break;
case DIF_SUBR_RW_READ_HELD: {
uintptr_t tmp;
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
r.rw = dtrace_loadptr(tupregs[0].dttk_value);
regs[rd] = _RW_READ_HELD(&r.ri, tmp);
break;
}
case DIF_SUBR_RW_WRITE_HELD:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
r.rw = dtrace_loadptr(tupregs[0].dttk_value);
regs[rd] = _RW_WRITE_HELD(&r.ri);
break;
case DIF_SUBR_RW_ISWRITER:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
r.rw = dtrace_loadptr(tupregs[0].dttk_value);
regs[rd] = _RW_ISWRITER(&r.ri);
break;
#else /* !illumos */
case DIF_SUBR_MUTEX_OWNED:
if (!dtrace_canload(tupregs[0].dttk_value,
sizeof (struct lock_object), mstate, vstate)) {
regs[rd] = 0;
break;
}
l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value);
regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner);
break;
case DIF_SUBR_MUTEX_OWNER:
if (!dtrace_canload(tupregs[0].dttk_value,
sizeof (struct lock_object), mstate, vstate)) {
regs[rd] = 0;
break;
}
l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value);
LOCK_CLASS(l.li)->lc_owner(l.li, &lowner);
regs[rd] = (uintptr_t)lowner;
break;
case DIF_SUBR_MUTEX_TYPE_ADAPTIVE:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (struct mtx),
mstate, vstate)) {
regs[rd] = 0;
break;
}
l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value);
/* XXX - should be only LC_SLEEPABLE? */
regs[rd] = (LOCK_CLASS(l.li)->lc_flags &
(LC_SLEEPLOCK | LC_SLEEPABLE)) != 0;
break;
case DIF_SUBR_MUTEX_TYPE_SPIN:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (struct mtx),
mstate, vstate)) {
regs[rd] = 0;
break;
}
l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value);
regs[rd] = (LOCK_CLASS(l.li)->lc_flags & LC_SPINLOCK) != 0;
break;
case DIF_SUBR_RW_READ_HELD:
case DIF_SUBR_SX_SHARED_HELD:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value);
regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner) &&
lowner == NULL;
break;
case DIF_SUBR_RW_WRITE_HELD:
case DIF_SUBR_SX_EXCLUSIVE_HELD:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
l.lx = dtrace_loadptr(tupregs[0].dttk_value);
LOCK_CLASS(l.li)->lc_owner(l.li, &lowner);
regs[rd] = (lowner == curthread);
break;
case DIF_SUBR_RW_ISWRITER:
case DIF_SUBR_SX_ISEXCLUSIVE:
if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t),
mstate, vstate)) {
regs[rd] = 0;
break;
}
l.lx = dtrace_loadptr(tupregs[0].dttk_value);
regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner) &&
lowner != NULL;
break;
#endif /* illumos */
case DIF_SUBR_BCOPY: {
/*
* We need to be sure that the destination is in the scratch
* region -- no other region is allowed.
*/
uintptr_t src = tupregs[0].dttk_value;
uintptr_t dest = tupregs[1].dttk_value;
size_t size = tupregs[2].dttk_value;
if (!dtrace_inscratch(dest, size, mstate)) {
*flags |= CPU_DTRACE_BADADDR;
*illval = regs[rd];
break;
}
if (!dtrace_canload(src, size, mstate, vstate)) {
regs[rd] = 0;
break;
}
dtrace_bcopy((void *)src, (void *)dest, size);
break;
}
case DIF_SUBR_ALLOCA:
case DIF_SUBR_COPYIN: {
uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
uint64_t size =
tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value;
size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size;
/*
* This action doesn't require any credential checks since
* probes will not activate in user contexts to which the
* enabling user does not have permissions.
*/
/*
* Rounding up the user allocation size could have overflowed
* a large, bogus allocation (like -1ULL) to 0.
*/
if (scratch_size < size ||
!DTRACE_INSCRATCH(mstate, scratch_size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
if (subr == DIF_SUBR_COPYIN) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
}
mstate->dtms_scratch_ptr += scratch_size;
regs[rd] = dest;
break;
}
case DIF_SUBR_COPYINTO: {
uint64_t size = tupregs[1].dttk_value;
uintptr_t dest = tupregs[2].dttk_value;
/*
* This action doesn't require any credential checks since
* probes will not activate in user contexts to which the
* enabling user does not have permissions.
*/
if (!dtrace_inscratch(dest, size, mstate)) {
*flags |= CPU_DTRACE_BADADDR;
*illval = regs[rd];
break;
}
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
break;
}
case DIF_SUBR_COPYINSTR: {
uintptr_t dest = mstate->dtms_scratch_ptr;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
if (nargs > 1 && tupregs[1].dttk_value < size)
size = tupregs[1].dttk_value + 1;
/*
* This action doesn't require any credential checks since
* probes will not activate in user contexts to which the
* enabling user does not have permissions.
*/
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
dtrace_copyinstr(tupregs[0].dttk_value, dest, size, flags);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
((char *)dest)[size - 1] = '\0';
mstate->dtms_scratch_ptr += size;
regs[rd] = dest;
break;
}
#ifdef illumos
case DIF_SUBR_MSGSIZE:
case DIF_SUBR_MSGDSIZE: {
uintptr_t baddr = tupregs[0].dttk_value, daddr;
uintptr_t wptr, rptr;
size_t count = 0;
int cont = 0;
while (baddr != 0 && !(*flags & CPU_DTRACE_FAULT)) {
if (!dtrace_canload(baddr, sizeof (mblk_t), mstate,
vstate)) {
regs[rd] = 0;
break;
}
wptr = dtrace_loadptr(baddr +
offsetof(mblk_t, b_wptr));
rptr = dtrace_loadptr(baddr +
offsetof(mblk_t, b_rptr));
if (wptr < rptr) {
*flags |= CPU_DTRACE_BADADDR;
*illval = tupregs[0].dttk_value;
break;
}
daddr = dtrace_loadptr(baddr +
offsetof(mblk_t, b_datap));
baddr = dtrace_loadptr(baddr +
offsetof(mblk_t, b_cont));
/*
* We want to prevent against denial-of-service here,
* so we're only going to search the list for
* dtrace_msgdsize_max mblks.
*/
if (cont++ > dtrace_msgdsize_max) {
*flags |= CPU_DTRACE_ILLOP;
break;
}
if (subr == DIF_SUBR_MSGDSIZE) {
if (dtrace_load8(daddr +
offsetof(dblk_t, db_type)) != M_DATA)
continue;
}
count += wptr - rptr;
}
if (!(*flags & CPU_DTRACE_FAULT))
regs[rd] = count;
break;
}
#endif
case DIF_SUBR_PROGENYOF: {
pid_t pid = tupregs[0].dttk_value;
proc_t *p;
int rval = 0;
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
for (p = curthread->t_procp; p != NULL; p = p->p_parent) {
#ifdef illumos
if (p->p_pidp->pid_id == pid) {
#else
if (p->p_pid == pid) {
#endif
rval = 1;
break;
}
}
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
regs[rd] = rval;
break;
}
case DIF_SUBR_SPECULATION:
regs[rd] = dtrace_speculation(state);
break;
case DIF_SUBR_COPYOUT: {
uintptr_t kaddr = tupregs[0].dttk_value;
uintptr_t uaddr = tupregs[1].dttk_value;
uint64_t size = tupregs[2].dttk_value;
if (!dtrace_destructive_disallow &&
dtrace_priv_proc_control(state) &&
!dtrace_istoxic(kaddr, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
dtrace_copyout(kaddr, uaddr, size, flags);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
}
break;
}
case DIF_SUBR_COPYOUTSTR: {
uintptr_t kaddr = tupregs[0].dttk_value;
uintptr_t uaddr = tupregs[1].dttk_value;
uint64_t size = tupregs[2].dttk_value;
if (!dtrace_destructive_disallow &&
dtrace_priv_proc_control(state) &&
!dtrace_istoxic(kaddr, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
dtrace_copyoutstr(kaddr, uaddr, size, flags);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
}
break;
}
case DIF_SUBR_STRLEN: {
size_t sz;
uintptr_t addr = (uintptr_t)tupregs[0].dttk_value;
sz = dtrace_strlen((char *)addr,
state->dts_options[DTRACEOPT_STRSIZE]);
if (!dtrace_canload(addr, sz + 1, mstate, vstate)) {
regs[rd] = 0;
break;
}
regs[rd] = sz;
break;
}
case DIF_SUBR_STRCHR:
case DIF_SUBR_STRRCHR: {
/*
* We're going to iterate over the string looking for the
* specified character. We will iterate until we have reached
* the string length or we have found the character. If this
* is DIF_SUBR_STRRCHR, we will look for the last occurrence
* of the specified character instead of the first.
*/
uintptr_t saddr = tupregs[0].dttk_value;
uintptr_t addr = tupregs[0].dttk_value;
uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE];
char c, target = (char)tupregs[1].dttk_value;
for (regs[rd] = 0; addr < limit; addr++) {
if ((c = dtrace_load8(addr)) == target) {
regs[rd] = addr;
if (subr == DIF_SUBR_STRCHR)
break;
}
if (c == '\0')
break;
}
if (!dtrace_canload(saddr, addr - saddr, mstate, vstate)) {
regs[rd] = 0;
break;
}
break;
}
case DIF_SUBR_STRSTR:
case DIF_SUBR_INDEX:
case DIF_SUBR_RINDEX: {
/*
* We're going to iterate over the string looking for the
* specified string. We will iterate until we have reached
* the string length or we have found the string. (Yes, this
* is done in the most naive way possible -- but considering
* that the string we're searching for is likely to be
* relatively short, the complexity of Rabin-Karp or similar
* hardly seems merited.)
*/
char *addr = (char *)(uintptr_t)tupregs[0].dttk_value;
char *substr = (char *)(uintptr_t)tupregs[1].dttk_value;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
size_t len = dtrace_strlen(addr, size);
size_t sublen = dtrace_strlen(substr, size);
char *limit = addr + len, *orig = addr;
int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1;
int inc = 1;
regs[rd] = notfound;
if (!dtrace_canload((uintptr_t)addr, len + 1, mstate, vstate)) {
regs[rd] = 0;
break;
}
if (!dtrace_canload((uintptr_t)substr, sublen + 1, mstate,
vstate)) {
regs[rd] = 0;
break;
}
/*
* strstr() and index()/rindex() have similar semantics if
* both strings are the empty string: strstr() returns a
* pointer to the (empty) string, and index() and rindex()
* both return index 0 (regardless of any position argument).
*/
if (sublen == 0 && len == 0) {
if (subr == DIF_SUBR_STRSTR)
regs[rd] = (uintptr_t)addr;
else
regs[rd] = 0;
break;
}
if (subr != DIF_SUBR_STRSTR) {
if (subr == DIF_SUBR_RINDEX) {
limit = orig - 1;
addr += len;
inc = -1;
}
/*
* Both index() and rindex() take an optional position
* argument that denotes the starting position.
*/
if (nargs == 3) {
int64_t pos = (int64_t)tupregs[2].dttk_value;
/*
* If the position argument to index() is
* negative, Perl implicitly clamps it at
* zero. This semantic is a little surprising
* given the special meaning of negative
* positions to similar Perl functions like
* substr(), but it appears to reflect a
* notion that index() can start from a
* negative index and increment its way up to
* the string. Given this notion, Perl's
* rindex() is at least self-consistent in
* that it implicitly clamps positions greater
* than the string length to be the string
* length. Where Perl completely loses
* coherence, however, is when the specified
* substring is the empty string (""). In
* this case, even if the position is
* negative, rindex() returns 0 -- and even if
* the position is greater than the length,
* index() returns the string length. These
* semantics violate the notion that index()
* should never return a value less than the
* specified position and that rindex() should
* never return a value greater than the
* specified position. (One assumes that
* these semantics are artifacts of Perl's
* implementation and not the results of
* deliberate design -- it beggars belief that
* even Larry Wall could desire such oddness.)
* While in the abstract one would wish for
* consistent position semantics across
* substr(), index() and rindex() -- or at the
* very least self-consistent position
* semantics for index() and rindex() -- we
* instead opt to keep with the extant Perl
* semantics, in all their broken glory. (Do
* we have more desire to maintain Perl's
* semantics than Perl does? Probably.)
*/
if (subr == DIF_SUBR_RINDEX) {
if (pos < 0) {
if (sublen == 0)
regs[rd] = 0;
break;
}
if (pos > len)
pos = len;
} else {
if (pos < 0)
pos = 0;
if (pos >= len) {
if (sublen == 0)
regs[rd] = len;
break;
}
}
addr = orig + pos;
}
}
for (regs[rd] = notfound; addr != limit; addr += inc) {
if (dtrace_strncmp(addr, substr, sublen) == 0) {
if (subr != DIF_SUBR_STRSTR) {
/*
* As D index() and rindex() are
* modeled on Perl (and not on awk),
* we return a zero-based (and not a
* one-based) index. (For you Perl
* weenies: no, we're not going to add
* $[ -- and shouldn't you be at a con
* or something?)
*/
regs[rd] = (uintptr_t)(addr - orig);
break;
}
ASSERT(subr == DIF_SUBR_STRSTR);
regs[rd] = (uintptr_t)addr;
break;
}
}
break;
}
case DIF_SUBR_STRTOK: {
uintptr_t addr = tupregs[0].dttk_value;
uintptr_t tokaddr = tupregs[1].dttk_value;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
uintptr_t limit, toklimit = tokaddr + size;
uint8_t c = 0, tokmap[32]; /* 256 / 8 */
char *dest = (char *)mstate->dtms_scratch_ptr;
int i;
/*
* Check both the token buffer and (later) the input buffer,
* since both could be non-scratch addresses.
*/
if (!dtrace_strcanload(tokaddr, size, mstate, vstate)) {
regs[rd] = 0;
break;
}
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
if (addr == 0) {
/*
* If the address specified is NULL, we use our saved
* strtok pointer from the mstate. Note that this
* means that the saved strtok pointer is _only_
* valid within multiple enablings of the same probe --
* it behaves like an implicit clause-local variable.
*/
addr = mstate->dtms_strtok;
} else {
/*
* If the user-specified address is non-NULL we must
* access check it. This is the only time we have
* a chance to do so, since this address may reside
* in the string table of this clause-- future calls
* (when we fetch addr from mstate->dtms_strtok)
* would fail this access check.
*/
if (!dtrace_strcanload(addr, size, mstate, vstate)) {
regs[rd] = 0;
break;
}
}
/*
* First, zero the token map, and then process the token
* string -- setting a bit in the map for every character
* found in the token string.
*/
for (i = 0; i < sizeof (tokmap); i++)
tokmap[i] = 0;
for (; tokaddr < toklimit; tokaddr++) {
if ((c = dtrace_load8(tokaddr)) == '\0')
break;
ASSERT((c >> 3) < sizeof (tokmap));
tokmap[c >> 3] |= (1 << (c & 0x7));
}
for (limit = addr + size; addr < limit; addr++) {
/*
* We're looking for a character that is _not_ contained
* in the token string.
*/
if ((c = dtrace_load8(addr)) == '\0')
break;
if (!(tokmap[c >> 3] & (1 << (c & 0x7))))
break;
}
if (c == '\0') {
/*
* We reached the end of the string without finding
* any character that was not in the token string.
* We return NULL in this case, and we set the saved
* address to NULL as well.
*/
regs[rd] = 0;
mstate->dtms_strtok = 0;
break;
}
/*
* From here on, we're copying into the destination string.
*/
for (i = 0; addr < limit && i < size - 1; addr++) {
if ((c = dtrace_load8(addr)) == '\0')
break;
if (tokmap[c >> 3] & (1 << (c & 0x7)))
break;
ASSERT(i < size);
dest[i++] = c;
}
ASSERT(i < size);
dest[i] = '\0';
regs[rd] = (uintptr_t)dest;
mstate->dtms_scratch_ptr += size;
mstate->dtms_strtok = addr;
break;
}
case DIF_SUBR_SUBSTR: {
uintptr_t s = tupregs[0].dttk_value;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
char *d = (char *)mstate->dtms_scratch_ptr;
int64_t index = (int64_t)tupregs[1].dttk_value;
int64_t remaining = (int64_t)tupregs[2].dttk_value;
size_t len = dtrace_strlen((char *)s, size);
int64_t i;
if (!dtrace_canload(s, len + 1, mstate, vstate)) {
regs[rd] = 0;
break;
}
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
if (nargs <= 2)
remaining = (int64_t)size;
if (index < 0) {
index += len;
if (index < 0 && index + remaining > 0) {
remaining += index;
index = 0;
}
}
if (index >= len || index < 0) {
remaining = 0;
} else if (remaining < 0) {
remaining += len - index;
} else if (index + remaining > size) {
remaining = size - index;
}
for (i = 0; i < remaining; i++) {
if ((d[i] = dtrace_load8(s + index + i)) == '\0')
break;
}
d[i] = '\0';
mstate->dtms_scratch_ptr += size;
regs[rd] = (uintptr_t)d;
break;
}
case DIF_SUBR_JSON: {
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
uintptr_t json = tupregs[0].dttk_value;
size_t jsonlen = dtrace_strlen((char *)json, size);
uintptr_t elem = tupregs[1].dttk_value;
size_t elemlen = dtrace_strlen((char *)elem, size);
char *dest = (char *)mstate->dtms_scratch_ptr;
char *elemlist = (char *)mstate->dtms_scratch_ptr + jsonlen + 1;
char *ee = elemlist;
int nelems = 1;
uintptr_t cur;
if (!dtrace_canload(json, jsonlen + 1, mstate, vstate) ||
!dtrace_canload(elem, elemlen + 1, mstate, vstate)) {
regs[rd] = 0;
break;
}
if (!DTRACE_INSCRATCH(mstate, jsonlen + 1 + elemlen + 1)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
/*
* Read the element selector and split it up into a packed list
* of strings.
*/
for (cur = elem; cur < elem + elemlen; cur++) {
char cc = dtrace_load8(cur);
if (cur == elem && cc == '[') {
/*
* If the first element selector key is
* actually an array index then ignore the
* bracket.
*/
continue;
}
if (cc == ']')
continue;
if (cc == '.' || cc == '[') {
nelems++;
cc = '\0';
}
*ee++ = cc;
}
*ee++ = '\0';
if ((regs[rd] = (uintptr_t)dtrace_json(size, json, elemlist,
nelems, dest)) != 0)
mstate->dtms_scratch_ptr += jsonlen + 1;
break;
}
case DIF_SUBR_TOUPPER:
case DIF_SUBR_TOLOWER: {
uintptr_t s = tupregs[0].dttk_value;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
char *dest = (char *)mstate->dtms_scratch_ptr, c;
size_t len = dtrace_strlen((char *)s, size);
char lower, upper, convert;
int64_t i;
if (subr == DIF_SUBR_TOUPPER) {
lower = 'a';
upper = 'z';
convert = 'A';
} else {
lower = 'A';
upper = 'Z';
convert = 'a';
}
if (!dtrace_canload(s, len + 1, mstate, vstate)) {
regs[rd] = 0;
break;
}
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
for (i = 0; i < size - 1; i++) {
if ((c = dtrace_load8(s + i)) == '\0')
break;
if (c >= lower && c <= upper)
c = convert + (c - lower);
dest[i] = c;
}
ASSERT(i < size);
dest[i] = '\0';
regs[rd] = (uintptr_t)dest;
mstate->dtms_scratch_ptr += size;
break;
}
#ifdef illumos
case DIF_SUBR_GETMAJOR:
#ifdef _LP64
regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64;
#else
regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ;
#endif
break;
case DIF_SUBR_GETMINOR:
#ifdef _LP64
regs[rd] = tupregs[0].dttk_value & MAXMIN64;
#else
regs[rd] = tupregs[0].dttk_value & MAXMIN;
#endif
break;
case DIF_SUBR_DDI_PATHNAME: {
/*
* This one is a galactic mess. We are going to roughly
* emulate ddi_pathname(), but it's made more complicated
* by the fact that we (a) want to include the minor name and
* (b) must proceed iteratively instead of recursively.
*/
uintptr_t dest = mstate->dtms_scratch_ptr;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
char *start = (char *)dest, *end = start + size - 1;
uintptr_t daddr = tupregs[0].dttk_value;
int64_t minor = (int64_t)tupregs[1].dttk_value;
char *s;
int i, len, depth = 0;
/*
* Due to all the pointer jumping we do and context we must
* rely upon, we just mandate that the user must have kernel
* read privileges to use this routine.
*/
if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) == 0) {
*flags |= CPU_DTRACE_KPRIV;
*illval = daddr;
regs[rd] = 0;
}
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
*end = '\0';
/*
* We want to have a name for the minor. In order to do this,
* we need to walk the minor list from the devinfo. We want
* to be sure that we don't infinitely walk a circular list,
* so we check for circularity by sending a scout pointer
* ahead two elements for every element that we iterate over;
* if the list is circular, these will ultimately point to the
* same element. You may recognize this little trick as the
* answer to a stupid interview question -- one that always
* seems to be asked by those who had to have it laboriously
* explained to them, and who can't even concisely describe
* the conditions under which one would be forced to resort to
* this technique. Needless to say, those conditions are
* found here -- and probably only here. Is this the only use
* of this infamous trick in shipping, production code? If it
* isn't, it probably should be...
*/
if (minor != -1) {
uintptr_t maddr = dtrace_loadptr(daddr +
offsetof(struct dev_info, devi_minor));
uintptr_t next = offsetof(struct ddi_minor_data, next);
uintptr_t name = offsetof(struct ddi_minor_data,
d_minor) + offsetof(struct ddi_minor, name);
uintptr_t dev = offsetof(struct ddi_minor_data,
d_minor) + offsetof(struct ddi_minor, dev);
uintptr_t scout;
if (maddr != NULL)
scout = dtrace_loadptr(maddr + next);
while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
uint64_t m;
#ifdef _LP64
m = dtrace_load64(maddr + dev) & MAXMIN64;
#else
m = dtrace_load32(maddr + dev) & MAXMIN;
#endif
if (m != minor) {
maddr = dtrace_loadptr(maddr + next);
if (scout == NULL)
continue;
scout = dtrace_loadptr(scout + next);
if (scout == NULL)
continue;
scout = dtrace_loadptr(scout + next);
if (scout == NULL)
continue;
if (scout == maddr) {
*flags |= CPU_DTRACE_ILLOP;
break;
}
continue;
}
/*
* We have the minor data. Now we need to
* copy the minor's name into the end of the
* pathname.
*/
s = (char *)dtrace_loadptr(maddr + name);
len = dtrace_strlen(s, size);
if (*flags & CPU_DTRACE_FAULT)
break;
if (len != 0) {
if ((end -= (len + 1)) < start)
break;
*end = ':';
}
for (i = 1; i <= len; i++)
end[i] = dtrace_load8((uintptr_t)s++);
break;
}
}
while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
ddi_node_state_t devi_state;
devi_state = dtrace_load32(daddr +
offsetof(struct dev_info, devi_node_state));
if (*flags & CPU_DTRACE_FAULT)
break;
if (devi_state >= DS_INITIALIZED) {
s = (char *)dtrace_loadptr(daddr +
offsetof(struct dev_info, devi_addr));
len = dtrace_strlen(s, size);
if (*flags & CPU_DTRACE_FAULT)
break;
if (len != 0) {
if ((end -= (len + 1)) < start)
break;
*end = '@';
}
for (i = 1; i <= len; i++)
end[i] = dtrace_load8((uintptr_t)s++);
}
/*
* Now for the node name...
*/
s = (char *)dtrace_loadptr(daddr +
offsetof(struct dev_info, devi_node_name));
daddr = dtrace_loadptr(daddr +
offsetof(struct dev_info, devi_parent));
/*
* If our parent is NULL (that is, if we're the root
* node), we're going to use the special path
* "devices".
*/
if (daddr == 0)
s = "devices";
len = dtrace_strlen(s, size);
if (*flags & CPU_DTRACE_FAULT)
break;
if ((end -= (len + 1)) < start)
break;
for (i = 1; i <= len; i++)
end[i] = dtrace_load8((uintptr_t)s++);
*end = '/';
if (depth++ > dtrace_devdepth_max) {
*flags |= CPU_DTRACE_ILLOP;
break;
}
}
if (end < start)
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
if (daddr == 0) {
regs[rd] = (uintptr_t)end;
mstate->dtms_scratch_ptr += size;
}
break;
}
#endif
case DIF_SUBR_STRJOIN: {
char *d = (char *)mstate->dtms_scratch_ptr;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
uintptr_t s1 = tupregs[0].dttk_value;
uintptr_t s2 = tupregs[1].dttk_value;
int i = 0;
if (!dtrace_strcanload(s1, size, mstate, vstate) ||
!dtrace_strcanload(s2, size, mstate, vstate)) {
regs[rd] = 0;
break;
}
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
for (;;) {
if (i >= size) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
if ((d[i++] = dtrace_load8(s1++)) == '\0') {
i--;
break;
}
}
for (;;) {
if (i >= size) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
if ((d[i++] = dtrace_load8(s2++)) == '\0')
break;
}
if (i < size) {
mstate->dtms_scratch_ptr += i;
regs[rd] = (uintptr_t)d;
}
break;
}
case DIF_SUBR_STRTOLL: {
uintptr_t s = tupregs[0].dttk_value;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
int base = 10;
if (nargs > 1) {
if ((base = tupregs[1].dttk_value) <= 1 ||
base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
*flags |= CPU_DTRACE_ILLOP;
break;
}
}
if (!dtrace_strcanload(s, size, mstate, vstate)) {
regs[rd] = INT64_MIN;
break;
}
regs[rd] = dtrace_strtoll((char *)s, base, size);
break;
}
case DIF_SUBR_LLTOSTR: {
int64_t i = (int64_t)tupregs[0].dttk_value;
uint64_t val, digit;
uint64_t size = 65; /* enough room for 2^64 in binary */
char *end = (char *)mstate->dtms_scratch_ptr + size - 1;
int base = 10;
if (nargs > 1) {
if ((base = tupregs[1].dttk_value) <= 1 ||
base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
*flags |= CPU_DTRACE_ILLOP;
break;
}
}
val = (base == 10 && i < 0) ? i * -1 : i;
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
for (*end-- = '\0'; val; val /= base) {
if ((digit = val % base) <= '9' - '0') {
*end-- = '0' + digit;
} else {
*end-- = 'a' + (digit - ('9' - '0') - 1);
}
}
if (i == 0 && base == 16)
*end-- = '0';
if (base == 16)
*end-- = 'x';
if (i == 0 || base == 8 || base == 16)
*end-- = '0';
if (i < 0 && base == 10)
*end-- = '-';
regs[rd] = (uintptr_t)end + 1;
mstate->dtms_scratch_ptr += size;
break;
}
case DIF_SUBR_HTONS:
case DIF_SUBR_NTOHS:
#if BYTE_ORDER == BIG_ENDIAN
regs[rd] = (uint16_t)tupregs[0].dttk_value;
#else
regs[rd] = DT_BSWAP_16((uint16_t)tupregs[0].dttk_value);
#endif
break;
case DIF_SUBR_HTONL:
case DIF_SUBR_NTOHL:
#if BYTE_ORDER == BIG_ENDIAN
regs[rd] = (uint32_t)tupregs[0].dttk_value;
#else
regs[rd] = DT_BSWAP_32((uint32_t)tupregs[0].dttk_value);
#endif
break;
case DIF_SUBR_HTONLL:
case DIF_SUBR_NTOHLL:
#if BYTE_ORDER == BIG_ENDIAN
regs[rd] = (uint64_t)tupregs[0].dttk_value;
#else
regs[rd] = DT_BSWAP_64((uint64_t)tupregs[0].dttk_value);
#endif
break;
case DIF_SUBR_DIRNAME:
case DIF_SUBR_BASENAME: {
char *dest = (char *)mstate->dtms_scratch_ptr;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
uintptr_t src = tupregs[0].dttk_value;
int i, j, len = dtrace_strlen((char *)src, size);
int lastbase = -1, firstbase = -1, lastdir = -1;
int start, end;
if (!dtrace_canload(src, len + 1, mstate, vstate)) {
regs[rd] = 0;
break;
}
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
/*
* The basename and dirname for a zero-length string is
* defined to be "."
*/
if (len == 0) {
len = 1;
src = (uintptr_t)".";
}
/*
* Start from the back of the string, moving back toward the
* front until we see a character that isn't a slash. That
* character is the last character in the basename.
*/
for (i = len - 1; i >= 0; i--) {
if (dtrace_load8(src + i) != '/')
break;
}
if (i >= 0)
lastbase = i;
/*
* Starting from the last character in the basename, move
* towards the front until we find a slash. The character
* that we processed immediately before that is the first
* character in the basename.
*/
for (; i >= 0; i--) {
if (dtrace_load8(src + i) == '/')
break;
}
if (i >= 0)
firstbase = i + 1;
/*
* Now keep going until we find a non-slash character. That
* character is the last character in the dirname.
*/
for (; i >= 0; i--) {
if (dtrace_load8(src + i) != '/')
break;
}
if (i >= 0)
lastdir = i;
ASSERT(!(lastbase == -1 && firstbase != -1));
ASSERT(!(firstbase == -1 && lastdir != -1));
if (lastbase == -1) {
/*
* We didn't find a non-slash character. We know that
* the length is non-zero, so the whole string must be
* slashes. In either the dirname or the basename
* case, we return '/'.
*/
ASSERT(firstbase == -1);
firstbase = lastbase = lastdir = 0;
}
if (firstbase == -1) {
/*
* The entire string consists only of a basename
* component. If we're looking for dirname, we need
* to change our string to be just "."; if we're
* looking for a basename, we'll just set the first
* character of the basename to be 0.
*/
if (subr == DIF_SUBR_DIRNAME) {
ASSERT(lastdir == -1);
src = (uintptr_t)".";
lastdir = 0;
} else {
firstbase = 0;
}
}
if (subr == DIF_SUBR_DIRNAME) {
if (lastdir == -1) {
/*
* We know that we have a slash in the name --
* or lastdir would be set to 0, above. And
* because lastdir is -1, we know that this
* slash must be the first character. (That
* is, the full string must be of the form
* "/basename".) In this case, the last
* character of the directory name is 0.
*/
lastdir = 0;
}
start = 0;
end = lastdir;
} else {
ASSERT(subr == DIF_SUBR_BASENAME);
ASSERT(firstbase != -1 && lastbase != -1);
start = firstbase;
end = lastbase;
}
for (i = start, j = 0; i <= end && j < size - 1; i++, j++)
dest[j] = dtrace_load8(src + i);
dest[j] = '\0';
regs[rd] = (uintptr_t)dest;
mstate->dtms_scratch_ptr += size;
break;
}
case DIF_SUBR_GETF: {
uintptr_t fd = tupregs[0].dttk_value;
struct filedesc *fdp;
file_t *fp;
if (!dtrace_priv_proc(state)) {
regs[rd] = 0;
break;
}
fdp = curproc->p_fd;
FILEDESC_SLOCK(fdp);
fp = fget_locked(fdp, fd);
mstate->dtms_getf = fp;
regs[rd] = (uintptr_t)fp;
FILEDESC_SUNLOCK(fdp);
break;
}
case DIF_SUBR_CLEANPATH: {
char *dest = (char *)mstate->dtms_scratch_ptr, c;
uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
uintptr_t src = tupregs[0].dttk_value;
int i = 0, j = 0;
#ifdef illumos
zone_t *z;
#endif
if (!dtrace_strcanload(src, size, mstate, vstate)) {
regs[rd] = 0;
break;
}
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
/*
* Move forward, loading each character.
*/
do {
c = dtrace_load8(src + i++);
next:
if (j + 5 >= size) /* 5 = strlen("/..c\0") */
break;
if (c != '/') {
dest[j++] = c;
continue;
}
c = dtrace_load8(src + i++);
if (c == '/') {
/*
* We have two slashes -- we can just advance
* to the next character.
*/
goto next;
}
if (c != '.') {
/*
* This is not "." and it's not ".." -- we can
* just store the "/" and this character and
* drive on.
*/
dest[j++] = '/';
dest[j++] = c;
continue;
}
c = dtrace_load8(src + i++);
if (c == '/') {
/*
* This is a "/./" component. We're not going
* to store anything in the destination buffer;
* we're just going to go to the next component.
*/
goto next;
}
if (c != '.') {
/*
* This is not ".." -- we can just store the
* "/." and this character and continue
* processing.
*/
dest[j++] = '/';
dest[j++] = '.';
dest[j++] = c;
continue;
}
c = dtrace_load8(src + i++);
if (c != '/' && c != '\0') {
/*
* This is not ".." -- it's "..[mumble]".
* We'll store the "/.." and this character
* and continue processing.
*/
dest[j++] = '/';
dest[j++] = '.';
dest[j++] = '.';
dest[j++] = c;
continue;
}
/*
* This is "/../" or "/..\0". We need to back up
* our destination pointer until we find a "/".
*/
i--;
while (j != 0 && dest[--j] != '/')
continue;
if (c == '\0')
dest[++j] = '/';
} while (c != '\0');
dest[j] = '\0';
#ifdef illumos
if (mstate->dtms_getf != NULL &&
!(mstate->dtms_access & DTRACE_ACCESS_KERNEL) &&
(z = state->dts_cred.dcr_cred->cr_zone) != kcred->cr_zone) {
/*
* If we've done a getf() as a part of this ECB and we
* don't have kernel access (and we're not in the global
* zone), check if the path we cleaned up begins with
* the zone's root path, and trim it off if so. Note
* that this is an output cleanliness issue, not a
* security issue: knowing one's zone root path does
* not enable privilege escalation.
*/
if (strstr(dest, z->zone_rootpath) == dest)
dest += strlen(z->zone_rootpath) - 1;
}
#endif
regs[rd] = (uintptr_t)dest;
mstate->dtms_scratch_ptr += size;
break;
}
case DIF_SUBR_INET_NTOA:
case DIF_SUBR_INET_NTOA6:
case DIF_SUBR_INET_NTOP: {
size_t size;
int af, argi, i;
char *base, *end;
if (subr == DIF_SUBR_INET_NTOP) {
af = (int)tupregs[0].dttk_value;
argi = 1;
} else {
af = subr == DIF_SUBR_INET_NTOA ? AF_INET: AF_INET6;
argi = 0;
}
if (af == AF_INET) {
ipaddr_t ip4;
uint8_t *ptr8, val;
/*
* Safely load the IPv4 address.
*/
ip4 = dtrace_load32(tupregs[argi].dttk_value);
/*
* Check an IPv4 string will fit in scratch.
*/
size = INET_ADDRSTRLEN;
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
base = (char *)mstate->dtms_scratch_ptr;
end = (char *)mstate->dtms_scratch_ptr + size - 1;
/*
* Stringify as a dotted decimal quad.
*/
*end-- = '\0';
ptr8 = (uint8_t *)&ip4;
for (i = 3; i >= 0; i--) {
val = ptr8[i];
if (val == 0) {
*end-- = '0';
} else {
for (; val; val /= 10) {
*end-- = '0' + (val % 10);
}
}
if (i > 0)
*end-- = '.';
}
ASSERT(end + 1 >= base);
} else if (af == AF_INET6) {
struct in6_addr ip6;
int firstzero, tryzero, numzero, v6end;
uint16_t val;
const char digits[] = "0123456789abcdef";
/*
* Stringify using RFC 1884 convention 2 - 16 bit
* hexadecimal values with a zero-run compression.
* Lower case hexadecimal digits are used.
* eg, fe80::214:4fff:fe0b:76c8.
* The IPv4 embedded form is returned for inet_ntop,
* just the IPv4 string is returned for inet_ntoa6.
*/
/*
* Safely load the IPv6 address.
*/
dtrace_bcopy(
(void *)(uintptr_t)tupregs[argi].dttk_value,
(void *)(uintptr_t)&ip6, sizeof (struct in6_addr));
/*
* Check an IPv6 string will fit in scratch.
*/
size = INET6_ADDRSTRLEN;
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
base = (char *)mstate->dtms_scratch_ptr;
end = (char *)mstate->dtms_scratch_ptr + size - 1;
*end-- = '\0';
/*
* Find the longest run of 16 bit zero values
* for the single allowed zero compression - "::".
*/
firstzero = -1;
tryzero = -1;
numzero = 1;
for (i = 0; i < sizeof (struct in6_addr); i++) {
#ifdef illumos
if (ip6._S6_un._S6_u8[i] == 0 &&
#else
if (ip6.__u6_addr.__u6_addr8[i] == 0 &&
#endif
tryzero == -1 && i % 2 == 0) {
tryzero = i;
continue;
}
if (tryzero != -1 &&
#ifdef illumos
(ip6._S6_un._S6_u8[i] != 0 ||
#else
(ip6.__u6_addr.__u6_addr8[i] != 0 ||
#endif
i == sizeof (struct in6_addr) - 1)) {
if (i - tryzero <= numzero) {
tryzero = -1;
continue;
}
firstzero = tryzero;
numzero = i - i % 2 - tryzero;
tryzero = -1;
#ifdef illumos
if (ip6._S6_un._S6_u8[i] == 0 &&
#else
if (ip6.__u6_addr.__u6_addr8[i] == 0 &&
#endif
i == sizeof (struct in6_addr) - 1)
numzero += 2;
}
}
ASSERT(firstzero + numzero <= sizeof (struct in6_addr));
/*
* Check for an IPv4 embedded address.
*/
v6end = sizeof (struct in6_addr) - 2;
if (IN6_IS_ADDR_V4MAPPED(&ip6) ||
IN6_IS_ADDR_V4COMPAT(&ip6)) {
for (i = sizeof (struct in6_addr) - 1;
i >= DTRACE_V4MAPPED_OFFSET; i--) {
ASSERT(end >= base);
#ifdef illumos
val = ip6._S6_un._S6_u8[i];
#else
val = ip6.__u6_addr.__u6_addr8[i];
#endif
if (val == 0) {
*end-- = '0';
} else {
for (; val; val /= 10) {
*end-- = '0' + val % 10;
}
}
if (i > DTRACE_V4MAPPED_OFFSET)
*end-- = '.';
}
if (subr == DIF_SUBR_INET_NTOA6)
goto inetout;
/*
* Set v6end to skip the IPv4 address that
* we have already stringified.
*/
v6end = 10;
}
/*
* Build the IPv6 string by working through the
* address in reverse.
*/
for (i = v6end; i >= 0; i -= 2) {
ASSERT(end >= base);
if (i == firstzero + numzero - 2) {
*end-- = ':';
*end-- = ':';
i -= numzero - 2;
continue;
}
if (i < 14 && i != firstzero - 2)
*end-- = ':';
#ifdef illumos
val = (ip6._S6_un._S6_u8[i] << 8) +
ip6._S6_un._S6_u8[i + 1];
#else
val = (ip6.__u6_addr.__u6_addr8[i] << 8) +
ip6.__u6_addr.__u6_addr8[i + 1];
#endif
if (val == 0) {
*end-- = '0';
} else {
for (; val; val /= 16) {
*end-- = digits[val % 16];
}
}
}
ASSERT(end + 1 >= base);
} else {
/*
* The user didn't use AH_INET or AH_INET6.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
regs[rd] = 0;
break;
}
inetout: regs[rd] = (uintptr_t)end + 1;
mstate->dtms_scratch_ptr += size;
break;
}
case DIF_SUBR_MEMREF: {
uintptr_t size = 2 * sizeof(uintptr_t);
uintptr_t *memref = (uintptr_t *) P2ROUNDUP(mstate->dtms_scratch_ptr, sizeof(uintptr_t));
size_t scratch_size = ((uintptr_t) memref - mstate->dtms_scratch_ptr) + size;
/* address and length */
memref[0] = tupregs[0].dttk_value;
memref[1] = tupregs[1].dttk_value;
regs[rd] = (uintptr_t) memref;
mstate->dtms_scratch_ptr += scratch_size;
break;
}
#ifndef illumos
case DIF_SUBR_MEMSTR: {
char *str = (char *)mstate->dtms_scratch_ptr;
uintptr_t mem = tupregs[0].dttk_value;
char c = tupregs[1].dttk_value;
size_t size = tupregs[2].dttk_value;
uint8_t n;
int i;
regs[rd] = 0;
if (size == 0)
break;
if (!dtrace_canload(mem, size - 1, mstate, vstate))
break;
if (!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
break;
}
if (dtrace_memstr_max != 0 && size > dtrace_memstr_max) {
*flags |= CPU_DTRACE_ILLOP;
break;
}
for (i = 0; i < size - 1; i++) {
n = dtrace_load8(mem++);
str[i] = (n == 0) ? c : n;
}
str[size - 1] = 0;
regs[rd] = (uintptr_t)str;
mstate->dtms_scratch_ptr += size;
break;
}
#endif
case DIF_SUBR_TYPEREF: {
uintptr_t size = 4 * sizeof(uintptr_t);
uintptr_t *typeref = (uintptr_t *) P2ROUNDUP(mstate->dtms_scratch_ptr, sizeof(uintptr_t));
size_t scratch_size = ((uintptr_t) typeref - mstate->dtms_scratch_ptr) + size;
/* address, num_elements, type_str, type_len */
typeref[0] = tupregs[0].dttk_value;
typeref[1] = tupregs[1].dttk_value;
typeref[2] = tupregs[2].dttk_value;
typeref[3] = tupregs[3].dttk_value;
regs[rd] = (uintptr_t) typeref;
mstate->dtms_scratch_ptr += scratch_size;
break;
}
}
}
/*
* Emulate the execution of DTrace IR instructions specified by the given
* DIF object. This function is deliberately void of assertions as all of
* the necessary checks are handled by a call to dtrace_difo_validate().
*/
static uint64_t
dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
dtrace_vstate_t *vstate, dtrace_state_t *state)
{
const dif_instr_t *text = difo->dtdo_buf;
const uint_t textlen = difo->dtdo_len;
const char *strtab = difo->dtdo_strtab;
const uint64_t *inttab = difo->dtdo_inttab;
uint64_t rval = 0;
dtrace_statvar_t *svar;
dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
dtrace_difv_t *v;
volatile uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags;
volatile uintptr_t *illval = &cpu_core[curcpu].cpuc_dtrace_illval;
dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
uint64_t regs[DIF_DIR_NREGS];
uint64_t *tmp;
uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0;
int64_t cc_r;
uint_t pc = 0, id, opc = 0;
uint8_t ttop = 0;
dif_instr_t instr;
uint_t r1, r2, rd;
/*
* We stash the current DIF object into the machine state: we need it
* for subsequent access checking.
*/
mstate->dtms_difo = difo;
regs[DIF_REG_R0] = 0; /* %r0 is fixed at zero */
while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) {
opc = pc;
instr = text[pc++];
r1 = DIF_INSTR_R1(instr);
r2 = DIF_INSTR_R2(instr);
rd = DIF_INSTR_RD(instr);
switch (DIF_INSTR_OP(instr)) {
case DIF_OP_OR:
regs[rd] = regs[r1] | regs[r2];
break;
case DIF_OP_XOR:
regs[rd] = regs[r1] ^ regs[r2];
break;
case DIF_OP_AND:
regs[rd] = regs[r1] & regs[r2];
break;
case DIF_OP_SLL:
regs[rd] = regs[r1] << regs[r2];
break;
case DIF_OP_SRL:
regs[rd] = regs[r1] >> regs[r2];
break;
case DIF_OP_SUB:
regs[rd] = regs[r1] - regs[r2];
break;
case DIF_OP_ADD:
regs[rd] = regs[r1] + regs[r2];
break;
case DIF_OP_MUL:
regs[rd] = regs[r1] * regs[r2];
break;
case DIF_OP_SDIV:
if (regs[r2] == 0) {
regs[rd] = 0;
*flags |= CPU_DTRACE_DIVZERO;
} else {
regs[rd] = (int64_t)regs[r1] /
(int64_t)regs[r2];
}
break;
case DIF_OP_UDIV:
if (regs[r2] == 0) {
regs[rd] = 0;
*flags |= CPU_DTRACE_DIVZERO;
} else {
regs[rd] = regs[r1] / regs[r2];
}
break;
case DIF_OP_SREM:
if (regs[r2] == 0) {
regs[rd] = 0;
*flags |= CPU_DTRACE_DIVZERO;
} else {
regs[rd] = (int64_t)regs[r1] %
(int64_t)regs[r2];
}
break;
case DIF_OP_UREM:
if (regs[r2] == 0) {
regs[rd] = 0;
*flags |= CPU_DTRACE_DIVZERO;
} else {
regs[rd] = regs[r1] % regs[r2];
}
break;
case DIF_OP_NOT:
regs[rd] = ~regs[r1];
break;
case DIF_OP_MOV:
regs[rd] = regs[r1];
break;
case DIF_OP_CMP:
cc_r = regs[r1] - regs[r2];
cc_n = cc_r < 0;
cc_z = cc_r == 0;
cc_v = 0;
cc_c = regs[r1] < regs[r2];
break;
case DIF_OP_TST:
cc_n = cc_v = cc_c = 0;
cc_z = regs[r1] == 0;
break;
case DIF_OP_BA:
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BE:
if (cc_z)
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BNE:
if (cc_z == 0)
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BG:
if ((cc_z | (cc_n ^ cc_v)) == 0)
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BGU:
if ((cc_c | cc_z) == 0)
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BGE:
if ((cc_n ^ cc_v) == 0)
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BGEU:
if (cc_c == 0)
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BL:
if (cc_n ^ cc_v)
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BLU:
if (cc_c)
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BLE:
if (cc_z | (cc_n ^ cc_v))
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_BLEU:
if (cc_c | cc_z)
pc = DIF_INSTR_LABEL(instr);
break;
case DIF_OP_RLDSB:
if (!dtrace_canload(regs[r1], 1, mstate, vstate))
break;
/*FALLTHROUGH*/
case DIF_OP_LDSB:
regs[rd] = (int8_t)dtrace_load8(regs[r1]);
break;
case DIF_OP_RLDSH:
if (!dtrace_canload(regs[r1], 2, mstate, vstate))
break;
/*FALLTHROUGH*/
case DIF_OP_LDSH:
regs[rd] = (int16_t)dtrace_load16(regs[r1]);
break;
case DIF_OP_RLDSW:
if (!dtrace_canload(regs[r1], 4, mstate, vstate))
break;
/*FALLTHROUGH*/
case DIF_OP_LDSW:
regs[rd] = (int32_t)dtrace_load32(regs[r1]);
break;
case DIF_OP_RLDUB:
if (!dtrace_canload(regs[r1], 1, mstate, vstate))
break;
/*FALLTHROUGH*/
case DIF_OP_LDUB:
regs[rd] = dtrace_load8(regs[r1]);
break;
case DIF_OP_RLDUH:
if (!dtrace_canload(regs[r1], 2, mstate, vstate))
break;
/*FALLTHROUGH*/
case DIF_OP_LDUH:
regs[rd] = dtrace_load16(regs[r1]);
break;
case DIF_OP_RLDUW:
if (!dtrace_canload(regs[r1], 4, mstate, vstate))
break;
/*FALLTHROUGH*/
case DIF_OP_LDUW:
regs[rd] = dtrace_load32(regs[r1]);
break;
case DIF_OP_RLDX:
if (!dtrace_canload(regs[r1], 8, mstate, vstate))
break;
/*FALLTHROUGH*/
case DIF_OP_LDX:
regs[rd] = dtrace_load64(regs[r1]);
break;
case DIF_OP_ULDSB:
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
regs[rd] = (int8_t)
dtrace_fuword8((void *)(uintptr_t)regs[r1]);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
break;
case DIF_OP_ULDSH:
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
regs[rd] = (int16_t)
dtrace_fuword16((void *)(uintptr_t)regs[r1]);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
break;
case DIF_OP_ULDSW:
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
regs[rd] = (int32_t)
dtrace_fuword32((void *)(uintptr_t)regs[r1]);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
break;
case DIF_OP_ULDUB:
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
regs[rd] =
dtrace_fuword8((void *)(uintptr_t)regs[r1]);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
break;
case DIF_OP_ULDUH:
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
regs[rd] =
dtrace_fuword16((void *)(uintptr_t)regs[r1]);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
break;
case DIF_OP_ULDUW:
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
regs[rd] =
dtrace_fuword32((void *)(uintptr_t)regs[r1]);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
break;
case DIF_OP_ULDX:
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
regs[rd] =
dtrace_fuword64((void *)(uintptr_t)regs[r1]);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
break;
case DIF_OP_RET:
rval = regs[rd];
pc = textlen;
break;
case DIF_OP_NOP:
break;
case DIF_OP_SETX:
regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
break;
case DIF_OP_SETS:
regs[rd] = (uint64_t)(uintptr_t)
(strtab + DIF_INSTR_STRING(instr));
break;
case DIF_OP_SCMP: {
size_t sz = state->dts_options[DTRACEOPT_STRSIZE];
uintptr_t s1 = regs[r1];
uintptr_t s2 = regs[r2];
if (s1 != 0 &&
!dtrace_strcanload(s1, sz, mstate, vstate))
break;
if (s2 != 0 &&
!dtrace_strcanload(s2, sz, mstate, vstate))
break;
cc_r = dtrace_strncmp((char *)s1, (char *)s2, sz);
cc_n = cc_r < 0;
cc_z = cc_r == 0;
cc_v = cc_c = 0;
break;
}
case DIF_OP_LDGA:
regs[rd] = dtrace_dif_variable(mstate, state,
r1, regs[r2]);
break;
case DIF_OP_LDGS:
id = DIF_INSTR_VAR(instr);
if (id >= DIF_VAR_OTHER_UBASE) {
uintptr_t a;
id -= DIF_VAR_OTHER_UBASE;
svar = vstate->dtvs_globals[id];
ASSERT(svar != NULL);
v = &svar->dtsv_var;
if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) {
regs[rd] = svar->dtsv_data;
break;
}
a = (uintptr_t)svar->dtsv_data;
if (*(uint8_t *)a == UINT8_MAX) {
/*
* If the 0th byte is set to UINT8_MAX
* then this is to be treated as a
* reference to a NULL variable.
*/
regs[rd] = 0;
} else {
regs[rd] = a + sizeof (uint64_t);
}
break;
}
regs[rd] = dtrace_dif_variable(mstate, state, id, 0);
break;
case DIF_OP_STGS:
id = DIF_INSTR_VAR(instr);
ASSERT(id >= DIF_VAR_OTHER_UBASE);
id -= DIF_VAR_OTHER_UBASE;
svar = vstate->dtvs_globals[id];
ASSERT(svar != NULL);
v = &svar->dtsv_var;
if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
uintptr_t a = (uintptr_t)svar->dtsv_data;
ASSERT(a != 0);
ASSERT(svar->dtsv_size != 0);
if (regs[rd] == 0) {
*(uint8_t *)a = UINT8_MAX;
break;
} else {
*(uint8_t *)a = 0;
a += sizeof (uint64_t);
}
if (!dtrace_vcanload(
(void *)(uintptr_t)regs[rd], &v->dtdv_type,
mstate, vstate))
break;
dtrace_vcopy((void *)(uintptr_t)regs[rd],
(void *)a, &v->dtdv_type);
break;
}
svar->dtsv_data = regs[rd];
break;
case DIF_OP_LDTA:
/*
* There are no DTrace built-in thread-local arrays at
* present. This opcode is saved for future work.
*/
*flags |= CPU_DTRACE_ILLOP;
regs[rd] = 0;
break;
case DIF_OP_LDLS:
id = DIF_INSTR_VAR(instr);
if (id < DIF_VAR_OTHER_UBASE) {
/*
* For now, this has no meaning.
*/
regs[rd] = 0;
break;
}
id -= DIF_VAR_OTHER_UBASE;
ASSERT(id < vstate->dtvs_nlocals);
ASSERT(vstate->dtvs_locals != NULL);
svar = vstate->dtvs_locals[id];
ASSERT(svar != NULL);
v = &svar->dtsv_var;
if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
uintptr_t a = (uintptr_t)svar->dtsv_data;
size_t sz = v->dtdv_type.dtdt_size;
sz += sizeof (uint64_t);
ASSERT(svar->dtsv_size == NCPU * sz);
a += curcpu * sz;
if (*(uint8_t *)a == UINT8_MAX) {
/*
* If the 0th byte is set to UINT8_MAX
* then this is to be treated as a
* reference to a NULL variable.
*/
regs[rd] = 0;
} else {
regs[rd] = a + sizeof (uint64_t);
}
break;
}
ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
regs[rd] = tmp[curcpu];
break;
case DIF_OP_STLS:
id = DIF_INSTR_VAR(instr);
ASSERT(id >= DIF_VAR_OTHER_UBASE);
id -= DIF_VAR_OTHER_UBASE;
ASSERT(id < vstate->dtvs_nlocals);
ASSERT(vstate->dtvs_locals != NULL);
svar = vstate->dtvs_locals[id];
ASSERT(svar != NULL);
v = &svar->dtsv_var;
if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
uintptr_t a = (uintptr_t)svar->dtsv_data;
size_t sz = v->dtdv_type.dtdt_size;
sz += sizeof (uint64_t);
ASSERT(svar->dtsv_size == NCPU * sz);
a += curcpu * sz;
if (regs[rd] == 0) {
*(uint8_t *)a = UINT8_MAX;
break;
} else {
*(uint8_t *)a = 0;
a += sizeof (uint64_t);
}
if (!dtrace_vcanload(
(void *)(uintptr_t)regs[rd], &v->dtdv_type,
mstate, vstate))
break;
dtrace_vcopy((void *)(uintptr_t)regs[rd],
(void *)a, &v->dtdv_type);
break;
}
ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
tmp[curcpu] = regs[rd];
break;
case DIF_OP_LDTS: {
dtrace_dynvar_t *dvar;
dtrace_key_t *key;
id = DIF_INSTR_VAR(instr);
ASSERT(id >= DIF_VAR_OTHER_UBASE);
id -= DIF_VAR_OTHER_UBASE;
v = &vstate->dtvs_tlocals[id];
key = &tupregs[DIF_DTR_NREGS];
key[0].dttk_value = (uint64_t)id;
key[0].dttk_size = 0;
DTRACE_TLS_THRKEY(key[1].dttk_value);
key[1].dttk_size = 0;
dvar = dtrace_dynvar(dstate, 2, key,
sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC,
mstate, vstate);
if (dvar == NULL) {
regs[rd] = 0;
break;
}
if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
} else {
regs[rd] = *((uint64_t *)dvar->dtdv_data);
}
break;
}
case DIF_OP_STTS: {
dtrace_dynvar_t *dvar;
dtrace_key_t *key;
id = DIF_INSTR_VAR(instr);
ASSERT(id >= DIF_VAR_OTHER_UBASE);
id -= DIF_VAR_OTHER_UBASE;
key = &tupregs[DIF_DTR_NREGS];
key[0].dttk_value = (uint64_t)id;
key[0].dttk_size = 0;
DTRACE_TLS_THRKEY(key[1].dttk_value);
key[1].dttk_size = 0;
v = &vstate->dtvs_tlocals[id];
dvar = dtrace_dynvar(dstate, 2, key,
v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
v->dtdv_type.dtdt_size : sizeof (uint64_t),
regs[rd] ? DTRACE_DYNVAR_ALLOC :
DTRACE_DYNVAR_DEALLOC, mstate, vstate);
/*
* Given that we're storing to thread-local data,
* we need to flush our predicate cache.
*/
curthread->t_predcache = 0;
if (dvar == NULL)
break;
if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
if (!dtrace_vcanload(
(void *)(uintptr_t)regs[rd],
&v->dtdv_type, mstate, vstate))
break;
dtrace_vcopy((void *)(uintptr_t)regs[rd],
dvar->dtdv_data, &v->dtdv_type);
} else {
*((uint64_t *)dvar->dtdv_data) = regs[rd];
}
break;
}
case DIF_OP_SRA:
regs[rd] = (int64_t)regs[r1] >> regs[r2];
break;
case DIF_OP_CALL:
dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd,
regs, tupregs, ttop, mstate, state);
break;
case DIF_OP_PUSHTR:
if (ttop == DIF_DTR_NREGS) {
*flags |= CPU_DTRACE_TUPOFLOW;
break;
}
if (r1 == DIF_TYPE_STRING) {
/*
* If this is a string type and the size is 0,
* we'll use the system-wide default string
* size. Note that we are _not_ looking at
* the value of the DTRACEOPT_STRSIZE option;
* had this been set, we would expect to have
* a non-zero size value in the "pushtr".
*/
tupregs[ttop].dttk_size =
dtrace_strlen((char *)(uintptr_t)regs[rd],
regs[r2] ? regs[r2] :
dtrace_strsize_default) + 1;
} else {
tupregs[ttop].dttk_size = regs[r2];
}
tupregs[ttop++].dttk_value = regs[rd];
break;
case DIF_OP_PUSHTV:
if (ttop == DIF_DTR_NREGS) {
*flags |= CPU_DTRACE_TUPOFLOW;
break;
}
tupregs[ttop].dttk_value = regs[rd];
tupregs[ttop++].dttk_size = 0;
break;
case DIF_OP_POPTS:
if (ttop != 0)
ttop--;
break;
case DIF_OP_FLUSHTS:
ttop = 0;
break;
case DIF_OP_LDGAA:
case DIF_OP_LDTAA: {
dtrace_dynvar_t *dvar;
dtrace_key_t *key = tupregs;
uint_t nkeys = ttop;
id = DIF_INSTR_VAR(instr);
ASSERT(id >= DIF_VAR_OTHER_UBASE);
id -= DIF_VAR_OTHER_UBASE;
key[nkeys].dttk_value = (uint64_t)id;
key[nkeys++].dttk_size = 0;
if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) {
DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
key[nkeys++].dttk_size = 0;
v = &vstate->dtvs_tlocals[id];
} else {
v = &vstate->dtvs_globals[id]->dtsv_var;
}
dvar = dtrace_dynvar(dstate, nkeys, key,
v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
v->dtdv_type.dtdt_size : sizeof (uint64_t),
DTRACE_DYNVAR_NOALLOC, mstate, vstate);
if (dvar == NULL) {
regs[rd] = 0;
break;
}
if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
} else {
regs[rd] = *((uint64_t *)dvar->dtdv_data);
}
break;
}
case DIF_OP_STGAA:
case DIF_OP_STTAA: {
dtrace_dynvar_t *dvar;
dtrace_key_t *key = tupregs;
uint_t nkeys = ttop;
id = DIF_INSTR_VAR(instr);
ASSERT(id >= DIF_VAR_OTHER_UBASE);
id -= DIF_VAR_OTHER_UBASE;
key[nkeys].dttk_value = (uint64_t)id;
key[nkeys++].dttk_size = 0;
if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) {
DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
key[nkeys++].dttk_size = 0;
v = &vstate->dtvs_tlocals[id];
} else {
v = &vstate->dtvs_globals[id]->dtsv_var;
}
dvar = dtrace_dynvar(dstate, nkeys, key,
v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
v->dtdv_type.dtdt_size : sizeof (uint64_t),
regs[rd] ? DTRACE_DYNVAR_ALLOC :
DTRACE_DYNVAR_DEALLOC, mstate, vstate);
if (dvar == NULL)
break;
if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
if (!dtrace_vcanload(
(void *)(uintptr_t)regs[rd], &v->dtdv_type,
mstate, vstate))
break;
dtrace_vcopy((void *)(uintptr_t)regs[rd],
dvar->dtdv_data, &v->dtdv_type);
} else {
*((uint64_t *)dvar->dtdv_data) = regs[rd];
}
break;
}
case DIF_OP_ALLOCS: {
uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1];
/*
* Rounding up the user allocation size could have
* overflowed large, bogus allocations (like -1ULL) to
* 0.
*/
if (size < regs[r1] ||
!DTRACE_INSCRATCH(mstate, size)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
regs[rd] = 0;
break;
}
dtrace_bzero((void *) mstate->dtms_scratch_ptr, size);
mstate->dtms_scratch_ptr += size;
regs[rd] = ptr;
break;
}
case DIF_OP_COPYS:
if (!dtrace_canstore(regs[rd], regs[r2],
mstate, vstate)) {
*flags |= CPU_DTRACE_BADADDR;
*illval = regs[rd];
break;
}
if (!dtrace_canload(regs[r1], regs[r2], mstate, vstate))
break;
dtrace_bcopy((void *)(uintptr_t)regs[r1],
(void *)(uintptr_t)regs[rd], (size_t)regs[r2]);
break;
case DIF_OP_STB:
if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) {
*flags |= CPU_DTRACE_BADADDR;
*illval = regs[rd];
break;
}
*((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1];
break;
case DIF_OP_STH:
if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) {
*flags |= CPU_DTRACE_BADADDR;
*illval = regs[rd];
break;
}
if (regs[rd] & 1) {
*flags |= CPU_DTRACE_BADALIGN;
*illval = regs[rd];
break;
}
*((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1];
break;
case DIF_OP_STW:
if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) {
*flags |= CPU_DTRACE_BADADDR;
*illval = regs[rd];
break;
}
if (regs[rd] & 3) {
*flags |= CPU_DTRACE_BADALIGN;
*illval = regs[rd];
break;
}
*((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1];
break;
case DIF_OP_STX:
if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) {
*flags |= CPU_DTRACE_BADADDR;
*illval = regs[rd];
break;
}
if (regs[rd] & 7) {
*flags |= CPU_DTRACE_BADALIGN;
*illval = regs[rd];
break;
}
*((uint64_t *)(uintptr_t)regs[rd]) = regs[r1];
break;
}
}
if (!(*flags & CPU_DTRACE_FAULT))
return (rval);
mstate->dtms_fltoffs = opc * sizeof (dif_instr_t);
mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS;
return (0);
}
static void
dtrace_action_breakpoint(dtrace_ecb_t *ecb)
{
dtrace_probe_t *probe = ecb->dte_probe;
dtrace_provider_t *prov = probe->dtpr_provider;
char c[DTRACE_FULLNAMELEN + 80], *str;
char *msg = "dtrace: breakpoint action at probe ";
char *ecbmsg = " (ecb ";
uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4));
uintptr_t val = (uintptr_t)ecb;
int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0;
if (dtrace_destructive_disallow)
return;
/*
* It's impossible to be taking action on the NULL probe.
*/
ASSERT(probe != NULL);
/*
* This is a poor man's (destitute man's?) sprintf(): we want to
* print the provider name, module name, function name and name of
* the probe, along with the hex address of the ECB with the breakpoint
* action -- all of which we must place in the character buffer by
* hand.
*/
while (*msg != '\0')
c[i++] = *msg++;
for (str = prov->dtpv_name; *str != '\0'; str++)
c[i++] = *str;
c[i++] = ':';
for (str = probe->dtpr_mod; *str != '\0'; str++)
c[i++] = *str;
c[i++] = ':';
for (str = probe->dtpr_func; *str != '\0'; str++)
c[i++] = *str;
c[i++] = ':';
for (str = probe->dtpr_name; *str != '\0'; str++)
c[i++] = *str;
while (*ecbmsg != '\0')
c[i++] = *ecbmsg++;
while (shift >= 0) {
mask = (uintptr_t)0xf << shift;
if (val >= ((uintptr_t)1 << shift))
c[i++] = "0123456789abcdef"[(val & mask) >> shift];
shift -= 4;
}
c[i++] = ')';
c[i] = '\0';
#ifdef illumos
debug_enter(c);
#else
kdb_enter(KDB_WHY_DTRACE, "breakpoint action");
#endif
}
static void
dtrace_action_panic(dtrace_ecb_t *ecb)
{
dtrace_probe_t *probe = ecb->dte_probe;
/*
* It's impossible to be taking action on the NULL probe.
*/
ASSERT(probe != NULL);
if (dtrace_destructive_disallow)
return;
if (dtrace_panicked != NULL)
return;
if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL)
return;
/*
* We won the right to panic. (We want to be sure that only one
* thread calls panic() from dtrace_probe(), and that panic() is
* called exactly once.)
*/
dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)",
probe->dtpr_provider->dtpv_name, probe->dtpr_mod,
probe->dtpr_func, probe->dtpr_name, (void *)ecb);
}
static void
dtrace_action_raise(uint64_t sig)
{
if (dtrace_destructive_disallow)
return;
if (sig >= NSIG) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
return;
}
#ifdef illumos
/*
* raise() has a queue depth of 1 -- we ignore all subsequent
* invocations of the raise() action.
*/
if (curthread->t_dtrace_sig == 0)
curthread->t_dtrace_sig = (uint8_t)sig;
curthread->t_sig_check = 1;
aston(curthread);
#else
struct proc *p = curproc;
PROC_LOCK(p);
kern_psignal(p, sig);
PROC_UNLOCK(p);
#endif
}
static void
dtrace_action_stop(void)
{
if (dtrace_destructive_disallow)
return;
#ifdef illumos
if (!curthread->t_dtrace_stop) {
curthread->t_dtrace_stop = 1;
curthread->t_sig_check = 1;
aston(curthread);
}
#else
struct proc *p = curproc;
PROC_LOCK(p);
kern_psignal(p, SIGSTOP);
PROC_UNLOCK(p);
#endif
}
static void
dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val)
{
hrtime_t now;
volatile uint16_t *flags;
#ifdef illumos
cpu_t *cpu = CPU;
#else
cpu_t *cpu = &solaris_cpu[curcpu];
#endif
if (dtrace_destructive_disallow)
return;
flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;
now = dtrace_gethrtime();
if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {
/*
* We need to advance the mark to the current time.
*/
cpu->cpu_dtrace_chillmark = now;
cpu->cpu_dtrace_chilled = 0;
}
/*
* Now check to see if the requested chill time would take us over
* the maximum amount of time allowed in the chill interval. (Or
* worse, if the calculation itself induces overflow.)
*/
if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max ||
cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) {
*flags |= CPU_DTRACE_ILLOP;
return;
}
while (dtrace_gethrtime() - now < val)
continue;
/*
* Normally, we assure that the value of the variable "timestamp" does
* not change within an ECB. The presence of chill() represents an
* exception to this rule, however.
*/
mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP;
cpu->cpu_dtrace_chilled += val;
}
static void
dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state,
uint64_t *buf, uint64_t arg)
{
int nframes = DTRACE_USTACK_NFRAMES(arg);
int strsize = DTRACE_USTACK_STRSIZE(arg);
uint64_t *pcs = &buf[1], *fps;
char *str = (char *)&pcs[nframes];
int size, offs = 0, i, j;
uintptr_t old = mstate->dtms_scratch_ptr, saved;
uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags;
char *sym;
/*
* Should be taking a faster path if string space has not been
* allocated.
*/
ASSERT(strsize != 0);
/*
* We will first allocate some temporary space for the frame pointers.
*/
fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
size = (uintptr_t)fps - mstate->dtms_scratch_ptr +
(nframes * sizeof (uint64_t));
if (!DTRACE_INSCRATCH(mstate, size)) {
/*
* Not enough room for our frame pointers -- need to indicate
* that we ran out of scratch space.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
return;
}
mstate->dtms_scratch_ptr += size;
saved = mstate->dtms_scratch_ptr;
/*
* Now get a stack with both program counters and frame pointers.
*/
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
dtrace_getufpstack(buf, fps, nframes + 1);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
/*
* If that faulted, we're cooked.
*/
if (*flags & CPU_DTRACE_FAULT)
goto out;
/*
* Now we want to walk up the stack, calling the USTACK helper. For
* each iteration, we restore the scratch pointer.
*/
for (i = 0; i < nframes; i++) {
mstate->dtms_scratch_ptr = saved;
if (offs >= strsize)
break;
sym = (char *)(uintptr_t)dtrace_helper(
DTRACE_HELPER_ACTION_USTACK,
mstate, state, pcs[i], fps[i]);
/*
* If we faulted while running the helper, we're going to
* clear the fault and null out the corresponding string.
*/
if (*flags & CPU_DTRACE_FAULT) {
*flags &= ~CPU_DTRACE_FAULT;
str[offs++] = '\0';
continue;
}
if (sym == NULL) {
str[offs++] = '\0';
continue;
}
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
/*
* Now copy in the string that the helper returned to us.
*/
for (j = 0; offs + j < strsize; j++) {
if ((str[offs + j] = sym[j]) == '\0')
break;
}
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
offs += j + 1;
}
if (offs >= strsize) {
/*
* If we didn't have room for all of the strings, we don't
* abort processing -- this needn't be a fatal error -- but we
* still want to increment a counter (dts_stkstroverflows) to
* allow this condition to be warned about. (If this is from
* a jstack() action, it is easily tuned via jstackstrsize.)
*/
dtrace_error(&state->dts_stkstroverflows);
}
while (offs < strsize)
str[offs++] = '\0';
out:
mstate->dtms_scratch_ptr = old;
}
static void
dtrace_store_by_ref(dtrace_difo_t *dp, caddr_t tomax, size_t size,
size_t *valoffsp, uint64_t *valp, uint64_t end, int intuple, int dtkind)
{
volatile uint16_t *flags;
uint64_t val = *valp;
size_t valoffs = *valoffsp;
flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;
ASSERT(dtkind == DIF_TF_BYREF || dtkind == DIF_TF_BYUREF);
/*
* If this is a string, we're going to only load until we find the zero
* byte -- after which we'll store zero bytes.
*/
if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
char c = '\0' + 1;
size_t s;
for (s = 0; s < size; s++) {
if (c != '\0' && dtkind == DIF_TF_BYREF) {
c = dtrace_load8(val++);
} else if (c != '\0' && dtkind == DIF_TF_BYUREF) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
c = dtrace_fuword8((void *)(uintptr_t)val++);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
if (*flags & CPU_DTRACE_FAULT)
break;
}
DTRACE_STORE(uint8_t, tomax, valoffs++, c);
if (c == '\0' && intuple)
break;
}
} else {
uint8_t c;
while (valoffs < end) {
if (dtkind == DIF_TF_BYREF) {
c = dtrace_load8(val++);
} else if (dtkind == DIF_TF_BYUREF) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
c = dtrace_fuword8((void *)(uintptr_t)val++);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
if (*flags & CPU_DTRACE_FAULT)
break;
}
DTRACE_STORE(uint8_t, tomax,
valoffs++, c);
}
}
*valp = val;
*valoffsp = valoffs;
}
/*
* If you're looking for the epicenter of DTrace, you just found it. This
* is the function called by the provider to fire a probe -- from which all
* subsequent probe-context DTrace activity emanates.
*/
void
dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
{
processorid_t cpuid;
dtrace_icookie_t cookie;
dtrace_probe_t *probe;
dtrace_mstate_t mstate;
dtrace_ecb_t *ecb;
dtrace_action_t *act;
intptr_t offs;
size_t size;
int vtime, onintr;
volatile uint16_t *flags;
hrtime_t now;
if (panicstr != NULL)
return;
#ifdef illumos
/*
* Kick out immediately if this CPU is still being born (in which case
* curthread will be set to -1) or the current thread can't allow
* probes in its current context.
*/
if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE))
return;
#endif
cookie = dtrace_interrupt_disable();
probe = dtrace_probes[id - 1];
cpuid = curcpu;
onintr = CPU_ON_INTR(CPU);
if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE &&
probe->dtpr_predcache == curthread->t_predcache) {
/*
* We have hit in the predicate cache; we know that
* this predicate would evaluate to be false.
*/
dtrace_interrupt_enable(cookie);
return;
}
#ifdef illumos
if (panic_quiesce) {
#else
if (panicstr != NULL) {
#endif
/*
* We don't trace anything if we're panicking.
*/
dtrace_interrupt_enable(cookie);
return;
}
now = mstate.dtms_timestamp = dtrace_gethrtime();
mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP;
vtime = dtrace_vtime_references != 0;
if (vtime && curthread->t_dtrace_start)
curthread->t_dtrace_vtime += now - curthread->t_dtrace_start;
mstate.dtms_difo = NULL;
mstate.dtms_probe = probe;
mstate.dtms_strtok = 0;
mstate.dtms_arg[0] = arg0;
mstate.dtms_arg[1] = arg1;
mstate.dtms_arg[2] = arg2;
mstate.dtms_arg[3] = arg3;
mstate.dtms_arg[4] = arg4;
flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags;
for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
dtrace_predicate_t *pred = ecb->dte_predicate;
dtrace_state_t *state = ecb->dte_state;
dtrace_buffer_t *buf = &state->dts_buffer[cpuid];
dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid];
dtrace_vstate_t *vstate = &state->dts_vstate;
dtrace_provider_t *prov = probe->dtpr_provider;
uint64_t tracememsize = 0;
int committed = 0;
caddr_t tomax;
/*
* A little subtlety with the following (seemingly innocuous)
* declaration of the automatic 'val': by looking at the
* code, you might think that it could be declared in the
* action processing loop, below. (That is, it's only used in
* the action processing loop.) However, it must be declared
* out of that scope because in the case of DIF expression
* arguments to aggregating actions, one iteration of the
* action loop will use the last iteration's value.
*/
uint64_t val = 0;
mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE;
mstate.dtms_getf = NULL;
*flags &= ~CPU_DTRACE_ERROR;
if (prov == dtrace_provider) {
/*
* If dtrace itself is the provider of this probe,
* we're only going to continue processing the ECB if
* arg0 (the dtrace_state_t) is equal to the ECB's
* creating state. (This prevents disjoint consumers
* from seeing one another's metaprobes.)
*/
if (arg0 != (uint64_t)(uintptr_t)state)
continue;
}
if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) {
/*
* We're not currently active. If our provider isn't
* the dtrace pseudo provider, we're not interested.
*/
if (prov != dtrace_provider)
continue;
/*
* Now we must further check if we are in the BEGIN
* probe. If we are, we will only continue processing
* if we're still in WARMUP -- if one BEGIN enabling
* has invoked the exit() action, we don't want to
* evaluate subsequent BEGIN enablings.
*/
if (probe->dtpr_id == dtrace_probeid_begin &&
state->dts_activity != DTRACE_ACTIVITY_WARMUP) {
ASSERT(state->dts_activity ==
DTRACE_ACTIVITY_DRAINING);
continue;
}
}
if (ecb->dte_cond) {
/*
* If the dte_cond bits indicate that this
* consumer is only allowed to see user-mode firings
* of this probe, call the provider's dtps_usermode()
* entry point to check that the probe was fired
* while in a user context. Skip this ECB if that's
* not the case.
*/
if ((ecb->dte_cond & DTRACE_COND_USERMODE) &&
prov->dtpv_pops.dtps_usermode(prov->dtpv_arg,
probe->dtpr_id, probe->dtpr_arg) == 0)
continue;
#ifdef illumos
/*
* This is more subtle than it looks. We have to be
* absolutely certain that CRED() isn't going to
* change out from under us so it's only legit to
* examine that structure if we're in constrained
* situations. Currently, the only times we'll this
* check is if a non-super-user has enabled the
* profile or syscall providers -- providers that
* allow visibility of all processes. For the
* profile case, the check above will ensure that
* we're examining a user context.
*/
if (ecb->dte_cond & DTRACE_COND_OWNER) {
cred_t *cr;
cred_t *s_cr =
ecb->dte_state->dts_cred.dcr_cred;
proc_t *proc;
ASSERT(s_cr != NULL);
if ((cr = CRED()) == NULL ||
s_cr->cr_uid != cr->cr_uid ||
s_cr->cr_uid != cr->cr_ruid ||
s_cr->cr_uid != cr->cr_suid ||
s_cr->cr_gid != cr->cr_gid ||
s_cr->cr_gid != cr->cr_rgid ||
s_cr->cr_gid != cr->cr_sgid ||
(proc = ttoproc(curthread)) == NULL ||
(proc->p_flag & SNOCD))
continue;
}
if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {
cred_t *cr;
cred_t *s_cr =
ecb->dte_state->dts_cred.dcr_cred;
ASSERT(s_cr != NULL);
if ((cr = CRED()) == NULL ||
s_cr->cr_zone->zone_id !=
cr->cr_zone->zone_id)
continue;
}
#endif
}
if (now - state->dts_alive > dtrace_deadman_timeout) {
/*
* We seem to be dead. Unless we (a) have kernel
* destructive permissions (b) have explicitly enabled
* destructive actions and (c) destructive actions have
* not been disabled, we're going to transition into
* the KILLED state, from which no further processing
* on this state will be performed.
*/
if (!dtrace_priv_kernel_destructive(state) ||
!state->dts_cred.dcr_destructive ||
dtrace_destructive_disallow) {
void *activity = &state->dts_activity;
dtrace_activity_t current;
do {
current = state->dts_activity;
} while (dtrace_cas32(activity, current,
DTRACE_ACTIVITY_KILLED) != current);
continue;
}
}
if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,
ecb->dte_alignment, state, &mstate)) < 0)
continue;
tomax = buf->dtb_tomax;
ASSERT(tomax != NULL);
if (ecb->dte_size != 0) {
dtrace_rechdr_t dtrh;
if (!(mstate.dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
mstate.dtms_timestamp = dtrace_gethrtime();
mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP;
}
ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t));
dtrh.dtrh_epid = ecb->dte_epid;
DTRACE_RECORD_STORE_TIMESTAMP(&dtrh,
mstate.dtms_timestamp);
*((dtrace_rechdr_t *)(tomax + offs)) = dtrh;
}
mstate.dtms_epid = ecb->dte_epid;
mstate.dtms_present |= DTRACE_MSTATE_EPID;
if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)
mstate.dtms_access = DTRACE_ACCESS_KERNEL;
else
mstate.dtms_access = 0;
if (pred != NULL) {
dtrace_difo_t *dp = pred->dtp_difo;
int rval;
rval = dtrace_dif_emulate(dp, &mstate, vstate, state);
if (!(*flags & CPU_DTRACE_ERROR) && !rval) {
dtrace_cacheid_t cid = probe->dtpr_predcache;
if (cid != DTRACE_CACHEIDNONE && !onintr) {
/*
* Update the predicate cache...
*/
ASSERT(cid == pred->dtp_cacheid);
curthread->t_predcache = cid;
}
continue;
}
}
for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&
act != NULL; act = act->dta_next) {
size_t valoffs;
dtrace_difo_t *dp;
dtrace_recdesc_t *rec = &act->dta_rec;
size = rec->dtrd_size;
valoffs = offs + rec->dtrd_offset;
if (DTRACEACT_ISAGG(act->dta_kind)) {
uint64_t v = 0xbad;
dtrace_aggregation_t *agg;
agg = (dtrace_aggregation_t *)act;
if ((dp = act->dta_difo) != NULL)
v = dtrace_dif_emulate(dp,
&mstate, vstate, state);
if (*flags & CPU_DTRACE_ERROR)
continue;
/*
* Note that we always pass the expression
* value from the previous iteration of the
* action loop. This value will only be used
* if there is an expression argument to the
* aggregating action, denoted by the
* dtag_hasarg field.
*/
dtrace_aggregate(agg, buf,
offs, aggbuf, v, val);
continue;
}
switch (act->dta_kind) {
case DTRACEACT_STOP:
if (dtrace_priv_proc_destructive(state))
dtrace_action_stop();
continue;
case DTRACEACT_BREAKPOINT:
if (dtrace_priv_kernel_destructive(state))
dtrace_action_breakpoint(ecb);
continue;
case DTRACEACT_PANIC:
if (dtrace_priv_kernel_destructive(state))
dtrace_action_panic(ecb);
continue;
case DTRACEACT_STACK:
if (!dtrace_priv_kernel(state))
continue;
dtrace_getpcstack((pc_t *)(tomax + valoffs),
size / sizeof (pc_t), probe->dtpr_aframes,
DTRACE_ANCHORED(probe) ? NULL :
(uint32_t *)arg0);
continue;
case DTRACEACT_JSTACK:
case DTRACEACT_USTACK:
if (!dtrace_priv_proc(state))
continue;
/*
* See comment in DIF_VAR_PID.
*/
if (DTRACE_ANCHORED(mstate.dtms_probe) &&
CPU_ON_INTR(CPU)) {
int depth = DTRACE_USTACK_NFRAMES(
rec->dtrd_arg) + 1;
dtrace_bzero((void *)(tomax + valoffs),
DTRACE_USTACK_STRSIZE(rec->dtrd_arg)
+ depth * sizeof (uint64_t));
continue;
}
if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 &&
curproc->p_dtrace_helpers != NULL) {
/*
* This is the slow path -- we have
* allocated string space, and we're
* getting the stack of a process that
* has helpers. Call into a separate
* routine to perform this processing.
*/
dtrace_action_ustack(&mstate, state,
(uint64_t *)(tomax + valoffs),
rec->dtrd_arg);
continue;
}
DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
dtrace_getupcstack((uint64_t *)
(tomax + valoffs),
DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1);
DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
continue;
default:
break;
}
dp = act->dta_difo;
ASSERT(dp != NULL);
val = dtrace_dif_emulate(dp, &mstate, vstate, state);
if (*flags & CPU_DTRACE_ERROR)
continue;
switch (act->dta_kind) {
case DTRACEACT_SPECULATE: {
dtrace_rechdr_t *dtrh;
ASSERT(buf == &state->dts_buffer[cpuid]);
buf = dtrace_speculation_buffer(state,
cpuid, val);
if (buf == NULL) {
*flags |= CPU_DTRACE_DROP;
continue;
}
offs = dtrace_buffer_reserve(buf,
ecb->dte_needed, ecb->dte_alignment,
state, NULL);
if (offs < 0) {
*flags |= CPU_DTRACE_DROP;
continue;
}
tomax = buf->dtb_tomax;
ASSERT(tomax != NULL);
if (ecb->dte_size == 0)
continue;
ASSERT3U(ecb->dte_size, >=,
sizeof (dtrace_rechdr_t));
dtrh = ((void *)(tomax + offs));
dtrh->dtrh_epid = ecb->dte_epid;
/*
* When the speculation is committed, all of
* the records in the speculative buffer will
* have their timestamps set to the commit
* time. Until then, it is set to a sentinel
* value, for debugability.
*/
DTRACE_RECORD_STORE_TIMESTAMP(dtrh, UINT64_MAX);
continue;
}
case DTRACEACT_PRINTM: {
/* The DIF returns a 'memref'. */
uintptr_t *memref = (uintptr_t *)(uintptr_t) val;
/* Get the size from the memref. */
size = memref[1];
/*
* Check if the size exceeds the allocated
* buffer size.
*/
if (size + sizeof(uintptr_t) > dp->dtdo_rtype.dtdt_size) {
/* Flag a drop! */
*flags |= CPU_DTRACE_DROP;
continue;
}
/* Store the size in the buffer first. */
DTRACE_STORE(uintptr_t, tomax,
valoffs, size);
/*
* Offset the buffer address to the start
* of the data.
*/
valoffs += sizeof(uintptr_t);
/*
* Reset to the memory address rather than
* the memref array, then let the BYREF
* code below do the work to store the
* memory data in the buffer.
*/
val = memref[0];
break;
}
case DTRACEACT_PRINTT: {
/* The DIF returns a 'typeref'. */
uintptr_t *typeref = (uintptr_t *)(uintptr_t) val;
char c = '\0' + 1;
size_t s;
/*
* Get the type string length and round it
* up so that the data that follows is
* aligned for easy access.
*/
size_t typs = strlen((char *) typeref[2]) + 1;
typs = roundup(typs, sizeof(uintptr_t));
/*
*Get the size from the typeref using the
* number of elements and the type size.
*/
size = typeref[1] * typeref[3];
/*
* Check if the size exceeds the allocated
* buffer size.
*/
if (size + typs + 2 * sizeof(uintptr_t) > dp->dtdo_rtype.dtdt_size) {
/* Flag a drop! */
*flags |= CPU_DTRACE_DROP;
}
/* Store the size in the buffer first. */
DTRACE_STORE(uintptr_t, tomax,
valoffs, size);
valoffs += sizeof(uintptr_t);
/* Store the type size in the buffer. */
DTRACE_STORE(uintptr_t, tomax,
valoffs, typeref[3]);
valoffs += sizeof(uintptr_t);
val = typeref[2];
for (s = 0; s < typs; s++) {
if (c != '\0')
c = dtrace_load8(val++);
DTRACE_STORE(uint8_t, tomax,
valoffs++, c);
}
/*
* Reset to the memory address rather than
* the typeref array, then let the BYREF
* code below do the work to store the
* memory data in the buffer.
*/
val = typeref[0];
break;
}
case DTRACEACT_CHILL:
if (dtrace_priv_kernel_destructive(state))
dtrace_action_chill(&mstate, val);
continue;
case DTRACEACT_RAISE:
if (dtrace_priv_proc_destructive(state))
dtrace_action_raise(val);
continue;
case DTRACEACT_COMMIT:
ASSERT(!committed);
/*
* We need to commit our buffer state.
*/
if (ecb->dte_size)
buf->dtb_offset = offs + ecb->dte_size;
buf = &state->dts_buffer[cpuid];
dtrace_speculation_commit(state, cpuid, val);
committed = 1;
continue;
case DTRACEACT_DISCARD:
dtrace_speculation_discard(state, cpuid, val);
continue;
case DTRACEACT_DIFEXPR:
case DTRACEACT_LIBACT:
case DTRACEACT_PRINTF:
case DTRACEACT_PRINTA:
case DTRACEACT_SYSTEM:
case DTRACEACT_FREOPEN:
case DTRACEACT_TRACEMEM:
break;
case DTRACEACT_TRACEMEM_DYNSIZE:
tracememsize = val;
break;
case DTRACEACT_SYM:
case DTRACEACT_MOD:
if (!dtrace_priv_kernel(state))
continue;
break;
case DTRACEACT_USYM:
case DTRACEACT_UMOD:
case DTRACEACT_UADDR: {
#ifdef illumos
struct pid *pid = curthread->t_procp->p_pidp;
#endif
if (!dtrace_priv_proc(state))
continue;
DTRACE_STORE(uint64_t, tomax,
#ifdef illumos
valoffs, (uint64_t)pid->pid_id);
#else
valoffs, (uint64_t) curproc->p_pid);
#endif
DTRACE_STORE(uint64_t, tomax,
valoffs + sizeof (uint64_t), val);
continue;
}
case DTRACEACT_EXIT: {
/*
* For the exit action, we are going to attempt
* to atomically set our activity to be
* draining. If this fails (either because
* another CPU has beat us to the exit action,
* or because our current activity is something
* other than ACTIVE or WARMUP), we will
* continue. This assures that the exit action
* can be successfully recorded at most once
* when we're in the ACTIVE state. If we're
* encountering the exit() action while in
* COOLDOWN, however, we want to honor the new
* status code. (We know that we're the only
* thread in COOLDOWN, so there is no race.)
*/
void *activity = &state->dts_activity;
dtrace_activity_t current = state->dts_activity;
if (current == DTRACE_ACTIVITY_COOLDOWN)
break;
if (current != DTRACE_ACTIVITY_WARMUP)
current = DTRACE_ACTIVITY_ACTIVE;
if (dtrace_cas32(activity, current,
DTRACE_ACTIVITY_DRAINING) != current) {
*flags |= CPU_DTRACE_DROP;
continue;
}
break;
}
default:
ASSERT(0);
}
if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ||
dp->dtdo_rtype.dtdt_flags & DIF_TF_BYUREF) {
uintptr_t end = valoffs + size;
if (tracememsize != 0 &&
valoffs + tracememsize < end) {
end = valoffs + tracememsize;
tracememsize = 0;
}
if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF &&
!dtrace_vcanload((void *)(uintptr_t)val,
&dp->dtdo_rtype, &mstate, vstate))
continue;
dtrace_store_by_ref(dp, tomax, size, &valoffs,
&val, end, act->dta_intuple,
dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ?
DIF_TF_BYREF: DIF_TF_BYUREF);
continue;
}
switch (size) {
case 0:
break;
case sizeof (uint8_t):
DTRACE_STORE(uint8_t, tomax, valoffs, val);
break;
case sizeof (uint16_t):
DTRACE_STORE(uint16_t, tomax, valoffs, val);
break;
case sizeof (uint32_t):
DTRACE_STORE(uint32_t, tomax, valoffs, val);
break;
case sizeof (uint64_t):
DTRACE_STORE(uint64_t, tomax, valoffs, val);
break;
default:
/*
* Any other size should have been returned by
* reference, not by value.
*/
ASSERT(0);
break;
}
}
if (*flags & CPU_DTRACE_DROP)
continue;
if (*flags & CPU_DTRACE_FAULT) {
int ndx;
dtrace_action_t *err;
buf->dtb_errors++;
if (probe->dtpr_id == dtrace_probeid_error) {
/*
* There's nothing we can do -- we had an
* error on the error probe. We bump an
* error counter to at least indicate that
* this condition happened.
*/
dtrace_error(&state->dts_dblerrors);
continue;
}
if (vtime) {
/*
* Before recursing on dtrace_probe(), we
* need to explicitly clear out our start
* time to prevent it from being accumulated
* into t_dtrace_vtime.
*/
curthread->t_dtrace_start = 0;
}
/*
* Iterate over the actions to figure out which action
* we were processing when we experienced the error.
* Note that act points _past_ the faulting action; if
* act is ecb->dte_action, the fault was in the
* predicate, if it's ecb->dte_action->dta_next it's
* in action #1, and so on.
*/
for (err = ecb->dte_action, ndx = 0;
err != act; err = err->dta_next, ndx++)
continue;
dtrace_probe_error(state, ecb->dte_epid, ndx,
(mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ?
mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags),
cpu_core[cpuid].cpuc_dtrace_illval);
continue;
}
if (!committed)
buf->dtb_offset = offs + ecb->dte_size;
}
if (vtime)
curthread->t_dtrace_start = dtrace_gethrtime();
dtrace_interrupt_enable(cookie);
}
/*
* DTrace Probe Hashing Functions
*
* The functions in this section (and indeed, the functions in remaining
* sections) are not _called_ from probe context. (Any exceptions to this are
* marked with a "Note:".) Rather, they are called from elsewhere in the
* DTrace framework to look-up probes in, add probes to and remove probes from
* the DTrace probe hashes. (Each probe is hashed by each element of the
* probe tuple -- allowing for fast lookups, regardless of what was
* specified.)
*/
static uint_t
dtrace_hash_str(const char *p)
{
unsigned int g;
uint_t hval = 0;
while (*p) {
hval = (hval << 4) + *p++;
if ((g = (hval & 0xf0000000)) != 0)
hval ^= g >> 24;
hval &= ~g;
}
return (hval);
}
static dtrace_hash_t *
dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs)
{
dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP);
hash->dth_stroffs = stroffs;
hash->dth_nextoffs = nextoffs;
hash->dth_prevoffs = prevoffs;
hash->dth_size = 1;
hash->dth_mask = hash->dth_size - 1;
hash->dth_tab = kmem_zalloc(hash->dth_size *
sizeof (dtrace_hashbucket_t *), KM_SLEEP);
return (hash);
}
static void
dtrace_hash_destroy(dtrace_hash_t *hash)
{
#ifdef DEBUG
int i;
for (i = 0; i < hash->dth_size; i++)
ASSERT(hash->dth_tab[i] == NULL);
#endif
kmem_free(hash->dth_tab,
hash->dth_size * sizeof (dtrace_hashbucket_t *));
kmem_free(hash, sizeof (dtrace_hash_t));
}
static void
dtrace_hash_resize(dtrace_hash_t *hash)
{
int size = hash->dth_size, i, ndx;
int new_size = hash->dth_size << 1;
int new_mask = new_size - 1;
dtrace_hashbucket_t **new_tab, *bucket, *next;
ASSERT((new_size & new_mask) == 0);
new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP);
for (i = 0; i < size; i++) {
for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) {
dtrace_probe_t *probe = bucket->dthb_chain;
ASSERT(probe != NULL);
ndx = DTRACE_HASHSTR(hash, probe) & new_mask;
next = bucket->dthb_next;
bucket->dthb_next = new_tab[ndx];
new_tab[ndx] = bucket;
}
}
kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *));
hash->dth_tab = new_tab;
hash->dth_size = new_size;
hash->dth_mask = new_mask;
}
static void
dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new)
{
int hashval = DTRACE_HASHSTR(hash, new);
int ndx = hashval & hash->dth_mask;
dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
dtrace_probe_t **nextp, **prevp;
for (; bucket != NULL; bucket = bucket->dthb_next) {
if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new))
goto add;
}
if ((hash->dth_nbuckets >> 1) > hash->dth_size) {
dtrace_hash_resize(hash);
dtrace_hash_add(hash, new);
return;
}
bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP);
bucket->dthb_next = hash->dth_tab[ndx];
hash->dth_tab[ndx] = bucket;
hash->dth_nbuckets++;
add:
nextp = DTRACE_HASHNEXT(hash, new);
ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL);
*nextp = bucket->dthb_chain;
if (bucket->dthb_chain != NULL) {
prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain);
ASSERT(*prevp == NULL);
*prevp = new;
}
bucket->dthb_chain = new;
bucket->dthb_len++;
}
static dtrace_probe_t *
dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template)
{
int hashval = DTRACE_HASHSTR(hash, template);
int ndx = hashval & hash->dth_mask;
dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
for (; bucket != NULL; bucket = bucket->dthb_next) {
if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
return (bucket->dthb_chain);
}
return (NULL);
}
static int
dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template)
{
int hashval = DTRACE_HASHSTR(hash, template);
int ndx = hashval & hash->dth_mask;
dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
for (; bucket != NULL; bucket = bucket->dthb_next) {
if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
return (bucket->dthb_len);
}
return (0);
}
static void
dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe)
{
int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask;
dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe);
dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe);
/*
* Find the bucket that we're removing this probe from.
*/
for (; bucket != NULL; bucket = bucket->dthb_next) {
if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe))
break;
}
ASSERT(bucket != NULL);
if (*prevp == NULL) {
if (*nextp == NULL) {
/*
* The removed probe was the only probe on this
* bucket; we need to remove the bucket.
*/
dtrace_hashbucket_t *b = hash->dth_tab[ndx];
ASSERT(bucket->dthb_chain == probe);
ASSERT(b != NULL);
if (b == bucket) {
hash->dth_tab[ndx] = bucket->dthb_next;
} else {
while (b->dthb_next != bucket)
b = b->dthb_next;
b->dthb_next = bucket->dthb_next;
}
ASSERT(hash->dth_nbuckets > 0);
hash->dth_nbuckets--;
kmem_free(bucket, sizeof (dtrace_hashbucket_t));
return;
}
bucket->dthb_chain = *nextp;
} else {
*(DTRACE_HASHNEXT(hash, *prevp)) = *nextp;
}
if (*nextp != NULL)
*(DTRACE_HASHPREV(hash, *nextp)) = *prevp;
}
/*
* DTrace Utility Functions
*
* These are random utility functions that are _not_ called from probe context.
*/
static int
dtrace_badattr(const dtrace_attribute_t *a)
{
return (a->dtat_name > DTRACE_STABILITY_MAX ||
a->dtat_data > DTRACE_STABILITY_MAX ||
a->dtat_class > DTRACE_CLASS_MAX);
}
/*
* Return a duplicate copy of a string. If the specified string is NULL,
* this function returns a zero-length string.
*/
static char *
dtrace_strdup(const char *str)
{
char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP);
if (str != NULL)
(void) strcpy(new, str);
return (new);
}
#define DTRACE_ISALPHA(c) \
(((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
static int
dtrace_badname(const char *s)
{
char c;
if (s == NULL || (c = *s++) == '\0')
return (0);
if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.')
return (1);
while ((c = *s++) != '\0') {
if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') &&
c != '-' && c != '_' && c != '.' && c != '`')
return (1);
}
return (0);
}
static void
dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp)
{
uint32_t priv;
#ifdef illumos
if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
/*
* For DTRACE_PRIV_ALL, the uid and zoneid don't matter.
*/
priv = DTRACE_PRIV_ALL;
} else {
*uidp = crgetuid(cr);
*zoneidp = crgetzoneid(cr);
priv = 0;
if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE))
priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER;
else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE))
priv |= DTRACE_PRIV_USER;
if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE))
priv |= DTRACE_PRIV_PROC;
if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
priv |= DTRACE_PRIV_OWNER;
if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
priv |= DTRACE_PRIV_ZONEOWNER;
}
#else
priv = DTRACE_PRIV_ALL;
#endif
*privp = priv;
}
#ifdef DTRACE_ERRDEBUG
static void
dtrace_errdebug(const char *str)
{
int hval = dtrace_hash_str(str) % DTRACE_ERRHASHSZ;
int occupied = 0;
mutex_enter(&dtrace_errlock);
dtrace_errlast = str;
dtrace_errthread = curthread;
while (occupied++ < DTRACE_ERRHASHSZ) {
if (dtrace_errhash[hval].dter_msg == str) {
dtrace_errhash[hval].dter_count++;
goto out;
}
if (dtrace_errhash[hval].dter_msg != NULL) {
hval = (hval + 1) % DTRACE_ERRHASHSZ;
continue;
}
dtrace_errhash[hval].dter_msg = str;
dtrace_errhash[hval].dter_count = 1;
goto out;
}
panic("dtrace: undersized error hash");
out:
mutex_exit(&dtrace_errlock);
}
#endif
/*
* DTrace Matching Functions
*
* These functions are used to match groups of probes, given some elements of
* a probe tuple, or some globbed expressions for elements of a probe tuple.
*/
static int
dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid,
zoneid_t zoneid)
{
if (priv != DTRACE_PRIV_ALL) {
uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags;
uint32_t match = priv & ppriv;
/*
* No PRIV_DTRACE_* privileges...
*/
if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER |
DTRACE_PRIV_KERNEL)) == 0)
return (0);
/*
* No matching bits, but there were bits to match...
*/
if (match == 0 && ppriv != 0)
return (0);
/*
* Need to have permissions to the process, but don't...
*/
if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 &&
uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) {
return (0);
}
/*
* Need to be in the same zone unless we possess the
* privilege to examine all zones.
*/
if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 &&
zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) {
return (0);
}
}
return (1);
}
/*
* dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which
* consists of input pattern strings and an ops-vector to evaluate them.
* This function returns >0 for match, 0 for no match, and <0 for error.
*/
static int
dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp,
uint32_t priv, uid_t uid, zoneid_t zoneid)
{
dtrace_provider_t *pvp = prp->dtpr_provider;
int rv;
if (pvp->dtpv_defunct)
return (0);
if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0)
return (rv);
if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0)
return (rv);
if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0)
return (rv);
if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0)
return (rv);
if (dtrace_match_priv(prp, priv, uid, zoneid) == 0)
return (0);
return (rv);
}
/*
* dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN)
* interface for matching a glob pattern 'p' to an input string 's'. Unlike
* libc's version, the kernel version only applies to 8-bit ASCII strings.
* In addition, all of the recursion cases except for '*' matching have been
* unwound. For '*', we still implement recursive evaluation, but a depth
* counter is maintained and matching is aborted if we recurse too deep.
* The function returns 0 if no match, >0 if match, and <0 if recursion error.
*/
static int
dtrace_match_glob(const char *s, const char *p, int depth)
{
const char *olds;
char s1, c;
int gs;
if (depth > DTRACE_PROBEKEY_MAXDEPTH)
return (-1);
if (s == NULL)
s = ""; /* treat NULL as empty string */
top:
olds = s;
s1 = *s++;
if (p == NULL)
return (0);
if ((c = *p++) == '\0')
return (s1 == '\0');
switch (c) {
case '[': {
int ok = 0, notflag = 0;
char lc = '\0';
if (s1 == '\0')
return (0);
if (*p == '!') {
notflag = 1;
p++;
}
if ((c = *p++) == '\0')
return (0);
do {
if (c == '-' && lc != '\0' && *p != ']') {
if ((c = *p++) == '\0')
return (0);
if (c == '\\' && (c = *p++) == '\0')
return (0);
if (notflag) {
if (s1 < lc || s1 > c)
ok++;
else
return (0);
} else if (lc <= s1 && s1 <= c)
ok++;
} else if (c == '\\' && (c = *p++) == '\0')
return (0);
lc = c; /* save left-hand 'c' for next iteration */
if (notflag) {
if (s1 != c)
ok++;
else
return (0);
} else if (s1 == c)
ok++;
if ((c = *p++) == '\0')
return (0);
} while (c != ']');
if (ok)
goto top;
return (0);
}
case '\\':
if ((c = *p++) == '\0')
return (0);
/*FALLTHRU*/
default:
if (c != s1)
return (0);
/*FALLTHRU*/
case '?':
if (s1 != '\0')
goto top;
return (0);
case '*':
while (*p == '*')
p++; /* consecutive *'s are identical to a single one */
if (*p == '\0')
return (1);
for (s = olds; *s != '\0'; s++) {
if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0)
return (gs);
}
return (0);
}
}
/*ARGSUSED*/
static int
dtrace_match_string(const char *s, const char *p, int depth)
{
return (s != NULL && strcmp(s, p) == 0);
}
/*ARGSUSED*/
static int
dtrace_match_nul(const char *s, const char *p, int depth)
{
return (1); /* always match the empty pattern */
}
/*ARGSUSED*/
static int
dtrace_match_nonzero(const char *s, const char *p, int depth)
{
return (s != NULL && s[0] != '\0');
}
static int
dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg)
{
dtrace_probe_t template, *probe;
dtrace_hash_t *hash = NULL;
int len, best = INT_MAX, nmatched = 0;
dtrace_id_t i;
ASSERT(MUTEX_HELD(&dtrace_lock));
/*
* If the probe ID is specified in the key, just lookup by ID and
* invoke the match callback once if a matching probe is found.
*/
if (pkp->dtpk_id != DTRACE_IDNONE) {
if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL &&
dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) {
(void) (*matched)(probe, arg);
nmatched++;
}
return (nmatched);
}
template.dtpr_mod = (char *)pkp->dtpk_mod;
template.dtpr_func = (char *)pkp->dtpk_func;
template.dtpr_name = (char *)pkp->dtpk_name;
/*
* We want to find the most distinct of the module name, function
* name, and name. So for each one that is not a glob pattern or
* empty string, we perform a lookup in the corresponding hash and
* use the hash table with the fewest collisions to do our search.
*/
if (pkp->dtpk_mmatch == &dtrace_match_string &&
(len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) {
best = len;
hash = dtrace_bymod;
}
if (pkp->dtpk_fmatch == &dtrace_match_string &&
(len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) {
best = len;
hash = dtrace_byfunc;
}
if (pkp->dtpk_nmatch == &dtrace_match_string &&
(len = dtrace_hash_collisions(dtrace_byname, &template)) < best) {
best = len;
hash = dtrace_byname;
}
/*
* If we did not select a hash table, iterate over every probe and
* invoke our callback for each one that matches our input probe key.
*/
if (hash == NULL) {
for (i = 0; i < dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i]) == NULL ||
dtrace_match_probe(probe, pkp, priv, uid,
zoneid) <= 0)
continue;
nmatched++;
if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT)
break;
}
return (nmatched);
}
/*
* If we selected a hash table, iterate over each probe of the same key
* name and invoke the callback for every probe that matches the other
* attributes of our input probe key.
*/
for (probe = dtrace_hash_lookup(hash, &template); probe != NULL;
probe = *(DTRACE_HASHNEXT(hash, probe))) {
if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0)
continue;
nmatched++;
if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT)
break;
}
return (nmatched);
}
/*
* Return the function pointer dtrace_probecmp() should use to compare the
* specified pattern with a string. For NULL or empty patterns, we select
* dtrace_match_nul(). For glob pattern strings, we use dtrace_match_glob().
* For non-empty non-glob strings, we use dtrace_match_string().
*/
static dtrace_probekey_f *
dtrace_probekey_func(const char *p)
{
char c;
if (p == NULL || *p == '\0')
return (&dtrace_match_nul);
while ((c = *p++) != '\0') {
if (c == '[' || c == '?' || c == '*' || c == '\\')
return (&dtrace_match_glob);
}
return (&dtrace_match_string);
}
/*
* Build a probe comparison key for use with dtrace_match_probe() from the
* given probe description. By convention, a null key only matches anchored
* probes: if each field is the empty string, reset dtpk_fmatch to
* dtrace_match_nonzero().
*/
static void
dtrace_probekey(dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp)
{
pkp->dtpk_prov = pdp->dtpd_provider;
pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider);
pkp->dtpk_mod = pdp->dtpd_mod;
pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod);
pkp->dtpk_func = pdp->dtpd_func;
pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func);
pkp->dtpk_name = pdp->dtpd_name;
pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name);
pkp->dtpk_id = pdp->dtpd_id;
if (pkp->dtpk_id == DTRACE_IDNONE &&
pkp->dtpk_pmatch == &dtrace_match_nul &&
pkp->dtpk_mmatch == &dtrace_match_nul &&
pkp->dtpk_fmatch == &dtrace_match_nul &&
pkp->dtpk_nmatch == &dtrace_match_nul)
pkp->dtpk_fmatch = &dtrace_match_nonzero;
}
/*
* DTrace Provider-to-Framework API Functions
*
* These functions implement much of the Provider-to-Framework API, as
* described in <sys/dtrace.h>. The parts of the API not in this section are
* the functions in the API for probe management (found below), and
* dtrace_probe() itself (found above).
*/
/*
* Register the calling provider with the DTrace framework. This should
* generally be called by DTrace providers in their attach(9E) entry point.
*/
int
dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv,
cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp)
{
dtrace_provider_t *provider;
if (name == NULL || pap == NULL || pops == NULL || idp == NULL) {
cmn_err(CE_WARN, "failed to register provider '%s': invalid "
"arguments", name ? name : "<NULL>");
return (EINVAL);
}
if (name[0] == '\0' || dtrace_badname(name)) {
cmn_err(CE_WARN, "failed to register provider '%s': invalid "
"provider name", name);
return (EINVAL);
}
if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) ||
pops->dtps_enable == NULL || pops->dtps_disable == NULL ||
pops->dtps_destroy == NULL ||
((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) {
cmn_err(CE_WARN, "failed to register provider '%s': invalid "
"provider ops", name);
return (EINVAL);
}
if (dtrace_badattr(&pap->dtpa_provider) ||
dtrace_badattr(&pap->dtpa_mod) ||
dtrace_badattr(&pap->dtpa_func) ||
dtrace_badattr(&pap->dtpa_name) ||
dtrace_badattr(&pap->dtpa_args)) {
cmn_err(CE_WARN, "failed to register provider '%s': invalid "
"provider attributes", name);
return (EINVAL);
}
if (priv & ~DTRACE_PRIV_ALL) {
cmn_err(CE_WARN, "failed to register provider '%s': invalid "
"privilege attributes", name);
return (EINVAL);
}
if ((priv & DTRACE_PRIV_KERNEL) &&
(priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) &&
pops->dtps_usermode == NULL) {
cmn_err(CE_WARN, "failed to register provider '%s': need "
"dtps_usermode() op for given privilege attributes", name);
return (EINVAL);
}
provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP);
provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
(void) strcpy(provider->dtpv_name, name);
provider->dtpv_attr = *pap;
provider->dtpv_priv.dtpp_flags = priv;
if (cr != NULL) {
provider->dtpv_priv.dtpp_uid = crgetuid(cr);
provider->dtpv_priv.dtpp_zoneid = crgetzoneid(cr);
}
provider->dtpv_pops = *pops;
if (pops->dtps_provide == NULL) {
ASSERT(pops->dtps_provide_module != NULL);
provider->dtpv_pops.dtps_provide =
(void (*)(void *, dtrace_probedesc_t *))dtrace_nullop;
}
if (pops->dtps_provide_module == NULL) {
ASSERT(pops->dtps_provide != NULL);
provider->dtpv_pops.dtps_provide_module =
(void (*)(void *, modctl_t *))dtrace_nullop;
}
if (pops->dtps_suspend == NULL) {
ASSERT(pops->dtps_resume == NULL);
provider->dtpv_pops.dtps_suspend =
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
provider->dtpv_pops.dtps_resume =
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
}
provider->dtpv_arg = arg;
*idp = (dtrace_provider_id_t)provider;
if (pops == &dtrace_provider_ops) {
ASSERT(MUTEX_HELD(&dtrace_provider_lock));
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(dtrace_anon.dta_enabling == NULL);
/*
* We make sure that the DTrace provider is at the head of
* the provider chain.
*/
provider->dtpv_next = dtrace_provider;
dtrace_provider = provider;
return (0);
}
mutex_enter(&dtrace_provider_lock);
mutex_enter(&dtrace_lock);
/*
* If there is at least one provider registered, we'll add this
* provider after the first provider.
*/
if (dtrace_provider != NULL) {
provider->dtpv_next = dtrace_provider->dtpv_next;
dtrace_provider->dtpv_next = provider;
} else {
dtrace_provider = provider;
}
if (dtrace_retained != NULL) {
dtrace_enabling_provide(provider);
/*
* Now we need to call dtrace_enabling_matchall() -- which
* will acquire cpu_lock and dtrace_lock. We therefore need
* to drop all of our locks before calling into it...
*/
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_provider_lock);
dtrace_enabling_matchall();
return (0);
}
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_provider_lock);
return (0);
}
/*
* Unregister the specified provider from the DTrace framework. This should
* generally be called by DTrace providers in their detach(9E) entry point.
*/
int
dtrace_unregister(dtrace_provider_id_t id)
{
dtrace_provider_t *old = (dtrace_provider_t *)id;
dtrace_provider_t *prev = NULL;
int i, self = 0, noreap = 0;
dtrace_probe_t *probe, *first = NULL;
if (old->dtpv_pops.dtps_enable ==
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop) {
/*
* If DTrace itself is the provider, we're called with locks
* already held.
*/
ASSERT(old == dtrace_provider);
#ifdef illumos
ASSERT(dtrace_devi != NULL);
#endif
ASSERT(MUTEX_HELD(&dtrace_provider_lock));
ASSERT(MUTEX_HELD(&dtrace_lock));
self = 1;
if (dtrace_provider->dtpv_next != NULL) {
/*
* There's another provider here; return failure.
*/
return (EBUSY);
}
} else {
mutex_enter(&dtrace_provider_lock);
#ifdef illumos
mutex_enter(&mod_lock);
#endif
mutex_enter(&dtrace_lock);
}
/*
* If anyone has /dev/dtrace open, or if there are anonymous enabled
* probes, we refuse to let providers slither away, unless this
* provider has already been explicitly invalidated.
*/
if (!old->dtpv_defunct &&
(dtrace_opens || (dtrace_anon.dta_state != NULL &&
dtrace_anon.dta_state->dts_necbs > 0))) {
if (!self) {
mutex_exit(&dtrace_lock);
#ifdef illumos
mutex_exit(&mod_lock);
#endif
mutex_exit(&dtrace_provider_lock);
}
return (EBUSY);
}
/*
* Attempt to destroy the probes associated with this provider.
*/
for (i = 0; i < dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i]) == NULL)
continue;
if (probe->dtpr_provider != old)
continue;
if (probe->dtpr_ecb == NULL)
continue;
/*
* If we are trying to unregister a defunct provider, and the
* provider was made defunct within the interval dictated by
* dtrace_unregister_defunct_reap, we'll (asynchronously)
* attempt to reap our enablings. To denote that the provider
* should reattempt to unregister itself at some point in the
* future, we will return a differentiable error code (EAGAIN
* instead of EBUSY) in this case.
*/
if (dtrace_gethrtime() - old->dtpv_defunct >
dtrace_unregister_defunct_reap)
noreap = 1;
if (!self) {
mutex_exit(&dtrace_lock);
#ifdef illumos
mutex_exit(&mod_lock);
#endif
mutex_exit(&dtrace_provider_lock);
}
if (noreap)
return (EBUSY);
(void) taskq_dispatch(dtrace_taskq,
(task_func_t *)dtrace_enabling_reap, NULL, TQ_SLEEP);
return (EAGAIN);
}
/*
* All of the probes for this provider are disabled; we can safely
* remove all of them from their hash chains and from the probe array.
*/
for (i = 0; i < dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i]) == NULL)
continue;
if (probe->dtpr_provider != old)
continue;
dtrace_probes[i] = NULL;
dtrace_hash_remove(dtrace_bymod, probe);
dtrace_hash_remove(dtrace_byfunc, probe);
dtrace_hash_remove(dtrace_byname, probe);
if (first == NULL) {
first = probe;
probe->dtpr_nextmod = NULL;
} else {
probe->dtpr_nextmod = first;
first = probe;
}
}
/*
* The provider's probes have been removed from the hash chains and
* from the probe array. Now issue a dtrace_sync() to be sure that
* everyone has cleared out from any probe array processing.
*/
dtrace_sync();
for (probe = first; probe != NULL; probe = first) {
first = probe->dtpr_nextmod;
old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id,
probe->dtpr_arg);
kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
#ifdef illumos
vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1);
#else
free_unr(dtrace_arena, probe->dtpr_id);
#endif
kmem_free(probe, sizeof (dtrace_probe_t));
}
if ((prev = dtrace_provider) == old) {
#ifdef illumos
ASSERT(self || dtrace_devi == NULL);
ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL);
#endif
dtrace_provider = old->dtpv_next;
} else {
while (prev != NULL && prev->dtpv_next != old)
prev = prev->dtpv_next;
if (prev == NULL) {
panic("attempt to unregister non-existent "
"dtrace provider %p\n", (void *)id);
}
prev->dtpv_next = old->dtpv_next;
}
if (!self) {
mutex_exit(&dtrace_lock);
#ifdef illumos
mutex_exit(&mod_lock);
#endif
mutex_exit(&dtrace_provider_lock);
}
kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1);
kmem_free(old, sizeof (dtrace_provider_t));
return (0);
}
/*
* Invalidate the specified provider. All subsequent probe lookups for the
* specified provider will fail, but its probes will not be removed.
*/
void
dtrace_invalidate(dtrace_provider_id_t id)
{
dtrace_provider_t *pvp = (dtrace_provider_t *)id;
ASSERT(pvp->dtpv_pops.dtps_enable !=
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop);
mutex_enter(&dtrace_provider_lock);
mutex_enter(&dtrace_lock);
pvp->dtpv_defunct = dtrace_gethrtime();
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_provider_lock);
}
/*
* Indicate whether or not DTrace has attached.
*/
int
dtrace_attached(void)
{
/*
* dtrace_provider will be non-NULL iff the DTrace driver has
* attached. (It's non-NULL because DTrace is always itself a
* provider.)
*/
return (dtrace_provider != NULL);
}
/*
* Remove all the unenabled probes for the given provider. This function is
* not unlike dtrace_unregister(), except that it doesn't remove the provider
* -- just as many of its associated probes as it can.
*/
int
dtrace_condense(dtrace_provider_id_t id)
{
dtrace_provider_t *prov = (dtrace_provider_t *)id;
int i;
dtrace_probe_t *probe;
/*
* Make sure this isn't the dtrace provider itself.
*/
ASSERT(prov->dtpv_pops.dtps_enable !=
(void (*)(void *, dtrace_id_t, void *))dtrace_nullop);
mutex_enter(&dtrace_provider_lock);
mutex_enter(&dtrace_lock);
/*
* Attempt to destroy the probes associated with this provider.
*/
for (i = 0; i < dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i]) == NULL)
continue;
if (probe->dtpr_provider != prov)
continue;
if (probe->dtpr_ecb != NULL)
continue;
dtrace_probes[i] = NULL;
dtrace_hash_remove(dtrace_bymod, probe);
dtrace_hash_remove(dtrace_byfunc, probe);
dtrace_hash_remove(dtrace_byname, probe);
prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1,
probe->dtpr_arg);
kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
kmem_free(probe, sizeof (dtrace_probe_t));
#ifdef illumos
vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1);
#else
free_unr(dtrace_arena, i + 1);
#endif
}
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_provider_lock);
return (0);
}
/*
* DTrace Probe Management Functions
*
* The functions in this section perform the DTrace probe management,
* including functions to create probes, look-up probes, and call into the
* providers to request that probes be provided. Some of these functions are
* in the Provider-to-Framework API; these functions can be identified by the
* fact that they are not declared "static".
*/
/*
* Create a probe with the specified module name, function name, and name.
*/
dtrace_id_t
dtrace_probe_create(dtrace_provider_id_t prov, const char *mod,
const char *func, const char *name, int aframes, void *arg)
{
dtrace_probe_t *probe, **probes;
dtrace_provider_t *provider = (dtrace_provider_t *)prov;
dtrace_id_t id;
if (provider == dtrace_provider) {
ASSERT(MUTEX_HELD(&dtrace_lock));
} else {
mutex_enter(&dtrace_lock);
}
#ifdef illumos
id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1,
VM_BESTFIT | VM_SLEEP);
#else
id = alloc_unr(dtrace_arena);
#endif
probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP);
probe->dtpr_id = id;
probe->dtpr_gen = dtrace_probegen++;
probe->dtpr_mod = dtrace_strdup(mod);
probe->dtpr_func = dtrace_strdup(func);
probe->dtpr_name = dtrace_strdup(name);
probe->dtpr_arg = arg;
probe->dtpr_aframes = aframes;
probe->dtpr_provider = provider;
dtrace_hash_add(dtrace_bymod, probe);
dtrace_hash_add(dtrace_byfunc, probe);
dtrace_hash_add(dtrace_byname, probe);
if (id - 1 >= dtrace_nprobes) {
size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *);
size_t nsize = osize << 1;
if (nsize == 0) {
ASSERT(osize == 0);
ASSERT(dtrace_probes == NULL);
nsize = sizeof (dtrace_probe_t *);
}
probes = kmem_zalloc(nsize, KM_SLEEP);
if (dtrace_probes == NULL) {
ASSERT(osize == 0);
dtrace_probes = probes;
dtrace_nprobes = 1;
} else {
dtrace_probe_t **oprobes = dtrace_probes;
bcopy(oprobes, probes, osize);
dtrace_membar_producer();
dtrace_probes = probes;
dtrace_sync();
/*
* All CPUs are now seeing the new probes array; we can
* safely free the old array.
*/
kmem_free(oprobes, osize);
dtrace_nprobes <<= 1;
}
ASSERT(id - 1 < dtrace_nprobes);
}
ASSERT(dtrace_probes[id - 1] == NULL);
dtrace_probes[id - 1] = probe;
if (provider != dtrace_provider)
mutex_exit(&dtrace_lock);
return (id);
}
static dtrace_probe_t *
dtrace_probe_lookup_id(dtrace_id_t id)
{
ASSERT(MUTEX_HELD(&dtrace_lock));
if (id == 0 || id > dtrace_nprobes)
return (NULL);
return (dtrace_probes[id - 1]);
}
static int
dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg)
{
*((dtrace_id_t *)arg) = probe->dtpr_id;
return (DTRACE_MATCH_DONE);
}
/*
* Look up a probe based on provider and one or more of module name, function
* name and probe name.
*/
dtrace_id_t
dtrace_probe_lookup(dtrace_provider_id_t prid, char *mod,
char *func, char *name)
{
dtrace_probekey_t pkey;
dtrace_id_t id;
int match;
pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name;
pkey.dtpk_pmatch = &dtrace_match_string;
pkey.dtpk_mod = mod;
pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul;
pkey.dtpk_func = func;
pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul;
pkey.dtpk_name = name;
pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul;
pkey.dtpk_id = DTRACE_IDNONE;
mutex_enter(&dtrace_lock);
match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0,
dtrace_probe_lookup_match, &id);
mutex_exit(&dtrace_lock);
ASSERT(match == 1 || match == 0);
return (match ? id : 0);
}
/*
* Returns the probe argument associated with the specified probe.
*/
void *
dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid)
{
dtrace_probe_t *probe;
void *rval = NULL;
mutex_enter(&dtrace_lock);
if ((probe = dtrace_probe_lookup_id(pid)) != NULL &&
probe->dtpr_provider == (dtrace_provider_t *)id)
rval = probe->dtpr_arg;
mutex_exit(&dtrace_lock);
return (rval);
}
/*
* Copy a probe into a probe description.
*/
static void
dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp)
{
bzero(pdp, sizeof (dtrace_probedesc_t));
pdp->dtpd_id = prp->dtpr_id;
(void) strncpy(pdp->dtpd_provider,
prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1);
(void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1);
(void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1);
(void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1);
}
/*
* Called to indicate that a probe -- or probes -- should be provided by a
* specfied provider. If the specified description is NULL, the provider will
* be told to provide all of its probes. (This is done whenever a new
* consumer comes along, or whenever a retained enabling is to be matched.) If
* the specified description is non-NULL, the provider is given the
* opportunity to dynamically provide the specified probe, allowing providers
* to support the creation of probes on-the-fly. (So-called _autocreated_
* probes.) If the provider is NULL, the operations will be applied to all
* providers; if the provider is non-NULL the operations will only be applied
* to the specified provider. The dtrace_provider_lock must be held, and the
* dtrace_lock must _not_ be held -- the provider's dtps_provide() operation
* will need to grab the dtrace_lock when it reenters the framework through
* dtrace_probe_lookup(), dtrace_probe_create(), etc.
*/
static void
dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv)
{
#ifdef illumos
modctl_t *ctl;
#endif
int all = 0;
ASSERT(MUTEX_HELD(&dtrace_provider_lock));
if (prv == NULL) {
all = 1;
prv = dtrace_provider;
}
do {
/*
* First, call the blanket provide operation.
*/
prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc);
#ifdef illumos
/*
* Now call the per-module provide operation. We will grab
* mod_lock to prevent the list from being modified. Note
* that this also prevents the mod_busy bits from changing.
* (mod_busy can only be changed with mod_lock held.)
*/
mutex_enter(&mod_lock);
ctl = &modules;
do {
if (ctl->mod_busy || ctl->mod_mp == NULL)
continue;
prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
} while ((ctl = ctl->mod_next) != &modules);
mutex_exit(&mod_lock);
#endif
} while (all && (prv = prv->dtpv_next) != NULL);
}
#ifdef illumos
/*
* Iterate over each probe, and call the Framework-to-Provider API function
* denoted by offs.
*/
static void
dtrace_probe_foreach(uintptr_t offs)
{
dtrace_provider_t *prov;
void (*func)(void *, dtrace_id_t, void *);
dtrace_probe_t *probe;
dtrace_icookie_t cookie;
int i;
/*
* We disable interrupts to walk through the probe array. This is
* safe -- the dtrace_sync() in dtrace_unregister() assures that we
* won't see stale data.
*/
cookie = dtrace_interrupt_disable();
for (i = 0; i < dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i]) == NULL)
continue;
if (probe->dtpr_ecb == NULL) {
/*
* This probe isn't enabled -- don't call the function.
*/
continue;
}
prov = probe->dtpr_provider;
func = *((void(**)(void *, dtrace_id_t, void *))
((uintptr_t)&prov->dtpv_pops + offs));
func(prov->dtpv_arg, i + 1, probe->dtpr_arg);
}
dtrace_interrupt_enable(cookie);
}
#endif
static int
dtrace_probe_enable(dtrace_probedesc_t *desc, dtrace_enabling_t *enab)
{
dtrace_probekey_t pkey;
uint32_t priv;
uid_t uid;
zoneid_t zoneid;
ASSERT(MUTEX_HELD(&dtrace_lock));
dtrace_ecb_create_cache = NULL;
if (desc == NULL) {
/*
* If we're passed a NULL description, we're being asked to
* create an ECB with a NULL probe.
*/
(void) dtrace_ecb_create_enable(NULL, enab);
return (0);
}
dtrace_probekey(desc, &pkey);
dtrace_cred2priv(enab->dten_vstate->dtvs_state->dts_cred.dcr_cred,
&priv, &uid, &zoneid);
return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable,
enab));
}
/*
* DTrace Helper Provider Functions
*/
static void
dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr)
{
attr->dtat_name = DOF_ATTR_NAME(dofattr);
attr->dtat_data = DOF_ATTR_DATA(dofattr);
attr->dtat_class = DOF_ATTR_CLASS(dofattr);
}
static void
dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov,
const dof_provider_t *dofprov, char *strtab)
{
hprov->dthpv_provname = strtab + dofprov->dofpv_name;
dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider,
dofprov->dofpv_provattr);
dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod,
dofprov->dofpv_modattr);
dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func,
dofprov->dofpv_funcattr);
dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name,
dofprov->dofpv_nameattr);
dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args,
dofprov->dofpv_argsattr);
}
static void
dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
{
uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
dof_hdr_t *dof = (dof_hdr_t *)daddr;
dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
dof_provider_t *provider;
dof_probe_t *probe;
uint32_t *off, *enoff;
uint8_t *arg;
char *strtab;
uint_t i, nprobes;
dtrace_helper_provdesc_t dhpv;
dtrace_helper_probedesc_t dhpb;
dtrace_meta_t *meta = dtrace_meta_pid;
dtrace_mops_t *mops = &meta->dtm_mops;
void *parg;
provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
provider->dofpv_strtab * dof->dofh_secsize);
prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
provider->dofpv_probes * dof->dofh_secsize);
arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
provider->dofpv_prargs * dof->dofh_secsize);
off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
provider->dofpv_proffs * dof->dofh_secsize);
strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset);
arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
enoff = NULL;
/*
* See dtrace_helper_provider_validate().
*/
if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
provider->dofpv_prenoffs != DOF_SECT_NONE) {
enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
provider->dofpv_prenoffs * dof->dofh_secsize);
enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset);
}
nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
/*
* Create the provider.
*/
dtrace_dofprov2hprov(&dhpv, provider, strtab);
if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL)
return;
meta->dtm_count++;
/*
* Create the probes.
*/
for (i = 0; i < nprobes; i++) {
probe = (dof_probe_t *)(uintptr_t)(daddr +
prb_sec->dofs_offset + i * prb_sec->dofs_entsize);
dhpb.dthpb_mod = dhp->dofhp_mod;
dhpb.dthpb_func = strtab + probe->dofpr_func;
dhpb.dthpb_name = strtab + probe->dofpr_name;
dhpb.dthpb_base = probe->dofpr_addr;
dhpb.dthpb_offs = off + probe->dofpr_offidx;
dhpb.dthpb_noffs = probe->dofpr_noffs;
if (enoff != NULL) {
dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx;
dhpb.dthpb_nenoffs = probe->dofpr_nenoffs;
} else {
dhpb.dthpb_enoffs = NULL;
dhpb.dthpb_nenoffs = 0;
}
dhpb.dthpb_args = arg + probe->dofpr_argidx;
dhpb.dthpb_nargc = probe->dofpr_nargc;
dhpb.dthpb_xargc = probe->dofpr_xargc;
dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv;
dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv;
mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb);
}
}
static void
dtrace_helper_provide(dof_helper_t *dhp, pid_t pid)
{
uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
dof_hdr_t *dof = (dof_hdr_t *)daddr;
int i;
ASSERT(MUTEX_HELD(&dtrace_meta_lock));
for (i = 0; i < dof->dofh_secnum; i++) {
dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
dof->dofh_secoff + i * dof->dofh_secsize);
if (sec->dofs_type != DOF_SECT_PROVIDER)
continue;
dtrace_helper_provide_one(dhp, sec, pid);
}
/*
* We may have just created probes, so we must now rematch against
* any retained enablings. Note that this call will acquire both
* cpu_lock and dtrace_lock; the fact that we are holding
* dtrace_meta_lock now is what defines the ordering with respect to
* these three locks.
*/
dtrace_enabling_matchall();
}
static void
dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
{
uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
dof_hdr_t *dof = (dof_hdr_t *)daddr;
dof_sec_t *str_sec;
dof_provider_t *provider;
char *strtab;
dtrace_helper_provdesc_t dhpv;
dtrace_meta_t *meta = dtrace_meta_pid;
dtrace_mops_t *mops = &meta->dtm_mops;
provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
provider->dofpv_strtab * dof->dofh_secsize);
strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
/*
* Create the provider.
*/
dtrace_dofprov2hprov(&dhpv, provider, strtab);
mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid);
meta->dtm_count--;
}
static void
dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid)
{
uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
dof_hdr_t *dof = (dof_hdr_t *)daddr;
int i;
ASSERT(MUTEX_HELD(&dtrace_meta_lock));
for (i = 0; i < dof->dofh_secnum; i++) {
dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
dof->dofh_secoff + i * dof->dofh_secsize);
if (sec->dofs_type != DOF_SECT_PROVIDER)
continue;
dtrace_helper_provider_remove_one(dhp, sec, pid);
}
}
/*
* DTrace Meta Provider-to-Framework API Functions
*
* These functions implement the Meta Provider-to-Framework API, as described
* in <sys/dtrace.h>.
*/
int
dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg,
dtrace_meta_provider_id_t *idp)
{
dtrace_meta_t *meta;
dtrace_helpers_t *help, *next;
int i;
*idp = DTRACE_METAPROVNONE;
/*
* We strictly don't need the name, but we hold onto it for
* debuggability. All hail error queues!
*/
if (name == NULL) {
cmn_err(CE_WARN, "failed to register meta-provider: "
"invalid name");
return (EINVAL);
}
if (mops == NULL ||
mops->dtms_create_probe == NULL ||
mops->dtms_provide_pid == NULL ||
mops->dtms_remove_pid == NULL) {
cmn_err(CE_WARN, "failed to register meta-register %s: "
"invalid ops", name);
return (EINVAL);
}
meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP);
meta->dtm_mops = *mops;
meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
(void) strcpy(meta->dtm_name, name);
meta->dtm_arg = arg;
mutex_enter(&dtrace_meta_lock);
mutex_enter(&dtrace_lock);
if (dtrace_meta_pid != NULL) {
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_meta_lock);
cmn_err(CE_WARN, "failed to register meta-register %s: "
"user-land meta-provider exists", name);
kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1);
kmem_free(meta, sizeof (dtrace_meta_t));
return (EINVAL);
}
dtrace_meta_pid = meta;
*idp = (dtrace_meta_provider_id_t)meta;
/*
* If there are providers and probes ready to go, pass them
* off to the new meta provider now.
*/
help = dtrace_deferred_pid;
dtrace_deferred_pid = NULL;
mutex_exit(&dtrace_lock);
while (help != NULL) {
for (i = 0; i < help->dthps_nprovs; i++) {
dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
help->dthps_pid);
}
next = help->dthps_next;
help->dthps_next = NULL;
help->dthps_prev = NULL;
help->dthps_deferred = 0;
help = next;
}
mutex_exit(&dtrace_meta_lock);
return (0);
}
int
dtrace_meta_unregister(dtrace_meta_provider_id_t id)
{
dtrace_meta_t **pp, *old = (dtrace_meta_t *)id;
mutex_enter(&dtrace_meta_lock);
mutex_enter(&dtrace_lock);
if (old == dtrace_meta_pid) {
pp = &dtrace_meta_pid;
} else {
panic("attempt to unregister non-existent "
"dtrace meta-provider %p\n", (void *)old);
}
if (old->dtm_count != 0) {
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_meta_lock);
return (EBUSY);
}
*pp = NULL;
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_meta_lock);
kmem_free(old->dtm_name, strlen(old->dtm_name) + 1);
kmem_free(old, sizeof (dtrace_meta_t));
return (0);
}
/*
* DTrace DIF Object Functions
*/
static int
dtrace_difo_err(uint_t pc, const char *format, ...)
{
if (dtrace_err_verbose) {
va_list alist;
(void) uprintf("dtrace DIF object error: [%u]: ", pc);
va_start(alist, format);
(void) vuprintf(format, alist);
va_end(alist);
}
#ifdef DTRACE_ERRDEBUG
dtrace_errdebug(format);
#endif
return (1);
}
/*
* Validate a DTrace DIF object by checking the IR instructions. The following
* rules are currently enforced by dtrace_difo_validate():
*
* 1. Each instruction must have a valid opcode
* 2. Each register, string, variable, or subroutine reference must be valid
* 3. No instruction can modify register %r0 (must be zero)
* 4. All instruction reserved bits must be set to zero
* 5. The last instruction must be a "ret" instruction
* 6. All branch targets must reference a valid instruction _after_ the branch
*/
static int
dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
cred_t *cr)
{
int err = 0, i;
int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
int kcheckload;
uint_t pc;
kcheckload = cr == NULL ||
(vstate->dtvs_state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) == 0;
dp->dtdo_destructive = 0;
for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) {
dif_instr_t instr = dp->dtdo_buf[pc];
uint_t r1 = DIF_INSTR_R1(instr);
uint_t r2 = DIF_INSTR_R2(instr);
uint_t rd = DIF_INSTR_RD(instr);
uint_t rs = DIF_INSTR_RS(instr);
uint_t label = DIF_INSTR_LABEL(instr);
uint_t v = DIF_INSTR_VAR(instr);
uint_t subr = DIF_INSTR_SUBR(instr);
uint_t type = DIF_INSTR_TYPE(instr);
uint_t op = DIF_INSTR_OP(instr);
switch (op) {
case DIF_OP_OR:
case DIF_OP_XOR:
case DIF_OP_AND:
case DIF_OP_SLL:
case DIF_OP_SRL:
case DIF_OP_SRA:
case DIF_OP_SUB:
case DIF_OP_ADD:
case DIF_OP_MUL:
case DIF_OP_SDIV:
case DIF_OP_UDIV:
case DIF_OP_SREM:
case DIF_OP_UREM:
case DIF_OP_COPYS:
if (r1 >= nregs)
err += efunc(pc, "invalid register %u\n", r1);
if (r2 >= nregs)
err += efunc(pc, "invalid register %u\n", r2);
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
break;
case DIF_OP_NOT:
case DIF_OP_MOV:
case DIF_OP_ALLOCS:
if (r1 >= nregs)
err += efunc(pc, "invalid register %u\n", r1);
if (r2 != 0)
err += efunc(pc, "non-zero reserved bits\n");
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
break;
case DIF_OP_LDSB:
case DIF_OP_LDSH:
case DIF_OP_LDSW:
case DIF_OP_LDUB:
case DIF_OP_LDUH:
case DIF_OP_LDUW:
case DIF_OP_LDX:
if (r1 >= nregs)
err += efunc(pc, "invalid register %u\n", r1);
if (r2 != 0)
err += efunc(pc, "non-zero reserved bits\n");
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
if (kcheckload)
dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op +
DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd);
break;
case DIF_OP_RLDSB:
case DIF_OP_RLDSH:
case DIF_OP_RLDSW:
case DIF_OP_RLDUB:
case DIF_OP_RLDUH:
case DIF_OP_RLDUW:
case DIF_OP_RLDX:
if (r1 >= nregs)
err += efunc(pc, "invalid register %u\n", r1);
if (r2 != 0)
err += efunc(pc, "non-zero reserved bits\n");
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
break;
case DIF_OP_ULDSB:
case DIF_OP_ULDSH:
case DIF_OP_ULDSW:
case DIF_OP_ULDUB:
case DIF_OP_ULDUH:
case DIF_OP_ULDUW:
case DIF_OP_ULDX:
if (r1 >= nregs)
err += efunc(pc, "invalid register %u\n", r1);
if (r2 != 0)
err += efunc(pc, "non-zero reserved bits\n");
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
break;
case DIF_OP_STB:
case DIF_OP_STH:
case DIF_OP_STW:
case DIF_OP_STX:
if (r1 >= nregs)
err += efunc(pc, "invalid register %u\n", r1);
if (r2 != 0)
err += efunc(pc, "non-zero reserved bits\n");
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to 0 address\n");
break;
case DIF_OP_CMP:
case DIF_OP_SCMP:
if (r1 >= nregs)
err += efunc(pc, "invalid register %u\n", r1);
if (r2 >= nregs)
err += efunc(pc, "invalid register %u\n", r2);
if (rd != 0)
err += efunc(pc, "non-zero reserved bits\n");
break;
case DIF_OP_TST:
if (r1 >= nregs)
err += efunc(pc, "invalid register %u\n", r1);
if (r2 != 0 || rd != 0)
err += efunc(pc, "non-zero reserved bits\n");
break;
case DIF_OP_BA:
case DIF_OP_BE:
case DIF_OP_BNE:
case DIF_OP_BG:
case DIF_OP_BGU:
case DIF_OP_BGE:
case DIF_OP_BGEU:
case DIF_OP_BL:
case DIF_OP_BLU:
case DIF_OP_BLE:
case DIF_OP_BLEU:
if (label >= dp->dtdo_len) {
err += efunc(pc, "invalid branch target %u\n",
label);
}
if (label <= pc) {
err += efunc(pc, "backward branch to %u\n",
label);
}
break;
case DIF_OP_RET:
if (r1 != 0 || r2 != 0)
err += efunc(pc, "non-zero reserved bits\n");
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
break;
case DIF_OP_NOP:
case DIF_OP_POPTS:
case DIF_OP_FLUSHTS:
if (r1 != 0 || r2 != 0 || rd != 0)
err += efunc(pc, "non-zero reserved bits\n");
break;
case DIF_OP_SETX:
if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) {
err += efunc(pc, "invalid integer ref %u\n",
DIF_INSTR_INTEGER(instr));
}
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
break;
case DIF_OP_SETS:
if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) {
err += efunc(pc, "invalid string ref %u\n",
DIF_INSTR_STRING(instr));
}
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
break;
case DIF_OP_LDGA:
case DIF_OP_LDTA:
if (r1 > DIF_VAR_ARRAY_MAX)
err += efunc(pc, "invalid array %u\n", r1);
if (r2 >= nregs)
err += efunc(pc, "invalid register %u\n", r2);
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
break;
case DIF_OP_LDGS:
case DIF_OP_LDTS:
case DIF_OP_LDLS:
case DIF_OP_LDGAA:
case DIF_OP_LDTAA:
if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX)
err += efunc(pc, "invalid variable %u\n", v);
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
break;
case DIF_OP_STGS:
case DIF_OP_STTS:
case DIF_OP_STLS:
case DIF_OP_STGAA:
case DIF_OP_STTAA:
if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX)
err += efunc(pc, "invalid variable %u\n", v);
if (rs >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
break;
case DIF_OP_CALL:
if (subr > DIF_SUBR_MAX)
err += efunc(pc, "invalid subr %u\n", subr);
if (rd >= nregs)
err += efunc(pc, "invalid register %u\n", rd);
if (rd == 0)
err += efunc(pc, "cannot write to %r0\n");
if (subr == DIF_SUBR_COPYOUT ||
subr == DIF_SUBR_COPYOUTSTR) {
dp->dtdo_destructive = 1;
}
if (subr == DIF_SUBR_GETF) {
/*
* If we have a getf() we need to record that
* in our state. Note that our state can be
* NULL if this is a helper -- but in that
* case, the call to getf() is itself illegal,
* and will be caught (slightly later) when
* the helper is validated.
*/
if (vstate->dtvs_state != NULL)
vstate->dtvs_state->dts_getf++;
}
break;
case DIF_OP_PUSHTR:
if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF)
err += efunc(pc, "invalid ref type %u\n", type);
if (r2 >= nregs)
err += efunc(pc, "invalid register %u\n", r2);
if (rs >= nregs)
err += efunc(pc, "invalid register %u\n", rs);
break;
case DIF_OP_PUSHTV:
if (type != DIF_TYPE_CTF)
err += efunc(pc, "invalid val type %u\n", type);
if (r2 >= nregs)
err += efunc(pc, "invalid register %u\n", r2);
if (rs >= nregs)
err += efunc(pc, "invalid register %u\n", rs);
break;
default:
err += efunc(pc, "invalid opcode %u\n",
DIF_INSTR_OP(instr));
}
}
if (dp->dtdo_len != 0 &&
DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
err += efunc(dp->dtdo_len - 1,
"expected 'ret' as last DIF instruction\n");
}
if (!(dp->dtdo_rtype.dtdt_flags & (DIF_TF_BYREF | DIF_TF_BYUREF))) {
/*
* If we're not returning by reference, the size must be either
* 0 or the size of one of the base types.
*/
switch (dp->dtdo_rtype.dtdt_size) {
case 0:
case sizeof (uint8_t):
case sizeof (uint16_t):
case sizeof (uint32_t):
case sizeof (uint64_t):
break;
default:
err += efunc(dp->dtdo_len - 1, "bad return size\n");
}
}
for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
dtrace_diftype_t *vt, *et;
uint_t id, ndx;
if (v->dtdv_scope != DIFV_SCOPE_GLOBAL &&
v->dtdv_scope != DIFV_SCOPE_THREAD &&
v->dtdv_scope != DIFV_SCOPE_LOCAL) {
err += efunc(i, "unrecognized variable scope %d\n",
v->dtdv_scope);
break;
}
if (v->dtdv_kind != DIFV_KIND_ARRAY &&
v->dtdv_kind != DIFV_KIND_SCALAR) {
err += efunc(i, "unrecognized variable type %d\n",
v->dtdv_kind);
break;
}
if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) {
err += efunc(i, "%d exceeds variable id limit\n", id);
break;
}
if (id < DIF_VAR_OTHER_UBASE)
continue;
/*
* For user-defined variables, we need to check that this
* definition is identical to any previous definition that we
* encountered.
*/
ndx = id - DIF_VAR_OTHER_UBASE;
switch (v->dtdv_scope) {
case DIFV_SCOPE_GLOBAL:
if (ndx < vstate->dtvs_nglobals) {
dtrace_statvar_t *svar;
if ((svar = vstate->dtvs_globals[ndx]) != NULL)
existing = &svar->dtsv_var;
}
break;
case DIFV_SCOPE_THREAD:
if (ndx < vstate->dtvs_ntlocals)
existing = &vstate->dtvs_tlocals[ndx];
break;
case DIFV_SCOPE_LOCAL:
if (ndx < vstate->dtvs_nlocals) {
dtrace_statvar_t *svar;
if ((svar = vstate->dtvs_locals[ndx]) != NULL)
existing = &svar->dtsv_var;
}
break;
}
vt = &v->dtdv_type;
if (vt->dtdt_flags & DIF_TF_BYREF) {
if (vt->dtdt_size == 0) {
err += efunc(i, "zero-sized variable\n");
break;
}
if (v->dtdv_scope == DIFV_SCOPE_GLOBAL &&
vt->dtdt_size > dtrace_global_maxsize) {
err += efunc(i, "oversized by-ref global\n");
break;
}
}
if (existing == NULL || existing->dtdv_id == 0)
continue;
ASSERT(existing->dtdv_id == v->dtdv_id);
ASSERT(existing->dtdv_scope == v->dtdv_scope);
if (existing->dtdv_kind != v->dtdv_kind)
err += efunc(i, "%d changed variable kind\n", id);
et = &existing->dtdv_type;
if (vt->dtdt_flags != et->dtdt_flags) {
err += efunc(i, "%d changed variable type flags\n", id);
break;
}
if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) {
err += efunc(i, "%d changed variable type size\n", id);
break;
}
}
return (err);
}
/*
* Validate a DTrace DIF object that it is to be used as a helper. Helpers
* are much more constrained than normal DIFOs. Specifically, they may
* not:
*
* 1. Make calls to subroutines other than copyin(), copyinstr() or
* miscellaneous string routines
* 2. Access DTrace variables other than the args[] array, and the
* curthread, pid, ppid, tid, execname, zonename, uid and gid variables.
* 3. Have thread-local variables.
* 4. Have dynamic variables.
*/
static int
dtrace_difo_validate_helper(dtrace_difo_t *dp)
{
int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
int err = 0;
uint_t pc;
for (pc = 0; pc < dp->dtdo_len; pc++) {
dif_instr_t instr = dp->dtdo_buf[pc];
uint_t v = DIF_INSTR_VAR(instr);
uint_t subr = DIF_INSTR_SUBR(instr);
uint_t op = DIF_INSTR_OP(instr);
switch (op) {
case DIF_OP_OR:
case DIF_OP_XOR:
case DIF_OP_AND:
case DIF_OP_SLL:
case DIF_OP_SRL:
case DIF_OP_SRA:
case DIF_OP_SUB:
case DIF_OP_ADD:
case DIF_OP_MUL:
case DIF_OP_SDIV:
case DIF_OP_UDIV:
case DIF_OP_SREM:
case DIF_OP_UREM:
case DIF_OP_COPYS:
case DIF_OP_NOT:
case DIF_OP_MOV:
case DIF_OP_RLDSB:
case DIF_OP_RLDSH:
case DIF_OP_RLDSW:
case DIF_OP_RLDUB:
case DIF_OP_RLDUH:
case DIF_OP_RLDUW:
case DIF_OP_RLDX:
case DIF_OP_ULDSB:
case DIF_OP_ULDSH:
case DIF_OP_ULDSW:
case DIF_OP_ULDUB:
case DIF_OP_ULDUH:
case DIF_OP_ULDUW:
case DIF_OP_ULDX:
case DIF_OP_STB:
case DIF_OP_STH:
case DIF_OP_STW:
case DIF_OP_STX:
case DIF_OP_ALLOCS:
case DIF_OP_CMP:
case DIF_OP_SCMP:
case DIF_OP_TST:
case DIF_OP_BA:
case DIF_OP_BE:
case DIF_OP_BNE:
case DIF_OP_BG:
case DIF_OP_BGU:
case DIF_OP_BGE:
case DIF_OP_BGEU:
case DIF_OP_BL:
case DIF_OP_BLU:
case DIF_OP_BLE:
case DIF_OP_BLEU:
case DIF_OP_RET:
case DIF_OP_NOP:
case DIF_OP_POPTS:
case DIF_OP_FLUSHTS:
case DIF_OP_SETX:
case DIF_OP_SETS:
case DIF_OP_LDGA:
case DIF_OP_LDLS:
case DIF_OP_STGS:
case DIF_OP_STLS:
case DIF_OP_PUSHTR:
case DIF_OP_PUSHTV:
break;
case DIF_OP_LDGS:
if (v >= DIF_VAR_OTHER_UBASE)
break;
if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9)
break;
if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID ||
v == DIF_VAR_PPID || v == DIF_VAR_TID ||
v == DIF_VAR_EXECARGS ||
v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME ||
v == DIF_VAR_UID || v == DIF_VAR_GID)
break;
err += efunc(pc, "illegal variable %u\n", v);
break;
case DIF_OP_LDTA:
case DIF_OP_LDTS:
case DIF_OP_LDGAA:
case DIF_OP_LDTAA:
err += efunc(pc, "illegal dynamic variable load\n");
break;
case DIF_OP_STTS:
case DIF_OP_STGAA:
case DIF_OP_STTAA:
err += efunc(pc, "illegal dynamic variable store\n");
break;
case DIF_OP_CALL:
if (subr == DIF_SUBR_ALLOCA ||
subr == DIF_SUBR_BCOPY ||
subr == DIF_SUBR_COPYIN ||
subr == DIF_SUBR_COPYINTO ||
subr == DIF_SUBR_COPYINSTR ||
subr == DIF_SUBR_INDEX ||
subr == DIF_SUBR_INET_NTOA ||
subr == DIF_SUBR_INET_NTOA6 ||
subr == DIF_SUBR_INET_NTOP ||
subr == DIF_SUBR_JSON ||
subr == DIF_SUBR_LLTOSTR ||
subr == DIF_SUBR_STRTOLL ||
subr == DIF_SUBR_RINDEX ||
subr == DIF_SUBR_STRCHR ||
subr == DIF_SUBR_STRJOIN ||
subr == DIF_SUBR_STRRCHR ||
subr == DIF_SUBR_STRSTR ||
subr == DIF_SUBR_HTONS ||
subr == DIF_SUBR_HTONL ||
subr == DIF_SUBR_HTONLL ||
subr == DIF_SUBR_NTOHS ||
subr == DIF_SUBR_NTOHL ||
subr == DIF_SUBR_NTOHLL ||
subr == DIF_SUBR_MEMREF ||
#ifndef illumos
subr == DIF_SUBR_MEMSTR ||
#endif
subr == DIF_SUBR_TYPEREF)
break;
err += efunc(pc, "invalid subr %u\n", subr);
break;
default:
err += efunc(pc, "invalid opcode %u\n",
DIF_INSTR_OP(instr));
}
}
return (err);
}
/*
* Returns 1 if the expression in the DIF object can be cached on a per-thread
* basis; 0 if not.
*/
static int
dtrace_difo_cacheable(dtrace_difo_t *dp)
{
int i;
if (dp == NULL)
return (0);
for (i = 0; i < dp->dtdo_varlen; i++) {
dtrace_difv_t *v = &dp->dtdo_vartab[i];
if (v->dtdv_scope != DIFV_SCOPE_GLOBAL)
continue;
switch (v->dtdv_id) {
case DIF_VAR_CURTHREAD:
case DIF_VAR_PID:
case DIF_VAR_TID:
case DIF_VAR_EXECARGS:
case DIF_VAR_EXECNAME:
case DIF_VAR_ZONENAME:
break;
default:
return (0);
}
}
/*
* This DIF object may be cacheable. Now we need to look for any
* array loading instructions, any memory loading instructions, or
* any stores to thread-local variables.
*/
for (i = 0; i < dp->dtdo_len; i++) {
uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]);
if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) ||
(op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) ||
(op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) ||
op == DIF_OP_LDGA || op == DIF_OP_STTS)
return (0);
}
return (1);
}
static void
dtrace_difo_hold(dtrace_difo_t *dp)
{
int i;
ASSERT(MUTEX_HELD(&dtrace_lock));
dp->dtdo_refcnt++;
ASSERT(dp->dtdo_refcnt != 0);
/*
* We need to check this DIF object for references to the variable
* DIF_VAR_VTIMESTAMP.
*/
for (i = 0; i < dp->dtdo_varlen; i++) {
dtrace_difv_t *v = &dp->dtdo_vartab[i];
if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
continue;
if (dtrace_vtime_references++ == 0)
dtrace_vtime_enable();
}
}
/*
* This routine calculates the dynamic variable chunksize for a given DIF
* object. The calculation is not fool-proof, and can probably be tricked by
* malicious DIF -- but it works for all compiler-generated DIF. Because this
* calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail
* if a dynamic variable size exceeds the chunksize.
*/
static void
dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
{
uint64_t sval = 0;
dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
const dif_instr_t *text = dp->dtdo_buf;
uint_t pc, srd = 0;
uint_t ttop = 0;
size_t size, ksize;
uint_t id, i;
for (pc = 0; pc < dp->dtdo_len; pc++) {
dif_instr_t instr = text[pc];
uint_t op = DIF_INSTR_OP(instr);
uint_t rd = DIF_INSTR_RD(instr);
uint_t r1 = DIF_INSTR_R1(instr);
uint_t nkeys = 0;
uchar_t scope = 0;
dtrace_key_t *key = tupregs;
switch (op) {
case DIF_OP_SETX:
sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)];
srd = rd;
continue;
case DIF_OP_STTS:
key = &tupregs[DIF_DTR_NREGS];
key[0].dttk_size = 0;
key[1].dttk_size = 0;
nkeys = 2;
scope = DIFV_SCOPE_THREAD;
break;
case DIF_OP_STGAA:
case DIF_OP_STTAA:
nkeys = ttop;
if (DIF_INSTR_OP(instr) == DIF_OP_STTAA)
key[nkeys++].dttk_size = 0;
key[nkeys++].dttk_size = 0;
if (op == DIF_OP_STTAA) {
scope = DIFV_SCOPE_THREAD;
} else {
scope = DIFV_SCOPE_GLOBAL;
}
break;
case DIF_OP_PUSHTR:
if (ttop == DIF_DTR_NREGS)
return;
if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) {
/*
* If the register for the size of the "pushtr"
* is %r0 (or the value is 0) and the type is
* a string, we'll use the system-wide default
* string size.
*/
tupregs[ttop++].dttk_size =
dtrace_strsize_default;
} else {
if (srd == 0)
return;
tupregs[ttop++].dttk_size = sval;
}
break;
case DIF_OP_PUSHTV:
if (ttop == DIF_DTR_NREGS)
return;
tupregs[ttop++].dttk_size = 0;
break;
case DIF_OP_FLUSHTS:
ttop = 0;
break;
case DIF_OP_POPTS:
if (ttop != 0)
ttop--;
break;
}
sval = 0;
srd = 0;
if (nkeys == 0)
continue;
/*
* We have a dynamic variable allocation; calculate its size.
*/
for (ksize = 0, i = 0; i < nkeys; i++)
ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
size = sizeof (dtrace_dynvar_t);
size += sizeof (dtrace_key_t) * (nkeys - 1);
size += ksize;
/*
* Now we need to determine the size of the stored data.
*/
id = DIF_INSTR_VAR(instr);
for (i = 0; i < dp->dtdo_varlen; i++) {
dtrace_difv_t *v = &dp->dtdo_vartab[i];
if (v->dtdv_id == id && v->dtdv_scope == scope) {
size += v->dtdv_type.dtdt_size;
break;
}
}
if (i == dp->dtdo_varlen)
return;
/*
* We have the size. If this is larger than the chunk size
* for our dynamic variable state, reset the chunk size.
*/
size = P2ROUNDUP(size, sizeof (uint64_t));
if (size > vstate->dtvs_dynvars.dtds_chunksize)
vstate->dtvs_dynvars.dtds_chunksize = size;
}
}
static void
dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
{
int i, oldsvars, osz, nsz, otlocals, ntlocals;
uint_t id;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0);
for (i = 0; i < dp->dtdo_varlen; i++) {
dtrace_difv_t *v = &dp->dtdo_vartab[i];
dtrace_statvar_t *svar, ***svarp = NULL;
size_t dsize = 0;
uint8_t scope = v->dtdv_scope;
int *np = NULL;
if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
continue;
id -= DIF_VAR_OTHER_UBASE;
switch (scope) {
case DIFV_SCOPE_THREAD:
while (id >= (otlocals = vstate->dtvs_ntlocals)) {
dtrace_difv_t *tlocals;
if ((ntlocals = (otlocals << 1)) == 0)
ntlocals = 1;
osz = otlocals * sizeof (dtrace_difv_t);
nsz = ntlocals * sizeof (dtrace_difv_t);
tlocals = kmem_zalloc(nsz, KM_SLEEP);
if (osz != 0) {
bcopy(vstate->dtvs_tlocals,
tlocals, osz);
kmem_free(vstate->dtvs_tlocals, osz);
}
vstate->dtvs_tlocals = tlocals;
vstate->dtvs_ntlocals = ntlocals;
}
vstate->dtvs_tlocals[id] = *v;
continue;
case DIFV_SCOPE_LOCAL:
np = &vstate->dtvs_nlocals;
svarp = &vstate->dtvs_locals;
if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
dsize = NCPU * (v->dtdv_type.dtdt_size +
sizeof (uint64_t));
else
dsize = NCPU * sizeof (uint64_t);
break;
case DIFV_SCOPE_GLOBAL:
np = &vstate->dtvs_nglobals;
svarp = &vstate->dtvs_globals;
if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
dsize = v->dtdv_type.dtdt_size +
sizeof (uint64_t);
break;
default:
ASSERT(0);
}
while (id >= (oldsvars = *np)) {
dtrace_statvar_t **statics;
int newsvars, oldsize, newsize;
if ((newsvars = (oldsvars << 1)) == 0)
newsvars = 1;
oldsize = oldsvars * sizeof (dtrace_statvar_t *);
newsize = newsvars * sizeof (dtrace_statvar_t *);
statics = kmem_zalloc(newsize, KM_SLEEP);
if (oldsize != 0) {
bcopy(*svarp, statics, oldsize);
kmem_free(*svarp, oldsize);
}
*svarp = statics;
*np = newsvars;
}
if ((svar = (*svarp)[id]) == NULL) {
svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP);
svar->dtsv_var = *v;
if ((svar->dtsv_size = dsize) != 0) {
svar->dtsv_data = (uint64_t)(uintptr_t)
kmem_zalloc(dsize, KM_SLEEP);
}
(*svarp)[id] = svar;
}
svar->dtsv_refcnt++;
}
dtrace_difo_chunksize(dp, vstate);
dtrace_difo_hold(dp);
}
static dtrace_difo_t *
dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
{
dtrace_difo_t *new;
size_t sz;
ASSERT(dp->dtdo_buf != NULL);
ASSERT(dp->dtdo_refcnt != 0);
new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
ASSERT(dp->dtdo_buf != NULL);
sz = dp->dtdo_len * sizeof (dif_instr_t);
new->dtdo_buf = kmem_alloc(sz, KM_SLEEP);
bcopy(dp->dtdo_buf, new->dtdo_buf, sz);
new->dtdo_len = dp->dtdo_len;
if (dp->dtdo_strtab != NULL) {
ASSERT(dp->dtdo_strlen != 0);
new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP);
bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen);
new->dtdo_strlen = dp->dtdo_strlen;
}
if (dp->dtdo_inttab != NULL) {
ASSERT(dp->dtdo_intlen != 0);
sz = dp->dtdo_intlen * sizeof (uint64_t);
new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP);
bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz);
new->dtdo_intlen = dp->dtdo_intlen;
}
if (dp->dtdo_vartab != NULL) {
ASSERT(dp->dtdo_varlen != 0);
sz = dp->dtdo_varlen * sizeof (dtrace_difv_t);
new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP);
bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz);
new->dtdo_varlen = dp->dtdo_varlen;
}
dtrace_difo_init(new, vstate);
return (new);
}
static void
dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
{
int i;
ASSERT(dp->dtdo_refcnt == 0);
for (i = 0; i < dp->dtdo_varlen; i++) {
dtrace_difv_t *v = &dp->dtdo_vartab[i];
dtrace_statvar_t *svar, **svarp = NULL;
uint_t id;
uint8_t scope = v->dtdv_scope;
int *np = NULL;
switch (scope) {
case DIFV_SCOPE_THREAD:
continue;
case DIFV_SCOPE_LOCAL:
np = &vstate->dtvs_nlocals;
svarp = vstate->dtvs_locals;
break;
case DIFV_SCOPE_GLOBAL:
np = &vstate->dtvs_nglobals;
svarp = vstate->dtvs_globals;
break;
default:
ASSERT(0);
}
if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
continue;
id -= DIF_VAR_OTHER_UBASE;
ASSERT(id < *np);
svar = svarp[id];
ASSERT(svar != NULL);
ASSERT(svar->dtsv_refcnt > 0);
if (--svar->dtsv_refcnt > 0)
continue;
if (svar->dtsv_size != 0) {
ASSERT(svar->dtsv_data != 0);
kmem_free((void *)(uintptr_t)svar->dtsv_data,
svar->dtsv_size);
}
kmem_free(svar, sizeof (dtrace_statvar_t));
svarp[id] = NULL;
}
if (dp->dtdo_buf != NULL)
kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
if (dp->dtdo_inttab != NULL)
kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
if (dp->dtdo_strtab != NULL)
kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
if (dp->dtdo_vartab != NULL)
kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
kmem_free(dp, sizeof (dtrace_difo_t));
}
static void
dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
{
int i;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(dp->dtdo_refcnt != 0);
for (i = 0; i < dp->dtdo_varlen; i++) {
dtrace_difv_t *v = &dp->dtdo_vartab[i];
if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
continue;
ASSERT(dtrace_vtime_references > 0);
if (--dtrace_vtime_references == 0)
dtrace_vtime_disable();
}
if (--dp->dtdo_refcnt == 0)
dtrace_difo_destroy(dp, vstate);
}
/*
* DTrace Format Functions
*/
static uint16_t
dtrace_format_add(dtrace_state_t *state, char *str)
{
char *fmt, **new;
uint16_t ndx, len = strlen(str) + 1;
fmt = kmem_zalloc(len, KM_SLEEP);
bcopy(str, fmt, len);
for (ndx = 0; ndx < state->dts_nformats; ndx++) {
if (state->dts_formats[ndx] == NULL) {
state->dts_formats[ndx] = fmt;
return (ndx + 1);
}
}
if (state->dts_nformats == USHRT_MAX) {
/*
* This is only likely if a denial-of-service attack is being
* attempted. As such, it's okay to fail silently here.
*/
kmem_free(fmt, len);
return (0);
}
/*
* For simplicity, we always resize the formats array to be exactly the
* number of formats.
*/
ndx = state->dts_nformats++;
new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP);
if (state->dts_formats != NULL) {
ASSERT(ndx != 0);
bcopy(state->dts_formats, new, ndx * sizeof (char *));
kmem_free(state->dts_formats, ndx * sizeof (char *));
}
state->dts_formats = new;
state->dts_formats[ndx] = fmt;
return (ndx + 1);
}
static void
dtrace_format_remove(dtrace_state_t *state, uint16_t format)
{
char *fmt;
ASSERT(state->dts_formats != NULL);
ASSERT(format <= state->dts_nformats);
ASSERT(state->dts_formats[format - 1] != NULL);
fmt = state->dts_formats[format - 1];
kmem_free(fmt, strlen(fmt) + 1);
state->dts_formats[format - 1] = NULL;
}
static void
dtrace_format_destroy(dtrace_state_t *state)
{
int i;
if (state->dts_nformats == 0) {
ASSERT(state->dts_formats == NULL);
return;
}
ASSERT(state->dts_formats != NULL);
for (i = 0; i < state->dts_nformats; i++) {
char *fmt = state->dts_formats[i];
if (fmt == NULL)
continue;
kmem_free(fmt, strlen(fmt) + 1);
}
kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *));
state->dts_nformats = 0;
state->dts_formats = NULL;
}
/*
* DTrace Predicate Functions
*/
static dtrace_predicate_t *
dtrace_predicate_create(dtrace_difo_t *dp)
{
dtrace_predicate_t *pred;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(dp->dtdo_refcnt != 0);
pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP);
pred->dtp_difo = dp;
pred->dtp_refcnt = 1;
if (!dtrace_difo_cacheable(dp))
return (pred);
if (dtrace_predcache_id == DTRACE_CACHEIDNONE) {
/*
* This is only theoretically possible -- we have had 2^32
* cacheable predicates on this machine. We cannot allow any
* more predicates to become cacheable: as unlikely as it is,
* there may be a thread caching a (now stale) predicate cache
* ID. (N.B.: the temptation is being successfully resisted to
* have this cmn_err() "Holy shit -- we executed this code!")
*/
return (pred);
}
pred->dtp_cacheid = dtrace_predcache_id++;
return (pred);
}
static void
dtrace_predicate_hold(dtrace_predicate_t *pred)
{
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0);
ASSERT(pred->dtp_refcnt > 0);
pred->dtp_refcnt++;
}
static void
dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate)
{
dtrace_difo_t *dp = pred->dtp_difo;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(dp != NULL && dp->dtdo_refcnt != 0);
ASSERT(pred->dtp_refcnt > 0);
if (--pred->dtp_refcnt == 0) {
dtrace_difo_release(pred->dtp_difo, vstate);
kmem_free(pred, sizeof (dtrace_predicate_t));
}
}
/*
* DTrace Action Description Functions
*/
static dtrace_actdesc_t *
dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple,
uint64_t uarg, uint64_t arg)
{
dtrace_actdesc_t *act;
#ifdef illumos
ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL &&
arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA));
#endif
act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP);
act->dtad_kind = kind;
act->dtad_ntuple = ntuple;
act->dtad_uarg = uarg;
act->dtad_arg = arg;
act->dtad_refcnt = 1;
return (act);
}
static void
dtrace_actdesc_hold(dtrace_actdesc_t *act)
{
ASSERT(act->dtad_refcnt >= 1);
act->dtad_refcnt++;
}
static void
dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate)
{
dtrace_actkind_t kind = act->dtad_kind;
dtrace_difo_t *dp;
ASSERT(act->dtad_refcnt >= 1);
if (--act->dtad_refcnt != 0)
return;
if ((dp = act->dtad_difo) != NULL)
dtrace_difo_release(dp, vstate);
if (DTRACEACT_ISPRINTFLIKE(kind)) {
char *str = (char *)(uintptr_t)act->dtad_arg;
#ifdef illumos
ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) ||
(str == NULL && act->dtad_kind == DTRACEACT_PRINTA));
#endif
if (str != NULL)
kmem_free(str, strlen(str) + 1);
}
kmem_free(act, sizeof (dtrace_actdesc_t));
}
/*
* DTrace ECB Functions
*/
static dtrace_ecb_t *
dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
{
dtrace_ecb_t *ecb;
dtrace_epid_t epid;
ASSERT(MUTEX_HELD(&dtrace_lock));
ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);
ecb->dte_predicate = NULL;
ecb->dte_probe = probe;
/*
* The default size is the size of the default action: recording
* the header.
*/
ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t);
ecb->dte_alignment = sizeof (dtrace_epid_t);
epid = state->dts_epid++;
if (epid - 1 >= state->dts_necbs) {
dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs;
int necbs = state->dts_necbs << 1;
ASSERT(epid == state->dts_necbs + 1);
if (necbs == 0) {
ASSERT(oecbs == NULL);
necbs = 1;
}
ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP);
if (oecbs != NULL)
bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs));
dtrace_membar_producer();
state->dts_ecbs = ecbs;
if (oecbs != NULL) {
/*
* If this state is active, we must dtrace_sync()
* before we can free the old dts_ecbs array: we're
* coming in hot, and there may be active ring
* buffer processing (which indexes into the dts_ecbs
* array) on another CPU.
*/
if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
dtrace_sync();
kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs));
}
dtrace_membar_producer();
state->dts_necbs = necbs;
}
ecb->dte_state = state;
ASSERT(state->dts_ecbs[epid - 1] == NULL);
dtrace_membar_producer();
state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
return (ecb);
}
static void
dtrace_ecb_enable(dtrace_ecb_t *ecb)
{
dtrace_probe_t *probe = ecb->dte_probe;
ASSERT(MUTEX_HELD(&cpu_lock));
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(ecb->dte_next == NULL);
if (probe == NULL) {
/*
* This is the NULL probe -- there's nothing to do.
*/
return;
}
if (probe->dtpr_ecb == NULL) {
dtrace_provider_t *prov = probe->dtpr_provider;
/*
* We're the first ECB on this probe.
*/
probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;
if (ecb->dte_predicate != NULL)
probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
probe->dtpr_id, probe->dtpr_arg);
} else {
/*
* This probe is already active. Swing the last pointer to
* point to the new ECB, and issue a dtrace_sync() to assure
* that all CPUs have seen the change.
*/
ASSERT(probe->dtpr_ecb_last != NULL);
probe->dtpr_ecb_last->dte_next = ecb;
probe->dtpr_ecb_last = ecb;
probe->dtpr_predcache = 0;
dtrace_sync();
}
}
static void
dtrace_ecb_resize(dtrace_ecb_t *ecb)
{
dtrace_action_t *act;
uint32_t curneeded = UINT32_MAX;
uint32_t aggbase = UINT32_MAX;
/*
* If we record anything, we always record the dtrace_rechdr_t. (And
* we always record it first.)
*/
ecb->dte_size = sizeof (dtrace_rechdr_t);
ecb->dte_alignment = sizeof (dtrace_epid_t);
for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
dtrace_recdesc_t *rec = &act->dta_rec;
ASSERT(rec->dtrd_size > 0 || rec->dtrd_alignment == 1);
ecb->dte_alignment = MAX(ecb->dte_alignment,
rec->dtrd_alignment);
if (DTRACEACT_ISAGG(act->dta_kind)) {
dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
ASSERT(rec->dtrd_size != 0);
ASSERT(agg->dtag_first != NULL);
ASSERT(act->dta_prev->dta_intuple);
ASSERT(aggbase != UINT32_MAX);
ASSERT(curneeded != UINT32_MAX);
agg->dtag_base = aggbase;
curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
rec->dtrd_offset = curneeded;
curneeded += rec->dtrd_size;
ecb->dte_needed = MAX(ecb->dte_needed, curneeded);
aggbase = UINT32_MAX;
curneeded = UINT32_MAX;
} else if (act->dta_intuple) {
if (curneeded == UINT32_MAX) {
/*
* This is the first record in a tuple. Align
* curneeded to be at offset 4 in an 8-byte
* aligned block.
*/
ASSERT(act->dta_prev == NULL ||
!act->dta_prev->dta_intuple);
ASSERT3U(aggbase, ==, UINT32_MAX);
curneeded = P2PHASEUP(ecb->dte_size,
sizeof (uint64_t), sizeof (dtrace_aggid_t));
aggbase = curneeded - sizeof (dtrace_aggid_t);
ASSERT(IS_P2ALIGNED(aggbase,
sizeof (uint64_t)));
}
curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
rec->dtrd_offset = curneeded;
curneeded += rec->dtrd_size;
} else {
/* tuples must be followed by an aggregation */
ASSERT(act->dta_prev == NULL ||
!act->dta_prev->dta_intuple);
ecb->dte_size = P2ROUNDUP(ecb->dte_size,
rec->dtrd_alignment);
rec->dtrd_offset = ecb->dte_size;
ecb->dte_size += rec->dtrd_size;
ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size);
}
}
if ((act = ecb->dte_action) != NULL &&
!(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) &&
ecb->dte_size == sizeof (dtrace_rechdr_t)) {
/*
* If the size is still sizeof (dtrace_rechdr_t), then all
* actions store no data; set the size to 0.
*/
ecb->dte_size = 0;
}
ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t));
ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t)));
ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed,
ecb->dte_needed);
}
static dtrace_action_t *
dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
{
dtrace_aggregation_t *agg;
size_t size = sizeof (uint64_t);
int ntuple = desc->dtad_ntuple;
dtrace_action_t *act;
dtrace_recdesc_t *frec;
dtrace_aggid_t aggid;
dtrace_state_t *state = ecb->dte_state;
agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP);
agg->dtag_ecb = ecb;
ASSERT(DTRACEACT_ISAGG(desc->dtad_kind));
switch (desc->dtad_kind) {
case DTRACEAGG_MIN:
agg->dtag_initial = INT64_MAX;
agg->dtag_aggregate = dtrace_aggregate_min;
break;
case DTRACEAGG_MAX:
agg->dtag_initial = INT64_MIN;
agg->dtag_aggregate = dtrace_aggregate_max;
break;
case DTRACEAGG_COUNT:
agg->dtag_aggregate = dtrace_aggregate_count;
break;
case DTRACEAGG_QUANTIZE:
agg->dtag_aggregate = dtrace_aggregate_quantize;
size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) *
sizeof (uint64_t);
break;
case DTRACEAGG_LQUANTIZE: {
uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg);
uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg);
agg->dtag_initial = desc->dtad_arg;
agg->dtag_aggregate = dtrace_aggregate_lquantize;
if (step == 0 || levels == 0)
goto err;
size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t);
break;
}
case DTRACEAGG_LLQUANTIZE: {
uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg);
uint16_t low = DTRACE_LLQUANTIZE_LOW(desc->dtad_arg);
uint16_t high = DTRACE_LLQUANTIZE_HIGH(desc->dtad_arg);
uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(desc->dtad_arg);
int64_t v;
agg->dtag_initial = desc->dtad_arg;
agg->dtag_aggregate = dtrace_aggregate_llquantize;
if (factor < 2 || low >= high || nsteps < factor)
goto err;
/*
* Now check that the number of steps evenly divides a power
* of the factor. (This assures both integer bucket size and
* linearity within each magnitude.)
*/
for (v = factor; v < nsteps; v *= factor)
continue;
if ((v % nsteps) || (nsteps % factor))
goto err;
size = (dtrace_aggregate_llquantize_bucket(factor,
low, high, nsteps, INT64_MAX) + 2) * sizeof (uint64_t);
break;
}
case DTRACEAGG_AVG:
agg->dtag_aggregate = dtrace_aggregate_avg;
size = sizeof (uint64_t) * 2;
break;
case DTRACEAGG_STDDEV:
agg->dtag_aggregate = dtrace_aggregate_stddev;
size = sizeof (uint64_t) * 4;
break;
case DTRACEAGG_SUM:
agg->dtag_aggregate = dtrace_aggregate_sum;
break;
default:
goto err;
}
agg->dtag_action.dta_rec.dtrd_size = size;
if (ntuple == 0)
goto err;
/*
* We must make sure that we have enough actions for the n-tuple.
*/
for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {
if (DTRACEACT_ISAGG(act->dta_kind))
break;
if (--ntuple == 0) {
/*
* This is the action with which our n-tuple begins.
*/
agg->dtag_first = act;
goto success;
}
}
/*
* This n-tuple is short by ntuple elements. Return failure.
*/
ASSERT(ntuple != 0);
err:
kmem_free(agg, sizeof (dtrace_aggregation_t));
return (NULL);
success:
/*
* If the last action in the tuple has a size of zero, it's actually
* an expression argument for the aggregating action.
*/
ASSERT(ecb->dte_action_last != NULL);
act = ecb->dte_action_last;
if (act->dta_kind == DTRACEACT_DIFEXPR) {
ASSERT(act->dta_difo != NULL);
if (act->dta_difo->dtdo_rtype.dtdt_size == 0)
agg->dtag_hasarg = 1;
}
/*
* We need to allocate an id for this aggregation.
*/
#ifdef illumos
aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1,
VM_BESTFIT | VM_SLEEP);
#else
aggid = alloc_unr(state->dts_aggid_arena);
#endif
if (aggid - 1 >= state->dts_naggregations) {
dtrace_aggregation_t **oaggs = state->dts_aggregations;
dtrace_aggregation_t **aggs;
int naggs = state->dts_naggregations << 1;
int onaggs = state->dts_naggregations;
ASSERT(aggid == state->dts_naggregations + 1);
if (naggs == 0) {
ASSERT(oaggs == NULL);
naggs = 1;
}
aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP);
if (oaggs != NULL) {
bcopy(oaggs, aggs, onaggs * sizeof (*aggs));
kmem_free(oaggs, onaggs * sizeof (*aggs));
}
state->dts_aggregations = aggs;
state->dts_naggregations = naggs;
}
ASSERT(state->dts_aggregations[aggid - 1] == NULL);
state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg;
frec = &agg->dtag_first->dta_rec;
if (frec->dtrd_alignment < sizeof (dtrace_aggid_t))
frec->dtrd_alignment = sizeof (dtrace_aggid_t);
for (act = agg->dtag_first; act != NULL; act = act->dta_next) {
ASSERT(!act->dta_intuple);
act->dta_intuple = 1;
}
return (&agg->dtag_action);
}
static void
dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)
{
dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
dtrace_state_t *state = ecb->dte_state;
dtrace_aggid_t aggid = agg->dtag_id;
ASSERT(DTRACEACT_ISAGG(act->dta_kind));
#ifdef illumos
vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1);
#else
free_unr(state->dts_aggid_arena, aggid);
#endif
ASSERT(state->dts_aggregations[aggid - 1] == agg);
state->dts_aggregations[aggid - 1] = NULL;
kmem_free(agg, sizeof (dtrace_aggregation_t));
}
static int
dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
{
dtrace_action_t *action, *last;
dtrace_difo_t *dp = desc->dtad_difo;
uint32_t size = 0, align = sizeof (uint8_t), mask;
uint16_t format = 0;
dtrace_recdesc_t *rec;
dtrace_state_t *state = ecb->dte_state;
dtrace_optval_t *opt = state->dts_options, nframes = 0, strsize;
uint64_t arg = desc->dtad_arg;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);
if (DTRACEACT_ISAGG(desc->dtad_kind)) {
/*
* If this is an aggregating action, there must be neither
* a speculate nor a commit on the action chain.
*/
dtrace_action_t *act;
for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
if (act->dta_kind == DTRACEACT_COMMIT)
return (EINVAL);
if (act->dta_kind == DTRACEACT_SPECULATE)
return (EINVAL);
}
action = dtrace_ecb_aggregation_create(ecb, desc);
if (action == NULL)
return (EINVAL);
} else {
if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) ||
(desc->dtad_kind == DTRACEACT_DIFEXPR &&
dp != NULL && dp->dtdo_destructive)) {
state->dts_destructive = 1;
}
switch (desc->dtad_kind) {
case DTRACEACT_PRINTF:
case DTRACEACT_PRINTA:
case DTRACEACT_SYSTEM:
case DTRACEACT_FREOPEN:
case DTRACEACT_DIFEXPR:
/*
* We know that our arg is a string -- turn it into a
* format.
*/
if (arg == 0) {
ASSERT(desc->dtad_kind == DTRACEACT_PRINTA ||
desc->dtad_kind == DTRACEACT_DIFEXPR);
format = 0;
} else {
ASSERT(arg != 0);
#ifdef illumos
ASSERT(arg > KERNELBASE);
#endif
format = dtrace_format_add(state,
(char *)(uintptr_t)arg);
}
/*FALLTHROUGH*/
case DTRACEACT_LIBACT:
case DTRACEACT_TRACEMEM:
case DTRACEACT_TRACEMEM_DYNSIZE:
if (dp == NULL)
return (EINVAL);
if ((size = dp->dtdo_rtype.dtdt_size) != 0)
break;
if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
return (EINVAL);
size = opt[DTRACEOPT_STRSIZE];
}
break;
case DTRACEACT_STACK:
if ((nframes = arg) == 0) {
nframes = opt[DTRACEOPT_STACKFRAMES];
ASSERT(nframes > 0);
arg = nframes;
}
size = nframes * sizeof (pc_t);
break;
case DTRACEACT_JSTACK:
if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0)
strsize = opt[DTRACEOPT_JSTACKSTRSIZE];
if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0)
nframes = opt[DTRACEOPT_JSTACKFRAMES];
arg = DTRACE_USTACK_ARG(nframes, strsize);
/*FALLTHROUGH*/
case DTRACEACT_USTACK:
if (desc->dtad_kind != DTRACEACT_JSTACK &&
(nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) {
strsize = DTRACE_USTACK_STRSIZE(arg);
nframes = opt[DTRACEOPT_USTACKFRAMES];
ASSERT(nframes > 0);
arg = DTRACE_USTACK_ARG(nframes, strsize);
}
/*
* Save a slot for the pid.
*/
size = (nframes + 1) * sizeof (uint64_t);
size += DTRACE_USTACK_STRSIZE(arg);
size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t)));
break;
case DTRACEACT_SYM:
case DTRACEACT_MOD:
if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) !=
sizeof (uint64_t)) ||
(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
return (EINVAL);
break;
case DTRACEACT_USYM:
case DTRACEACT_UMOD:
case DTRACEACT_UADDR:
if (dp == NULL ||
(dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) ||
(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
return (EINVAL);
/*
* We have a slot for the pid, plus a slot for the
* argument. To keep things simple (aligned with
* bitness-neutral sizing), we store each as a 64-bit
* quantity.
*/
size = 2 * sizeof (uint64_t);
break;
case DTRACEACT_STOP:
case DTRACEACT_BREAKPOINT:
case DTRACEACT_PANIC:
break;
case DTRACEACT_CHILL:
case DTRACEACT_DISCARD:
case DTRACEACT_RAISE:
if (dp == NULL)
return (EINVAL);
break;
case DTRACEACT_EXIT:
if (dp == NULL ||
(size = dp->dtdo_rtype.dtdt_size) != sizeof (int) ||
(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
return (EINVAL);
break;
case DTRACEACT_SPECULATE:
if (ecb->dte_size > sizeof (dtrace_rechdr_t))
return (EINVAL);
if (dp == NULL)
return (EINVAL);
state->dts_speculates = 1;
break;
case DTRACEACT_PRINTM:
size = dp->dtdo_rtype.dtdt_size;
break;
case DTRACEACT_PRINTT:
size = dp->dtdo_rtype.dtdt_size;
break;
case DTRACEACT_COMMIT: {
dtrace_action_t *act = ecb->dte_action;
for (; act != NULL; act = act->dta_next) {
if (act->dta_kind == DTRACEACT_COMMIT)
return (EINVAL);
}
if (dp == NULL)
return (EINVAL);
break;
}
default:
return (EINVAL);
}
if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) {
/*
* If this is a data-storing action or a speculate,
* we must be sure that there isn't a commit on the
* action chain.
*/
dtrace_action_t *act = ecb->dte_action;
for (; act != NULL; act = act->dta_next) {
if (act->dta_kind == DTRACEACT_COMMIT)
return (EINVAL);
}
}
action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP);
action->dta_rec.dtrd_size = size;
}
action->dta_refcnt = 1;
rec = &action->dta_rec;
size = rec->dtrd_size;
for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) {
if (!(size & mask)) {
align = mask + 1;
break;
}
}
action->dta_kind = desc->dtad_kind;
if ((action->dta_difo = dp) != NULL)
dtrace_difo_hold(dp);
rec->dtrd_action = action->dta_kind;
rec->dtrd_arg = arg;
rec->dtrd_uarg = desc->dtad_uarg;
rec->dtrd_alignment = (uint16_t)align;
rec->dtrd_format = format;
if ((last = ecb->dte_action_last) != NULL) {
ASSERT(ecb->dte_action != NULL);
action->dta_prev = last;
last->dta_next = action;
} else {
ASSERT(ecb->dte_action == NULL);
ecb->dte_action = action;
}
ecb->dte_action_last = action;
return (0);
}
static void
dtrace_ecb_action_remove(dtrace_ecb_t *ecb)
{
dtrace_action_t *act = ecb->dte_action, *next;
dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;
dtrace_difo_t *dp;
uint16_t format;
if (act != NULL && act->dta_refcnt > 1) {
ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1);
act->dta_refcnt--;
} else {
for (; act != NULL; act = next) {
next = act->dta_next;
ASSERT(next != NULL || act == ecb->dte_action_last);
ASSERT(act->dta_refcnt == 1);
if ((format = act->dta_rec.dtrd_format) != 0)
dtrace_format_remove(ecb->dte_state, format);
if ((dp = act->dta_difo) != NULL)
dtrace_difo_release(dp, vstate);
if (DTRACEACT_ISAGG(act->dta_kind)) {
dtrace_ecb_aggregation_destroy(ecb, act);
} else {
kmem_free(act, sizeof (dtrace_action_t));
}
}
}
ecb->dte_action = NULL;
ecb->dte_action_last = NULL;
ecb->dte_size = 0;
}
static void
dtrace_ecb_disable(dtrace_ecb_t *ecb)
{
/*
* We disable the ECB by removing it from its probe.
*/
dtrace_ecb_t *pecb, *prev = NULL;
dtrace_probe_t *probe = ecb->dte_probe;
ASSERT(MUTEX_HELD(&dtrace_lock));
if (probe == NULL) {
/*
* This is the NULL probe; there is nothing to disable.
*/
return;
}
for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) {
if (pecb == ecb)
break;
prev = pecb;
}
ASSERT(pecb != NULL);
if (prev == NULL) {
probe->dtpr_ecb = ecb->dte_next;
} else {
prev->dte_next = ecb->dte_next;
}
if (ecb == probe->dtpr_ecb_last) {
ASSERT(ecb->dte_next == NULL);
probe->dtpr_ecb_last = prev;
}
/*
* The ECB has been disconnected from the probe; now sync to assure
* that all CPUs have seen the change before returning.
*/
dtrace_sync();
if (probe->dtpr_ecb == NULL) {
/*
* That was the last ECB on the probe; clear the predicate
* cache ID for the probe, disable it and sync one more time
* to assure that we'll never hit it again.
*/
dtrace_provider_t *prov = probe->dtpr_provider;
ASSERT(ecb->dte_next == NULL);
ASSERT(probe->dtpr_ecb_last == NULL);
probe->dtpr_predcache = DTRACE_CACHEIDNONE;
prov->dtpv_pops.dtps_disable(prov->dtpv_arg,
probe->dtpr_id, probe->dtpr_arg);
dtrace_sync();
} else {
/*
* There is at least one ECB remaining on the probe. If there
* is _exactly_ one, set the probe's predicate cache ID to be
* the predicate cache ID of the remaining ECB.
*/
ASSERT(probe->dtpr_ecb_last != NULL);
ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE);
if (probe->dtpr_ecb == probe->dtpr_ecb_last) {
dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate;
ASSERT(probe->dtpr_ecb->dte_next == NULL);
if (p != NULL)
probe->dtpr_predcache = p->dtp_cacheid;
}
ecb->dte_next = NULL;
}
}
static void
dtrace_ecb_destroy(dtrace_ecb_t *ecb)
{
dtrace_state_t *state = ecb->dte_state;
dtrace_vstate_t *vstate = &state->dts_vstate;
dtrace_predicate_t *pred;
dtrace_epid_t epid = ecb->dte_epid;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(ecb->dte_next == NULL);
ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);
if ((pred = ecb->dte_predicate) != NULL)
dtrace_predicate_release(pred, vstate);
dtrace_ecb_action_remove(ecb);
ASSERT(state->dts_ecbs[epid - 1] == ecb);
state->dts_ecbs[epid - 1] = NULL;
kmem_free(ecb, sizeof (dtrace_ecb_t));
}
static dtrace_ecb_t *
dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe,
dtrace_enabling_t *enab)
{
dtrace_ecb_t *ecb;
dtrace_predicate_t *pred;
dtrace_actdesc_t *act;
dtrace_provider_t *prov;
dtrace_ecbdesc_t *desc = enab->dten_current;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(state != NULL);
ecb = dtrace_ecb_add(state, probe);
ecb->dte_uarg = desc->dted_uarg;
if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) {
dtrace_predicate_hold(pred);
ecb->dte_predicate = pred;
}
if (probe != NULL) {
/*
* If the provider shows more leg than the consumer is old
* enough to see, we need to enable the appropriate implicit
* predicate bits to prevent the ecb from activating at
* revealing times.
*
* Providers specifying DTRACE_PRIV_USER at register time
* are stating that they need the /proc-style privilege
* model to be enforced, and this is what DTRACE_COND_OWNER
* and DTRACE_COND_ZONEOWNER will then do at probe time.
*/
prov = probe->dtpr_provider;
if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) &&
(prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
ecb->dte_cond |= DTRACE_COND_OWNER;
if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) &&
(prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
ecb->dte_cond |= DTRACE_COND_ZONEOWNER;
/*
* If the provider shows us kernel innards and the user
* is lacking sufficient privilege, enable the
* DTRACE_COND_USERMODE implicit predicate.
*/
if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) &&
(prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL))
ecb->dte_cond |= DTRACE_COND_USERMODE;
}
if (dtrace_ecb_create_cache != NULL) {
/*
* If we have a cached ecb, we'll use its action list instead
* of creating our own (saving both time and space).
*/
dtrace_ecb_t *cached = dtrace_ecb_create_cache;
dtrace_action_t *act = cached->dte_action;
if (act != NULL) {
ASSERT(act->dta_refcnt > 0);
act->dta_refcnt++;
ecb->dte_action = act;
ecb->dte_action_last = cached->dte_action_last;
ecb->dte_needed = cached->dte_needed;
ecb->dte_size = cached->dte_size;
ecb->dte_alignment = cached->dte_alignment;
}
return (ecb);
}
for (act = desc->dted_action; act != NULL; act = act->dtad_next) {
if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {
dtrace_ecb_destroy(ecb);
return (NULL);
}
}
dtrace_ecb_resize(ecb);
return (dtrace_ecb_create_cache = ecb);
}
static int
dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg)
{
dtrace_ecb_t *ecb;
dtrace_enabling_t *enab = arg;
dtrace_state_t *state = enab->dten_vstate->dtvs_state;
ASSERT(state != NULL);
if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) {
/*
* This probe was created in a generation for which this
* enabling has previously created ECBs; we don't want to
* enable it again, so just kick out.
*/
return (DTRACE_MATCH_NEXT);
}
if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
return (DTRACE_MATCH_DONE);
dtrace_ecb_enable(ecb);
return (DTRACE_MATCH_NEXT);
}
static dtrace_ecb_t *
dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id)
{
dtrace_ecb_t *ecb;
ASSERT(MUTEX_HELD(&dtrace_lock));
if (id == 0 || id > state->dts_necbs)
return (NULL);
ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL);
ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
return (state->dts_ecbs[id - 1]);
}
static dtrace_aggregation_t *
dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id)
{
dtrace_aggregation_t *agg;
ASSERT(MUTEX_HELD(&dtrace_lock));
if (id == 0 || id > state->dts_naggregations)
return (NULL);
ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL);
ASSERT((agg = state->dts_aggregations[id - 1]) == NULL ||
agg->dtag_id == id);
return (state->dts_aggregations[id - 1]);
}
/*
* DTrace Buffer Functions
*
* The following functions manipulate DTrace buffers. Most of these functions
* are called in the context of establishing or processing consumer state;
* exceptions are explicitly noted.
*/
/*
* Note: called from cross call context. This function switches the two
* buffers on a given CPU. The atomicity of this operation is assured by
* disabling interrupts while the actual switch takes place; the disabling of
* interrupts serializes the execution with any execution of dtrace_probe() on
* the same CPU.
*/
static void
dtrace_buffer_switch(dtrace_buffer_t *buf)
{
caddr_t tomax = buf->dtb_tomax;
caddr_t xamot = buf->dtb_xamot;
dtrace_icookie_t cookie;
hrtime_t now;
ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
ASSERT(!(buf->dtb_flags & DTRACEBUF_RING));
cookie = dtrace_interrupt_disable();
now = dtrace_gethrtime();
buf->dtb_tomax = xamot;
buf->dtb_xamot = tomax;
buf->dtb_xamot_drops = buf->dtb_drops;
buf->dtb_xamot_offset = buf->dtb_offset;
buf->dtb_xamot_errors = buf->dtb_errors;
buf->dtb_xamot_flags = buf->dtb_flags;
buf->dtb_offset = 0;
buf->dtb_drops = 0;
buf->dtb_errors = 0;
buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED);
buf->dtb_interval = now - buf->dtb_switched;
buf->dtb_switched = now;
dtrace_interrupt_enable(cookie);
}
/*
* Note: called from cross call context. This function activates a buffer
* on a CPU. As with dtrace_buffer_switch(), the atomicity of the operation
* is guaranteed by the disabling of interrupts.
*/
static void
dtrace_buffer_activate(dtrace_state_t *state)
{
dtrace_buffer_t *buf;
dtrace_icookie_t cookie = dtrace_interrupt_disable();
buf = &state->dts_buffer[curcpu];
if (buf->dtb_tomax != NULL) {
/*
* We might like to assert that the buffer is marked inactive,
* but this isn't necessarily true: the buffer for the CPU
* that processes the BEGIN probe has its buffer activated
* manually. In this case, we take the (harmless) action
* re-clearing the bit INACTIVE bit.
*/
buf->dtb_flags &= ~DTRACEBUF_INACTIVE;
}
dtrace_interrupt_enable(cookie);
}
static int
dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags,
processorid_t cpu, int *factor)
{
#ifdef illumos
cpu_t *cp;
#endif
dtrace_buffer_t *buf;
int allocated = 0, desired = 0;
#ifdef illumos
ASSERT(MUTEX_HELD(&cpu_lock));
ASSERT(MUTEX_HELD(&dtrace_lock));
*factor = 1;
if (size > dtrace_nonroot_maxsize &&
!PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE))
return (EFBIG);
cp = cpu_list;
do {
if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
continue;
buf = &bufs[cp->cpu_id];
/*
* If there is already a buffer allocated for this CPU, it
* is only possible that this is a DR event. In this case,
*/
if (buf->dtb_tomax != NULL) {
ASSERT(buf->dtb_size == size);
continue;
}
ASSERT(buf->dtb_xamot == NULL);
if ((buf->dtb_tomax = kmem_zalloc(size,
KM_NOSLEEP | KM_NORMALPRI)) == NULL)
goto err;
buf->dtb_size = size;
buf->dtb_flags = flags;
buf->dtb_offset = 0;
buf->dtb_drops = 0;
if (flags & DTRACEBUF_NOSWITCH)
continue;
if ((buf->dtb_xamot = kmem_zalloc(size,
KM_NOSLEEP | KM_NORMALPRI)) == NULL)
goto err;
} while ((cp = cp->cpu_next) != cpu_list);
return (0);
err:
cp = cpu_list;
do {
if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
continue;
buf = &bufs[cp->cpu_id];
desired += 2;
if (buf->dtb_xamot != NULL) {
ASSERT(buf->dtb_tomax != NULL);
ASSERT(buf->dtb_size == size);
kmem_free(buf->dtb_xamot, size);
allocated++;
}
if (buf->dtb_tomax != NULL) {
ASSERT(buf->dtb_size == size);
kmem_free(buf->dtb_tomax, size);
allocated++;
}
buf->dtb_tomax = NULL;
buf->dtb_xamot = NULL;
buf->dtb_size = 0;
} while ((cp = cp->cpu_next) != cpu_list);
#else
int i;
*factor = 1;
#if defined(__amd64__) || defined(__arm__) || defined(__mips__) || defined(__powerpc__)
/*
* FreeBSD isn't good at limiting the amount of memory we
* ask to malloc, so let's place a limit here before trying
* to do something that might well end in tears at bedtime.
*/
if (size > physmem * PAGE_SIZE / (128 * (mp_maxid + 1)))
return (ENOMEM);
#endif
ASSERT(MUTEX_HELD(&dtrace_lock));
CPU_FOREACH(i) {
if (cpu != DTRACE_CPUALL && cpu != i)
continue;
buf = &bufs[i];
/*
* If there is already a buffer allocated for this CPU, it
* is only possible that this is a DR event. In this case,
* the buffer size must match our specified size.
*/
if (buf->dtb_tomax != NULL) {
ASSERT(buf->dtb_size == size);
continue;
}
ASSERT(buf->dtb_xamot == NULL);
if ((buf->dtb_tomax = kmem_zalloc(size,
KM_NOSLEEP | KM_NORMALPRI)) == NULL)
goto err;
buf->dtb_size = size;
buf->dtb_flags = flags;
buf->dtb_offset = 0;
buf->dtb_drops = 0;
if (flags & DTRACEBUF_NOSWITCH)
continue;
if ((buf->dtb_xamot = kmem_zalloc(size,
KM_NOSLEEP | KM_NORMALPRI)) == NULL)
goto err;
}
return (0);
err:
/*
* Error allocating memory, so free the buffers that were
* allocated before the failed allocation.
*/
CPU_FOREACH(i) {
if (cpu != DTRACE_CPUALL && cpu != i)
continue;
buf = &bufs[i];
desired += 2;
if (buf->dtb_xamot != NULL) {
ASSERT(buf->dtb_tomax != NULL);
ASSERT(buf->dtb_size == size);
kmem_free(buf->dtb_xamot, size);
allocated++;
}
if (buf->dtb_tomax != NULL) {
ASSERT(buf->dtb_size == size);
kmem_free(buf->dtb_tomax, size);
allocated++;
}
buf->dtb_tomax = NULL;
buf->dtb_xamot = NULL;
buf->dtb_size = 0;
}
#endif
*factor = desired / (allocated > 0 ? allocated : 1);
return (ENOMEM);
}
/*
* Note: called from probe context. This function just increments the drop
* count on a buffer. It has been made a function to allow for the
* possibility of understanding the source of mysterious drop counts. (A
* problem for which one may be particularly disappointed that DTrace cannot
* be used to understand DTrace.)
*/
static void
dtrace_buffer_drop(dtrace_buffer_t *buf)
{
buf->dtb_drops++;
}
/*
* Note: called from probe context. This function is called to reserve space
* in a buffer. If mstate is non-NULL, sets the scratch base and size in the
* mstate. Returns the new offset in the buffer, or a negative value if an
* error has occurred.
*/
static intptr_t
dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align,
dtrace_state_t *state, dtrace_mstate_t *mstate)
{
intptr_t offs = buf->dtb_offset, soffs;
intptr_t woffs;
caddr_t tomax;
size_t total;
if (buf->dtb_flags & DTRACEBUF_INACTIVE)
return (-1);
if ((tomax = buf->dtb_tomax) == NULL) {
dtrace_buffer_drop(buf);
return (-1);
}
if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) {
while (offs & (align - 1)) {
/*
* Assert that our alignment is off by a number which
* is itself sizeof (uint32_t) aligned.
*/
ASSERT(!((align - (offs & (align - 1))) &
(sizeof (uint32_t) - 1)));
DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
offs += sizeof (uint32_t);
}
if ((soffs = offs + needed) > buf->dtb_size) {
dtrace_buffer_drop(buf);
return (-1);
}
if (mstate == NULL)
return (offs);
mstate->dtms_scratch_base = (uintptr_t)tomax + soffs;
mstate->dtms_scratch_size = buf->dtb_size - soffs;
mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
return (offs);
}
if (buf->dtb_flags & DTRACEBUF_FILL) {
if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN &&
(buf->dtb_flags & DTRACEBUF_FULL))
return (-1);
goto out;
}
total = needed + (offs & (align - 1));
/*
* For a ring buffer, life is quite a bit more complicated. Before
* we can store any padding, we need to adjust our wrapping offset.
* (If we've never before wrapped or we're not about to, no adjustment
* is required.)
*/
if ((buf->dtb_flags & DTRACEBUF_WRAPPED) ||
offs + total > buf->dtb_size) {
woffs = buf->dtb_xamot_offset;
if (offs + total > buf->dtb_size) {
/*
* We can't fit in the end of the buffer. First, a
* sanity check that we can fit in the buffer at all.
*/
if (total > buf->dtb_size) {
dtrace_buffer_drop(buf);
return (-1);
}
/*
* We're going to be storing at the top of the buffer,
* so now we need to deal with the wrapped offset. We
* only reset our wrapped offset to 0 if it is
* currently greater than the current offset. If it
* is less than the current offset, it is because a
* previous allocation induced a wrap -- but the
* allocation didn't subsequently take the space due
* to an error or false predicate evaluation. In this
* case, we'll just leave the wrapped offset alone: if
* the wrapped offset hasn't been advanced far enough
* for this allocation, it will be adjusted in the
* lower loop.
*/
if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
if (woffs >= offs)
woffs = 0;
} else {
woffs = 0;
}
/*
* Now we know that we're going to be storing to the
* top of the buffer and that there is room for us
* there. We need to clear the buffer from the current
* offset to the end (there may be old gunk there).
*/
while (offs < buf->dtb_size)
tomax[offs++] = 0;
/*
* We need to set our offset to zero. And because we
* are wrapping, we need to set the bit indicating as
* much. We can also adjust our needed space back
* down to the space required by the ECB -- we know
* that the top of the buffer is aligned.
*/
offs = 0;
total = needed;
buf->dtb_flags |= DTRACEBUF_WRAPPED;
} else {
/*
* There is room for us in the buffer, so we simply
* need to check the wrapped offset.
*/
if (woffs < offs) {
/*
* The wrapped offset is less than the offset.
* This can happen if we allocated buffer space
* that induced a wrap, but then we didn't
* subsequently take the space due to an error
* or false predicate evaluation. This is
* okay; we know that _this_ allocation isn't
* going to induce a wrap. We still can't
* reset the wrapped offset to be zero,
* however: the space may have been trashed in
* the previous failed probe attempt. But at
* least the wrapped offset doesn't need to
* be adjusted at all...
*/
goto out;
}
}
while (offs + total > woffs) {
dtrace_epid_t epid = *(uint32_t *)(tomax + woffs);
size_t size;
if (epid == DTRACE_EPIDNONE) {
size = sizeof (uint32_t);
} else {
ASSERT3U(epid, <=, state->dts_necbs);
ASSERT(state->dts_ecbs[epid - 1] != NULL);
size = state->dts_ecbs[epid - 1]->dte_size;
}
ASSERT(woffs + size <= buf->dtb_size);
ASSERT(size != 0);
if (woffs + size == buf->dtb_size) {
/*
* We've reached the end of the buffer; we want
* to set the wrapped offset to 0 and break
* out. However, if the offs is 0, then we're
* in a strange edge-condition: the amount of
* space that we want to reserve plus the size
* of the record that we're overwriting is
* greater than the size of the buffer. This
* is problematic because if we reserve the
* space but subsequently don't consume it (due
* to a failed predicate or error) the wrapped
* offset will be 0 -- yet the EPID at offset 0
* will not be committed. This situation is
* relatively easy to deal with: if we're in
* this case, the buffer is indistinguishable
* from one that hasn't wrapped; we need only
* finish the job by clearing the wrapped bit,
* explicitly setting the offset to be 0, and
* zero'ing out the old data in the buffer.
*/
if (offs == 0) {
buf->dtb_flags &= ~DTRACEBUF_WRAPPED;
buf->dtb_offset = 0;
woffs = total;
while (woffs < buf->dtb_size)
tomax[woffs++] = 0;
}
woffs = 0;
break;
}
woffs += size;
}
/*
* We have a wrapped offset. It may be that the wrapped offset
* has become zero -- that's okay.
*/
buf->dtb_xamot_offset = woffs;
}
out:
/*
* Now we can plow the buffer with any necessary padding.
*/
while (offs & (align - 1)) {
/*
* Assert that our alignment is off by a number which
* is itself sizeof (uint32_t) aligned.
*/
ASSERT(!((align - (offs & (align - 1))) &
(sizeof (uint32_t) - 1)));
DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
offs += sizeof (uint32_t);
}
if (buf->dtb_flags & DTRACEBUF_FILL) {
if (offs + needed > buf->dtb_size - state->dts_reserve) {
buf->dtb_flags |= DTRACEBUF_FULL;
return (-1);
}
}
if (mstate == NULL)
return (offs);
/*
* For ring buffers and fill buffers, the scratch space is always
* the inactive buffer.
*/
mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot;
mstate->dtms_scratch_size = buf->dtb_size;
mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
return (offs);
}
static void
dtrace_buffer_polish(dtrace_buffer_t *buf)
{
ASSERT(buf->dtb_flags & DTRACEBUF_RING);
ASSERT(MUTEX_HELD(&dtrace_lock));
if (!(buf->dtb_flags & DTRACEBUF_WRAPPED))
return;
/*
* We need to polish the ring buffer. There are three cases:
*
* - The first (and presumably most common) is that there is no gap
* between the buffer offset and the wrapped offset. In this case,
* there is nothing in the buffer that isn't valid data; we can
* mark the buffer as polished and return.
*
* - The second (less common than the first but still more common
* than the third) is that there is a gap between the buffer offset
* and the wrapped offset, and the wrapped offset is larger than the
* buffer offset. This can happen because of an alignment issue, or
* can happen because of a call to dtrace_buffer_reserve() that
* didn't subsequently consume the buffer space. In this case,
* we need to zero the data from the buffer offset to the wrapped
* offset.
*
* - The third (and least common) is that there is a gap between the
* buffer offset and the wrapped offset, but the wrapped offset is
* _less_ than the buffer offset. This can only happen because a
* call to dtrace_buffer_reserve() induced a wrap, but the space
* was not subsequently consumed. In this case, we need to zero the
* space from the offset to the end of the buffer _and_ from the
* top of the buffer to the wrapped offset.
*/
if (buf->dtb_offset < buf->dtb_xamot_offset) {
bzero(buf->dtb_tomax + buf->dtb_offset,
buf->dtb_xamot_offset - buf->dtb_offset);
}
if (buf->dtb_offset > buf->dtb_xamot_offset) {
bzero(buf->dtb_tomax + buf->dtb_offset,
buf->dtb_size - buf->dtb_offset);
bzero(buf->dtb_tomax, buf->dtb_xamot_offset);
}
}
/*
* This routine determines if data generated at the specified time has likely
* been entirely consumed at user-level. This routine is called to determine
* if an ECB on a defunct probe (but for an active enabling) can be safely
* disabled and destroyed.
*/
static int
dtrace_buffer_consumed(dtrace_buffer_t *bufs, hrtime_t when)
{
int i;
for (i = 0; i < NCPU; i++) {
dtrace_buffer_t *buf = &bufs[i];
if (buf->dtb_size == 0)
continue;
if (buf->dtb_flags & DTRACEBUF_RING)
return (0);
if (!buf->dtb_switched && buf->dtb_offset != 0)
return (0);
if (buf->dtb_switched - buf->dtb_interval < when)
return (0);
}
return (1);
}
static void
dtrace_buffer_free(dtrace_buffer_t *bufs)
{
int i;
for (i = 0; i < NCPU; i++) {
dtrace_buffer_t *buf = &bufs[i];
if (buf->dtb_tomax == NULL) {
ASSERT(buf->dtb_xamot == NULL);
ASSERT(buf->dtb_size == 0);
continue;
}
if (buf->dtb_xamot != NULL) {
ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
kmem_free(buf->dtb_xamot, buf->dtb_size);
}
kmem_free(buf->dtb_tomax, buf->dtb_size);
buf->dtb_size = 0;
buf->dtb_tomax = NULL;
buf->dtb_xamot = NULL;
}
}
/*
* DTrace Enabling Functions
*/
static dtrace_enabling_t *
dtrace_enabling_create(dtrace_vstate_t *vstate)
{
dtrace_enabling_t *enab;
enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP);
enab->dten_vstate = vstate;
return (enab);
}
static void
dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)
{
dtrace_ecbdesc_t **ndesc;
size_t osize, nsize;
/*
* We can't add to enablings after we've enabled them, or after we've
* retained them.
*/
ASSERT(enab->dten_probegen == 0);
ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
if (enab->dten_ndesc < enab->dten_maxdesc) {
enab->dten_desc[enab->dten_ndesc++] = ecb;
return;
}
osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
if (enab->dten_maxdesc == 0) {
enab->dten_maxdesc = 1;
} else {
enab->dten_maxdesc <<= 1;
}
ASSERT(enab->dten_ndesc < enab->dten_maxdesc);
nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
ndesc = kmem_zalloc(nsize, KM_SLEEP);
bcopy(enab->dten_desc, ndesc, osize);
if (enab->dten_desc != NULL)
kmem_free(enab->dten_desc, osize);
enab->dten_desc = ndesc;
enab->dten_desc[enab->dten_ndesc++] = ecb;
}
static void
dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,
dtrace_probedesc_t *pd)
{
dtrace_ecbdesc_t *new;
dtrace_predicate_t *pred;
dtrace_actdesc_t *act;
/*
* We're going to create a new ECB description that matches the
* specified ECB in every way, but has the specified probe description.
*/
new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)
dtrace_predicate_hold(pred);
for (act = ecb->dted_action; act != NULL; act = act->dtad_next)
dtrace_actdesc_hold(act);
new->dted_action = ecb->dted_action;
new->dted_pred = ecb->dted_pred;
new->dted_probe = *pd;
new->dted_uarg = ecb->dted_uarg;
dtrace_enabling_add(enab, new);
}
static void
dtrace_enabling_dump(dtrace_enabling_t *enab)
{
int i;
for (i = 0; i < enab->dten_ndesc; i++) {
dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe;
cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i,
desc->dtpd_provider, desc->dtpd_mod,
desc->dtpd_func, desc->dtpd_name);
}
}
static void
dtrace_enabling_destroy(dtrace_enabling_t *enab)
{
int i;
dtrace_ecbdesc_t *ep;
dtrace_vstate_t *vstate = enab->dten_vstate;
ASSERT(MUTEX_HELD(&dtrace_lock));
for (i = 0; i < enab->dten_ndesc; i++) {
dtrace_actdesc_t *act, *next;
dtrace_predicate_t *pred;
ep = enab->dten_desc[i];
if ((pred = ep->dted_pred.dtpdd_predicate) != NULL)
dtrace_predicate_release(pred, vstate);
for (act = ep->dted_action; act != NULL; act = next) {
next = act->dtad_next;
dtrace_actdesc_release(act, vstate);
}
kmem_free(ep, sizeof (dtrace_ecbdesc_t));
}
if (enab->dten_desc != NULL)
kmem_free(enab->dten_desc,
enab->dten_maxdesc * sizeof (dtrace_enabling_t *));
/*
* If this was a retained enabling, decrement the dts_nretained count
* and take it off of the dtrace_retained list.
*/
if (enab->dten_prev != NULL || enab->dten_next != NULL ||
dtrace_retained == enab) {
ASSERT(enab->dten_vstate->dtvs_state != NULL);
ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0);
enab->dten_vstate->dtvs_state->dts_nretained--;
dtrace_retained_gen++;
}
if (enab->dten_prev == NULL) {
if (dtrace_retained == enab) {
dtrace_retained = enab->dten_next;
if (dtrace_retained != NULL)
dtrace_retained->dten_prev = NULL;
}
} else {
ASSERT(enab != dtrace_retained);
ASSERT(dtrace_retained != NULL);
enab->dten_prev->dten_next = enab->dten_next;
}
if (enab->dten_next != NULL) {
ASSERT(dtrace_retained != NULL);
enab->dten_next->dten_prev = enab->dten_prev;
}
kmem_free(enab, sizeof (dtrace_enabling_t));
}
static int
dtrace_enabling_retain(dtrace_enabling_t *enab)
{
dtrace_state_t *state;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
ASSERT(enab->dten_vstate != NULL);
state = enab->dten_vstate->dtvs_state;
ASSERT(state != NULL);
/*
* We only allow each state to retain dtrace_retain_max enablings.
*/
if (state->dts_nretained >= dtrace_retain_max)
return (ENOSPC);
state->dts_nretained++;
dtrace_retained_gen++;
if (dtrace_retained == NULL) {
dtrace_retained = enab;
return (0);
}
enab->dten_next = dtrace_retained;
dtrace_retained->dten_prev = enab;
dtrace_retained = enab;
return (0);
}
static int
dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match,
dtrace_probedesc_t *create)
{
dtrace_enabling_t *new, *enab;
int found = 0, err = ENOENT;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN);
ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN);
ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN);
ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN);
new = dtrace_enabling_create(&state->dts_vstate);
/*
* Iterate over all retained enablings, looking for enablings that
* match the specified state.
*/
for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
int i;
/*
* dtvs_state can only be NULL for helper enablings -- and
* helper enablings can't be retained.
*/
ASSERT(enab->dten_vstate->dtvs_state != NULL);
if (enab->dten_vstate->dtvs_state != state)
continue;
/*
* Now iterate over each probe description; we're looking for
* an exact match to the specified probe description.
*/
for (i = 0; i < enab->dten_ndesc; i++) {
dtrace_ecbdesc_t *ep = enab->dten_desc[i];
dtrace_probedesc_t *pd = &ep->dted_probe;
if (strcmp(pd->dtpd_provider, match->dtpd_provider))
continue;
if (strcmp(pd->dtpd_mod, match->dtpd_mod))
continue;
if (strcmp(pd->dtpd_func, match->dtpd_func))
continue;
if (strcmp(pd->dtpd_name, match->dtpd_name))
continue;
/*
* We have a winning probe! Add it to our growing
* enabling.
*/
found = 1;
dtrace_enabling_addlike(new, ep, create);
}
}
if (!found || (err = dtrace_enabling_retain(new)) != 0) {
dtrace_enabling_destroy(new);
return (err);
}
return (0);
}
static void
dtrace_enabling_retract(dtrace_state_t *state)
{
dtrace_enabling_t *enab, *next;
ASSERT(MUTEX_HELD(&dtrace_lock));
/*
* Iterate over all retained enablings, destroy the enablings retained
* for the specified state.
*/
for (enab = dtrace_retained; enab != NULL; enab = next) {
next = enab->dten_next;
/*
* dtvs_state can only be NULL for helper enablings -- and
* helper enablings can't be retained.
*/
ASSERT(enab->dten_vstate->dtvs_state != NULL);
if (enab->dten_vstate->dtvs_state == state) {
ASSERT(state->dts_nretained > 0);
dtrace_enabling_destroy(enab);
}
}
ASSERT(state->dts_nretained == 0);
}
static int
dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
{
int i = 0;
int matched = 0;
ASSERT(MUTEX_HELD(&cpu_lock));
ASSERT(MUTEX_HELD(&dtrace_lock));
for (i = 0; i < enab->dten_ndesc; i++) {
dtrace_ecbdesc_t *ep = enab->dten_desc[i];
enab->dten_current = ep;
enab->dten_error = 0;
matched += dtrace_probe_enable(&ep->dted_probe, enab);
if (enab->dten_error != 0) {
/*
* If we get an error half-way through enabling the
* probes, we kick out -- perhaps with some number of
* them enabled. Leaving enabled probes enabled may
* be slightly confusing for user-level, but we expect
* that no one will attempt to actually drive on in
* the face of such errors. If this is an anonymous
* enabling (indicated with a NULL nmatched pointer),
* we cmn_err() a message. We aren't expecting to
* get such an error -- such as it can exist at all,
* it would be a result of corrupted DOF in the driver
* properties.
*/
if (nmatched == NULL) {
cmn_err(CE_WARN, "dtrace_enabling_match() "
"error on %p: %d", (void *)ep,
enab->dten_error);
}
return (enab->dten_error);
}
}
enab->dten_probegen = dtrace_probegen;
if (nmatched != NULL)
*nmatched = matched;
return (0);
}
static void
dtrace_enabling_matchall(void)
{
dtrace_enabling_t *enab;
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_lock);
/*
* Iterate over all retained enablings to see if any probes match
* against them. We only perform this operation on enablings for which
* we have sufficient permissions by virtue of being in the global zone
* or in the same zone as the DTrace client. Because we can be called
* after dtrace_detach() has been called, we cannot assert that there
* are retained enablings. We can safely load from dtrace_retained,
* however: the taskq_destroy() at the end of dtrace_detach() will
* block pending our completion.
*/
for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
#ifdef illumos
cred_t *cr = enab->dten_vstate->dtvs_state->dts_cred.dcr_cred;
if (INGLOBALZONE(curproc) ||
cr != NULL && getzoneid() == crgetzoneid(cr))
#endif
(void) dtrace_enabling_match(enab, NULL);
}
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
}
/*
* If an enabling is to be enabled without having matched probes (that is, if
* dtrace_state_go() is to be called on the underlying dtrace_state_t), the
* enabling must be _primed_ by creating an ECB for every ECB description.
* This must be done to assure that we know the number of speculations, the
* number of aggregations, the minimum buffer size needed, etc. before we
* transition out of DTRACE_ACTIVITY_INACTIVE. To do this without actually
* enabling any probes, we create ECBs for every ECB decription, but with a
* NULL probe -- which is exactly what this function does.
*/
static void
dtrace_enabling_prime(dtrace_state_t *state)
{
dtrace_enabling_t *enab;
int i;
for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
ASSERT(enab->dten_vstate->dtvs_state != NULL);
if (enab->dten_vstate->dtvs_state != state)
continue;
/*
* We don't want to prime an enabling more than once, lest
* we allow a malicious user to induce resource exhaustion.
* (The ECBs that result from priming an enabling aren't
* leaked -- but they also aren't deallocated until the
* consumer state is destroyed.)
*/
if (enab->dten_primed)
continue;
for (i = 0; i < enab->dten_ndesc; i++) {
enab->dten_current = enab->dten_desc[i];
(void) dtrace_probe_enable(NULL, enab);
}
enab->dten_primed = 1;
}
}
/*
* Called to indicate that probes should be provided due to retained
* enablings. This is implemented in terms of dtrace_probe_provide(), but it
* must take an initial lap through the enabling calling the dtps_provide()
* entry point explicitly to allow for autocreated probes.
*/
static void
dtrace_enabling_provide(dtrace_provider_t *prv)
{
int i, all = 0;
dtrace_probedesc_t desc;
dtrace_genid_t gen;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(MUTEX_HELD(&dtrace_provider_lock));
if (prv == NULL) {
all = 1;
prv = dtrace_provider;
}
do {
dtrace_enabling_t *enab;
void *parg = prv->dtpv_arg;
retry:
gen = dtrace_retained_gen;
for (enab = dtrace_retained; enab != NULL;
enab = enab->dten_next) {
for (i = 0; i < enab->dten_ndesc; i++) {
desc = enab->dten_desc[i]->dted_probe;
mutex_exit(&dtrace_lock);
prv->dtpv_pops.dtps_provide(parg, &desc);
mutex_enter(&dtrace_lock);
/*
* Process the retained enablings again if
* they have changed while we weren't holding
* dtrace_lock.
*/
if (gen != dtrace_retained_gen)
goto retry;
}
}
} while (all && (prv = prv->dtpv_next) != NULL);
mutex_exit(&dtrace_lock);
dtrace_probe_provide(NULL, all ? NULL : prv);
mutex_enter(&dtrace_lock);
}
/*
* Called to reap ECBs that are attached to probes from defunct providers.
*/
static void
dtrace_enabling_reap(void)
{
dtrace_provider_t *prov;
dtrace_probe_t *probe;
dtrace_ecb_t *ecb;
hrtime_t when;
int i;
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_lock);
for (i = 0; i < dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i]) == NULL)
continue;
if (probe->dtpr_ecb == NULL)
continue;
prov = probe->dtpr_provider;
if ((when = prov->dtpv_defunct) == 0)
continue;
/*
* We have ECBs on a defunct provider: we want to reap these
* ECBs to allow the provider to unregister. The destruction
* of these ECBs must be done carefully: if we destroy the ECB
* and the consumer later wishes to consume an EPID that
* corresponds to the destroyed ECB (and if the EPID metadata
* has not been previously consumed), the consumer will abort
* processing on the unknown EPID. To reduce (but not, sadly,
* eliminate) the possibility of this, we will only destroy an
* ECB for a defunct provider if, for the state that
* corresponds to the ECB:
*
* (a) There is no speculative tracing (which can effectively
* cache an EPID for an arbitrary amount of time).
*
* (b) The principal buffers have been switched twice since the
* provider became defunct.
*
* (c) The aggregation buffers are of zero size or have been
* switched twice since the provider became defunct.
*
* We use dts_speculates to determine (a) and call a function
* (dtrace_buffer_consumed()) to determine (b) and (c). Note
* that as soon as we've been unable to destroy one of the ECBs
* associated with the probe, we quit trying -- reaping is only
* fruitful in as much as we can destroy all ECBs associated
* with the defunct provider's probes.
*/
while ((ecb = probe->dtpr_ecb) != NULL) {
dtrace_state_t *state = ecb->dte_state;
dtrace_buffer_t *buf = state->dts_buffer;
dtrace_buffer_t *aggbuf = state->dts_aggbuffer;
if (state->dts_speculates)
break;
if (!dtrace_buffer_consumed(buf, when))
break;
if (!dtrace_buffer_consumed(aggbuf, when))
break;
dtrace_ecb_disable(ecb);
ASSERT(probe->dtpr_ecb != ecb);
dtrace_ecb_destroy(ecb);
}
}
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
}
/*
* DTrace DOF Functions
*/
/*ARGSUSED*/
static void
dtrace_dof_error(dof_hdr_t *dof, const char *str)
{
if (dtrace_err_verbose)
cmn_err(CE_WARN, "failed to process DOF: %s", str);
#ifdef DTRACE_ERRDEBUG
dtrace_errdebug(str);
#endif
}
/*
* Create DOF out of a currently enabled state. Right now, we only create
* DOF containing the run-time options -- but this could be expanded to create
* complete DOF representing the enabled state.
*/
static dof_hdr_t *
dtrace_dof_create(dtrace_state_t *state)
{
dof_hdr_t *dof;
dof_sec_t *sec;
dof_optdesc_t *opt;
int i, len = sizeof (dof_hdr_t) +
roundup(sizeof (dof_sec_t), sizeof (uint64_t)) +
sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
ASSERT(MUTEX_HELD(&dtrace_lock));
dof = kmem_zalloc(len, KM_SLEEP);
dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0;
dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1;
dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2;
dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3;
dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE;
dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE;
dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION;
dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION;
dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS;
dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS;
dof->dofh_flags = 0;
dof->dofh_hdrsize = sizeof (dof_hdr_t);
dof->dofh_secsize = sizeof (dof_sec_t);
dof->dofh_secnum = 1; /* only DOF_SECT_OPTDESC */
dof->dofh_secoff = sizeof (dof_hdr_t);
dof->dofh_loadsz = len;
dof->dofh_filesz = len;
dof->dofh_pad = 0;
/*
* Fill in the option section header...
*/
sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t));
sec->dofs_type = DOF_SECT_OPTDESC;
sec->dofs_align = sizeof (uint64_t);
sec->dofs_flags = DOF_SECF_LOAD;
sec->dofs_entsize = sizeof (dof_optdesc_t);
opt = (dof_optdesc_t *)((uintptr_t)sec +
roundup(sizeof (dof_sec_t), sizeof (uint64_t)));
sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof;
sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
for (i = 0; i < DTRACEOPT_MAX; i++) {
opt[i].dofo_option = i;
opt[i].dofo_strtab = DOF_SECIDX_NONE;
opt[i].dofo_value = state->dts_options[i];
}
return (dof);
}
static dof_hdr_t *
dtrace_dof_copyin(uintptr_t uarg, int *errp)
{
dof_hdr_t hdr, *dof;
ASSERT(!MUTEX_HELD(&dtrace_lock));
/*
* First, we're going to copyin() the sizeof (dof_hdr_t).
*/
if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) {
dtrace_dof_error(NULL, "failed to copyin DOF header");
*errp = EFAULT;
return (NULL);
}
/*
* Now we'll allocate the entire DOF and copy it in -- provided
* that the length isn't outrageous.
*/
if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
dtrace_dof_error(&hdr, "load size exceeds maximum");
*errp = E2BIG;
return (NULL);
}
if (hdr.dofh_loadsz < sizeof (hdr)) {
dtrace_dof_error(&hdr, "invalid load size");
*errp = EINVAL;
return (NULL);
}
dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP);
if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0 ||
dof->dofh_loadsz != hdr.dofh_loadsz) {
kmem_free(dof, hdr.dofh_loadsz);
*errp = EFAULT;
return (NULL);
}
return (dof);
}
#ifndef illumos
static __inline uchar_t
dtrace_dof_char(char c) {
switch (c) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return (c - '0');
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
return (c - 'A' + 10);
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
return (c - 'a' + 10);
}
/* Should not reach here. */
return (0);
}
#endif
static dof_hdr_t *
dtrace_dof_property(const char *name)
{
uchar_t *buf;
uint64_t loadsz;
unsigned int len, i;
dof_hdr_t *dof;
#ifdef illumos
/*
* Unfortunately, array of values in .conf files are always (and
* only) interpreted to be integer arrays. We must read our DOF
* as an integer array, and then squeeze it into a byte array.
*/
if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0,
(char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS)
return (NULL);
for (i = 0; i < len; i++)
buf[i] = (uchar_t)(((int *)buf)[i]);
if (len < sizeof (dof_hdr_t)) {
ddi_prop_free(buf);
dtrace_dof_error(NULL, "truncated header");
return (NULL);
}
if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) {
ddi_prop_free(buf);
dtrace_dof_error(NULL, "truncated DOF");
return (NULL);
}
if (loadsz >= dtrace_dof_maxsize) {
ddi_prop_free(buf);
dtrace_dof_error(NULL, "oversized DOF");
return (NULL);
}
dof = kmem_alloc(loadsz, KM_SLEEP);
bcopy(buf, dof, loadsz);
ddi_prop_free(buf);
#else
char *p;
char *p_env;
if ((p_env = kern_getenv(name)) == NULL)
return (NULL);
len = strlen(p_env) / 2;
buf = kmem_alloc(len, KM_SLEEP);
dof = (dof_hdr_t *) buf;
p = p_env;
for (i = 0; i < len; i++) {
buf[i] = (dtrace_dof_char(p[0]) << 4) |
dtrace_dof_char(p[1]);
p += 2;
}
freeenv(p_env);
if (len < sizeof (dof_hdr_t)) {
kmem_free(buf, 0);
dtrace_dof_error(NULL, "truncated header");
return (NULL);
}
if (len < (loadsz = dof->dofh_loadsz)) {
kmem_free(buf, 0);
dtrace_dof_error(NULL, "truncated DOF");
return (NULL);
}
if (loadsz >= dtrace_dof_maxsize) {
kmem_free(buf, 0);
dtrace_dof_error(NULL, "oversized DOF");
return (NULL);
}
#endif
return (dof);
}
static void
dtrace_dof_destroy(dof_hdr_t *dof)
{
kmem_free(dof, dof->dofh_loadsz);
}
/*
* Return the dof_sec_t pointer corresponding to a given section index. If the
* index is not valid, dtrace_dof_error() is called and NULL is returned. If
* a type other than DOF_SECT_NONE is specified, the header is checked against
* this type and NULL is returned if the types do not match.
*/
static dof_sec_t *
dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i)
{
dof_sec_t *sec = (dof_sec_t *)(uintptr_t)
((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize);
if (i >= dof->dofh_secnum) {
dtrace_dof_error(dof, "referenced section index is invalid");
return (NULL);
}
if (!(sec->dofs_flags & DOF_SECF_LOAD)) {
dtrace_dof_error(dof, "referenced section is not loadable");
return (NULL);
}
if (type != DOF_SECT_NONE && type != sec->dofs_type) {
dtrace_dof_error(dof, "referenced section is the wrong type");
return (NULL);
}
return (sec);
}
static dtrace_probedesc_t *
dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc)
{
dof_probedesc_t *probe;
dof_sec_t *strtab;
uintptr_t daddr = (uintptr_t)dof;
uintptr_t str;
size_t size;
if (sec->dofs_type != DOF_SECT_PROBEDESC) {
dtrace_dof_error(dof, "invalid probe section");
return (NULL);
}
if (sec->dofs_align != sizeof (dof_secidx_t)) {
dtrace_dof_error(dof, "bad alignment in probe description");
return (NULL);
}
if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) {
dtrace_dof_error(dof, "truncated probe description");
return (NULL);
}
probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset);
strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab);
if (strtab == NULL)
return (NULL);
str = daddr + strtab->dofs_offset;
size = strtab->dofs_size;
if (probe->dofp_provider >= strtab->dofs_size) {
dtrace_dof_error(dof, "corrupt probe provider");
return (NULL);
}
(void) strncpy(desc->dtpd_provider,
(char *)(str + probe->dofp_provider),
MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider));
if (probe->dofp_mod >= strtab->dofs_size) {
dtrace_dof_error(dof, "corrupt probe module");
return (NULL);
}
(void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod),
MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod));
if (probe->dofp_func >= strtab->dofs_size) {
dtrace_dof_error(dof, "corrupt probe function");
return (NULL);
}
(void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func),
MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func));
if (probe->dofp_name >= strtab->dofs_size) {
dtrace_dof_error(dof, "corrupt probe name");
return (NULL);
}
(void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name),
MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name));
return (desc);
}
static dtrace_difo_t *
dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
cred_t *cr)
{
dtrace_difo_t *dp;
size_t ttl = 0;
dof_difohdr_t *dofd;
uintptr_t daddr = (uintptr_t)dof;
size_t max = dtrace_difo_maxsize;
int i, l, n;
static const struct {
int section;
int bufoffs;
int lenoffs;
int entsize;
int align;
const char *msg;
} difo[] = {
{ DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf),
offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t),
sizeof (dif_instr_t), "multiple DIF sections" },
{ DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab),
offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t),
sizeof (uint64_t), "multiple integer tables" },
{ DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab),
offsetof(dtrace_difo_t, dtdo_strlen), 0,
sizeof (char), "multiple string tables" },
{ DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab),
offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t),
sizeof (uint_t), "multiple variable tables" },
{ DOF_SECT_NONE, 0, 0, 0, 0, NULL }
};
if (sec->dofs_type != DOF_SECT_DIFOHDR) {
dtrace_dof_error(dof, "invalid DIFO header section");
return (NULL);
}
if (sec->dofs_align != sizeof (dof_secidx_t)) {
dtrace_dof_error(dof, "bad alignment in DIFO header");
return (NULL);
}
if (sec->dofs_size < sizeof (dof_difohdr_t) ||
sec->dofs_size % sizeof (dof_secidx_t)) {
dtrace_dof_error(dof, "bad size in DIFO header");
return (NULL);
}
dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1;
dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
dp->dtdo_rtype = dofd->dofd_rtype;
for (l = 0; l < n; l++) {
dof_sec_t *subsec;
void **bufp;
uint32_t *lenp;
if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE,
dofd->dofd_links[l])) == NULL)
goto err; /* invalid section link */
if (ttl + subsec->dofs_size > max) {
dtrace_dof_error(dof, "exceeds maximum size");
goto err;
}
ttl += subsec->dofs_size;
for (i = 0; difo[i].section != DOF_SECT_NONE; i++) {
if (subsec->dofs_type != difo[i].section)
continue;
if (!(subsec->dofs_flags & DOF_SECF_LOAD)) {
dtrace_dof_error(dof, "section not loaded");
goto err;
}
if (subsec->dofs_align != difo[i].align) {
dtrace_dof_error(dof, "bad alignment");
goto err;
}
bufp = (void **)((uintptr_t)dp + difo[i].bufoffs);
lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs);
if (*bufp != NULL) {
dtrace_dof_error(dof, difo[i].msg);
goto err;
}
if (difo[i].entsize != subsec->dofs_entsize) {
dtrace_dof_error(dof, "entry size mismatch");
goto err;
}
if (subsec->dofs_entsize != 0 &&
(subsec->dofs_size % subsec->dofs_entsize) != 0) {
dtrace_dof_error(dof, "corrupt entry size");
goto err;
}
*lenp = subsec->dofs_size;
*bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP);
bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset),
*bufp, subsec->dofs_size);
if (subsec->dofs_entsize != 0)
*lenp /= subsec->dofs_entsize;
break;
}
/*
* If we encounter a loadable DIFO sub-section that is not
* known to us, assume this is a broken program and fail.
*/
if (difo[i].section == DOF_SECT_NONE &&
(subsec->dofs_flags & DOF_SECF_LOAD)) {
dtrace_dof_error(dof, "unrecognized DIFO subsection");
goto err;
}
}
if (dp->dtdo_buf == NULL) {
/*
* We can't have a DIF object without DIF text.
*/
dtrace_dof_error(dof, "missing DIF text");
goto err;
}
/*
* Before we validate the DIF object, run through the variable table
* looking for the strings -- if any of their size are under, we'll set
* their size to be the system-wide default string size. Note that
* this should _not_ happen if the "strsize" option has been set --
* in this case, the compiler should have set the size to reflect the
* setting of the option.
*/
for (i = 0; i < dp->dtdo_varlen; i++) {
dtrace_difv_t *v = &dp->dtdo_vartab[i];
dtrace_diftype_t *t = &v->dtdv_type;
if (v->dtdv_id < DIF_VAR_OTHER_UBASE)
continue;
if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0)
t->dtdt_size = dtrace_strsize_default;
}
if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0)
goto err;
dtrace_difo_init(dp, vstate);
return (dp);
err:
kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
kmem_free(dp, sizeof (dtrace_difo_t));
return (NULL);
}
static dtrace_predicate_t *
dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
cred_t *cr)
{
dtrace_difo_t *dp;
if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL)
return (NULL);
return (dtrace_predicate_create(dp));
}
static dtrace_actdesc_t *
dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
cred_t *cr)
{
dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next;
dof_actdesc_t *desc;
dof_sec_t *difosec;
size_t offs;
uintptr_t daddr = (uintptr_t)dof;
uint64_t arg;
dtrace_actkind_t kind;
if (sec->dofs_type != DOF_SECT_ACTDESC) {
dtrace_dof_error(dof, "invalid action section");
return (NULL);
}
if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) {
dtrace_dof_error(dof, "truncated action description");
return (NULL);
}
if (sec->dofs_align != sizeof (uint64_t)) {
dtrace_dof_error(dof, "bad alignment in action description");
return (NULL);
}
if (sec->dofs_size < sec->dofs_entsize) {
dtrace_dof_error(dof, "section entry size exceeds total size");
return (NULL);
}
if (sec->dofs_entsize != sizeof (dof_actdesc_t)) {
dtrace_dof_error(dof, "bad entry size in action description");
return (NULL);
}
if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) {
dtrace_dof_error(dof, "actions exceed dtrace_actions_max");
return (NULL);
}
for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) {
desc = (dof_actdesc_t *)(daddr +
(uintptr_t)sec->dofs_offset + offs);
kind = (dtrace_actkind_t)desc->dofa_kind;
if ((DTRACEACT_ISPRINTFLIKE(kind) &&
(kind != DTRACEACT_PRINTA ||
desc->dofa_strtab != DOF_SECIDX_NONE)) ||
(kind == DTRACEACT_DIFEXPR &&
desc->dofa_strtab != DOF_SECIDX_NONE)) {
dof_sec_t *strtab;
char *str, *fmt;
uint64_t i;
/*
* The argument to these actions is an index into the
* DOF string table. For printf()-like actions, this
* is the format string. For print(), this is the
* CTF type of the expression result.
*/
if ((strtab = dtrace_dof_sect(dof,
DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL)
goto err;
str = (char *)((uintptr_t)dof +
(uintptr_t)strtab->dofs_offset);
for (i = desc->dofa_arg; i < strtab->dofs_size; i++) {
if (str[i] == '\0')
break;
}
if (i >= strtab->dofs_size) {
dtrace_dof_error(dof, "bogus format string");
goto err;
}
if (i == desc->dofa_arg) {
dtrace_dof_error(dof, "empty format string");
goto err;
}
i -= desc->dofa_arg;
fmt = kmem_alloc(i + 1, KM_SLEEP);
bcopy(&str[desc->dofa_arg], fmt, i + 1);
arg = (uint64_t)(uintptr_t)fmt;
} else {
if (kind == DTRACEACT_PRINTA) {
ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE);
arg = 0;
} else {
arg = desc->dofa_arg;
}
}
act = dtrace_actdesc_create(kind, desc->dofa_ntuple,
desc->dofa_uarg, arg);
if (last != NULL) {
last->dtad_next = act;
} else {
first = act;
}
last = act;
if (desc->dofa_difo == DOF_SECIDX_NONE)
continue;
if ((difosec = dtrace_dof_sect(dof,
DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL)
goto err;
act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr);
if (act->dtad_difo == NULL)
goto err;
}
ASSERT(first != NULL);
return (first);
err:
for (act = first; act != NULL; act = next) {
next = act->dtad_next;
dtrace_actdesc_release(act, vstate);
}
return (NULL);
}
static dtrace_ecbdesc_t *
dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
cred_t *cr)
{
dtrace_ecbdesc_t *ep;
dof_ecbdesc_t *ecb;
dtrace_probedesc_t *desc;
dtrace_predicate_t *pred = NULL;
if (sec->dofs_size < sizeof (dof_ecbdesc_t)) {
dtrace_dof_error(dof, "truncated ECB description");
return (NULL);
}
if (sec->dofs_align != sizeof (uint64_t)) {
dtrace_dof_error(dof, "bad alignment in ECB description");
return (NULL);
}
ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);
sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);
if (sec == NULL)
return (NULL);
ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
ep->dted_uarg = ecb->dofe_uarg;
desc = &ep->dted_probe;
if (dtrace_dof_probedesc(dof, sec, desc) == NULL)
goto err;
if (ecb->dofe_pred != DOF_SECIDX_NONE) {
if ((sec = dtrace_dof_sect(dof,
DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)
goto err;
if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL)
goto err;
ep->dted_pred.dtpdd_predicate = pred;
}
if (ecb->dofe_actions != DOF_SECIDX_NONE) {
if ((sec = dtrace_dof_sect(dof,
DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)
goto err;
ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr);
if (ep->dted_action == NULL)
goto err;
}
return (ep);
err:
if (pred != NULL)
dtrace_predicate_release(pred, vstate);
kmem_free(ep, sizeof (dtrace_ecbdesc_t));
return (NULL);
}
/*
* Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the
* specified DOF. At present, this amounts to simply adding 'ubase' to the
* site of any user SETX relocations to account for load object base address.
* In the future, if we need other relocations, this function can be extended.
*/
static int
dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
{
uintptr_t daddr = (uintptr_t)dof;
dof_relohdr_t *dofr =
(dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
dof_sec_t *ss, *rs, *ts;
dof_relodesc_t *r;
uint_t i, n;
if (sec->dofs_size < sizeof (dof_relohdr_t) ||
sec->dofs_align != sizeof (dof_secidx_t)) {
dtrace_dof_error(dof, "invalid relocation header");
return (-1);
}
ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab);
rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec);
ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec);
if (ss == NULL || rs == NULL || ts == NULL)
return (-1); /* dtrace_dof_error() has been called already */
if (rs->dofs_entsize < sizeof (dof_relodesc_t) ||
rs->dofs_align != sizeof (uint64_t)) {
dtrace_dof_error(dof, "invalid relocation section");
return (-1);
}
r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset);
n = rs->dofs_size / rs->dofs_entsize;
for (i = 0; i < n; i++) {
uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset;
switch (r->dofr_type) {
case DOF_RELO_NONE:
break;
case DOF_RELO_SETX:
if (r->dofr_offset >= ts->dofs_size || r->dofr_offset +
sizeof (uint64_t) > ts->dofs_size) {
dtrace_dof_error(dof, "bad relocation offset");
return (-1);
}
if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) {
dtrace_dof_error(dof, "misaligned setx relo");
return (-1);
}
*(uint64_t *)taddr += ubase;
break;
default:
dtrace_dof_error(dof, "invalid relocation type");
return (-1);
}
r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize);
}
return (0);
}
/*
* The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
* header: it should be at the front of a memory region that is at least
* sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
* size. It need not be validated in any other way.
*/
static int
dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
dtrace_enabling_t **enabp, uint64_t ubase, int noprobes)
{
uint64_t len = dof->dofh_loadsz, seclen;
uintptr_t daddr = (uintptr_t)dof;
dtrace_ecbdesc_t *ep;
dtrace_enabling_t *enab;
uint_t i;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t));
/*
* Check the DOF header identification bytes. In addition to checking
* valid settings, we also verify that unused bits/bytes are zeroed so
* we can use them later without fear of regressing existing binaries.
*/
if (bcmp(&dof->dofh_ident[DOF_ID_MAG0],
DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) {
dtrace_dof_error(dof, "DOF magic string mismatch");
return (-1);
}
if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 &&
dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) {
dtrace_dof_error(dof, "DOF has invalid data model");
return (-1);
}
if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) {
dtrace_dof_error(dof, "DOF encoding mismatch");
return (-1);
}
if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) {
dtrace_dof_error(dof, "DOF version mismatch");
return (-1);
}
if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) {
dtrace_dof_error(dof, "DOF uses unsupported instruction set");
return (-1);
}
if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) {
dtrace_dof_error(dof, "DOF uses too many integer registers");
return (-1);
}
if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) {
dtrace_dof_error(dof, "DOF uses too many tuple registers");
return (-1);
}
for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) {
if (dof->dofh_ident[i] != 0) {
dtrace_dof_error(dof, "DOF has invalid ident byte set");
return (-1);
}
}
if (dof->dofh_flags & ~DOF_FL_VALID) {
dtrace_dof_error(dof, "DOF has invalid flag bits set");
return (-1);
}
if (dof->dofh_secsize == 0) {
dtrace_dof_error(dof, "zero section header size");
return (-1);
}
/*
* Check that the section headers don't exceed the amount of DOF
* data. Note that we cast the section size and number of sections
* to uint64_t's to prevent possible overflow in the multiplication.
*/
seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize;
if (dof->dofh_secoff > len || seclen > len ||
dof->dofh_secoff + seclen > len) {
dtrace_dof_error(dof, "truncated section headers");
return (-1);
}
if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) {
dtrace_dof_error(dof, "misaligned section headers");
return (-1);
}
if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) {
dtrace_dof_error(dof, "misaligned section size");
return (-1);
}
/*
* Take an initial pass through the section headers to be sure that
* the headers don't have stray offsets. If the 'noprobes' flag is
* set, do not permit sections relating to providers, probes, or args.
*/
for (i = 0; i < dof->dofh_secnum; i++) {
dof_sec_t *sec = (dof_sec_t *)(daddr +
(uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
if (noprobes) {
switch (sec->dofs_type) {
case DOF_SECT_PROVIDER:
case DOF_SECT_PROBES:
case DOF_SECT_PRARGS:
case DOF_SECT_PROFFS:
dtrace_dof_error(dof, "illegal sections "
"for enabling");
return (-1);
}
}
if (DOF_SEC_ISLOADABLE(sec->dofs_type) &&
!(sec->dofs_flags & DOF_SECF_LOAD)) {
dtrace_dof_error(dof, "loadable section with load "
"flag unset");
return (-1);
}
if (!(sec->dofs_flags & DOF_SECF_LOAD))
continue; /* just ignore non-loadable sections */
if (!ISP2(sec->dofs_align)) {
dtrace_dof_error(dof, "bad section alignment");
return (-1);
}
if (sec->dofs_offset & (sec->dofs_align - 1)) {
dtrace_dof_error(dof, "misaligned section");
return (-1);
}
if (sec->dofs_offset > len || sec->dofs_size > len ||
sec->dofs_offset + sec->dofs_size > len) {
dtrace_dof_error(dof, "corrupt section header");
return (-1);
}
if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr +
sec->dofs_offset + sec->dofs_size - 1) != '\0') {
dtrace_dof_error(dof, "non-terminating string table");
return (-1);
}
}
/*
* Take a second pass through the sections and locate and perform any
* relocations that are present. We do this after the first pass to
* be sure that all sections have had their headers validated.
*/
for (i = 0; i < dof->dofh_secnum; i++) {
dof_sec_t *sec = (dof_sec_t *)(daddr +
(uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
if (!(sec->dofs_flags & DOF_SECF_LOAD))
continue; /* skip sections that are not loadable */
switch (sec->dofs_type) {
case DOF_SECT_URELHDR:
if (dtrace_dof_relocate(dof, sec, ubase) != 0)
return (-1);
break;
}
}
if ((enab = *enabp) == NULL)
enab = *enabp = dtrace_enabling_create(vstate);
for (i = 0; i < dof->dofh_secnum; i++) {
dof_sec_t *sec = (dof_sec_t *)(daddr +
(uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
if (sec->dofs_type != DOF_SECT_ECBDESC)
continue;
if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) {
dtrace_enabling_destroy(enab);
*enabp = NULL;
return (-1);
}
dtrace_enabling_add(enab, ep);
}
return (0);
}
/*
* Process DOF for any options. This routine assumes that the DOF has been
* at least processed by dtrace_dof_slurp().
*/
static int
dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state)
{
int i, rval;
uint32_t entsize;
size_t offs;
dof_optdesc_t *desc;
for (i = 0; i < dof->dofh_secnum; i++) {
dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof +
(uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
if (sec->dofs_type != DOF_SECT_OPTDESC)
continue;
if (sec->dofs_align != sizeof (uint64_t)) {
dtrace_dof_error(dof, "bad alignment in "
"option description");
return (EINVAL);
}
if ((entsize = sec->dofs_entsize) == 0) {
dtrace_dof_error(dof, "zeroed option entry size");
return (EINVAL);
}
if (entsize < sizeof (dof_optdesc_t)) {
dtrace_dof_error(dof, "bad option entry size");
return (EINVAL);
}
for (offs = 0; offs < sec->dofs_size; offs += entsize) {
desc = (dof_optdesc_t *)((uintptr_t)dof +
(uintptr_t)sec->dofs_offset + offs);
if (desc->dofo_strtab != DOF_SECIDX_NONE) {
dtrace_dof_error(dof, "non-zero option string");
return (EINVAL);
}
if (desc->dofo_value == DTRACEOPT_UNSET) {
dtrace_dof_error(dof, "unset option");
return (EINVAL);
}
if ((rval = dtrace_state_option(state,
desc->dofo_option, desc->dofo_value)) != 0) {
dtrace_dof_error(dof, "rejected option");
return (rval);
}
}
}
return (0);
}
/*
* DTrace Consumer State Functions
*/
static int
dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
{
size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize;
void *base;
uintptr_t limit;
dtrace_dynvar_t *dvar, *next, *start;
int i;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL);
bzero(dstate, sizeof (dtrace_dstate_t));
if ((dstate->dtds_chunksize = chunksize) == 0)
dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
size = min;
if ((base = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL)
return (ENOMEM);
dstate->dtds_size = size;
dstate->dtds_base = base;
dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP);
bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t));
hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t));
if (hashsize != 1 && (hashsize & 1))
hashsize--;
dstate->dtds_hashsize = hashsize;
dstate->dtds_hash = dstate->dtds_base;
/*
* Set all of our hash buckets to point to the single sink, and (if
* it hasn't already been set), set the sink's hash value to be the
* sink sentinel value. The sink is needed for dynamic variable
* lookups to know that they have iterated over an entire, valid hash
* chain.
*/
for (i = 0; i < hashsize; i++)
dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink;
if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK)
dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK;
/*
* Determine number of active CPUs. Divide free list evenly among
* active CPUs.
*/
start = (dtrace_dynvar_t *)
((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
limit = (uintptr_t)base + size;
maxper = (limit - (uintptr_t)start) / NCPU;
maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
#ifndef illumos
CPU_FOREACH(i) {
#else
for (i = 0; i < NCPU; i++) {
#endif
dstate->dtds_percpu[i].dtdsc_free = dvar = start;
/*
* If we don't even have enough chunks to make it once through
* NCPUs, we're just going to allocate everything to the first
* CPU. And if we're on the last CPU, we're going to allocate
* whatever is left over. In either case, we set the limit to
* be the limit of the dynamic variable space.
*/
if (maxper == 0 || i == NCPU - 1) {
limit = (uintptr_t)base + size;
start = NULL;
} else {
limit = (uintptr_t)start + maxper;
start = (dtrace_dynvar_t *)limit;
}
ASSERT(limit <= (uintptr_t)base + size);
for (;;) {
next = (dtrace_dynvar_t *)((uintptr_t)dvar +
dstate->dtds_chunksize);
if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
break;
dvar->dtdv_next = next;
dvar = next;
}
if (maxper == 0)
break;
}
return (0);
}
static void
dtrace_dstate_fini(dtrace_dstate_t *dstate)
{
ASSERT(MUTEX_HELD(&cpu_lock));
if (dstate->dtds_base == NULL)
return;
kmem_free(dstate->dtds_base, dstate->dtds_size);
kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu);
}
static void
dtrace_vstate_fini(dtrace_vstate_t *vstate)
{
/*
* Logical XOR, where are you?
*/
ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL));
if (vstate->dtvs_nglobals > 0) {
kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals *
sizeof (dtrace_statvar_t *));
}
if (vstate->dtvs_ntlocals > 0) {
kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals *
sizeof (dtrace_difv_t));
}
ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL));
if (vstate->dtvs_nlocals > 0) {
kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals *
sizeof (dtrace_statvar_t *));
}
}
#ifdef illumos
static void
dtrace_state_clean(dtrace_state_t *state)
{
if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
return;
dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
dtrace_speculation_clean(state);
}
static void
dtrace_state_deadman(dtrace_state_t *state)
{
hrtime_t now;
dtrace_sync();
now = dtrace_gethrtime();
if (state != dtrace_anon.dta_state &&
now - state->dts_laststatus >= dtrace_deadman_user)
return;
/*
* We must be sure that dts_alive never appears to be less than the
* value upon entry to dtrace_state_deadman(), and because we lack a
* dtrace_cas64(), we cannot store to it atomically. We thus instead
* store INT64_MAX to it, followed by a memory barrier, followed by
* the new value. This assures that dts_alive never appears to be
* less than its true value, regardless of the order in which the
* stores to the underlying storage are issued.
*/
state->dts_alive = INT64_MAX;
dtrace_membar_producer();
state->dts_alive = now;
}
#else /* !illumos */
static void
dtrace_state_clean(void *arg)
{
dtrace_state_t *state = arg;
dtrace_optval_t *opt = state->dts_options;
if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
return;
dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
dtrace_speculation_clean(state);
callout_reset(&state->dts_cleaner, hz * opt[DTRACEOPT_CLEANRATE] / NANOSEC,
dtrace_state_clean, state);
}
static void
dtrace_state_deadman(void *arg)
{
dtrace_state_t *state = arg;
hrtime_t now;
dtrace_sync();
dtrace_debug_output();
now = dtrace_gethrtime();
if (state != dtrace_anon.dta_state &&
now - state->dts_laststatus >= dtrace_deadman_user)
return;
/*
* We must be sure that dts_alive never appears to be less than the
* value upon entry to dtrace_state_deadman(), and because we lack a
* dtrace_cas64(), we cannot store to it atomically. We thus instead
* store INT64_MAX to it, followed by a memory barrier, followed by
* the new value. This assures that dts_alive never appears to be
* less than its true value, regardless of the order in which the
* stores to the underlying storage are issued.
*/
state->dts_alive = INT64_MAX;
dtrace_membar_producer();
state->dts_alive = now;
callout_reset(&state->dts_deadman, hz * dtrace_deadman_interval / NANOSEC,
dtrace_state_deadman, state);
}
#endif /* illumos */
static dtrace_state_t *
#ifdef illumos
dtrace_state_create(dev_t *devp, cred_t *cr)
#else
dtrace_state_create(struct cdev *dev)
#endif
{
#ifdef illumos
minor_t minor;
major_t major;
#else
cred_t *cr = NULL;
int m = 0;
#endif
char c[30];
dtrace_state_t *state;
dtrace_optval_t *opt;
int bufsize = NCPU * sizeof (dtrace_buffer_t), i;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(MUTEX_HELD(&cpu_lock));
#ifdef illumos
minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1,
VM_BESTFIT | VM_SLEEP);
if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) {
vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
return (NULL);
}
state = ddi_get_soft_state(dtrace_softstate, minor);
#else
if (dev != NULL) {
cr = dev->si_cred;
m = dev2unit(dev);
}
/* Allocate memory for the state. */
state = kmem_zalloc(sizeof(dtrace_state_t), KM_SLEEP);
#endif
state->dts_epid = DTRACE_EPIDNONE + 1;
(void) snprintf(c, sizeof (c), "dtrace_aggid_%d", m);
#ifdef illumos
state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1,
NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
if (devp != NULL) {
major = getemajor(*devp);
} else {
major = ddi_driver_major(dtrace_devi);
}
state->dts_dev = makedevice(major, minor);
if (devp != NULL)
*devp = state->dts_dev;
#else
state->dts_aggid_arena = new_unrhdr(1, INT_MAX, &dtrace_unr_mtx);
state->dts_dev = dev;
#endif
/*
* We allocate NCPU buffers. On the one hand, this can be quite
* a bit of memory per instance (nearly 36K on a Starcat). On the
* other hand, it saves an additional memory reference in the probe
* path.
*/
state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP);
state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP);
#ifdef illumos
state->dts_cleaner = CYCLIC_NONE;
state->dts_deadman = CYCLIC_NONE;
#else
- callout_init(&state->dts_cleaner, CALLOUT_MPSAFE);
- callout_init(&state->dts_deadman, CALLOUT_MPSAFE);
+ callout_init(&state->dts_cleaner, 1);
+ callout_init(&state->dts_deadman, 1);
#endif
state->dts_vstate.dtvs_state = state;
for (i = 0; i < DTRACEOPT_MAX; i++)
state->dts_options[i] = DTRACEOPT_UNSET;
/*
* Set the default options.
*/
opt = state->dts_options;
opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH;
opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO;
opt[DTRACEOPT_NSPEC] = dtrace_nspec_default;
opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default;
opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL;
opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default;
opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default;
opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default;
opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default;
opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default;
opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default;
opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default;
opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default;
opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default;
state->dts_activity = DTRACE_ACTIVITY_INACTIVE;
/*
* Depending on the user credentials, we set flag bits which alter probe
* visibility or the amount of destructiveness allowed. In the case of
* actual anonymous tracing, or the possession of all privileges, all of
* the normal checks are bypassed.
*/
if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
state->dts_cred.dcr_visible = DTRACE_CRV_ALL;
state->dts_cred.dcr_action = DTRACE_CRA_ALL;
} else {
/*
* Set up the credentials for this instantiation. We take a
* hold on the credential to prevent it from disappearing on
* us; this in turn prevents the zone_t referenced by this
* credential from disappearing. This means that we can
* examine the credential and the zone from probe context.
*/
crhold(cr);
state->dts_cred.dcr_cred = cr;
/*
* CRA_PROC means "we have *some* privilege for dtrace" and
* unlocks the use of variables like pid, zonename, etc.
*/
if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) ||
PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
state->dts_cred.dcr_action |= DTRACE_CRA_PROC;
}
/*
* dtrace_user allows use of syscall and profile providers.
* If the user also has proc_owner and/or proc_zone, we
* extend the scope to include additional visibility and
* destructive power.
*/
if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) {
if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) {
state->dts_cred.dcr_visible |=
DTRACE_CRV_ALLPROC;
state->dts_cred.dcr_action |=
DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
}
if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) {
state->dts_cred.dcr_visible |=
DTRACE_CRV_ALLZONE;
state->dts_cred.dcr_action |=
DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
}
/*
* If we have all privs in whatever zone this is,
* we can do destructive things to processes which
* have altered credentials.
*/
#ifdef illumos
if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
cr->cr_zone->zone_privset)) {
state->dts_cred.dcr_action |=
DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
}
#endif
}
/*
* Holding the dtrace_kernel privilege also implies that
* the user has the dtrace_user privilege from a visibility
* perspective. But without further privileges, some
* destructive actions are not available.
*/
if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) {
/*
* Make all probes in all zones visible. However,
* this doesn't mean that all actions become available
* to all zones.
*/
state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL |
DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE;
state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL |
DTRACE_CRA_PROC;
/*
* Holding proc_owner means that destructive actions
* for *this* zone are allowed.
*/
if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
state->dts_cred.dcr_action |=
DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
/*
* Holding proc_zone means that destructive actions
* for this user/group ID in all zones is allowed.
*/
if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
state->dts_cred.dcr_action |=
DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
#ifdef illumos
/*
* If we have all privs in whatever zone this is,
* we can do destructive things to processes which
* have altered credentials.
*/
if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
cr->cr_zone->zone_privset)) {
state->dts_cred.dcr_action |=
DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
}
#endif
}
/*
* Holding the dtrace_proc privilege gives control over fasttrap
* and pid providers. We need to grant wider destructive
* privileges in the event that the user has proc_owner and/or
* proc_zone.
*/
if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
state->dts_cred.dcr_action |=
DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
state->dts_cred.dcr_action |=
DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
}
}
return (state);
}
static int
dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which)
{
dtrace_optval_t *opt = state->dts_options, size;
processorid_t cpu = 0;;
int flags = 0, rval, factor, divisor = 1;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(MUTEX_HELD(&cpu_lock));
ASSERT(which < DTRACEOPT_MAX);
ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE ||
(state == dtrace_anon.dta_state &&
state->dts_activity == DTRACE_ACTIVITY_ACTIVE));
if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0)
return (0);
if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET)
cpu = opt[DTRACEOPT_CPU];
if (which == DTRACEOPT_SPECSIZE)
flags |= DTRACEBUF_NOSWITCH;
if (which == DTRACEOPT_BUFSIZE) {
if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING)
flags |= DTRACEBUF_RING;
if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL)
flags |= DTRACEBUF_FILL;
if (state != dtrace_anon.dta_state ||
state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
flags |= DTRACEBUF_INACTIVE;
}
for (size = opt[which]; size >= sizeof (uint64_t); size /= divisor) {
/*
* The size must be 8-byte aligned. If the size is not 8-byte
* aligned, drop it down by the difference.
*/
if (size & (sizeof (uint64_t) - 1))
size -= size & (sizeof (uint64_t) - 1);
if (size < state->dts_reserve) {
/*
* Buffers always must be large enough to accommodate
* their prereserved space. We return E2BIG instead
* of ENOMEM in this case to allow for user-level
* software to differentiate the cases.
*/
return (E2BIG);
}
rval = dtrace_buffer_alloc(buf, size, flags, cpu, &factor);
if (rval != ENOMEM) {
opt[which] = size;
return (rval);
}
if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
return (rval);
for (divisor = 2; divisor < factor; divisor <<= 1)
continue;
}
return (ENOMEM);
}
static int
dtrace_state_buffers(dtrace_state_t *state)
{
dtrace_speculation_t *spec = state->dts_speculations;
int rval, i;
if ((rval = dtrace_state_buffer(state, state->dts_buffer,
DTRACEOPT_BUFSIZE)) != 0)
return (rval);
if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer,
DTRACEOPT_AGGSIZE)) != 0)
return (rval);
for (i = 0; i < state->dts_nspeculations; i++) {
if ((rval = dtrace_state_buffer(state,
spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0)
return (rval);
}
return (0);
}
static void
dtrace_state_prereserve(dtrace_state_t *state)
{
dtrace_ecb_t *ecb;
dtrace_probe_t *probe;
state->dts_reserve = 0;
if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL)
return;
/*
* If our buffer policy is a "fill" buffer policy, we need to set the
* prereserved space to be the space required by the END probes.
*/
probe = dtrace_probes[dtrace_probeid_end - 1];
ASSERT(probe != NULL);
for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
if (ecb->dte_state != state)
continue;
state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
}
}
static int
dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
{
dtrace_optval_t *opt = state->dts_options, sz, nspec;
dtrace_speculation_t *spec;
dtrace_buffer_t *buf;
#ifdef illumos
cyc_handler_t hdlr;
cyc_time_t when;
#endif
int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t);
dtrace_icookie_t cookie;
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_lock);
if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
rval = EBUSY;
goto out;
}
/*
* Before we can perform any checks, we must prime all of the
* retained enablings that correspond to this state.
*/
dtrace_enabling_prime(state);
if (state->dts_destructive && !state->dts_cred.dcr_destructive) {
rval = EACCES;
goto out;
}
dtrace_state_prereserve(state);
/*
* Now we want to do is try to allocate our speculations.
* We do not automatically resize the number of speculations; if
* this fails, we will fail the operation.
*/
nspec = opt[DTRACEOPT_NSPEC];
ASSERT(nspec != DTRACEOPT_UNSET);
if (nspec > INT_MAX) {
rval = ENOMEM;
goto out;
}
spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t),
KM_NOSLEEP | KM_NORMALPRI);
if (spec == NULL) {
rval = ENOMEM;
goto out;
}
state->dts_speculations = spec;
state->dts_nspeculations = (int)nspec;
for (i = 0; i < nspec; i++) {
if ((buf = kmem_zalloc(bufsize,
KM_NOSLEEP | KM_NORMALPRI)) == NULL) {
rval = ENOMEM;
goto err;
}
spec[i].dtsp_buffer = buf;
}
if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) {
if (dtrace_anon.dta_state == NULL) {
rval = ENOENT;
goto out;
}
if (state->dts_necbs != 0) {
rval = EALREADY;
goto out;
}
state->dts_anon = dtrace_anon_grab();
ASSERT(state->dts_anon != NULL);
state = state->dts_anon;
/*
* We want "grabanon" to be set in the grabbed state, so we'll
* copy that option value from the grabbing state into the
* grabbed state.
*/
state->dts_options[DTRACEOPT_GRABANON] =
opt[DTRACEOPT_GRABANON];
*cpu = dtrace_anon.dta_beganon;
/*
* If the anonymous state is active (as it almost certainly
* is if the anonymous enabling ultimately matched anything),
* we don't allow any further option processing -- but we
* don't return failure.
*/
if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
goto out;
}
if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET &&
opt[DTRACEOPT_AGGSIZE] != 0) {
if (state->dts_aggregations == NULL) {
/*
* We're not going to create an aggregation buffer
* because we don't have any ECBs that contain
* aggregations -- set this option to 0.
*/
opt[DTRACEOPT_AGGSIZE] = 0;
} else {
/*
* If we have an aggregation buffer, we must also have
* a buffer to use as scratch.
*/
if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET ||
opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
}
}
}
if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET &&
opt[DTRACEOPT_SPECSIZE] != 0) {
if (!state->dts_speculates) {
/*
* We're not going to create speculation buffers
* because we don't have any ECBs that actually
* speculate -- set the speculation size to 0.
*/
opt[DTRACEOPT_SPECSIZE] = 0;
}
}
/*
* The bare minimum size for any buffer that we're actually going to
* do anything to is sizeof (uint64_t).
*/
sz = sizeof (uint64_t);
if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) ||
(state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) ||
(state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) {
/*
* A buffer size has been explicitly set to 0 (or to a size
* that will be adjusted to 0) and we need the space -- we
* need to return failure. We return ENOSPC to differentiate
* it from failing to allocate a buffer due to failure to meet
* the reserve (for which we return E2BIG).
*/
rval = ENOSPC;
goto out;
}
if ((rval = dtrace_state_buffers(state)) != 0)
goto err;
if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET)
sz = dtrace_dstate_defsize;
do {
rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz);
if (rval == 0)
break;
if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
goto err;
} while (sz >>= 1);
opt[DTRACEOPT_DYNVARSIZE] = sz;
if (rval != 0)
goto err;
if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max)
opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max;
if (opt[DTRACEOPT_CLEANRATE] == 0)
opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min)
opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min;
if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max)
opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
#ifdef illumos
hdlr.cyh_func = (cyc_func_t)dtrace_state_clean;
hdlr.cyh_arg = state;
hdlr.cyh_level = CY_LOW_LEVEL;
when.cyt_when = 0;
when.cyt_interval = opt[DTRACEOPT_CLEANRATE];
state->dts_cleaner = cyclic_add(&hdlr, &when);
hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman;
hdlr.cyh_arg = state;
hdlr.cyh_level = CY_LOW_LEVEL;
when.cyt_when = 0;
when.cyt_interval = dtrace_deadman_interval;
state->dts_deadman = cyclic_add(&hdlr, &when);
#else
callout_reset(&state->dts_cleaner, hz * opt[DTRACEOPT_CLEANRATE] / NANOSEC,
dtrace_state_clean, state);
callout_reset(&state->dts_deadman, hz * dtrace_deadman_interval / NANOSEC,
dtrace_state_deadman, state);
#endif
state->dts_activity = DTRACE_ACTIVITY_WARMUP;
#ifdef illumos
if (state->dts_getf != 0 &&
!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
/*
* We don't have kernel privs but we have at least one call
* to getf(); we need to bump our zone's count, and (if
* this is the first enabling to have an unprivileged call
* to getf()) we need to hook into closef().
*/
state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf++;
if (dtrace_getf++ == 0) {
ASSERT(dtrace_closef == NULL);
dtrace_closef = dtrace_getf_barrier;
}
}
#endif
/*
* Now it's time to actually fire the BEGIN probe. We need to disable
* interrupts here both to record the CPU on which we fired the BEGIN
* probe (the data from this CPU will be processed first at user
* level) and to manually activate the buffer for this CPU.
*/
cookie = dtrace_interrupt_disable();
*cpu = curcpu;
ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE);
state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE;
dtrace_probe(dtrace_probeid_begin,
(uint64_t)(uintptr_t)state, 0, 0, 0, 0);
dtrace_interrupt_enable(cookie);
/*
* We may have had an exit action from a BEGIN probe; only change our
* state to ACTIVE if we're still in WARMUP.
*/
ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP ||
state->dts_activity == DTRACE_ACTIVITY_DRAINING);
if (state->dts_activity == DTRACE_ACTIVITY_WARMUP)
state->dts_activity = DTRACE_ACTIVITY_ACTIVE;
/*
* Regardless of whether or not now we're in ACTIVE or DRAINING, we
* want each CPU to transition its principal buffer out of the
* INACTIVE state. Doing this assures that no CPU will suddenly begin
* processing an ECB halfway down a probe's ECB chain; all CPUs will
* atomically transition from processing none of a state's ECBs to
* processing all of them.
*/
dtrace_xcall(DTRACE_CPUALL,
(dtrace_xcall_t)dtrace_buffer_activate, state);
goto out;
err:
dtrace_buffer_free(state->dts_buffer);
dtrace_buffer_free(state->dts_aggbuffer);
if ((nspec = state->dts_nspeculations) == 0) {
ASSERT(state->dts_speculations == NULL);
goto out;
}
spec = state->dts_speculations;
ASSERT(spec != NULL);
for (i = 0; i < state->dts_nspeculations; i++) {
if ((buf = spec[i].dtsp_buffer) == NULL)
break;
dtrace_buffer_free(buf);
kmem_free(buf, bufsize);
}
kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
state->dts_nspeculations = 0;
state->dts_speculations = NULL;
out:
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
return (rval);
}
static int
dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu)
{
dtrace_icookie_t cookie;
ASSERT(MUTEX_HELD(&dtrace_lock));
if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE &&
state->dts_activity != DTRACE_ACTIVITY_DRAINING)
return (EINVAL);
/*
* We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync
* to be sure that every CPU has seen it. See below for the details
* on why this is done.
*/
state->dts_activity = DTRACE_ACTIVITY_DRAINING;
dtrace_sync();
/*
* By this point, it is impossible for any CPU to be still processing
* with DTRACE_ACTIVITY_ACTIVE. We can thus set our activity to
* DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any
* other CPU in dtrace_buffer_reserve(). This allows dtrace_probe()
* and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN
* iff we're in the END probe.
*/
state->dts_activity = DTRACE_ACTIVITY_COOLDOWN;
dtrace_sync();
ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN);
/*
* Finally, we can release the reserve and call the END probe. We
* disable interrupts across calling the END probe to allow us to
* return the CPU on which we actually called the END probe. This
* allows user-land to be sure that this CPU's principal buffer is
* processed last.
*/
state->dts_reserve = 0;
cookie = dtrace_interrupt_disable();
*cpu = curcpu;
dtrace_probe(dtrace_probeid_end,
(uint64_t)(uintptr_t)state, 0, 0, 0, 0);
dtrace_interrupt_enable(cookie);
state->dts_activity = DTRACE_ACTIVITY_STOPPED;
dtrace_sync();
#ifdef illumos
if (state->dts_getf != 0 &&
!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
/*
* We don't have kernel privs but we have at least one call
* to getf(); we need to lower our zone's count, and (if
* this is the last enabling to have an unprivileged call
* to getf()) we need to clear the closef() hook.
*/
ASSERT(state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf > 0);
ASSERT(dtrace_closef == dtrace_getf_barrier);
ASSERT(dtrace_getf > 0);
state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf--;
if (--dtrace_getf == 0)
dtrace_closef = NULL;
}
#endif
return (0);
}
static int
dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option,
dtrace_optval_t val)
{
ASSERT(MUTEX_HELD(&dtrace_lock));
if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
return (EBUSY);
if (option >= DTRACEOPT_MAX)
return (EINVAL);
if (option != DTRACEOPT_CPU && val < 0)
return (EINVAL);
switch (option) {
case DTRACEOPT_DESTRUCTIVE:
if (dtrace_destructive_disallow)
return (EACCES);
state->dts_cred.dcr_destructive = 1;
break;
case DTRACEOPT_BUFSIZE:
case DTRACEOPT_DYNVARSIZE:
case DTRACEOPT_AGGSIZE:
case DTRACEOPT_SPECSIZE:
case DTRACEOPT_STRSIZE:
if (val < 0)
return (EINVAL);
if (val >= LONG_MAX) {
/*
* If this is an otherwise negative value, set it to
* the highest multiple of 128m less than LONG_MAX.
* Technically, we're adjusting the size without
* regard to the buffer resizing policy, but in fact,
* this has no effect -- if we set the buffer size to
* ~LONG_MAX and the buffer policy is ultimately set to
* be "manual", the buffer allocation is guaranteed to
* fail, if only because the allocation requires two
* buffers. (We set the the size to the highest
* multiple of 128m because it ensures that the size
* will remain a multiple of a megabyte when
* repeatedly halved -- all the way down to 15m.)
*/
val = LONG_MAX - (1 << 27) + 1;
}
}
state->dts_options[option] = val;
return (0);
}
static void
dtrace_state_destroy(dtrace_state_t *state)
{
dtrace_ecb_t *ecb;
dtrace_vstate_t *vstate = &state->dts_vstate;
#ifdef illumos
minor_t minor = getminor(state->dts_dev);
#endif
int i, bufsize = NCPU * sizeof (dtrace_buffer_t);
dtrace_speculation_t *spec = state->dts_speculations;
int nspec = state->dts_nspeculations;
uint32_t match;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(MUTEX_HELD(&cpu_lock));
/*
* First, retract any retained enablings for this state.
*/
dtrace_enabling_retract(state);
ASSERT(state->dts_nretained == 0);
if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE ||
state->dts_activity == DTRACE_ACTIVITY_DRAINING) {
/*
* We have managed to come into dtrace_state_destroy() on a
* hot enabling -- almost certainly because of a disorderly
* shutdown of a consumer. (That is, a consumer that is
* exiting without having called dtrace_stop().) In this case,
* we're going to set our activity to be KILLED, and then
* issue a sync to be sure that everyone is out of probe
* context before we start blowing away ECBs.
*/
state->dts_activity = DTRACE_ACTIVITY_KILLED;
dtrace_sync();
}
/*
* Release the credential hold we took in dtrace_state_create().
*/
if (state->dts_cred.dcr_cred != NULL)
crfree(state->dts_cred.dcr_cred);
/*
* Now we can safely disable and destroy any enabled probes. Because
* any DTRACE_PRIV_KERNEL probes may actually be slowing our progress
* (especially if they're all enabled), we take two passes through the
* ECBs: in the first, we disable just DTRACE_PRIV_KERNEL probes, and
* in the second we disable whatever is left over.
*/
for (match = DTRACE_PRIV_KERNEL; ; match = 0) {
for (i = 0; i < state->dts_necbs; i++) {
if ((ecb = state->dts_ecbs[i]) == NULL)
continue;
if (match && ecb->dte_probe != NULL) {
dtrace_probe_t *probe = ecb->dte_probe;
dtrace_provider_t *prov = probe->dtpr_provider;
if (!(prov->dtpv_priv.dtpp_flags & match))
continue;
}
dtrace_ecb_disable(ecb);
dtrace_ecb_destroy(ecb);
}
if (!match)
break;
}
/*
* Before we free the buffers, perform one more sync to assure that
* every CPU is out of probe context.
*/
dtrace_sync();
dtrace_buffer_free(state->dts_buffer);
dtrace_buffer_free(state->dts_aggbuffer);
for (i = 0; i < nspec; i++)
dtrace_buffer_free(spec[i].dtsp_buffer);
#ifdef illumos
if (state->dts_cleaner != CYCLIC_NONE)
cyclic_remove(state->dts_cleaner);
if (state->dts_deadman != CYCLIC_NONE)
cyclic_remove(state->dts_deadman);
#else
callout_stop(&state->dts_cleaner);
callout_drain(&state->dts_cleaner);
callout_stop(&state->dts_deadman);
callout_drain(&state->dts_deadman);
#endif
dtrace_dstate_fini(&vstate->dtvs_dynvars);
dtrace_vstate_fini(vstate);
if (state->dts_ecbs != NULL)
kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *));
if (state->dts_aggregations != NULL) {
#ifdef DEBUG
for (i = 0; i < state->dts_naggregations; i++)
ASSERT(state->dts_aggregations[i] == NULL);
#endif
ASSERT(state->dts_naggregations > 0);
kmem_free(state->dts_aggregations,
state->dts_naggregations * sizeof (dtrace_aggregation_t *));
}
kmem_free(state->dts_buffer, bufsize);
kmem_free(state->dts_aggbuffer, bufsize);
for (i = 0; i < nspec; i++)
kmem_free(spec[i].dtsp_buffer, bufsize);
if (spec != NULL)
kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
dtrace_format_destroy(state);
if (state->dts_aggid_arena != NULL) {
#ifdef illumos
vmem_destroy(state->dts_aggid_arena);
#else
delete_unrhdr(state->dts_aggid_arena);
#endif
state->dts_aggid_arena = NULL;
}
#ifdef illumos
ddi_soft_state_free(dtrace_softstate, minor);
vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
#endif
}
/*
* DTrace Anonymous Enabling Functions
*/
static dtrace_state_t *
dtrace_anon_grab(void)
{
dtrace_state_t *state;
ASSERT(MUTEX_HELD(&dtrace_lock));
if ((state = dtrace_anon.dta_state) == NULL) {
ASSERT(dtrace_anon.dta_enabling == NULL);
return (NULL);
}
ASSERT(dtrace_anon.dta_enabling != NULL);
ASSERT(dtrace_retained != NULL);
dtrace_enabling_destroy(dtrace_anon.dta_enabling);
dtrace_anon.dta_enabling = NULL;
dtrace_anon.dta_state = NULL;
return (state);
}
static void
dtrace_anon_property(void)
{
int i, rv;
dtrace_state_t *state;
dof_hdr_t *dof;
char c[32]; /* enough for "dof-data-" + digits */
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(MUTEX_HELD(&cpu_lock));
for (i = 0; ; i++) {
(void) snprintf(c, sizeof (c), "dof-data-%d", i);
dtrace_err_verbose = 1;
if ((dof = dtrace_dof_property(c)) == NULL) {
dtrace_err_verbose = 0;
break;
}
#ifdef illumos
/*
* We want to create anonymous state, so we need to transition
* the kernel debugger to indicate that DTrace is active. If
* this fails (e.g. because the debugger has modified text in
* some way), we won't continue with the processing.
*/
if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
cmn_err(CE_NOTE, "kernel debugger active; anonymous "
"enabling ignored.");
dtrace_dof_destroy(dof);
break;
}
#endif
/*
* If we haven't allocated an anonymous state, we'll do so now.
*/
if ((state = dtrace_anon.dta_state) == NULL) {
#ifdef illumos
state = dtrace_state_create(NULL, NULL);
#else
state = dtrace_state_create(NULL);
#endif
dtrace_anon.dta_state = state;
if (state == NULL) {
/*
* This basically shouldn't happen: the only
* failure mode from dtrace_state_create() is a
* failure of ddi_soft_state_zalloc() that
* itself should never happen. Still, the
* interface allows for a failure mode, and
* we want to fail as gracefully as possible:
* we'll emit an error message and cease
* processing anonymous state in this case.
*/
cmn_err(CE_WARN, "failed to create "
"anonymous state");
dtrace_dof_destroy(dof);
break;
}
}
rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(),
&dtrace_anon.dta_enabling, 0, B_TRUE);
if (rv == 0)
rv = dtrace_dof_options(dof, state);
dtrace_err_verbose = 0;
dtrace_dof_destroy(dof);
if (rv != 0) {
/*
* This is malformed DOF; chuck any anonymous state
* that we created.
*/
ASSERT(dtrace_anon.dta_enabling == NULL);
dtrace_state_destroy(state);
dtrace_anon.dta_state = NULL;
break;
}
ASSERT(dtrace_anon.dta_enabling != NULL);
}
if (dtrace_anon.dta_enabling != NULL) {
int rval;
/*
* dtrace_enabling_retain() can only fail because we are
* trying to retain more enablings than are allowed -- but
* we only have one anonymous enabling, and we are guaranteed
* to be allowed at least one retained enabling; we assert
* that dtrace_enabling_retain() returns success.
*/
rval = dtrace_enabling_retain(dtrace_anon.dta_enabling);
ASSERT(rval == 0);
dtrace_enabling_dump(dtrace_anon.dta_enabling);
}
}
/*
* DTrace Helper Functions
*/
static void
dtrace_helper_trace(dtrace_helper_action_t *helper,
dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where)
{
uint32_t size, next, nnext, i;
dtrace_helptrace_t *ent, *buffer;
uint16_t flags = cpu_core[curcpu].cpuc_dtrace_flags;
if ((buffer = dtrace_helptrace_buffer) == NULL)
return;
ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals);
/*
* What would a tracing framework be without its own tracing
* framework? (Well, a hell of a lot simpler, for starters...)
*/
size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals *
sizeof (uint64_t) - sizeof (uint64_t);
/*
* Iterate until we can allocate a slot in the trace buffer.
*/
do {
next = dtrace_helptrace_next;
if (next + size < dtrace_helptrace_bufsize) {
nnext = next + size;
} else {
nnext = size;
}
} while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next);
/*
* We have our slot; fill it in.
*/
if (nnext == size) {
dtrace_helptrace_wrapped++;
next = 0;
}
ent = (dtrace_helptrace_t *)((uintptr_t)buffer + next);
ent->dtht_helper = helper;
ent->dtht_where = where;
ent->dtht_nlocals = vstate->dtvs_nlocals;
ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ?
mstate->dtms_fltoffs : -1;
ent->dtht_fault = DTRACE_FLAGS2FLT(flags);
ent->dtht_illval = cpu_core[curcpu].cpuc_dtrace_illval;
for (i = 0; i < vstate->dtvs_nlocals; i++) {
dtrace_statvar_t *svar;
if ((svar = vstate->dtvs_locals[i]) == NULL)
continue;
ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t));
ent->dtht_locals[i] =
((uint64_t *)(uintptr_t)svar->dtsv_data)[curcpu];
}
}
static uint64_t
dtrace_helper(int which, dtrace_mstate_t *mstate,
dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
{
uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags;
uint64_t sarg0 = mstate->dtms_arg[0];
uint64_t sarg1 = mstate->dtms_arg[1];
uint64_t rval = 0;
dtrace_helpers_t *helpers = curproc->p_dtrace_helpers;
dtrace_helper_action_t *helper;
dtrace_vstate_t *vstate;
dtrace_difo_t *pred;
int i, trace = dtrace_helptrace_buffer != NULL;
ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS);
if (helpers == NULL)
return (0);
if ((helper = helpers->dthps_actions[which]) == NULL)
return (0);
vstate = &helpers->dthps_vstate;
mstate->dtms_arg[0] = arg0;
mstate->dtms_arg[1] = arg1;
/*
* Now iterate over each helper. If its predicate evaluates to 'true',
* we'll call the corresponding actions. Note that the below calls
* to dtrace_dif_emulate() may set faults in machine state. This is
* okay: our caller (the outer dtrace_dif_emulate()) will simply plow
* the stored DIF offset with its own (which is the desired behavior).
* Also, note the calls to dtrace_dif_emulate() may allocate scratch
* from machine state; this is okay, too.
*/
for (; helper != NULL; helper = helper->dtha_next) {
if ((pred = helper->dtha_predicate) != NULL) {
if (trace)
dtrace_helper_trace(helper, mstate, vstate, 0);
if (!dtrace_dif_emulate(pred, mstate, vstate, state))
goto next;
if (*flags & CPU_DTRACE_FAULT)
goto err;
}
for (i = 0; i < helper->dtha_nactions; i++) {
if (trace)
dtrace_helper_trace(helper,
mstate, vstate, i + 1);
rval = dtrace_dif_emulate(helper->dtha_actions[i],
mstate, vstate, state);
if (*flags & CPU_DTRACE_FAULT)
goto err;
}
next:
if (trace)
dtrace_helper_trace(helper, mstate, vstate,
DTRACE_HELPTRACE_NEXT);
}
if (trace)
dtrace_helper_trace(helper, mstate, vstate,
DTRACE_HELPTRACE_DONE);
/*
* Restore the arg0 that we saved upon entry.
*/
mstate->dtms_arg[0] = sarg0;
mstate->dtms_arg[1] = sarg1;
return (rval);
err:
if (trace)
dtrace_helper_trace(helper, mstate, vstate,
DTRACE_HELPTRACE_ERR);
/*
* Restore the arg0 that we saved upon entry.
*/
mstate->dtms_arg[0] = sarg0;
mstate->dtms_arg[1] = sarg1;
return (0);
}
static void
dtrace_helper_action_destroy(dtrace_helper_action_t *helper,
dtrace_vstate_t *vstate)
{
int i;
if (helper->dtha_predicate != NULL)
dtrace_difo_release(helper->dtha_predicate, vstate);
for (i = 0; i < helper->dtha_nactions; i++) {
ASSERT(helper->dtha_actions[i] != NULL);
dtrace_difo_release(helper->dtha_actions[i], vstate);
}
kmem_free(helper->dtha_actions,
helper->dtha_nactions * sizeof (dtrace_difo_t *));
kmem_free(helper, sizeof (dtrace_helper_action_t));
}
static int
dtrace_helper_destroygen(dtrace_helpers_t *help, int gen)
{
proc_t *p = curproc;
dtrace_vstate_t *vstate;
int i;
if (help == NULL)
help = p->p_dtrace_helpers;
ASSERT(MUTEX_HELD(&dtrace_lock));
if (help == NULL || gen > help->dthps_generation)
return (EINVAL);
vstate = &help->dthps_vstate;
for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
dtrace_helper_action_t *last = NULL, *h, *next;
for (h = help->dthps_actions[i]; h != NULL; h = next) {
next = h->dtha_next;
if (h->dtha_generation == gen) {
if (last != NULL) {
last->dtha_next = next;
} else {
help->dthps_actions[i] = next;
}
dtrace_helper_action_destroy(h, vstate);
} else {
last = h;
}
}
}
/*
* Interate until we've cleared out all helper providers with the
* given generation number.
*/
for (;;) {
dtrace_helper_provider_t *prov;
/*
* Look for a helper provider with the right generation. We
* have to start back at the beginning of the list each time
* because we drop dtrace_lock. It's unlikely that we'll make
* more than two passes.
*/
for (i = 0; i < help->dthps_nprovs; i++) {
prov = help->dthps_provs[i];
if (prov->dthp_generation == gen)
break;
}
/*
* If there were no matches, we're done.
*/
if (i == help->dthps_nprovs)
break;
/*
* Move the last helper provider into this slot.
*/
help->dthps_nprovs--;
help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs];
help->dthps_provs[help->dthps_nprovs] = NULL;
mutex_exit(&dtrace_lock);
/*
* If we have a meta provider, remove this helper provider.
*/
mutex_enter(&dtrace_meta_lock);
if (dtrace_meta_pid != NULL) {
ASSERT(dtrace_deferred_pid == NULL);
dtrace_helper_provider_remove(&prov->dthp_prov,
p->p_pid);
}
mutex_exit(&dtrace_meta_lock);
dtrace_helper_provider_destroy(prov);
mutex_enter(&dtrace_lock);
}
return (0);
}
static int
dtrace_helper_validate(dtrace_helper_action_t *helper)
{
int err = 0, i;
dtrace_difo_t *dp;
if ((dp = helper->dtha_predicate) != NULL)
err += dtrace_difo_validate_helper(dp);
for (i = 0; i < helper->dtha_nactions; i++)
err += dtrace_difo_validate_helper(helper->dtha_actions[i]);
return (err == 0);
}
static int
dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep,
dtrace_helpers_t *help)
{
dtrace_helper_action_t *helper, *last;
dtrace_actdesc_t *act;
dtrace_vstate_t *vstate;
dtrace_predicate_t *pred;
int count = 0, nactions = 0, i;
if (which < 0 || which >= DTRACE_NHELPER_ACTIONS)
return (EINVAL);
last = help->dthps_actions[which];
vstate = &help->dthps_vstate;
for (count = 0; last != NULL; last = last->dtha_next) {
count++;
if (last->dtha_next == NULL)
break;
}
/*
* If we already have dtrace_helper_actions_max helper actions for this
* helper action type, we'll refuse to add a new one.
*/
if (count >= dtrace_helper_actions_max)
return (ENOSPC);
helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP);
helper->dtha_generation = help->dthps_generation;
if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) {
ASSERT(pred->dtp_difo != NULL);
dtrace_difo_hold(pred->dtp_difo);
helper->dtha_predicate = pred->dtp_difo;
}
for (act = ep->dted_action; act != NULL; act = act->dtad_next) {
if (act->dtad_kind != DTRACEACT_DIFEXPR)
goto err;
if (act->dtad_difo == NULL)
goto err;
nactions++;
}
helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) *
(helper->dtha_nactions = nactions), KM_SLEEP);
for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) {
dtrace_difo_hold(act->dtad_difo);
helper->dtha_actions[i++] = act->dtad_difo;
}
if (!dtrace_helper_validate(helper))
goto err;
if (last == NULL) {
help->dthps_actions[which] = helper;
} else {
last->dtha_next = helper;
}
if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) {
dtrace_helptrace_nlocals = vstate->dtvs_nlocals;
dtrace_helptrace_next = 0;
}
return (0);
err:
dtrace_helper_action_destroy(helper, vstate);
return (EINVAL);
}
static void
dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help,
dof_helper_t *dofhp)
{
ASSERT(MUTEX_NOT_HELD(&dtrace_lock));
mutex_enter(&dtrace_meta_lock);
mutex_enter(&dtrace_lock);
if (!dtrace_attached() || dtrace_meta_pid == NULL) {
/*
* If the dtrace module is loaded but not attached, or if
* there aren't isn't a meta provider registered to deal with
* these provider descriptions, we need to postpone creating
* the actual providers until later.
*/
if (help->dthps_next == NULL && help->dthps_prev == NULL &&
dtrace_deferred_pid != help) {
help->dthps_deferred = 1;
help->dthps_pid = p->p_pid;
help->dthps_next = dtrace_deferred_pid;
help->dthps_prev = NULL;
if (dtrace_deferred_pid != NULL)
dtrace_deferred_pid->dthps_prev = help;
dtrace_deferred_pid = help;
}
mutex_exit(&dtrace_lock);
} else if (dofhp != NULL) {
/*
* If the dtrace module is loaded and we have a particular
* helper provider description, pass that off to the
* meta provider.
*/
mutex_exit(&dtrace_lock);
dtrace_helper_provide(dofhp, p->p_pid);
} else {
/*
* Otherwise, just pass all the helper provider descriptions
* off to the meta provider.
*/
int i;
mutex_exit(&dtrace_lock);
for (i = 0; i < help->dthps_nprovs; i++) {
dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
p->p_pid);
}
}
mutex_exit(&dtrace_meta_lock);
}
static int
dtrace_helper_provider_add(dof_helper_t *dofhp, dtrace_helpers_t *help, int gen)
{
dtrace_helper_provider_t *hprov, **tmp_provs;
uint_t tmp_maxprovs, i;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(help != NULL);
/*
* If we already have dtrace_helper_providers_max helper providers,
* we're refuse to add a new one.
*/
if (help->dthps_nprovs >= dtrace_helper_providers_max)
return (ENOSPC);
/*
* Check to make sure this isn't a duplicate.
*/
for (i = 0; i < help->dthps_nprovs; i++) {
if (dofhp->dofhp_dof ==
help->dthps_provs[i]->dthp_prov.dofhp_dof)
return (EALREADY);
}
hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP);
hprov->dthp_prov = *dofhp;
hprov->dthp_ref = 1;
hprov->dthp_generation = gen;
/*
* Allocate a bigger table for helper providers if it's already full.
*/
if (help->dthps_maxprovs == help->dthps_nprovs) {
tmp_maxprovs = help->dthps_maxprovs;
tmp_provs = help->dthps_provs;
if (help->dthps_maxprovs == 0)
help->dthps_maxprovs = 2;
else
help->dthps_maxprovs *= 2;
if (help->dthps_maxprovs > dtrace_helper_providers_max)
help->dthps_maxprovs = dtrace_helper_providers_max;
ASSERT(tmp_maxprovs < help->dthps_maxprovs);
help->dthps_provs = kmem_zalloc(help->dthps_maxprovs *
sizeof (dtrace_helper_provider_t *), KM_SLEEP);
if (tmp_provs != NULL) {
bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs *
sizeof (dtrace_helper_provider_t *));
kmem_free(tmp_provs, tmp_maxprovs *
sizeof (dtrace_helper_provider_t *));
}
}
help->dthps_provs[help->dthps_nprovs] = hprov;
help->dthps_nprovs++;
return (0);
}
static void
dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov)
{
mutex_enter(&dtrace_lock);
if (--hprov->dthp_ref == 0) {
dof_hdr_t *dof;
mutex_exit(&dtrace_lock);
dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof;
dtrace_dof_destroy(dof);
kmem_free(hprov, sizeof (dtrace_helper_provider_t));
} else {
mutex_exit(&dtrace_lock);
}
}
static int
dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec)
{
uintptr_t daddr = (uintptr_t)dof;
dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
dof_provider_t *provider;
dof_probe_t *probe;
uint8_t *arg;
char *strtab, *typestr;
dof_stridx_t typeidx;
size_t typesz;
uint_t nprobes, j, k;
ASSERT(sec->dofs_type == DOF_SECT_PROVIDER);
if (sec->dofs_offset & (sizeof (uint_t) - 1)) {
dtrace_dof_error(dof, "misaligned section offset");
return (-1);
}
/*
* The section needs to be large enough to contain the DOF provider
* structure appropriate for the given version.
*/
if (sec->dofs_size <
((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ?
offsetof(dof_provider_t, dofpv_prenoffs) :
sizeof (dof_provider_t))) {
dtrace_dof_error(dof, "provider section too small");
return (-1);
}
provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab);
prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes);
arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs);
off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs);
if (str_sec == NULL || prb_sec == NULL ||
arg_sec == NULL || off_sec == NULL)
return (-1);
enoff_sec = NULL;
if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
provider->dofpv_prenoffs != DOF_SECT_NONE &&
(enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS,
provider->dofpv_prenoffs)) == NULL)
return (-1);
strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
if (provider->dofpv_name >= str_sec->dofs_size ||
strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) {
dtrace_dof_error(dof, "invalid provider name");
return (-1);
}
if (prb_sec->dofs_entsize == 0 ||
prb_sec->dofs_entsize > prb_sec->dofs_size) {
dtrace_dof_error(dof, "invalid entry size");
return (-1);
}
if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) {
dtrace_dof_error(dof, "misaligned entry size");
return (-1);
}
if (off_sec->dofs_entsize != sizeof (uint32_t)) {
dtrace_dof_error(dof, "invalid entry size");
return (-1);
}
if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) {
dtrace_dof_error(dof, "misaligned section offset");
return (-1);
}
if (arg_sec->dofs_entsize != sizeof (uint8_t)) {
dtrace_dof_error(dof, "invalid entry size");
return (-1);
}
arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
/*
* Take a pass through the probes to check for errors.
*/
for (j = 0; j < nprobes; j++) {
probe = (dof_probe_t *)(uintptr_t)(daddr +
prb_sec->dofs_offset + j * prb_sec->dofs_entsize);
if (probe->dofpr_func >= str_sec->dofs_size) {
dtrace_dof_error(dof, "invalid function name");
return (-1);
}
if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) {
dtrace_dof_error(dof, "function name too long");
return (-1);
}
if (probe->dofpr_name >= str_sec->dofs_size ||
strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) {
dtrace_dof_error(dof, "invalid probe name");
return (-1);
}
/*
* The offset count must not wrap the index, and the offsets
* must also not overflow the section's data.
*/
if (probe->dofpr_offidx + probe->dofpr_noffs <
probe->dofpr_offidx ||
(probe->dofpr_offidx + probe->dofpr_noffs) *
off_sec->dofs_entsize > off_sec->dofs_size) {
dtrace_dof_error(dof, "invalid probe offset");
return (-1);
}
if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) {
/*
* If there's no is-enabled offset section, make sure
* there aren't any is-enabled offsets. Otherwise
* perform the same checks as for probe offsets
* (immediately above).
*/
if (enoff_sec == NULL) {
if (probe->dofpr_enoffidx != 0 ||
probe->dofpr_nenoffs != 0) {
dtrace_dof_error(dof, "is-enabled "
"offsets with null section");
return (-1);
}
} else if (probe->dofpr_enoffidx +
probe->dofpr_nenoffs < probe->dofpr_enoffidx ||
(probe->dofpr_enoffidx + probe->dofpr_nenoffs) *
enoff_sec->dofs_entsize > enoff_sec->dofs_size) {
dtrace_dof_error(dof, "invalid is-enabled "
"offset");
return (-1);
}
if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) {
dtrace_dof_error(dof, "zero probe and "
"is-enabled offsets");
return (-1);
}
} else if (probe->dofpr_noffs == 0) {
dtrace_dof_error(dof, "zero probe offsets");
return (-1);
}
if (probe->dofpr_argidx + probe->dofpr_xargc <
probe->dofpr_argidx ||
(probe->dofpr_argidx + probe->dofpr_xargc) *
arg_sec->dofs_entsize > arg_sec->dofs_size) {
dtrace_dof_error(dof, "invalid args");
return (-1);
}
typeidx = probe->dofpr_nargv;
typestr = strtab + probe->dofpr_nargv;
for (k = 0; k < probe->dofpr_nargc; k++) {
if (typeidx >= str_sec->dofs_size) {
dtrace_dof_error(dof, "bad "
"native argument type");
return (-1);
}
typesz = strlen(typestr) + 1;
if (typesz > DTRACE_ARGTYPELEN) {
dtrace_dof_error(dof, "native "
"argument type too long");
return (-1);
}
typeidx += typesz;
typestr += typesz;
}
typeidx = probe->dofpr_xargv;
typestr = strtab + probe->dofpr_xargv;
for (k = 0; k < probe->dofpr_xargc; k++) {
if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) {
dtrace_dof_error(dof, "bad "
"native argument index");
return (-1);
}
if (typeidx >= str_sec->dofs_size) {
dtrace_dof_error(dof, "bad "
"translated argument type");
return (-1);
}
typesz = strlen(typestr) + 1;
if (typesz > DTRACE_ARGTYPELEN) {
dtrace_dof_error(dof, "translated argument "
"type too long");
return (-1);
}
typeidx += typesz;
typestr += typesz;
}
}
return (0);
}
static int
dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
{
dtrace_helpers_t *help;
dtrace_vstate_t *vstate;
dtrace_enabling_t *enab = NULL;
proc_t *p = curproc;
int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1;
uintptr_t daddr = (uintptr_t)dof;
ASSERT(MUTEX_HELD(&dtrace_lock));
#ifdef __FreeBSD__
if (dhp->dofhp_pid != p->p_pid) {
if ((p = pfind(dhp->dofhp_pid)) == NULL)
return (-1);
if (!P_SHOULDSTOP(p) ||
(p->p_flag & P_TRACED) == 0 ||
p->p_pptr->p_pid != curproc->p_pid) {
PROC_UNLOCK(p);
return (-1);
}
PROC_UNLOCK(p);
}
#endif
if ((help = p->p_dtrace_helpers) == NULL)
help = dtrace_helpers_create(p);
vstate = &help->dthps_vstate;
if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab,
dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) {
dtrace_dof_destroy(dof);
return (rv);
}
/*
* Look for helper providers and validate their descriptions.
*/
if (dhp != NULL) {
for (i = 0; i < dof->dofh_secnum; i++) {
dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
dof->dofh_secoff + i * dof->dofh_secsize);
if (sec->dofs_type != DOF_SECT_PROVIDER)
continue;
if (dtrace_helper_provider_validate(dof, sec) != 0) {
dtrace_enabling_destroy(enab);
dtrace_dof_destroy(dof);
return (-1);
}
nprovs++;
}
}
/*
* Now we need to walk through the ECB descriptions in the enabling.
*/
for (i = 0; i < enab->dten_ndesc; i++) {
dtrace_ecbdesc_t *ep = enab->dten_desc[i];
dtrace_probedesc_t *desc = &ep->dted_probe;
if (strcmp(desc->dtpd_provider, "dtrace") != 0)
continue;
if (strcmp(desc->dtpd_mod, "helper") != 0)
continue;
if (strcmp(desc->dtpd_func, "ustack") != 0)
continue;
if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK,
ep, help)) != 0) {
/*
* Adding this helper action failed -- we are now going
* to rip out the entire generation and return failure.
*/
(void) dtrace_helper_destroygen(help,
help->dthps_generation);
dtrace_enabling_destroy(enab);
dtrace_dof_destroy(dof);
return (-1);
}
nhelpers++;
}
if (nhelpers < enab->dten_ndesc)
dtrace_dof_error(dof, "unmatched helpers");
gen = help->dthps_generation++;
dtrace_enabling_destroy(enab);
if (dhp != NULL && nprovs > 0) {
dhp->dofhp_dof = (uint64_t)(uintptr_t)dof;
if (dtrace_helper_provider_add(dhp, help, gen) == 0) {
mutex_exit(&dtrace_lock);
dtrace_helper_provider_register(p, help, dhp);
mutex_enter(&dtrace_lock);
destroy = 0;
}
}
if (destroy)
dtrace_dof_destroy(dof);
return (gen);
}
static dtrace_helpers_t *
dtrace_helpers_create(proc_t *p)
{
dtrace_helpers_t *help;
ASSERT(MUTEX_HELD(&dtrace_lock));
ASSERT(p->p_dtrace_helpers == NULL);
help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP);
help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) *
DTRACE_NHELPER_ACTIONS, KM_SLEEP);
p->p_dtrace_helpers = help;
dtrace_helpers++;
return (help);
}
#ifdef illumos
static
#endif
void
dtrace_helpers_destroy(proc_t *p)
{
dtrace_helpers_t *help;
dtrace_vstate_t *vstate;
#ifdef illumos
proc_t *p = curproc;
#endif
int i;
mutex_enter(&dtrace_lock);
ASSERT(p->p_dtrace_helpers != NULL);
ASSERT(dtrace_helpers > 0);
help = p->p_dtrace_helpers;
vstate = &help->dthps_vstate;
/*
* We're now going to lose the help from this process.
*/
p->p_dtrace_helpers = NULL;
dtrace_sync();
/*
* Destory the helper actions.
*/
for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
dtrace_helper_action_t *h, *next;
for (h = help->dthps_actions[i]; h != NULL; h = next) {
next = h->dtha_next;
dtrace_helper_action_destroy(h, vstate);
h = next;
}
}
mutex_exit(&dtrace_lock);
/*
* Destroy the helper providers.
*/
if (help->dthps_maxprovs > 0) {
mutex_enter(&dtrace_meta_lock);
if (dtrace_meta_pid != NULL) {
ASSERT(dtrace_deferred_pid == NULL);
for (i = 0; i < help->dthps_nprovs; i++) {
dtrace_helper_provider_remove(
&help->dthps_provs[i]->dthp_prov, p->p_pid);
}
} else {
mutex_enter(&dtrace_lock);
ASSERT(help->dthps_deferred == 0 ||
help->dthps_next != NULL ||
help->dthps_prev != NULL ||
help == dtrace_deferred_pid);
/*
* Remove the helper from the deferred list.
*/
if (help->dthps_next != NULL)
help->dthps_next->dthps_prev = help->dthps_prev;
if (help->dthps_prev != NULL)
help->dthps_prev->dthps_next = help->dthps_next;
if (dtrace_deferred_pid == help) {
dtrace_deferred_pid = help->dthps_next;
ASSERT(help->dthps_prev == NULL);
}
mutex_exit(&dtrace_lock);
}
mutex_exit(&dtrace_meta_lock);
for (i = 0; i < help->dthps_nprovs; i++) {
dtrace_helper_provider_destroy(help->dthps_provs[i]);
}
kmem_free(help->dthps_provs, help->dthps_maxprovs *
sizeof (dtrace_helper_provider_t *));
}
mutex_enter(&dtrace_lock);
dtrace_vstate_fini(&help->dthps_vstate);
kmem_free(help->dthps_actions,
sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS);
kmem_free(help, sizeof (dtrace_helpers_t));
--dtrace_helpers;
mutex_exit(&dtrace_lock);
}
#ifdef illumos
static
#endif
void
dtrace_helpers_duplicate(proc_t *from, proc_t *to)
{
dtrace_helpers_t *help, *newhelp;
dtrace_helper_action_t *helper, *new, *last;
dtrace_difo_t *dp;
dtrace_vstate_t *vstate;
int i, j, sz, hasprovs = 0;
mutex_enter(&dtrace_lock);
ASSERT(from->p_dtrace_helpers != NULL);
ASSERT(dtrace_helpers > 0);
help = from->p_dtrace_helpers;
newhelp = dtrace_helpers_create(to);
ASSERT(to->p_dtrace_helpers != NULL);
newhelp->dthps_generation = help->dthps_generation;
vstate = &newhelp->dthps_vstate;
/*
* Duplicate the helper actions.
*/
for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
if ((helper = help->dthps_actions[i]) == NULL)
continue;
for (last = NULL; helper != NULL; helper = helper->dtha_next) {
new = kmem_zalloc(sizeof (dtrace_helper_action_t),
KM_SLEEP);
new->dtha_generation = helper->dtha_generation;
if ((dp = helper->dtha_predicate) != NULL) {
dp = dtrace_difo_duplicate(dp, vstate);
new->dtha_predicate = dp;
}
new->dtha_nactions = helper->dtha_nactions;
sz = sizeof (dtrace_difo_t *) * new->dtha_nactions;
new->dtha_actions = kmem_alloc(sz, KM_SLEEP);
for (j = 0; j < new->dtha_nactions; j++) {
dtrace_difo_t *dp = helper->dtha_actions[j];
ASSERT(dp != NULL);
dp = dtrace_difo_duplicate(dp, vstate);
new->dtha_actions[j] = dp;
}
if (last != NULL) {
last->dtha_next = new;
} else {
newhelp->dthps_actions[i] = new;
}
last = new;
}
}
/*
* Duplicate the helper providers and register them with the
* DTrace framework.
*/
if (help->dthps_nprovs > 0) {
newhelp->dthps_nprovs = help->dthps_nprovs;
newhelp->dthps_maxprovs = help->dthps_nprovs;
newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs *
sizeof (dtrace_helper_provider_t *), KM_SLEEP);
for (i = 0; i < newhelp->dthps_nprovs; i++) {
newhelp->dthps_provs[i] = help->dthps_provs[i];
newhelp->dthps_provs[i]->dthp_ref++;
}
hasprovs = 1;
}
mutex_exit(&dtrace_lock);
if (hasprovs)
dtrace_helper_provider_register(to, newhelp, NULL);
}
/*
* DTrace Hook Functions
*/
static void
dtrace_module_loaded(modctl_t *ctl)
{
dtrace_provider_t *prv;
mutex_enter(&dtrace_provider_lock);
#ifdef illumos
mutex_enter(&mod_lock);
#endif
#ifdef illumos
ASSERT(ctl->mod_busy);
#endif
/*
* We're going to call each providers per-module provide operation
* specifying only this module.
*/
for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next)
prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
#ifdef illumos
mutex_exit(&mod_lock);
#endif
mutex_exit(&dtrace_provider_lock);
/*
* If we have any retained enablings, we need to match against them.
* Enabling probes requires that cpu_lock be held, and we cannot hold
* cpu_lock here -- it is legal for cpu_lock to be held when loading a
* module. (In particular, this happens when loading scheduling
* classes.) So if we have any retained enablings, we need to dispatch
* our task queue to do the match for us.
*/
mutex_enter(&dtrace_lock);
if (dtrace_retained == NULL) {
mutex_exit(&dtrace_lock);
return;
}
(void) taskq_dispatch(dtrace_taskq,
(task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP);
mutex_exit(&dtrace_lock);
/*
* And now, for a little heuristic sleaze: in general, we want to
* match modules as soon as they load. However, we cannot guarantee
* this, because it would lead us to the lock ordering violation
* outlined above. The common case, of course, is that cpu_lock is
* _not_ held -- so we delay here for a clock tick, hoping that that's
* long enough for the task queue to do its work. If it's not, it's
* not a serious problem -- it just means that the module that we
* just loaded may not be immediately instrumentable.
*/
delay(1);
}
static void
#ifdef illumos
dtrace_module_unloaded(modctl_t *ctl)
#else
dtrace_module_unloaded(modctl_t *ctl, int *error)
#endif
{
dtrace_probe_t template, *probe, *first, *next;
dtrace_provider_t *prov;
#ifndef illumos
char modname[DTRACE_MODNAMELEN];
size_t len;
#endif
#ifdef illumos
template.dtpr_mod = ctl->mod_modname;
#else
/* Handle the fact that ctl->filename may end in ".ko". */
strlcpy(modname, ctl->filename, sizeof(modname));
len = strlen(ctl->filename);
if (len > 3 && strcmp(modname + len - 3, ".ko") == 0)
modname[len - 3] = '\0';
template.dtpr_mod = modname;
#endif
mutex_enter(&dtrace_provider_lock);
#ifdef illumos
mutex_enter(&mod_lock);
#endif
mutex_enter(&dtrace_lock);
#ifndef illumos
if (ctl->nenabled > 0) {
/* Don't allow unloads if a probe is enabled. */
mutex_exit(&dtrace_provider_lock);
mutex_exit(&dtrace_lock);
*error = -1;
printf(
"kldunload: attempt to unload module that has DTrace probes enabled\n");
return;
}
#endif
if (dtrace_bymod == NULL) {
/*
* The DTrace module is loaded (obviously) but not attached;
* we don't have any work to do.
*/
mutex_exit(&dtrace_provider_lock);
#ifdef illumos
mutex_exit(&mod_lock);
#endif
mutex_exit(&dtrace_lock);
return;
}
for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template);
probe != NULL; probe = probe->dtpr_nextmod) {
if (probe->dtpr_ecb != NULL) {
mutex_exit(&dtrace_provider_lock);
#ifdef illumos
mutex_exit(&mod_lock);
#endif
mutex_exit(&dtrace_lock);
/*
* This shouldn't _actually_ be possible -- we're
* unloading a module that has an enabled probe in it.
* (It's normally up to the provider to make sure that
* this can't happen.) However, because dtps_enable()
* doesn't have a failure mode, there can be an
* enable/unload race. Upshot: we don't want to
* assert, but we're not going to disable the
* probe, either.
*/
if (dtrace_err_verbose) {
#ifdef illumos
cmn_err(CE_WARN, "unloaded module '%s' had "
"enabled probes", ctl->mod_modname);
#else
cmn_err(CE_WARN, "unloaded module '%s' had "
"enabled probes", modname);
#endif
}
return;
}
}
probe = first;
for (first = NULL; probe != NULL; probe = next) {
ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe);
dtrace_probes[probe->dtpr_id - 1] = NULL;
next = probe->dtpr_nextmod;
dtrace_hash_remove(dtrace_bymod, probe);
dtrace_hash_remove(dtrace_byfunc, probe);
dtrace_hash_remove(dtrace_byname, probe);
if (first == NULL) {
first = probe;
probe->dtpr_nextmod = NULL;
} else {
probe->dtpr_nextmod = first;
first = probe;
}
}
/*
* We've removed all of the module's probes from the hash chains and
* from the probe array. Now issue a dtrace_sync() to be sure that
* everyone has cleared out from any probe array processing.
*/
dtrace_sync();
for (probe = first; probe != NULL; probe = first) {
first = probe->dtpr_nextmod;
prov = probe->dtpr_provider;
prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id,
probe->dtpr_arg);
kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
#ifdef illumos
vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1);
#else
free_unr(dtrace_arena, probe->dtpr_id);
#endif
kmem_free(probe, sizeof (dtrace_probe_t));
}
mutex_exit(&dtrace_lock);
#ifdef illumos
mutex_exit(&mod_lock);
#endif
mutex_exit(&dtrace_provider_lock);
}
#ifndef illumos
static void
dtrace_kld_load(void *arg __unused, linker_file_t lf)
{
dtrace_module_loaded(lf);
}
static void
dtrace_kld_unload_try(void *arg __unused, linker_file_t lf, int *error)
{
if (*error != 0)
/* We already have an error, so don't do anything. */
return;
dtrace_module_unloaded(lf, error);
}
#endif
#ifdef illumos
static void
dtrace_suspend(void)
{
dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend));
}
static void
dtrace_resume(void)
{
dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume));
}
#endif
static int
dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu)
{
ASSERT(MUTEX_HELD(&cpu_lock));
mutex_enter(&dtrace_lock);
switch (what) {
case CPU_CONFIG: {
dtrace_state_t *state;
dtrace_optval_t *opt, rs, c;
/*
* For now, we only allocate a new buffer for anonymous state.
*/
if ((state = dtrace_anon.dta_state) == NULL)
break;
if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
break;
opt = state->dts_options;
c = opt[DTRACEOPT_CPU];
if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu)
break;
/*
* Regardless of what the actual policy is, we're going to
* temporarily set our resize policy to be manual. We're
* also going to temporarily set our CPU option to denote
* the newly configured CPU.
*/
rs = opt[DTRACEOPT_BUFRESIZE];
opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL;
opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu;
(void) dtrace_state_buffers(state);
opt[DTRACEOPT_BUFRESIZE] = rs;
opt[DTRACEOPT_CPU] = c;
break;
}
case CPU_UNCONFIG:
/*
* We don't free the buffer in the CPU_UNCONFIG case. (The
* buffer will be freed when the consumer exits.)
*/
break;
default:
break;
}
mutex_exit(&dtrace_lock);
return (0);
}
#ifdef illumos
static void
dtrace_cpu_setup_initial(processorid_t cpu)
{
(void) dtrace_cpu_setup(CPU_CONFIG, cpu);
}
#endif
static void
dtrace_toxrange_add(uintptr_t base, uintptr_t limit)
{
if (dtrace_toxranges >= dtrace_toxranges_max) {
int osize, nsize;
dtrace_toxrange_t *range;
osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
if (osize == 0) {
ASSERT(dtrace_toxrange == NULL);
ASSERT(dtrace_toxranges_max == 0);
dtrace_toxranges_max = 1;
} else {
dtrace_toxranges_max <<= 1;
}
nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
range = kmem_zalloc(nsize, KM_SLEEP);
if (dtrace_toxrange != NULL) {
ASSERT(osize != 0);
bcopy(dtrace_toxrange, range, osize);
kmem_free(dtrace_toxrange, osize);
}
dtrace_toxrange = range;
}
ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == 0);
ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == 0);
dtrace_toxrange[dtrace_toxranges].dtt_base = base;
dtrace_toxrange[dtrace_toxranges].dtt_limit = limit;
dtrace_toxranges++;
}
static void
dtrace_getf_barrier()
{
#ifdef illumos
/*
* When we have unprivileged (that is, non-DTRACE_CRV_KERNEL) enablings
* that contain calls to getf(), this routine will be called on every
* closef() before either the underlying vnode is released or the
* file_t itself is freed. By the time we are here, it is essential
* that the file_t can no longer be accessed from a call to getf()
* in probe context -- that assures that a dtrace_sync() can be used
* to clear out any enablings referring to the old structures.
*/
if (curthread->t_procp->p_zone->zone_dtrace_getf != 0 ||
kcred->cr_zone->zone_dtrace_getf != 0)
dtrace_sync();
#endif
}
/*
* DTrace Driver Cookbook Functions
*/
#ifdef illumos
/*ARGSUSED*/
static int
dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
dtrace_provider_id_t id;
dtrace_state_t *state = NULL;
dtrace_enabling_t *enab;
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_provider_lock);
mutex_enter(&dtrace_lock);
if (ddi_soft_state_init(&dtrace_softstate,
sizeof (dtrace_state_t), 0) != 0) {
cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state");
mutex_exit(&cpu_lock);
mutex_exit(&dtrace_provider_lock);
mutex_exit(&dtrace_lock);
return (DDI_FAILURE);
}
if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR,
DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE ||
ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR,
DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) {
cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes");
ddi_remove_minor_node(devi, NULL);
ddi_soft_state_fini(&dtrace_softstate);
mutex_exit(&cpu_lock);
mutex_exit(&dtrace_provider_lock);
mutex_exit(&dtrace_lock);
return (DDI_FAILURE);
}
ddi_report_dev(devi);
dtrace_devi = devi;
dtrace_modload = dtrace_module_loaded;
dtrace_modunload = dtrace_module_unloaded;
dtrace_cpu_init = dtrace_cpu_setup_initial;
dtrace_helpers_cleanup = dtrace_helpers_destroy;
dtrace_helpers_fork = dtrace_helpers_duplicate;
dtrace_cpustart_init = dtrace_suspend;
dtrace_cpustart_fini = dtrace_resume;
dtrace_debugger_init = dtrace_suspend;
dtrace_debugger_fini = dtrace_resume;
register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
ASSERT(MUTEX_HELD(&cpu_lock));
dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1,
NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE,
UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0,
VM_SLEEP | VMC_IDENTIFIER);
dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri,
1, INT_MAX, 0);
dtrace_state_cache = kmem_cache_create("dtrace_state_cache",
sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN,
NULL, NULL, NULL, NULL, NULL, 0);
ASSERT(MUTEX_HELD(&cpu_lock));
dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod),
offsetof(dtrace_probe_t, dtpr_nextmod),
offsetof(dtrace_probe_t, dtpr_prevmod));
dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func),
offsetof(dtrace_probe_t, dtpr_nextfunc),
offsetof(dtrace_probe_t, dtpr_prevfunc));
dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name),
offsetof(dtrace_probe_t, dtpr_nextname),
offsetof(dtrace_probe_t, dtpr_prevname));
if (dtrace_retain_max < 1) {
cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; "
"setting to 1", dtrace_retain_max);
dtrace_retain_max = 1;
}
/*
* Now discover our toxic ranges.
*/
dtrace_toxic_ranges(dtrace_toxrange_add);
/*
* Before we register ourselves as a provider to our own framework,
* we would like to assert that dtrace_provider is NULL -- but that's
* not true if we were loaded as a dependency of a DTrace provider.
* Once we've registered, we can assert that dtrace_provider is our
* pseudo provider.
*/
(void) dtrace_register("dtrace", &dtrace_provider_attr,
DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id);
ASSERT(dtrace_provider != NULL);
ASSERT((dtrace_provider_id_t)dtrace_provider == id);
dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
dtrace_provider, NULL, NULL, "BEGIN", 0, NULL);
dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
dtrace_provider, NULL, NULL, "END", 0, NULL);
dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
dtrace_provider, NULL, NULL, "ERROR", 1, NULL);
dtrace_anon_property();
mutex_exit(&cpu_lock);
/*
* If there are already providers, we must ask them to provide their
* probes, and then match any anonymous enabling against them. Note
* that there should be no other retained enablings at this time:
* the only retained enablings at this time should be the anonymous
* enabling.
*/
if (dtrace_anon.dta_enabling != NULL) {
ASSERT(dtrace_retained == dtrace_anon.dta_enabling);
dtrace_enabling_provide(NULL);
state = dtrace_anon.dta_state;
/*
* We couldn't hold cpu_lock across the above call to
* dtrace_enabling_provide(), but we must hold it to actually
* enable the probes. We have to drop all of our locks, pick
* up cpu_lock, and regain our locks before matching the
* retained anonymous enabling.
*/
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_provider_lock);
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_provider_lock);
mutex_enter(&dtrace_lock);
if ((enab = dtrace_anon.dta_enabling) != NULL)
(void) dtrace_enabling_match(enab, NULL);
mutex_exit(&cpu_lock);
}
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_provider_lock);
if (state != NULL) {
/*
* If we created any anonymous state, set it going now.
*/
(void) dtrace_state_go(state, &dtrace_anon.dta_beganon);
}
return (DDI_SUCCESS);
}
#endif /* illumos */
#ifndef illumos
static void dtrace_dtr(void *);
#endif
/*ARGSUSED*/
static int
#ifdef illumos
dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
#else
dtrace_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
#endif
{
dtrace_state_t *state;
uint32_t priv;
uid_t uid;
zoneid_t zoneid;
#ifdef illumos
if (getminor(*devp) == DTRACEMNRN_HELPER)
return (0);
/*
* If this wasn't an open with the "helper" minor, then it must be
* the "dtrace" minor.
*/
if (getminor(*devp) == DTRACEMNRN_DTRACE)
return (ENXIO);
#else
cred_t *cred_p = NULL;
cred_p = dev->si_cred;
/*
* If no DTRACE_PRIV_* bits are set in the credential, then the
* caller lacks sufficient permission to do anything with DTrace.
*/
dtrace_cred2priv(cred_p, &priv, &uid, &zoneid);
if (priv == DTRACE_PRIV_NONE) {
#endif
return (EACCES);
}
/*
* Ask all providers to provide all their probes.
*/
mutex_enter(&dtrace_provider_lock);
dtrace_probe_provide(NULL, NULL);
mutex_exit(&dtrace_provider_lock);
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_lock);
dtrace_opens++;
dtrace_membar_producer();
#ifdef illumos
/*
* If the kernel debugger is active (that is, if the kernel debugger
* modified text in some way), we won't allow the open.
*/
if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
dtrace_opens--;
mutex_exit(&cpu_lock);
mutex_exit(&dtrace_lock);
return (EBUSY);
}
if (dtrace_helptrace_enable && dtrace_helptrace_buffer == NULL) {
/*
* If DTrace helper tracing is enabled, we need to allocate the
* trace buffer and initialize the values.
*/
dtrace_helptrace_buffer =
kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP);
dtrace_helptrace_next = 0;
dtrace_helptrace_wrapped = 0;
dtrace_helptrace_enable = 0;
}
state = dtrace_state_create(devp, cred_p);
#else
state = dtrace_state_create(dev);
devfs_set_cdevpriv(state, dtrace_dtr);
#endif
mutex_exit(&cpu_lock);
if (state == NULL) {
#ifdef illumos
if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
(void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
#else
--dtrace_opens;
#endif
mutex_exit(&dtrace_lock);
return (EAGAIN);
}
mutex_exit(&dtrace_lock);
return (0);
}
/*ARGSUSED*/
#ifdef illumos
static int
dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
#else
static void
dtrace_dtr(void *data)
#endif
{
#ifdef illumos
minor_t minor = getminor(dev);
dtrace_state_t *state;
#endif
dtrace_helptrace_t *buf = NULL;
#ifdef illumos
if (minor == DTRACEMNRN_HELPER)
return (0);
state = ddi_get_soft_state(dtrace_softstate, minor);
#else
dtrace_state_t *state = data;
#endif
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_lock);
#ifdef illumos
if (state->dts_anon)
#else
if (state != NULL && state->dts_anon)
#endif
{
/*
* There is anonymous state. Destroy that first.
*/
ASSERT(dtrace_anon.dta_state == NULL);
dtrace_state_destroy(state->dts_anon);
}
if (dtrace_helptrace_disable) {
/*
* If we have been told to disable helper tracing, set the
* buffer to NULL before calling into dtrace_state_destroy();
* we take advantage of its dtrace_sync() to know that no
* CPU is in probe context with enabled helper tracing
* after it returns.
*/
buf = dtrace_helptrace_buffer;
dtrace_helptrace_buffer = NULL;
}
#ifdef illumos
dtrace_state_destroy(state);
#else
if (state != NULL) {
dtrace_state_destroy(state);
kmem_free(state, 0);
}
#endif
ASSERT(dtrace_opens > 0);
#ifdef illumos
/*
* Only relinquish control of the kernel debugger interface when there
* are no consumers and no anonymous enablings.
*/
if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
(void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
#else
--dtrace_opens;
#endif
if (buf != NULL) {
kmem_free(buf, dtrace_helptrace_bufsize);
dtrace_helptrace_disable = 0;
}
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
#ifdef illumos
return (0);
#endif
}
#ifdef illumos
/*ARGSUSED*/
static int
dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv)
{
int rval;
dof_helper_t help, *dhp = NULL;
switch (cmd) {
case DTRACEHIOC_ADDDOF:
if (copyin((void *)arg, &help, sizeof (help)) != 0) {
dtrace_dof_error(NULL, "failed to copyin DOF helper");
return (EFAULT);
}
dhp = &help;
arg = (intptr_t)help.dofhp_dof;
/*FALLTHROUGH*/
case DTRACEHIOC_ADD: {
dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval);
if (dof == NULL)
return (rval);
mutex_enter(&dtrace_lock);
/*
* dtrace_helper_slurp() takes responsibility for the dof --
* it may free it now or it may save it and free it later.
*/
if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) {
*rv = rval;
rval = 0;
} else {
rval = EINVAL;
}
mutex_exit(&dtrace_lock);
return (rval);
}
case DTRACEHIOC_REMOVE: {
mutex_enter(&dtrace_lock);
rval = dtrace_helper_destroygen(NULL, arg);
mutex_exit(&dtrace_lock);
return (rval);
}
default:
break;
}
return (ENOTTY);
}
/*ARGSUSED*/
static int
dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
{
minor_t minor = getminor(dev);
dtrace_state_t *state;
int rval;
if (minor == DTRACEMNRN_HELPER)
return (dtrace_ioctl_helper(cmd, arg, rv));
state = ddi_get_soft_state(dtrace_softstate, minor);
if (state->dts_anon) {
ASSERT(dtrace_anon.dta_state == NULL);
state = state->dts_anon;
}
switch (cmd) {
case DTRACEIOC_PROVIDER: {
dtrace_providerdesc_t pvd;
dtrace_provider_t *pvp;
if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0)
return (EFAULT);
pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0';
mutex_enter(&dtrace_provider_lock);
for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) {
if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0)
break;
}
mutex_exit(&dtrace_provider_lock);
if (pvp == NULL)
return (ESRCH);
bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t));
bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t));
if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0)
return (EFAULT);
return (0);
}
case DTRACEIOC_EPROBE: {
dtrace_eprobedesc_t epdesc;
dtrace_ecb_t *ecb;
dtrace_action_t *act;
void *buf;
size_t size;
uintptr_t dest;
int nrecs;
if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0)
return (EFAULT);
mutex_enter(&dtrace_lock);
if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
if (ecb->dte_probe == NULL) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
epdesc.dtepd_uarg = ecb->dte_uarg;
epdesc.dtepd_size = ecb->dte_size;
nrecs = epdesc.dtepd_nrecs;
epdesc.dtepd_nrecs = 0;
for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
continue;
epdesc.dtepd_nrecs++;
}
/*
* Now that we have the size, we need to allocate a temporary
* buffer in which to store the complete description. We need
* the temporary buffer to be able to drop dtrace_lock()
* across the copyout(), below.
*/
size = sizeof (dtrace_eprobedesc_t) +
(epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t));
buf = kmem_alloc(size, KM_SLEEP);
dest = (uintptr_t)buf;
bcopy(&epdesc, (void *)dest, sizeof (epdesc));
dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]);
for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
continue;
if (nrecs-- == 0)
break;
bcopy(&act->dta_rec, (void *)dest,
sizeof (dtrace_recdesc_t));
dest += sizeof (dtrace_recdesc_t);
}
mutex_exit(&dtrace_lock);
if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
kmem_free(buf, size);
return (EFAULT);
}
kmem_free(buf, size);
return (0);
}
case DTRACEIOC_AGGDESC: {
dtrace_aggdesc_t aggdesc;
dtrace_action_t *act;
dtrace_aggregation_t *agg;
int nrecs;
uint32_t offs;
dtrace_recdesc_t *lrec;
void *buf;
size_t size;
uintptr_t dest;
if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0)
return (EFAULT);
mutex_enter(&dtrace_lock);
if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid;
nrecs = aggdesc.dtagd_nrecs;
aggdesc.dtagd_nrecs = 0;
offs = agg->dtag_base;
lrec = &agg->dtag_action.dta_rec;
aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs;
for (act = agg->dtag_first; ; act = act->dta_next) {
ASSERT(act->dta_intuple ||
DTRACEACT_ISAGG(act->dta_kind));
/*
* If this action has a record size of zero, it
* denotes an argument to the aggregating action.
* Because the presence of this record doesn't (or
* shouldn't) affect the way the data is interpreted,
* we don't copy it out to save user-level the
* confusion of dealing with a zero-length record.
*/
if (act->dta_rec.dtrd_size == 0) {
ASSERT(agg->dtag_hasarg);
continue;
}
aggdesc.dtagd_nrecs++;
if (act == &agg->dtag_action)
break;
}
/*
* Now that we have the size, we need to allocate a temporary
* buffer in which to store the complete description. We need
* the temporary buffer to be able to drop dtrace_lock()
* across the copyout(), below.
*/
size = sizeof (dtrace_aggdesc_t) +
(aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t));
buf = kmem_alloc(size, KM_SLEEP);
dest = (uintptr_t)buf;
bcopy(&aggdesc, (void *)dest, sizeof (aggdesc));
dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]);
for (act = agg->dtag_first; ; act = act->dta_next) {
dtrace_recdesc_t rec = act->dta_rec;
/*
* See the comment in the above loop for why we pass
* over zero-length records.
*/
if (rec.dtrd_size == 0) {
ASSERT(agg->dtag_hasarg);
continue;
}
if (nrecs-- == 0)
break;
rec.dtrd_offset -= offs;
bcopy(&rec, (void *)dest, sizeof (rec));
dest += sizeof (dtrace_recdesc_t);
if (act == &agg->dtag_action)
break;
}
mutex_exit(&dtrace_lock);
if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
kmem_free(buf, size);
return (EFAULT);
}
kmem_free(buf, size);
return (0);
}
case DTRACEIOC_ENABLE: {
dof_hdr_t *dof;
dtrace_enabling_t *enab = NULL;
dtrace_vstate_t *vstate;
int err = 0;
*rv = 0;
/*
* If a NULL argument has been passed, we take this as our
* cue to reevaluate our enablings.
*/
if (arg == NULL) {
dtrace_enabling_matchall();
return (0);
}
if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL)
return (rval);
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_lock);
vstate = &state->dts_vstate;
if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
dtrace_dof_destroy(dof);
return (EBUSY);
}
if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) {
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
dtrace_dof_destroy(dof);
return (EINVAL);
}
if ((rval = dtrace_dof_options(dof, state)) != 0) {
dtrace_enabling_destroy(enab);
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
dtrace_dof_destroy(dof);
return (rval);
}
if ((err = dtrace_enabling_match(enab, rv)) == 0) {
err = dtrace_enabling_retain(enab);
} else {
dtrace_enabling_destroy(enab);
}
mutex_exit(&cpu_lock);
mutex_exit(&dtrace_lock);
dtrace_dof_destroy(dof);
return (err);
}
case DTRACEIOC_REPLICATE: {
dtrace_repldesc_t desc;
dtrace_probedesc_t *match = &desc.dtrpd_match;
dtrace_probedesc_t *create = &desc.dtrpd_create;
int err;
if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
return (EFAULT);
match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
match->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
create->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
mutex_enter(&dtrace_lock);
err = dtrace_enabling_replicate(state, match, create);
mutex_exit(&dtrace_lock);
return (err);
}
case DTRACEIOC_PROBEMATCH:
case DTRACEIOC_PROBES: {
dtrace_probe_t *probe = NULL;
dtrace_probedesc_t desc;
dtrace_probekey_t pkey;
dtrace_id_t i;
int m = 0;
uint32_t priv;
uid_t uid;
zoneid_t zoneid;
if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
return (EFAULT);
desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0';
/*
* Before we attempt to match this probe, we want to give
* all providers the opportunity to provide it.
*/
if (desc.dtpd_id == DTRACE_IDNONE) {
mutex_enter(&dtrace_provider_lock);
dtrace_probe_provide(&desc, NULL);
mutex_exit(&dtrace_provider_lock);
desc.dtpd_id++;
}
if (cmd == DTRACEIOC_PROBEMATCH) {
dtrace_probekey(&desc, &pkey);
pkey.dtpk_id = DTRACE_IDNONE;
}
dtrace_cred2priv(cr, &priv, &uid, &zoneid);
mutex_enter(&dtrace_lock);
if (cmd == DTRACEIOC_PROBEMATCH) {
for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i - 1]) != NULL &&
(m = dtrace_match_probe(probe, &pkey,
priv, uid, zoneid)) != 0)
break;
}
if (m < 0) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
} else {
for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
if ((probe = dtrace_probes[i - 1]) != NULL &&
dtrace_match_priv(probe, priv, uid, zoneid))
break;
}
}
if (probe == NULL) {
mutex_exit(&dtrace_lock);
return (ESRCH);
}
dtrace_probe_description(probe, &desc);
mutex_exit(&dtrace_lock);
if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
return (EFAULT);
return (0);
}
case DTRACEIOC_PROBEARG: {
dtrace_argdesc_t desc;
dtrace_probe_t *probe;
dtrace_provider_t *prov;
if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
return (EFAULT);
if (desc.dtargd_id == DTRACE_IDNONE)
return (EINVAL);
if (desc.dtargd_ndx == DTRACE_ARGNONE)
return (EINVAL);
mutex_enter(&dtrace_provider_lock);
mutex_enter(&mod_lock);
mutex_enter(&dtrace_lock);
if (desc.dtargd_id > dtrace_nprobes) {
mutex_exit(&dtrace_lock);
mutex_exit(&mod_lock);
mutex_exit(&dtrace_provider_lock);
return (EINVAL);
}
if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) {
mutex_exit(&dtrace_lock);
mutex_exit(&mod_lock);
mutex_exit(&dtrace_provider_lock);
return (EINVAL);
}
mutex_exit(&dtrace_lock);
prov = probe->dtpr_provider;
if (prov->dtpv_pops.dtps_getargdesc == NULL) {
/*
* There isn't any typed information for this probe.
* Set the argument number to DTRACE_ARGNONE.
*/
desc.dtargd_ndx = DTRACE_ARGNONE;
} else {
desc.dtargd_native[0] = '\0';
desc.dtargd_xlate[0] = '\0';
desc.dtargd_mapping = desc.dtargd_ndx;
prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg,
probe->dtpr_id, probe->dtpr_arg, &desc);
}
mutex_exit(&mod_lock);
mutex_exit(&dtrace_provider_lock);
if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
return (EFAULT);
return (0);
}
case DTRACEIOC_GO: {
processorid_t cpuid;
rval = dtrace_state_go(state, &cpuid);
if (rval != 0)
return (rval);
if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
return (EFAULT);
return (0);
}
case DTRACEIOC_STOP: {
processorid_t cpuid;
mutex_enter(&dtrace_lock);
rval = dtrace_state_stop(state, &cpuid);
mutex_exit(&dtrace_lock);
if (rval != 0)
return (rval);
if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
return (EFAULT);
return (0);
}
case DTRACEIOC_DOFGET: {
dof_hdr_t hdr, *dof;
uint64_t len;
if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0)
return (EFAULT);
mutex_enter(&dtrace_lock);
dof = dtrace_dof_create(state);
mutex_exit(&dtrace_lock);
len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz);
rval = copyout(dof, (void *)arg, len);
dtrace_dof_destroy(dof);
return (rval == 0 ? 0 : EFAULT);
}
case DTRACEIOC_AGGSNAP:
case DTRACEIOC_BUFSNAP: {
dtrace_bufdesc_t desc;
caddr_t cached;
dtrace_buffer_t *buf;
if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
return (EFAULT);
if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU)
return (EINVAL);
mutex_enter(&dtrace_lock);
if (cmd == DTRACEIOC_BUFSNAP) {
buf = &state->dts_buffer[desc.dtbd_cpu];
} else {
buf = &state->dts_aggbuffer[desc.dtbd_cpu];
}
if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) {
size_t sz = buf->dtb_offset;
if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
mutex_exit(&dtrace_lock);
return (EBUSY);
}
/*
* If this buffer has already been consumed, we're
* going to indicate that there's nothing left here
* to consume.
*/
if (buf->dtb_flags & DTRACEBUF_CONSUMED) {
mutex_exit(&dtrace_lock);
desc.dtbd_size = 0;
desc.dtbd_drops = 0;
desc.dtbd_errors = 0;
desc.dtbd_oldest = 0;
sz = sizeof (desc);
if (copyout(&desc, (void *)arg, sz) != 0)
return (EFAULT);
return (0);
}
/*
* If this is a ring buffer that has wrapped, we want
* to copy the whole thing out.
*/
if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
dtrace_buffer_polish(buf);
sz = buf->dtb_size;
}
if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) {
mutex_exit(&dtrace_lock);
return (EFAULT);
}
desc.dtbd_size = sz;
desc.dtbd_drops = buf->dtb_drops;
desc.dtbd_errors = buf->dtb_errors;
desc.dtbd_oldest = buf->dtb_xamot_offset;
desc.dtbd_timestamp = dtrace_gethrtime();
mutex_exit(&dtrace_lock);
if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
return (EFAULT);
buf->dtb_flags |= DTRACEBUF_CONSUMED;
return (0);
}
if (buf->dtb_tomax == NULL) {
ASSERT(buf->dtb_xamot == NULL);
mutex_exit(&dtrace_lock);
return (ENOENT);
}
cached = buf->dtb_tomax;
ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
dtrace_xcall(desc.dtbd_cpu,
(dtrace_xcall_t)dtrace_buffer_switch, buf);
state->dts_errors += buf->dtb_xamot_errors;
/*
* If the buffers did not actually switch, then the cross call
* did not take place -- presumably because the given CPU is
* not in the ready set. If this is the case, we'll return
* ENOENT.
*/
if (buf->dtb_tomax == cached) {
ASSERT(buf->dtb_xamot != cached);
mutex_exit(&dtrace_lock);
return (ENOENT);
}
ASSERT(cached == buf->dtb_xamot);
/*
* We have our snapshot; now copy it out.
*/
if (copyout(buf->dtb_xamot, desc.dtbd_data,
buf->dtb_xamot_offset) != 0) {
mutex_exit(&dtrace_lock);
return (EFAULT);
}
desc.dtbd_size = buf->dtb_xamot_offset;
desc.dtbd_drops = buf->dtb_xamot_drops;
desc.dtbd_errors = buf->dtb_xamot_errors;
desc.dtbd_oldest = 0;
desc.dtbd_timestamp = buf->dtb_switched;
mutex_exit(&dtrace_lock);
/*
* Finally, copy out the buffer description.
*/
if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
return (EFAULT);
return (0);
}
case DTRACEIOC_CONF: {
dtrace_conf_t conf;
bzero(&conf, sizeof (conf));
conf.dtc_difversion = DIF_VERSION;
conf.dtc_difintregs = DIF_DIR_NREGS;
conf.dtc_diftupregs = DIF_DTR_NREGS;
conf.dtc_ctfmodel = CTF_MODEL_NATIVE;
if (copyout(&conf, (void *)arg, sizeof (conf)) != 0)
return (EFAULT);
return (0);
}
case DTRACEIOC_STATUS: {
dtrace_status_t stat;
dtrace_dstate_t *dstate;
int i, j;
uint64_t nerrs;
/*
* See the comment in dtrace_state_deadman() for the reason
* for setting dts_laststatus to INT64_MAX before setting
* it to the correct value.
*/
state->dts_laststatus = INT64_MAX;
dtrace_membar_producer();
state->dts_laststatus = dtrace_gethrtime();
bzero(&stat, sizeof (stat));
mutex_enter(&dtrace_lock);
if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
mutex_exit(&dtrace_lock);
return (ENOENT);
}
if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
stat.dtst_exiting = 1;
nerrs = state->dts_errors;
dstate = &state->dts_vstate.dtvs_dynvars;
for (i = 0; i < NCPU; i++) {
dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i];
stat.dtst_dyndrops += dcpu->dtdsc_drops;
stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops;
stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops;
if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
stat.dtst_filled++;
nerrs += state->dts_buffer[i].dtb_errors;
for (j = 0; j < state->dts_nspeculations; j++) {
dtrace_speculation_t *spec;
dtrace_buffer_t *buf;
spec = &state->dts_speculations[j];
buf = &spec->dtsp_buffer[i];
stat.dtst_specdrops += buf->dtb_xamot_drops;
}
}
stat.dtst_specdrops_busy = state->dts_speculations_busy;
stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
stat.dtst_stkstroverflows = state->dts_stkstroverflows;
stat.dtst_dblerrors = state->dts_dblerrors;
stat.dtst_killed =
(state->dts_activity == DTRACE_ACTIVITY_KILLED);
stat.dtst_errors = nerrs;
mutex_exit(&dtrace_lock);
if (copyout(&stat, (void *)arg, sizeof (stat)) != 0)
return (EFAULT);
return (0);
}
case DTRACEIOC_FORMAT: {
dtrace_fmtdesc_t fmt;
char *str;
int len;
if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0)
return (EFAULT);
mutex_enter(&dtrace_lock);
if (fmt.dtfd_format == 0 ||
fmt.dtfd_format > state->dts_nformats) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
/*
* Format strings are allocated contiguously and they are
* never freed; if a format index is less than the number
* of formats, we can assert that the format map is non-NULL
* and that the format for the specified index is non-NULL.
*/
ASSERT(state->dts_formats != NULL);
str = state->dts_formats[fmt.dtfd_format - 1];
ASSERT(str != NULL);
len = strlen(str) + 1;
if (len > fmt.dtfd_length) {
fmt.dtfd_length = len;
if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
} else {
if (copyout(str, fmt.dtfd_string, len) != 0) {
mutex_exit(&dtrace_lock);
return (EINVAL);
}
}
mutex_exit(&dtrace_lock);
return (0);
}
default:
break;
}
return (ENOTTY);
}
/*ARGSUSED*/
static int
dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
dtrace_state_t *state;
switch (cmd) {
case DDI_DETACH:
break;
case DDI_SUSPEND:
return (DDI_SUCCESS);
default:
return (DDI_FAILURE);
}
mutex_enter(&cpu_lock);
mutex_enter(&dtrace_provider_lock);
mutex_enter(&dtrace_lock);
ASSERT(dtrace_opens == 0);
if (dtrace_helpers > 0) {
mutex_exit(&dtrace_provider_lock);
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
return (DDI_FAILURE);
}
if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) {
mutex_exit(&dtrace_provider_lock);
mutex_exit(&dtrace_lock);
mutex_exit(&cpu_lock);
return (DDI_FAILURE);
}
dtrace_provider = NULL;
if ((state = dtrace_anon_grab()) != NULL) {
/*
* If there were ECBs on this state, the provider should
* have not been allowed to detach; assert that there is
* none.
*/
ASSERT(state->dts_necbs == 0);
dtrace_state_destroy(state);
/*
* If we're being detached with anonymous state, we need to
* indicate to the kernel debugger that DTrace is now inactive.
*/
(void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
}
bzero(&dtrace_anon, sizeof (dtrace_anon_t));
unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
dtrace_cpu_init = NULL;
dtrace_helpers_cleanup = NULL;
dtrace_helpers_fork = NULL;
dtrace_cpustart_init = NULL;
dtrace_cpustart_fini = NULL;
dtrace_debugger_init = NULL;
dtrace_debugger_fini = NULL;
dtrace_modload = NULL;
dtrace_modunload = NULL;
ASSERT(dtrace_getf == 0);
ASSERT(dtrace_closef == NULL);
mutex_exit(&cpu_lock);
kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *));
dtrace_probes = NULL;
dtrace_nprobes = 0;
dtrace_hash_destroy(dtrace_bymod);
dtrace_hash_destroy(dtrace_byfunc);
dtrace_hash_destroy(dtrace_byname);
dtrace_bymod = NULL;
dtrace_byfunc = NULL;
dtrace_byname = NULL;
kmem_cache_destroy(dtrace_state_cache);
vmem_destroy(dtrace_minor);
vmem_destroy(dtrace_arena);
if (dtrace_toxrange != NULL) {
kmem_free(dtrace_toxrange,
dtrace_toxranges_max * sizeof (dtrace_toxrange_t));
dtrace_toxrange = NULL;
dtrace_toxranges = 0;
dtrace_toxranges_max = 0;
}
ddi_remove_minor_node(dtrace_devi, NULL);
dtrace_devi = NULL;
ddi_soft_state_fini(&dtrace_softstate);
ASSERT(dtrace_vtime_references == 0);
ASSERT(dtrace_opens == 0);
ASSERT(dtrace_retained == NULL);
mutex_exit(&dtrace_lock);
mutex_exit(&dtrace_provider_lock);
/*
* We don't destroy the task queue until after we have dropped our
* locks (taskq_destroy() may block on running tasks). To prevent
* attempting to do work after we have effectively detached but before
* the task queue has been destroyed, all tasks dispatched via the
* task queue must check that DTrace is still attached before
* performing any operation.
*/
taskq_destroy(dtrace_taskq);
dtrace_taskq = NULL;
return (DDI_SUCCESS);
}
#endif
#ifdef illumos
/*ARGSUSED*/
static int
dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
{
int error;
switch (infocmd) {
case DDI_INFO_DEVT2DEVINFO:
*result = (void *)dtrace_devi;
error = DDI_SUCCESS;
break;
case DDI_INFO_DEVT2INSTANCE:
*result = (void *)0;
error = DDI_SUCCESS;
break;
default:
error = DDI_FAILURE;
}
return (error);
}
#endif
#ifdef illumos
static struct cb_ops dtrace_cb_ops = {
dtrace_open, /* open */
dtrace_close, /* close */
nulldev, /* strategy */
nulldev, /* print */
nodev, /* dump */
nodev, /* read */
nodev, /* write */
dtrace_ioctl, /* ioctl */
nodev, /* devmap */
nodev, /* mmap */
nodev, /* segmap */
nochpoll, /* poll */
ddi_prop_op, /* cb_prop_op */
0, /* streamtab */
D_NEW | D_MP /* Driver compatibility flag */
};
static struct dev_ops dtrace_ops = {
DEVO_REV, /* devo_rev */
0, /* refcnt */
dtrace_info, /* get_dev_info */
nulldev, /* identify */
nulldev, /* probe */
dtrace_attach, /* attach */
dtrace_detach, /* detach */
nodev, /* reset */
&dtrace_cb_ops, /* driver operations */
NULL, /* bus operations */
nodev /* dev power */
};
static struct modldrv modldrv = {
&mod_driverops, /* module type (this is a pseudo driver) */
"Dynamic Tracing", /* name of module */
&dtrace_ops, /* driver ops */
};
static struct modlinkage modlinkage = {
MODREV_1,
(void *)&modldrv,
NULL
};
int
_init(void)
{
return (mod_install(&modlinkage));
}
int
_info(struct modinfo *modinfop)
{
return (mod_info(&modlinkage, modinfop));
}
int
_fini(void)
{
return (mod_remove(&modlinkage));
}
#else
static d_ioctl_t dtrace_ioctl;
static d_ioctl_t dtrace_ioctl_helper;
static void dtrace_load(void *);
static int dtrace_unload(void);
static struct cdev *dtrace_dev;
static struct cdev *helper_dev;
void dtrace_invop_init(void);
void dtrace_invop_uninit(void);
static struct cdevsw dtrace_cdevsw = {
.d_version = D_VERSION,
.d_ioctl = dtrace_ioctl,
.d_open = dtrace_open,
.d_name = "dtrace",
};
static struct cdevsw helper_cdevsw = {
.d_version = D_VERSION,
.d_ioctl = dtrace_ioctl_helper,
.d_name = "helper",
};
#include <dtrace_anon.c>
#include <dtrace_ioctl.c>
#include <dtrace_load.c>
#include <dtrace_modevent.c>
#include <dtrace_sysctl.c>
#include <dtrace_unload.c>
#include <dtrace_vtime.c>
#include <dtrace_hacks.c>
#include <dtrace_isa.c>
SYSINIT(dtrace_load, SI_SUB_DTRACE, SI_ORDER_FIRST, dtrace_load, NULL);
SYSUNINIT(dtrace_unload, SI_SUB_DTRACE, SI_ORDER_FIRST, dtrace_unload, NULL);
SYSINIT(dtrace_anon_init, SI_SUB_DTRACE_ANON, SI_ORDER_FIRST, dtrace_anon_init, NULL);
DEV_MODULE(dtrace, dtrace_modevent, NULL);
MODULE_VERSION(dtrace, 1);
MODULE_DEPEND(dtrace, opensolaris, 1, 1, 1);
#endif
Index: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c
===================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c (revision 283290)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c (revision 283291)
@@ -1,2062 +1,2062 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2014 by Delphix. All rights reserved.
* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
* Copyright 2013 Martin Matuska <mm@FreeBSD.org>. All rights reserved.
*/
#include <sys/zfs_context.h>
#include <sys/spa_impl.h>
#include <sys/spa_boot.h>
#include <sys/zio.h>
#include <sys/zio_checksum.h>
#include <sys/zio_compress.h>
#include <sys/dmu.h>
#include <sys/dmu_tx.h>
#include <sys/zap.h>
#include <sys/zil.h>
#include <sys/vdev_impl.h>
#include <sys/metaslab.h>
#include <sys/uberblock_impl.h>
#include <sys/txg.h>
#include <sys/avl.h>
#include <sys/unique.h>
#include <sys/dsl_pool.h>
#include <sys/dsl_dir.h>
#include <sys/dsl_prop.h>
#include <sys/dsl_scan.h>
#include <sys/fs/zfs.h>
#include <sys/metaslab_impl.h>
#include <sys/arc.h>
#include <sys/ddt.h>
#include "zfs_prop.h"
#include "zfeature_common.h"
/*
* SPA locking
*
* There are four basic locks for managing spa_t structures:
*
* spa_namespace_lock (global mutex)
*
* This lock must be acquired to do any of the following:
*
* - Lookup a spa_t by name
* - Add or remove a spa_t from the namespace
* - Increase spa_refcount from non-zero
* - Check if spa_refcount is zero
* - Rename a spa_t
* - add/remove/attach/detach devices
* - Held for the duration of create/destroy/import/export
*
* It does not need to handle recursion. A create or destroy may
* reference objects (files or zvols) in other pools, but by
* definition they must have an existing reference, and will never need
* to lookup a spa_t by name.
*
* spa_refcount (per-spa refcount_t protected by mutex)
*
* This reference count keep track of any active users of the spa_t. The
* spa_t cannot be destroyed or freed while this is non-zero. Internally,
* the refcount is never really 'zero' - opening a pool implicitly keeps
* some references in the DMU. Internally we check against spa_minref, but
* present the image of a zero/non-zero value to consumers.
*
* spa_config_lock[] (per-spa array of rwlocks)
*
* This protects the spa_t from config changes, and must be held in
* the following circumstances:
*
* - RW_READER to perform I/O to the spa
* - RW_WRITER to change the vdev config
*
* The locking order is fairly straightforward:
*
* spa_namespace_lock -> spa_refcount
*
* The namespace lock must be acquired to increase the refcount from 0
* or to check if it is zero.
*
* spa_refcount -> spa_config_lock[]
*
* There must be at least one valid reference on the spa_t to acquire
* the config lock.
*
* spa_namespace_lock -> spa_config_lock[]
*
* The namespace lock must always be taken before the config lock.
*
*
* The spa_namespace_lock can be acquired directly and is globally visible.
*
* The namespace is manipulated using the following functions, all of which
* require the spa_namespace_lock to be held.
*
* spa_lookup() Lookup a spa_t by name.
*
* spa_add() Create a new spa_t in the namespace.
*
* spa_remove() Remove a spa_t from the namespace. This also
* frees up any memory associated with the spa_t.
*
* spa_next() Returns the next spa_t in the system, or the
* first if NULL is passed.
*
* spa_evict_all() Shutdown and remove all spa_t structures in
* the system.
*
* spa_guid_exists() Determine whether a pool/device guid exists.
*
* The spa_refcount is manipulated using the following functions:
*
* spa_open_ref() Adds a reference to the given spa_t. Must be
* called with spa_namespace_lock held if the
* refcount is currently zero.
*
* spa_close() Remove a reference from the spa_t. This will
* not free the spa_t or remove it from the
* namespace. No locking is required.
*
* spa_refcount_zero() Returns true if the refcount is currently
* zero. Must be called with spa_namespace_lock
* held.
*
* The spa_config_lock[] is an array of rwlocks, ordered as follows:
* SCL_CONFIG > SCL_STATE > SCL_ALLOC > SCL_ZIO > SCL_FREE > SCL_VDEV.
* spa_config_lock[] is manipulated with spa_config_{enter,exit,held}().
*
* To read the configuration, it suffices to hold one of these locks as reader.
* To modify the configuration, you must hold all locks as writer. To modify
* vdev state without altering the vdev tree's topology (e.g. online/offline),
* you must hold SCL_STATE and SCL_ZIO as writer.
*
* We use these distinct config locks to avoid recursive lock entry.
* For example, spa_sync() (which holds SCL_CONFIG as reader) induces
* block allocations (SCL_ALLOC), which may require reading space maps
* from disk (dmu_read() -> zio_read() -> SCL_ZIO).
*
* The spa config locks cannot be normal rwlocks because we need the
* ability to hand off ownership. For example, SCL_ZIO is acquired
* by the issuing thread and later released by an interrupt thread.
* They do, however, obey the usual write-wanted semantics to prevent
* writer (i.e. system administrator) starvation.
*
* The lock acquisition rules are as follows:
*
* SCL_CONFIG
* Protects changes to the vdev tree topology, such as vdev
* add/remove/attach/detach. Protects the dirty config list
* (spa_config_dirty_list) and the set of spares and l2arc devices.
*
* SCL_STATE
* Protects changes to pool state and vdev state, such as vdev
* online/offline/fault/degrade/clear. Protects the dirty state list
* (spa_state_dirty_list) and global pool state (spa_state).
*
* SCL_ALLOC
* Protects changes to metaslab groups and classes.
* Held as reader by metaslab_alloc() and metaslab_claim().
*
* SCL_ZIO
* Held by bp-level zios (those which have no io_vd upon entry)
* to prevent changes to the vdev tree. The bp-level zio implicitly
* protects all of its vdev child zios, which do not hold SCL_ZIO.
*
* SCL_FREE
* Protects changes to metaslab groups and classes.
* Held as reader by metaslab_free(). SCL_FREE is distinct from
* SCL_ALLOC, and lower than SCL_ZIO, so that we can safely free
* blocks in zio_done() while another i/o that holds either
* SCL_ALLOC or SCL_ZIO is waiting for this i/o to complete.
*
* SCL_VDEV
* Held as reader to prevent changes to the vdev tree during trivial
* inquiries such as bp_get_dsize(). SCL_VDEV is distinct from the
* other locks, and lower than all of them, to ensure that it's safe
* to acquire regardless of caller context.
*
* In addition, the following rules apply:
*
* (a) spa_props_lock protects pool properties, spa_config and spa_config_list.
* The lock ordering is SCL_CONFIG > spa_props_lock.
*
* (b) I/O operations on leaf vdevs. For any zio operation that takes
* an explicit vdev_t argument -- such as zio_ioctl(), zio_read_phys(),
* or zio_write_phys() -- the caller must ensure that the config cannot
* cannot change in the interim, and that the vdev cannot be reopened.
* SCL_STATE as reader suffices for both.
*
* The vdev configuration is protected by spa_vdev_enter() / spa_vdev_exit().
*
* spa_vdev_enter() Acquire the namespace lock and the config lock
* for writing.
*
* spa_vdev_exit() Release the config lock, wait for all I/O
* to complete, sync the updated configs to the
* cache, and release the namespace lock.
*
* vdev state is protected by spa_vdev_state_enter() / spa_vdev_state_exit().
* Like spa_vdev_enter/exit, these are convenience wrappers -- the actual
* locking is, always, based on spa_namespace_lock and spa_config_lock[].
*
* spa_rename() is also implemented within this file since it requires
* manipulation of the namespace.
*/
static avl_tree_t spa_namespace_avl;
kmutex_t spa_namespace_lock;
static kcondvar_t spa_namespace_cv;
static int spa_active_count;
int spa_max_replication_override = SPA_DVAS_PER_BP;
static kmutex_t spa_spare_lock;
static avl_tree_t spa_spare_avl;
static kmutex_t spa_l2cache_lock;
static avl_tree_t spa_l2cache_avl;
kmem_cache_t *spa_buffer_pool;
int spa_mode_global;
#ifdef ZFS_DEBUG
/* Everything except dprintf and spa is on by default in debug builds */
int zfs_flags = ~(ZFS_DEBUG_DPRINTF | ZFS_DEBUG_SPA);
#else
int zfs_flags = 0;
#endif
/*
* zfs_recover can be set to nonzero to attempt to recover from
* otherwise-fatal errors, typically caused by on-disk corruption. When
* set, calls to zfs_panic_recover() will turn into warning messages.
* This should only be used as a last resort, as it typically results
* in leaked space, or worse.
*/
boolean_t zfs_recover = B_FALSE;
SYSCTL_DECL(_vfs_zfs);
SYSCTL_INT(_vfs_zfs, OID_AUTO, recover, CTLFLAG_RDTUN, &zfs_recover, 0,
"Try to recover from otherwise-fatal errors.");
static int
sysctl_vfs_zfs_debug_flags(SYSCTL_HANDLER_ARGS)
{
int err, val;
val = zfs_flags;
err = sysctl_handle_int(oidp, &val, 0, req);
if (err != 0 || req->newptr == NULL)
return (err);
/*
* ZFS_DEBUG_MODIFY must be enabled prior to boot so all
* arc buffers in the system have the necessary additional
* checksum data. However, it is safe to disable at any
* time.
*/
if (!(zfs_flags & ZFS_DEBUG_MODIFY))
val &= ~ZFS_DEBUG_MODIFY;
zfs_flags = val;
return (0);
}
SYSCTL_PROC(_vfs_zfs, OID_AUTO, debug_flags,
CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, 0, sizeof(int),
sysctl_vfs_zfs_debug_flags, "IU", "Debug flags for ZFS testing.");
/*
* If destroy encounters an EIO while reading metadata (e.g. indirect
* blocks), space referenced by the missing metadata can not be freed.
* Normally this causes the background destroy to become "stalled", as
* it is unable to make forward progress. While in this stalled state,
* all remaining space to free from the error-encountering filesystem is
* "temporarily leaked". Set this flag to cause it to ignore the EIO,
* permanently leak the space from indirect blocks that can not be read,
* and continue to free everything else that it can.
*
* The default, "stalling" behavior is useful if the storage partially
* fails (i.e. some but not all i/os fail), and then later recovers. In
* this case, we will be able to continue pool operations while it is
* partially failed, and when it recovers, we can continue to free the
* space, with no leaks. However, note that this case is actually
* fairly rare.
*
* Typically pools either (a) fail completely (but perhaps temporarily,
* e.g. a top-level vdev going offline), or (b) have localized,
* permanent errors (e.g. disk returns the wrong data due to bit flip or
* firmware bug). In case (a), this setting does not matter because the
* pool will be suspended and the sync thread will not be able to make
* forward progress regardless. In case (b), because the error is
* permanent, the best we can do is leak the minimum amount of space,
* which is what setting this flag will do. Therefore, it is reasonable
* for this flag to normally be set, but we chose the more conservative
* approach of not setting it, so that there is no possibility of
* leaking space in the "partial temporary" failure case.
*/
boolean_t zfs_free_leak_on_eio = B_FALSE;
/*
* Expiration time in milliseconds. This value has two meanings. First it is
* used to determine when the spa_deadman() logic should fire. By default the
* spa_deadman() will fire if spa_sync() has not completed in 1000 seconds.
* Secondly, the value determines if an I/O is considered "hung". Any I/O that
* has not completed in zfs_deadman_synctime_ms is considered "hung" resulting
* in a system panic.
*/
uint64_t zfs_deadman_synctime_ms = 1000000ULL;
SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_synctime_ms, CTLFLAG_RDTUN,
&zfs_deadman_synctime_ms, 0,
"Stalled ZFS I/O expiration time in milliseconds");
/*
* Check time in milliseconds. This defines the frequency at which we check
* for hung I/O.
*/
uint64_t zfs_deadman_checktime_ms = 5000ULL;
SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_checktime_ms, CTLFLAG_RDTUN,
&zfs_deadman_checktime_ms, 0,
"Period of checks for stalled ZFS I/O in milliseconds");
/*
* Default value of -1 for zfs_deadman_enabled is resolved in
* zfs_deadman_init()
*/
int zfs_deadman_enabled = -1;
SYSCTL_INT(_vfs_zfs, OID_AUTO, deadman_enabled, CTLFLAG_RDTUN,
&zfs_deadman_enabled, 0, "Kernel panic on stalled ZFS I/O");
/*
* The worst case is single-sector max-parity RAID-Z blocks, in which
* case the space requirement is exactly (VDEV_RAIDZ_MAXPARITY + 1)
* times the size; so just assume that. Add to this the fact that
* we can have up to 3 DVAs per bp, and one more factor of 2 because
* the block may be dittoed with up to 3 DVAs by ddt_sync(). All together,
* the worst case is:
* (VDEV_RAIDZ_MAXPARITY + 1) * SPA_DVAS_PER_BP * 2 == 24
*/
int spa_asize_inflation = 24;
SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_asize_inflation, CTLFLAG_RWTUN,
&spa_asize_inflation, 0, "Worst case inflation factor for single sector writes");
#ifndef illumos
#ifdef _KERNEL
static void
zfs_deadman_init()
{
/*
* If we are not i386 or amd64 or in a virtual machine,
* disable ZFS deadman thread by default
*/
if (zfs_deadman_enabled == -1) {
#if defined(__amd64__) || defined(__i386__)
zfs_deadman_enabled = (vm_guest == VM_GUEST_NO) ? 1 : 0;
#else
zfs_deadman_enabled = 0;
#endif
}
}
#endif /* _KERNEL */
#endif /* !illumos */
/*
* Normally, we don't allow the last 3.2% (1/(2^spa_slop_shift)) of space in
* the pool to be consumed. This ensures that we don't run the pool
* completely out of space, due to unaccounted changes (e.g. to the MOS).
* It also limits the worst-case time to allocate space. If we have
* less than this amount of free space, most ZPL operations (e.g. write,
* create) will return ENOSPC.
*
* Certain operations (e.g. file removal, most administrative actions) can
* use half the slop space. They will only return ENOSPC if less than half
* the slop space is free. Typically, once the pool has less than the slop
* space free, the user will use these operations to free up space in the pool.
* These are the operations that call dsl_pool_adjustedsize() with the netfree
* argument set to TRUE.
*
* A very restricted set of operations are always permitted, regardless of
* the amount of free space. These are the operations that call
* dsl_sync_task(ZFS_SPACE_CHECK_NONE), e.g. "zfs destroy". If these
* operations result in a net increase in the amount of space used,
* it is possible to run the pool completely out of space, causing it to
* be permanently read-only.
*
* See also the comments in zfs_space_check_t.
*/
int spa_slop_shift = 5;
SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_slop_shift, CTLFLAG_RWTUN,
&spa_slop_shift, 0,
"Shift value of reserved space (1/(2^spa_slop_shift)).");
/*
* ==========================================================================
* SPA config locking
* ==========================================================================
*/
static void
spa_config_lock_init(spa_t *spa)
{
for (int i = 0; i < SCL_LOCKS; i++) {
spa_config_lock_t *scl = &spa->spa_config_lock[i];
mutex_init(&scl->scl_lock, NULL, MUTEX_DEFAULT, NULL);
cv_init(&scl->scl_cv, NULL, CV_DEFAULT, NULL);
refcount_create_untracked(&scl->scl_count);
scl->scl_writer = NULL;
scl->scl_write_wanted = 0;
}
}
static void
spa_config_lock_destroy(spa_t *spa)
{
for (int i = 0; i < SCL_LOCKS; i++) {
spa_config_lock_t *scl = &spa->spa_config_lock[i];
mutex_destroy(&scl->scl_lock);
cv_destroy(&scl->scl_cv);
refcount_destroy(&scl->scl_count);
ASSERT(scl->scl_writer == NULL);
ASSERT(scl->scl_write_wanted == 0);
}
}
int
spa_config_tryenter(spa_t *spa, int locks, void *tag, krw_t rw)
{
for (int i = 0; i < SCL_LOCKS; i++) {
spa_config_lock_t *scl = &spa->spa_config_lock[i];
if (!(locks & (1 << i)))
continue;
mutex_enter(&scl->scl_lock);
if (rw == RW_READER) {
if (scl->scl_writer || scl->scl_write_wanted) {
mutex_exit(&scl->scl_lock);
spa_config_exit(spa, locks ^ (1 << i), tag);
return (0);
}
} else {
ASSERT(scl->scl_writer != curthread);
if (!refcount_is_zero(&scl->scl_count)) {
mutex_exit(&scl->scl_lock);
spa_config_exit(spa, locks ^ (1 << i), tag);
return (0);
}
scl->scl_writer = curthread;
}
(void) refcount_add(&scl->scl_count, tag);
mutex_exit(&scl->scl_lock);
}
return (1);
}
void
spa_config_enter(spa_t *spa, int locks, void *tag, krw_t rw)
{
int wlocks_held = 0;
ASSERT3U(SCL_LOCKS, <, sizeof (wlocks_held) * NBBY);
for (int i = 0; i < SCL_LOCKS; i++) {
spa_config_lock_t *scl = &spa->spa_config_lock[i];
if (scl->scl_writer == curthread)
wlocks_held |= (1 << i);
if (!(locks & (1 << i)))
continue;
mutex_enter(&scl->scl_lock);
if (rw == RW_READER) {
while (scl->scl_writer || scl->scl_write_wanted) {
cv_wait(&scl->scl_cv, &scl->scl_lock);
}
} else {
ASSERT(scl->scl_writer != curthread);
while (!refcount_is_zero(&scl->scl_count)) {
scl->scl_write_wanted++;
cv_wait(&scl->scl_cv, &scl->scl_lock);
scl->scl_write_wanted--;
}
scl->scl_writer = curthread;
}
(void) refcount_add(&scl->scl_count, tag);
mutex_exit(&scl->scl_lock);
}
ASSERT(wlocks_held <= locks);
}
void
spa_config_exit(spa_t *spa, int locks, void *tag)
{
for (int i = SCL_LOCKS - 1; i >= 0; i--) {
spa_config_lock_t *scl = &spa->spa_config_lock[i];
if (!(locks & (1 << i)))
continue;
mutex_enter(&scl->scl_lock);
ASSERT(!refcount_is_zero(&scl->scl_count));
if (refcount_remove(&scl->scl_count, tag) == 0) {
ASSERT(scl->scl_writer == NULL ||
scl->scl_writer == curthread);
scl->scl_writer = NULL; /* OK in either case */
cv_broadcast(&scl->scl_cv);
}
mutex_exit(&scl->scl_lock);
}
}
int
spa_config_held(spa_t *spa, int locks, krw_t rw)
{
int locks_held = 0;
for (int i = 0; i < SCL_LOCKS; i++) {
spa_config_lock_t *scl = &spa->spa_config_lock[i];
if (!(locks & (1 << i)))
continue;
if ((rw == RW_READER && !refcount_is_zero(&scl->scl_count)) ||
(rw == RW_WRITER && scl->scl_writer == curthread))
locks_held |= 1 << i;
}
return (locks_held);
}
/*
* ==========================================================================
* SPA namespace functions
* ==========================================================================
*/
/*
* Lookup the named spa_t in the AVL tree. The spa_namespace_lock must be held.
* Returns NULL if no matching spa_t is found.
*/
spa_t *
spa_lookup(const char *name)
{
static spa_t search; /* spa_t is large; don't allocate on stack */
spa_t *spa;
avl_index_t where;
char *cp;
ASSERT(MUTEX_HELD(&spa_namespace_lock));
(void) strlcpy(search.spa_name, name, sizeof (search.spa_name));
/*
* If it's a full dataset name, figure out the pool name and
* just use that.
*/
cp = strpbrk(search.spa_name, "/@#");
if (cp != NULL)
*cp = '\0';
spa = avl_find(&spa_namespace_avl, &search, &where);
return (spa);
}
/*
* Fires when spa_sync has not completed within zfs_deadman_synctime_ms.
* If the zfs_deadman_enabled flag is set then it inspects all vdev queues
* looking for potentially hung I/Os.
*/
void
spa_deadman(void *arg)
{
spa_t *spa = arg;
/*
* Disable the deadman timer if the pool is suspended.
*/
if (spa_suspended(spa)) {
#ifdef illumos
VERIFY(cyclic_reprogram(spa->spa_deadman_cycid, CY_INFINITY));
#else
/* Nothing. just don't schedule any future callouts. */
#endif
return;
}
zfs_dbgmsg("slow spa_sync: started %llu seconds ago, calls %llu",
(gethrtime() - spa->spa_sync_starttime) / NANOSEC,
++spa->spa_deadman_calls);
if (zfs_deadman_enabled)
vdev_deadman(spa->spa_root_vdev);
#ifdef __FreeBSD__
#ifdef _KERNEL
callout_schedule(&spa->spa_deadman_cycid,
hz * zfs_deadman_checktime_ms / MILLISEC);
#endif
#endif
}
/*
* Create an uninitialized spa_t with the given name. Requires
* spa_namespace_lock. The caller must ensure that the spa_t doesn't already
* exist by calling spa_lookup() first.
*/
spa_t *
spa_add(const char *name, nvlist_t *config, const char *altroot)
{
spa_t *spa;
spa_config_dirent_t *dp;
#ifdef illumos
cyc_handler_t hdlr;
cyc_time_t when;
#endif
ASSERT(MUTEX_HELD(&spa_namespace_lock));
spa = kmem_zalloc(sizeof (spa_t), KM_SLEEP);
mutex_init(&spa->spa_async_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa->spa_errlist_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa->spa_errlog_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa->spa_history_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa->spa_proc_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa->spa_props_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa->spa_scrub_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa->spa_suspend_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa->spa_vdev_top_lock, NULL, MUTEX_DEFAULT, NULL);
cv_init(&spa->spa_async_cv, NULL, CV_DEFAULT, NULL);
cv_init(&spa->spa_proc_cv, NULL, CV_DEFAULT, NULL);
cv_init(&spa->spa_scrub_io_cv, NULL, CV_DEFAULT, NULL);
cv_init(&spa->spa_suspend_cv, NULL, CV_DEFAULT, NULL);
for (int t = 0; t < TXG_SIZE; t++)
bplist_create(&spa->spa_free_bplist[t]);
(void) strlcpy(spa->spa_name, name, sizeof (spa->spa_name));
spa->spa_state = POOL_STATE_UNINITIALIZED;
spa->spa_freeze_txg = UINT64_MAX;
spa->spa_final_txg = UINT64_MAX;
spa->spa_load_max_txg = UINT64_MAX;
spa->spa_proc = &p0;
spa->spa_proc_state = SPA_PROC_NONE;
#ifdef illumos
hdlr.cyh_func = spa_deadman;
hdlr.cyh_arg = spa;
hdlr.cyh_level = CY_LOW_LEVEL;
#endif
spa->spa_deadman_synctime = MSEC2NSEC(zfs_deadman_synctime_ms);
#ifdef illumos
/*
* This determines how often we need to check for hung I/Os after
* the cyclic has already fired. Since checking for hung I/Os is
* an expensive operation we don't want to check too frequently.
* Instead wait for 5 seconds before checking again.
*/
when.cyt_interval = MSEC2NSEC(zfs_deadman_checktime_ms);
when.cyt_when = CY_INFINITY;
mutex_enter(&cpu_lock);
spa->spa_deadman_cycid = cyclic_add(&hdlr, &when);
mutex_exit(&cpu_lock);
#else /* !illumos */
#ifdef _KERNEL
- callout_init(&spa->spa_deadman_cycid, CALLOUT_MPSAFE);
+ callout_init(&spa->spa_deadman_cycid, 1);
#endif
#endif
refcount_create(&spa->spa_refcount);
spa_config_lock_init(spa);
avl_add(&spa_namespace_avl, spa);
/*
* Set the alternate root, if there is one.
*/
if (altroot) {
spa->spa_root = spa_strdup(altroot);
spa_active_count++;
}
/*
* Every pool starts with the default cachefile
*/
list_create(&spa->spa_config_list, sizeof (spa_config_dirent_t),
offsetof(spa_config_dirent_t, scd_link));
dp = kmem_zalloc(sizeof (spa_config_dirent_t), KM_SLEEP);
dp->scd_path = altroot ? NULL : spa_strdup(spa_config_path);
list_insert_head(&spa->spa_config_list, dp);
VERIFY(nvlist_alloc(&spa->spa_load_info, NV_UNIQUE_NAME,
KM_SLEEP) == 0);
if (config != NULL) {
nvlist_t *features;
if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_FEATURES_FOR_READ,
&features) == 0) {
VERIFY(nvlist_dup(features, &spa->spa_label_features,
0) == 0);
}
VERIFY(nvlist_dup(config, &spa->spa_config, 0) == 0);
}
if (spa->spa_label_features == NULL) {
VERIFY(nvlist_alloc(&spa->spa_label_features, NV_UNIQUE_NAME,
KM_SLEEP) == 0);
}
spa->spa_debug = ((zfs_flags & ZFS_DEBUG_SPA) != 0);
/*
* As a pool is being created, treat all features as disabled by
* setting SPA_FEATURE_DISABLED for all entries in the feature
* refcount cache.
*/
for (int i = 0; i < SPA_FEATURES; i++) {
spa->spa_feat_refcount_cache[i] = SPA_FEATURE_DISABLED;
}
return (spa);
}
/*
* Removes a spa_t from the namespace, freeing up any memory used. Requires
* spa_namespace_lock. This is called only after the spa_t has been closed and
* deactivated.
*/
void
spa_remove(spa_t *spa)
{
spa_config_dirent_t *dp;
ASSERT(MUTEX_HELD(&spa_namespace_lock));
ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
nvlist_free(spa->spa_config_splitting);
avl_remove(&spa_namespace_avl, spa);
cv_broadcast(&spa_namespace_cv);
if (spa->spa_root) {
spa_strfree(spa->spa_root);
spa_active_count--;
}
while ((dp = list_head(&spa->spa_config_list)) != NULL) {
list_remove(&spa->spa_config_list, dp);
if (dp->scd_path != NULL)
spa_strfree(dp->scd_path);
kmem_free(dp, sizeof (spa_config_dirent_t));
}
list_destroy(&spa->spa_config_list);
nvlist_free(spa->spa_label_features);
nvlist_free(spa->spa_load_info);
spa_config_set(spa, NULL);
#ifdef illumos
mutex_enter(&cpu_lock);
if (spa->spa_deadman_cycid != CYCLIC_NONE)
cyclic_remove(spa->spa_deadman_cycid);
mutex_exit(&cpu_lock);
spa->spa_deadman_cycid = CYCLIC_NONE;
#else /* !illumos */
#ifdef _KERNEL
callout_drain(&spa->spa_deadman_cycid);
#endif
#endif
refcount_destroy(&spa->spa_refcount);
spa_config_lock_destroy(spa);
for (int t = 0; t < TXG_SIZE; t++)
bplist_destroy(&spa->spa_free_bplist[t]);
cv_destroy(&spa->spa_async_cv);
cv_destroy(&spa->spa_proc_cv);
cv_destroy(&spa->spa_scrub_io_cv);
cv_destroy(&spa->spa_suspend_cv);
mutex_destroy(&spa->spa_async_lock);
mutex_destroy(&spa->spa_errlist_lock);
mutex_destroy(&spa->spa_errlog_lock);
mutex_destroy(&spa->spa_history_lock);
mutex_destroy(&spa->spa_proc_lock);
mutex_destroy(&spa->spa_props_lock);
mutex_destroy(&spa->spa_scrub_lock);
mutex_destroy(&spa->spa_suspend_lock);
mutex_destroy(&spa->spa_vdev_top_lock);
kmem_free(spa, sizeof (spa_t));
}
/*
* Given a pool, return the next pool in the namespace, or NULL if there is
* none. If 'prev' is NULL, return the first pool.
*/
spa_t *
spa_next(spa_t *prev)
{
ASSERT(MUTEX_HELD(&spa_namespace_lock));
if (prev)
return (AVL_NEXT(&spa_namespace_avl, prev));
else
return (avl_first(&spa_namespace_avl));
}
/*
* ==========================================================================
* SPA refcount functions
* ==========================================================================
*/
/*
* Add a reference to the given spa_t. Must have at least one reference, or
* have the namespace lock held.
*/
void
spa_open_ref(spa_t *spa, void *tag)
{
ASSERT(refcount_count(&spa->spa_refcount) >= spa->spa_minref ||
MUTEX_HELD(&spa_namespace_lock));
(void) refcount_add(&spa->spa_refcount, tag);
}
/*
* Remove a reference to the given spa_t. Must have at least one reference, or
* have the namespace lock held.
*/
void
spa_close(spa_t *spa, void *tag)
{
ASSERT(refcount_count(&spa->spa_refcount) > spa->spa_minref ||
MUTEX_HELD(&spa_namespace_lock));
(void) refcount_remove(&spa->spa_refcount, tag);
}
/*
* Check to see if the spa refcount is zero. Must be called with
* spa_namespace_lock held. We really compare against spa_minref, which is the
* number of references acquired when opening a pool
*/
boolean_t
spa_refcount_zero(spa_t *spa)
{
ASSERT(MUTEX_HELD(&spa_namespace_lock));
return (refcount_count(&spa->spa_refcount) == spa->spa_minref);
}
/*
* ==========================================================================
* SPA spare and l2cache tracking
* ==========================================================================
*/
/*
* Hot spares and cache devices are tracked using the same code below,
* for 'auxiliary' devices.
*/
typedef struct spa_aux {
uint64_t aux_guid;
uint64_t aux_pool;
avl_node_t aux_avl;
int aux_count;
} spa_aux_t;
static int
spa_aux_compare(const void *a, const void *b)
{
const spa_aux_t *sa = a;
const spa_aux_t *sb = b;
if (sa->aux_guid < sb->aux_guid)
return (-1);
else if (sa->aux_guid > sb->aux_guid)
return (1);
else
return (0);
}
void
spa_aux_add(vdev_t *vd, avl_tree_t *avl)
{
avl_index_t where;
spa_aux_t search;
spa_aux_t *aux;
search.aux_guid = vd->vdev_guid;
if ((aux = avl_find(avl, &search, &where)) != NULL) {
aux->aux_count++;
} else {
aux = kmem_zalloc(sizeof (spa_aux_t), KM_SLEEP);
aux->aux_guid = vd->vdev_guid;
aux->aux_count = 1;
avl_insert(avl, aux, where);
}
}
void
spa_aux_remove(vdev_t *vd, avl_tree_t *avl)
{
spa_aux_t search;
spa_aux_t *aux;
avl_index_t where;
search.aux_guid = vd->vdev_guid;
aux = avl_find(avl, &search, &where);
ASSERT(aux != NULL);
if (--aux->aux_count == 0) {
avl_remove(avl, aux);
kmem_free(aux, sizeof (spa_aux_t));
} else if (aux->aux_pool == spa_guid(vd->vdev_spa)) {
aux->aux_pool = 0ULL;
}
}
boolean_t
spa_aux_exists(uint64_t guid, uint64_t *pool, int *refcnt, avl_tree_t *avl)
{
spa_aux_t search, *found;
search.aux_guid = guid;
found = avl_find(avl, &search, NULL);
if (pool) {
if (found)
*pool = found->aux_pool;
else
*pool = 0ULL;
}
if (refcnt) {
if (found)
*refcnt = found->aux_count;
else
*refcnt = 0;
}
return (found != NULL);
}
void
spa_aux_activate(vdev_t *vd, avl_tree_t *avl)
{
spa_aux_t search, *found;
avl_index_t where;
search.aux_guid = vd->vdev_guid;
found = avl_find(avl, &search, &where);
ASSERT(found != NULL);
ASSERT(found->aux_pool == 0ULL);
found->aux_pool = spa_guid(vd->vdev_spa);
}
/*
* Spares are tracked globally due to the following constraints:
*
* - A spare may be part of multiple pools.
* - A spare may be added to a pool even if it's actively in use within
* another pool.
* - A spare in use in any pool can only be the source of a replacement if
* the target is a spare in the same pool.
*
* We keep track of all spares on the system through the use of a reference
* counted AVL tree. When a vdev is added as a spare, or used as a replacement
* spare, then we bump the reference count in the AVL tree. In addition, we set
* the 'vdev_isspare' member to indicate that the device is a spare (active or
* inactive). When a spare is made active (used to replace a device in the
* pool), we also keep track of which pool its been made a part of.
*
* The 'spa_spare_lock' protects the AVL tree. These functions are normally
* called under the spa_namespace lock as part of vdev reconfiguration. The
* separate spare lock exists for the status query path, which does not need to
* be completely consistent with respect to other vdev configuration changes.
*/
static int
spa_spare_compare(const void *a, const void *b)
{
return (spa_aux_compare(a, b));
}
void
spa_spare_add(vdev_t *vd)
{
mutex_enter(&spa_spare_lock);
ASSERT(!vd->vdev_isspare);
spa_aux_add(vd, &spa_spare_avl);
vd->vdev_isspare = B_TRUE;
mutex_exit(&spa_spare_lock);
}
void
spa_spare_remove(vdev_t *vd)
{
mutex_enter(&spa_spare_lock);
ASSERT(vd->vdev_isspare);
spa_aux_remove(vd, &spa_spare_avl);
vd->vdev_isspare = B_FALSE;
mutex_exit(&spa_spare_lock);
}
boolean_t
spa_spare_exists(uint64_t guid, uint64_t *pool, int *refcnt)
{
boolean_t found;
mutex_enter(&spa_spare_lock);
found = spa_aux_exists(guid, pool, refcnt, &spa_spare_avl);
mutex_exit(&spa_spare_lock);
return (found);
}
void
spa_spare_activate(vdev_t *vd)
{
mutex_enter(&spa_spare_lock);
ASSERT(vd->vdev_isspare);
spa_aux_activate(vd, &spa_spare_avl);
mutex_exit(&spa_spare_lock);
}
/*
* Level 2 ARC devices are tracked globally for the same reasons as spares.
* Cache devices currently only support one pool per cache device, and so
* for these devices the aux reference count is currently unused beyond 1.
*/
static int
spa_l2cache_compare(const void *a, const void *b)
{
return (spa_aux_compare(a, b));
}
void
spa_l2cache_add(vdev_t *vd)
{
mutex_enter(&spa_l2cache_lock);
ASSERT(!vd->vdev_isl2cache);
spa_aux_add(vd, &spa_l2cache_avl);
vd->vdev_isl2cache = B_TRUE;
mutex_exit(&spa_l2cache_lock);
}
void
spa_l2cache_remove(vdev_t *vd)
{
mutex_enter(&spa_l2cache_lock);
ASSERT(vd->vdev_isl2cache);
spa_aux_remove(vd, &spa_l2cache_avl);
vd->vdev_isl2cache = B_FALSE;
mutex_exit(&spa_l2cache_lock);
}
boolean_t
spa_l2cache_exists(uint64_t guid, uint64_t *pool)
{
boolean_t found;
mutex_enter(&spa_l2cache_lock);
found = spa_aux_exists(guid, pool, NULL, &spa_l2cache_avl);
mutex_exit(&spa_l2cache_lock);
return (found);
}
void
spa_l2cache_activate(vdev_t *vd)
{
mutex_enter(&spa_l2cache_lock);
ASSERT(vd->vdev_isl2cache);
spa_aux_activate(vd, &spa_l2cache_avl);
mutex_exit(&spa_l2cache_lock);
}
/*
* ==========================================================================
* SPA vdev locking
* ==========================================================================
*/
/*
* Lock the given spa_t for the purpose of adding or removing a vdev.
* Grabs the global spa_namespace_lock plus the spa config lock for writing.
* It returns the next transaction group for the spa_t.
*/
uint64_t
spa_vdev_enter(spa_t *spa)
{
mutex_enter(&spa->spa_vdev_top_lock);
mutex_enter(&spa_namespace_lock);
return (spa_vdev_config_enter(spa));
}
/*
* Internal implementation for spa_vdev_enter(). Used when a vdev
* operation requires multiple syncs (i.e. removing a device) while
* keeping the spa_namespace_lock held.
*/
uint64_t
spa_vdev_config_enter(spa_t *spa)
{
ASSERT(MUTEX_HELD(&spa_namespace_lock));
spa_config_enter(spa, SCL_ALL, spa, RW_WRITER);
return (spa_last_synced_txg(spa) + 1);
}
/*
* Used in combination with spa_vdev_config_enter() to allow the syncing
* of multiple transactions without releasing the spa_namespace_lock.
*/
void
spa_vdev_config_exit(spa_t *spa, vdev_t *vd, uint64_t txg, int error, char *tag)
{
ASSERT(MUTEX_HELD(&spa_namespace_lock));
int config_changed = B_FALSE;
ASSERT(txg > spa_last_synced_txg(spa));
spa->spa_pending_vdev = NULL;
/*
* Reassess the DTLs.
*/
vdev_dtl_reassess(spa->spa_root_vdev, 0, 0, B_FALSE);
if (error == 0 && !list_is_empty(&spa->spa_config_dirty_list)) {
config_changed = B_TRUE;
spa->spa_config_generation++;
}
/*
* Verify the metaslab classes.
*/
ASSERT(metaslab_class_validate(spa_normal_class(spa)) == 0);
ASSERT(metaslab_class_validate(spa_log_class(spa)) == 0);
spa_config_exit(spa, SCL_ALL, spa);
/*
* Panic the system if the specified tag requires it. This
* is useful for ensuring that configurations are updated
* transactionally.
*/
if (zio_injection_enabled)
zio_handle_panic_injection(spa, tag, 0);
/*
* Note: this txg_wait_synced() is important because it ensures
* that there won't be more than one config change per txg.
* This allows us to use the txg as the generation number.
*/
if (error == 0)
txg_wait_synced(spa->spa_dsl_pool, txg);
if (vd != NULL) {
ASSERT(!vd->vdev_detached || vd->vdev_dtl_sm == NULL);
spa_config_enter(spa, SCL_ALL, spa, RW_WRITER);
vdev_free(vd);
spa_config_exit(spa, SCL_ALL, spa);
}
/*
* If the config changed, update the config cache.
*/
if (config_changed)
spa_config_sync(spa, B_FALSE, B_TRUE);
}
/*
* Unlock the spa_t after adding or removing a vdev. Besides undoing the
* locking of spa_vdev_enter(), we also want make sure the transactions have
* synced to disk, and then update the global configuration cache with the new
* information.
*/
int
spa_vdev_exit(spa_t *spa, vdev_t *vd, uint64_t txg, int error)
{
spa_vdev_config_exit(spa, vd, txg, error, FTAG);
mutex_exit(&spa_namespace_lock);
mutex_exit(&spa->spa_vdev_top_lock);
return (error);
}
/*
* Lock the given spa_t for the purpose of changing vdev state.
*/
void
spa_vdev_state_enter(spa_t *spa, int oplocks)
{
int locks = SCL_STATE_ALL | oplocks;
/*
* Root pools may need to read of the underlying devfs filesystem
* when opening up a vdev. Unfortunately if we're holding the
* SCL_ZIO lock it will result in a deadlock when we try to issue
* the read from the root filesystem. Instead we "prefetch"
* the associated vnodes that we need prior to opening the
* underlying devices and cache them so that we can prevent
* any I/O when we are doing the actual open.
*/
if (spa_is_root(spa)) {
int low = locks & ~(SCL_ZIO - 1);
int high = locks & ~low;
spa_config_enter(spa, high, spa, RW_WRITER);
vdev_hold(spa->spa_root_vdev);
spa_config_enter(spa, low, spa, RW_WRITER);
} else {
spa_config_enter(spa, locks, spa, RW_WRITER);
}
spa->spa_vdev_locks = locks;
}
int
spa_vdev_state_exit(spa_t *spa, vdev_t *vd, int error)
{
boolean_t config_changed = B_FALSE;
if (vd != NULL || error == 0)
vdev_dtl_reassess(vd ? vd->vdev_top : spa->spa_root_vdev,
0, 0, B_FALSE);
if (vd != NULL) {
vdev_state_dirty(vd->vdev_top);
config_changed = B_TRUE;
spa->spa_config_generation++;
}
if (spa_is_root(spa))
vdev_rele(spa->spa_root_vdev);
ASSERT3U(spa->spa_vdev_locks, >=, SCL_STATE_ALL);
spa_config_exit(spa, spa->spa_vdev_locks, spa);
/*
* If anything changed, wait for it to sync. This ensures that,
* from the system administrator's perspective, zpool(1M) commands
* are synchronous. This is important for things like zpool offline:
* when the command completes, you expect no further I/O from ZFS.
*/
if (vd != NULL)
txg_wait_synced(spa->spa_dsl_pool, 0);
/*
* If the config changed, update the config cache.
*/
if (config_changed) {
mutex_enter(&spa_namespace_lock);
spa_config_sync(spa, B_FALSE, B_TRUE);
mutex_exit(&spa_namespace_lock);
}
return (error);
}
/*
* ==========================================================================
* Miscellaneous functions
* ==========================================================================
*/
void
spa_activate_mos_feature(spa_t *spa, const char *feature, dmu_tx_t *tx)
{
if (!nvlist_exists(spa->spa_label_features, feature)) {
fnvlist_add_boolean(spa->spa_label_features, feature);
/*
* When we are creating the pool (tx_txg==TXG_INITIAL), we can't
* dirty the vdev config because lock SCL_CONFIG is not held.
* Thankfully, in this case we don't need to dirty the config
* because it will be written out anyway when we finish
* creating the pool.
*/
if (tx->tx_txg != TXG_INITIAL)
vdev_config_dirty(spa->spa_root_vdev);
}
}
void
spa_deactivate_mos_feature(spa_t *spa, const char *feature)
{
if (nvlist_remove_all(spa->spa_label_features, feature) == 0)
vdev_config_dirty(spa->spa_root_vdev);
}
/*
* Rename a spa_t.
*/
int
spa_rename(const char *name, const char *newname)
{
spa_t *spa;
int err;
/*
* Lookup the spa_t and grab the config lock for writing. We need to
* actually open the pool so that we can sync out the necessary labels.
* It's OK to call spa_open() with the namespace lock held because we
* allow recursive calls for other reasons.
*/
mutex_enter(&spa_namespace_lock);
if ((err = spa_open(name, &spa, FTAG)) != 0) {
mutex_exit(&spa_namespace_lock);
return (err);
}
spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
avl_remove(&spa_namespace_avl, spa);
(void) strlcpy(spa->spa_name, newname, sizeof (spa->spa_name));
avl_add(&spa_namespace_avl, spa);
/*
* Sync all labels to disk with the new names by marking the root vdev
* dirty and waiting for it to sync. It will pick up the new pool name
* during the sync.
*/
vdev_config_dirty(spa->spa_root_vdev);
spa_config_exit(spa, SCL_ALL, FTAG);
txg_wait_synced(spa->spa_dsl_pool, 0);
/*
* Sync the updated config cache.
*/
spa_config_sync(spa, B_FALSE, B_TRUE);
spa_close(spa, FTAG);
mutex_exit(&spa_namespace_lock);
return (0);
}
/*
* Return the spa_t associated with given pool_guid, if it exists. If
* device_guid is non-zero, determine whether the pool exists *and* contains
* a device with the specified device_guid.
*/
spa_t *
spa_by_guid(uint64_t pool_guid, uint64_t device_guid)
{
spa_t *spa;
avl_tree_t *t = &spa_namespace_avl;
ASSERT(MUTEX_HELD(&spa_namespace_lock));
for (spa = avl_first(t); spa != NULL; spa = AVL_NEXT(t, spa)) {
if (spa->spa_state == POOL_STATE_UNINITIALIZED)
continue;
if (spa->spa_root_vdev == NULL)
continue;
if (spa_guid(spa) == pool_guid) {
if (device_guid == 0)
break;
if (vdev_lookup_by_guid(spa->spa_root_vdev,
device_guid) != NULL)
break;
/*
* Check any devices we may be in the process of adding.
*/
if (spa->spa_pending_vdev) {
if (vdev_lookup_by_guid(spa->spa_pending_vdev,
device_guid) != NULL)
break;
}
}
}
return (spa);
}
/*
* Determine whether a pool with the given pool_guid exists.
*/
boolean_t
spa_guid_exists(uint64_t pool_guid, uint64_t device_guid)
{
return (spa_by_guid(pool_guid, device_guid) != NULL);
}
char *
spa_strdup(const char *s)
{
size_t len;
char *new;
len = strlen(s);
new = kmem_alloc(len + 1, KM_SLEEP);
bcopy(s, new, len);
new[len] = '\0';
return (new);
}
void
spa_strfree(char *s)
{
kmem_free(s, strlen(s) + 1);
}
uint64_t
spa_get_random(uint64_t range)
{
uint64_t r;
ASSERT(range != 0);
(void) random_get_pseudo_bytes((void *)&r, sizeof (uint64_t));
return (r % range);
}
uint64_t
spa_generate_guid(spa_t *spa)
{
uint64_t guid = spa_get_random(-1ULL);
if (spa != NULL) {
while (guid == 0 || spa_guid_exists(spa_guid(spa), guid))
guid = spa_get_random(-1ULL);
} else {
while (guid == 0 || spa_guid_exists(guid, 0))
guid = spa_get_random(-1ULL);
}
return (guid);
}
void
snprintf_blkptr(char *buf, size_t buflen, const blkptr_t *bp)
{
char type[256];
char *checksum = NULL;
char *compress = NULL;
if (bp != NULL) {
if (BP_GET_TYPE(bp) & DMU_OT_NEWTYPE) {
dmu_object_byteswap_t bswap =
DMU_OT_BYTESWAP(BP_GET_TYPE(bp));
(void) snprintf(type, sizeof (type), "bswap %s %s",
DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) ?
"metadata" : "data",
dmu_ot_byteswap[bswap].ob_name);
} else {
(void) strlcpy(type, dmu_ot[BP_GET_TYPE(bp)].ot_name,
sizeof (type));
}
if (!BP_IS_EMBEDDED(bp)) {
checksum =
zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_name;
}
compress = zio_compress_table[BP_GET_COMPRESS(bp)].ci_name;
}
SNPRINTF_BLKPTR(snprintf, ' ', buf, buflen, bp, type, checksum,
compress);
}
void
spa_freeze(spa_t *spa)
{
uint64_t freeze_txg = 0;
spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
if (spa->spa_freeze_txg == UINT64_MAX) {
freeze_txg = spa_last_synced_txg(spa) + TXG_SIZE;
spa->spa_freeze_txg = freeze_txg;
}
spa_config_exit(spa, SCL_ALL, FTAG);
if (freeze_txg != 0)
txg_wait_synced(spa_get_dsl(spa), freeze_txg);
}
void
zfs_panic_recover(const char *fmt, ...)
{
va_list adx;
va_start(adx, fmt);
vcmn_err(zfs_recover ? CE_WARN : CE_PANIC, fmt, adx);
va_end(adx);
}
/*
* This is a stripped-down version of strtoull, suitable only for converting
* lowercase hexadecimal numbers that don't overflow.
*/
uint64_t
zfs_strtonum(const char *str, char **nptr)
{
uint64_t val = 0;
char c;
int digit;
while ((c = *str) != '\0') {
if (c >= '0' && c <= '9')
digit = c - '0';
else if (c >= 'a' && c <= 'f')
digit = 10 + c - 'a';
else
break;
val *= 16;
val += digit;
str++;
}
if (nptr)
*nptr = (char *)str;
return (val);
}
/*
* ==========================================================================
* Accessor functions
* ==========================================================================
*/
boolean_t
spa_shutting_down(spa_t *spa)
{
return (spa->spa_async_suspended);
}
dsl_pool_t *
spa_get_dsl(spa_t *spa)
{
return (spa->spa_dsl_pool);
}
boolean_t
spa_is_initializing(spa_t *spa)
{
return (spa->spa_is_initializing);
}
blkptr_t *
spa_get_rootblkptr(spa_t *spa)
{
return (&spa->spa_ubsync.ub_rootbp);
}
void
spa_set_rootblkptr(spa_t *spa, const blkptr_t *bp)
{
spa->spa_uberblock.ub_rootbp = *bp;
}
void
spa_altroot(spa_t *spa, char *buf, size_t buflen)
{
if (spa->spa_root == NULL)
buf[0] = '\0';
else
(void) strncpy(buf, spa->spa_root, buflen);
}
int
spa_sync_pass(spa_t *spa)
{
return (spa->spa_sync_pass);
}
char *
spa_name(spa_t *spa)
{
return (spa->spa_name);
}
uint64_t
spa_guid(spa_t *spa)
{
dsl_pool_t *dp = spa_get_dsl(spa);
uint64_t guid;
/*
* If we fail to parse the config during spa_load(), we can go through
* the error path (which posts an ereport) and end up here with no root
* vdev. We stash the original pool guid in 'spa_config_guid' to handle
* this case.
*/
if (spa->spa_root_vdev == NULL)
return (spa->spa_config_guid);
guid = spa->spa_last_synced_guid != 0 ?
spa->spa_last_synced_guid : spa->spa_root_vdev->vdev_guid;
/*
* Return the most recently synced out guid unless we're
* in syncing context.
*/
if (dp && dsl_pool_sync_context(dp))
return (spa->spa_root_vdev->vdev_guid);
else
return (guid);
}
uint64_t
spa_load_guid(spa_t *spa)
{
/*
* This is a GUID that exists solely as a reference for the
* purposes of the arc. It is generated at load time, and
* is never written to persistent storage.
*/
return (spa->spa_load_guid);
}
uint64_t
spa_last_synced_txg(spa_t *spa)
{
return (spa->spa_ubsync.ub_txg);
}
uint64_t
spa_first_txg(spa_t *spa)
{
return (spa->spa_first_txg);
}
uint64_t
spa_syncing_txg(spa_t *spa)
{
return (spa->spa_syncing_txg);
}
pool_state_t
spa_state(spa_t *spa)
{
return (spa->spa_state);
}
spa_load_state_t
spa_load_state(spa_t *spa)
{
return (spa->spa_load_state);
}
uint64_t
spa_freeze_txg(spa_t *spa)
{
return (spa->spa_freeze_txg);
}
/* ARGSUSED */
uint64_t
spa_get_asize(spa_t *spa, uint64_t lsize)
{
return (lsize * spa_asize_inflation);
}
/*
* Return the amount of slop space in bytes. It is 1/32 of the pool (3.2%),
* or at least 32MB.
*
* See the comment above spa_slop_shift for details.
*/
uint64_t
spa_get_slop_space(spa_t *spa) {
uint64_t space = spa_get_dspace(spa);
return (MAX(space >> spa_slop_shift, SPA_MINDEVSIZE >> 1));
}
uint64_t
spa_get_dspace(spa_t *spa)
{
return (spa->spa_dspace);
}
void
spa_update_dspace(spa_t *spa)
{
spa->spa_dspace = metaslab_class_get_dspace(spa_normal_class(spa)) +
ddt_get_dedup_dspace(spa);
}
/*
* Return the failure mode that has been set to this pool. The default
* behavior will be to block all I/Os when a complete failure occurs.
*/
uint8_t
spa_get_failmode(spa_t *spa)
{
return (spa->spa_failmode);
}
boolean_t
spa_suspended(spa_t *spa)
{
return (spa->spa_suspended);
}
uint64_t
spa_version(spa_t *spa)
{
return (spa->spa_ubsync.ub_version);
}
boolean_t
spa_deflate(spa_t *spa)
{
return (spa->spa_deflate);
}
metaslab_class_t *
spa_normal_class(spa_t *spa)
{
return (spa->spa_normal_class);
}
metaslab_class_t *
spa_log_class(spa_t *spa)
{
return (spa->spa_log_class);
}
int
spa_max_replication(spa_t *spa)
{
/*
* As of SPA_VERSION == SPA_VERSION_DITTO_BLOCKS, we are able to
* handle BPs with more than one DVA allocated. Set our max
* replication level accordingly.
*/
if (spa_version(spa) < SPA_VERSION_DITTO_BLOCKS)
return (1);
return (MIN(SPA_DVAS_PER_BP, spa_max_replication_override));
}
int
spa_prev_software_version(spa_t *spa)
{
return (spa->spa_prev_software_version);
}
uint64_t
spa_deadman_synctime(spa_t *spa)
{
return (spa->spa_deadman_synctime);
}
uint64_t
dva_get_dsize_sync(spa_t *spa, const dva_t *dva)
{
uint64_t asize = DVA_GET_ASIZE(dva);
uint64_t dsize = asize;
ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0);
if (asize != 0 && spa->spa_deflate) {
vdev_t *vd = vdev_lookup_top(spa, DVA_GET_VDEV(dva));
dsize = (asize >> SPA_MINBLOCKSHIFT) * vd->vdev_deflate_ratio;
}
return (dsize);
}
uint64_t
bp_get_dsize_sync(spa_t *spa, const blkptr_t *bp)
{
uint64_t dsize = 0;
for (int d = 0; d < BP_GET_NDVAS(bp); d++)
dsize += dva_get_dsize_sync(spa, &bp->blk_dva[d]);
return (dsize);
}
uint64_t
bp_get_dsize(spa_t *spa, const blkptr_t *bp)
{
uint64_t dsize = 0;
spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
for (int d = 0; d < BP_GET_NDVAS(bp); d++)
dsize += dva_get_dsize_sync(spa, &bp->blk_dva[d]);
spa_config_exit(spa, SCL_VDEV, FTAG);
return (dsize);
}
/*
* ==========================================================================
* Initialization and Termination
* ==========================================================================
*/
static int
spa_name_compare(const void *a1, const void *a2)
{
const spa_t *s1 = a1;
const spa_t *s2 = a2;
int s;
s = strcmp(s1->spa_name, s2->spa_name);
if (s > 0)
return (1);
if (s < 0)
return (-1);
return (0);
}
int
spa_busy(void)
{
return (spa_active_count);
}
void
spa_boot_init()
{
spa_config_load();
}
#ifdef _KERNEL
EVENTHANDLER_DEFINE(mountroot, spa_boot_init, NULL, 0);
#endif
void
spa_init(int mode)
{
mutex_init(&spa_namespace_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa_spare_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&spa_l2cache_lock, NULL, MUTEX_DEFAULT, NULL);
cv_init(&spa_namespace_cv, NULL, CV_DEFAULT, NULL);
avl_create(&spa_namespace_avl, spa_name_compare, sizeof (spa_t),
offsetof(spa_t, spa_avl));
avl_create(&spa_spare_avl, spa_spare_compare, sizeof (spa_aux_t),
offsetof(spa_aux_t, aux_avl));
avl_create(&spa_l2cache_avl, spa_l2cache_compare, sizeof (spa_aux_t),
offsetof(spa_aux_t, aux_avl));
spa_mode_global = mode;
#ifdef illumos
#ifdef _KERNEL
spa_arch_init();
#else
if (spa_mode_global != FREAD && dprintf_find_string("watch")) {
arc_procfd = open("/proc/self/ctl", O_WRONLY);
if (arc_procfd == -1) {
perror("could not enable watchpoints: "
"opening /proc/self/ctl failed: ");
} else {
arc_watch = B_TRUE;
}
}
#endif
#endif /* illumos */
refcount_sysinit();
unique_init();
range_tree_init();
zio_init();
lz4_init();
dmu_init();
zil_init();
vdev_cache_stat_init();
zfs_prop_init();
zpool_prop_init();
zpool_feature_init();
spa_config_load();
l2arc_start();
#ifndef illumos
#ifdef _KERNEL
zfs_deadman_init();
#endif
#endif /* !illumos */
}
void
spa_fini(void)
{
l2arc_stop();
spa_evict_all();
vdev_cache_stat_fini();
zil_fini();
dmu_fini();
lz4_fini();
zio_fini();
range_tree_fini();
unique_fini();
refcount_fini();
avl_destroy(&spa_namespace_avl);
avl_destroy(&spa_spare_avl);
avl_destroy(&spa_l2cache_avl);
cv_destroy(&spa_namespace_cv);
mutex_destroy(&spa_namespace_lock);
mutex_destroy(&spa_spare_lock);
mutex_destroy(&spa_l2cache_lock);
}
/*
* Return whether this pool has slogs. No locking needed.
* It's not a problem if the wrong answer is returned as it's only for
* performance and not correctness
*/
boolean_t
spa_has_slogs(spa_t *spa)
{
return (spa->spa_log_class->mc_rotor != NULL);
}
spa_log_state_t
spa_get_log_state(spa_t *spa)
{
return (spa->spa_log_state);
}
void
spa_set_log_state(spa_t *spa, spa_log_state_t state)
{
spa->spa_log_state = state;
}
boolean_t
spa_is_root(spa_t *spa)
{
return (spa->spa_is_root);
}
boolean_t
spa_writeable(spa_t *spa)
{
return (!!(spa->spa_mode & FWRITE));
}
/*
* Returns true if there is a pending sync task in any of the current
* syncing txg, the current quiescing txg, or the current open txg.
*/
boolean_t
spa_has_pending_synctask(spa_t *spa)
{
return (!txg_all_lists_empty(&spa->spa_dsl_pool->dp_sync_tasks));
}
int
spa_mode(spa_t *spa)
{
return (spa->spa_mode);
}
uint64_t
spa_bootfs(spa_t *spa)
{
return (spa->spa_bootfs);
}
uint64_t
spa_delegation(spa_t *spa)
{
return (spa->spa_delegation);
}
objset_t *
spa_meta_objset(spa_t *spa)
{
return (spa->spa_meta_objset);
}
enum zio_checksum
spa_dedup_checksum(spa_t *spa)
{
return (spa->spa_dedup_checksum);
}
/*
* Reset pool scan stat per scan pass (or reboot).
*/
void
spa_scan_stat_init(spa_t *spa)
{
/* data not stored on disk */
spa->spa_scan_pass_start = gethrestime_sec();
spa->spa_scan_pass_exam = 0;
vdev_scan_stat_init(spa->spa_root_vdev);
}
/*
* Get scan stats for zpool status reports
*/
int
spa_scan_get_stats(spa_t *spa, pool_scan_stat_t *ps)
{
dsl_scan_t *scn = spa->spa_dsl_pool ? spa->spa_dsl_pool->dp_scan : NULL;
if (scn == NULL || scn->scn_phys.scn_func == POOL_SCAN_NONE)
return (SET_ERROR(ENOENT));
bzero(ps, sizeof (pool_scan_stat_t));
/* data stored on disk */
ps->pss_func = scn->scn_phys.scn_func;
ps->pss_start_time = scn->scn_phys.scn_start_time;
ps->pss_end_time = scn->scn_phys.scn_end_time;
ps->pss_to_examine = scn->scn_phys.scn_to_examine;
ps->pss_examined = scn->scn_phys.scn_examined;
ps->pss_to_process = scn->scn_phys.scn_to_process;
ps->pss_processed = scn->scn_phys.scn_processed;
ps->pss_errors = scn->scn_phys.scn_errors;
ps->pss_state = scn->scn_phys.scn_state;
/* data not stored on disk */
ps->pss_pass_start = spa->spa_scan_pass_start;
ps->pss_pass_exam = spa->spa_scan_pass_exam;
return (0);
}
boolean_t
spa_debug_enabled(spa_t *spa)
{
return (spa->spa_debug);
}
int
spa_maxblocksize(spa_t *spa)
{
if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS))
return (SPA_MAXBLOCKSIZE);
else
return (SPA_OLD_MAXBLOCKSIZE);
}
Index: head/sys/cddl/dev/profile/profile.c
===================================================================
--- head/sys/cddl/dev/profile/profile.c (revision 283290)
+++ head/sys/cddl/dev/profile/profile.c (revision 283291)
@@ -1,716 +1,716 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*
* Portions Copyright 2006-2008 John Birrell jb@freebsd.org
*
* $FreeBSD$
*
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/conf.h>
#include <sys/cpuvar.h>
#include <sys/fcntl.h>
#include <sys/filio.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/kthread.h>
#include <sys/limits.h>
#include <sys/linker.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/poll.h>
#include <sys/proc.h>
#include <sys/selinfo.h>
#include <sys/smp.h>
#include <sys/uio.h>
#include <sys/unistd.h>
#include <machine/cpu.h>
#include <machine/stdarg.h>
#include <sys/dtrace.h>
#include <sys/dtrace_bsd.h>
#define PROF_NAMELEN 15
#define PROF_PROFILE 0
#define PROF_TICK 1
#define PROF_PREFIX_PROFILE "profile-"
#define PROF_PREFIX_TICK "tick-"
/*
* Regardless of platform, there are five artificial frames in the case of the
* profile provider:
*
* profile_fire
* cyclic_expire
* cyclic_fire
* [ cbe ]
* [ locore ]
*
* On amd64, there are two frames associated with locore: one in locore, and
* another in common interrupt dispatch code. (i386 has not been modified to
* use this common layer.) Further, on i386, the interrupted instruction
* appears as its own stack frame. All of this means that we need to add one
* frame for amd64, and then take one away for both amd64 and i386.
*
* On SPARC, the picture is further complicated because the compiler
* optimizes away tail-calls -- so the following frames are optimized away:
*
* profile_fire
* cyclic_expire
*
* This gives three frames. However, on DEBUG kernels, the cyclic_expire
* frame cannot be tail-call eliminated, yielding four frames in this case.
*
* All of the above constraints lead to the mess below. Yes, the profile
* provider should ideally figure this out on-the-fly by hiting one of its own
* probes and then walking its own stack trace. This is complicated, however,
* and the static definition doesn't seem to be overly brittle. Still, we
* allow for a manual override in case we get it completely wrong.
*/
#ifdef __amd64
#define PROF_ARTIFICIAL_FRAMES 10
#else
#ifdef __i386
#define PROF_ARTIFICIAL_FRAMES 6
#else
#ifdef __sparc
#ifdef DEBUG
#define PROF_ARTIFICIAL_FRAMES 4
#else
#define PROF_ARTIFICIAL_FRAMES 3
#endif
#endif
#endif
#endif
#ifdef __mips
/*
* This value is bogus just to make module compilable on mips
*/
#define PROF_ARTIFICIAL_FRAMES 3
#endif
#ifdef __powerpc__
/*
* This value is bogus just to make module compilable on powerpc
*/
#define PROF_ARTIFICIAL_FRAMES 3
#endif
struct profile_probe_percpu;
#ifdef __mips
/* bogus */
#define PROF_ARTIFICIAL_FRAMES 3
#endif
#ifdef __arm__
/*
* At least on ARMv7, this appears to work quite well.
*/
#define PROF_ARTIFICIAL_FRAMES 10
#endif
typedef struct profile_probe {
char prof_name[PROF_NAMELEN];
dtrace_id_t prof_id;
int prof_kind;
#ifdef illumos
hrtime_t prof_interval;
cyclic_id_t prof_cyclic;
#else
sbintime_t prof_interval;
struct callout prof_cyclic;
sbintime_t prof_expected;
struct profile_probe_percpu **prof_pcpus;
#endif
} profile_probe_t;
typedef struct profile_probe_percpu {
hrtime_t profc_expected;
hrtime_t profc_interval;
profile_probe_t *profc_probe;
#ifdef __FreeBSD__
struct callout profc_cyclic;
#endif
} profile_probe_percpu_t;
static d_open_t profile_open;
static int profile_unload(void);
static void profile_create(hrtime_t, char *, int);
static void profile_destroy(void *, dtrace_id_t, void *);
static void profile_enable(void *, dtrace_id_t, void *);
static void profile_disable(void *, dtrace_id_t, void *);
static void profile_load(void *);
static void profile_provide(void *, dtrace_probedesc_t *);
static int profile_rates[] = {
97, 199, 499, 997, 1999,
4001, 4999, 0, 0, 0,
0, 0, 0, 0, 0,
0, 0, 0, 0, 0
};
static int profile_ticks[] = {
1, 10, 100, 500, 1000,
5000, 0, 0, 0, 0,
0, 0, 0, 0, 0
};
/*
* profile_max defines the upper bound on the number of profile probes that
* can exist (this is to prevent malicious or clumsy users from exhausing
* system resources by creating a slew of profile probes). At mod load time,
* this gets its value from PROFILE_MAX_DEFAULT or profile-max-probes if it's
* present in the profile.conf file.
*/
#define PROFILE_MAX_DEFAULT 1000 /* default max. number of probes */
static uint32_t profile_max = PROFILE_MAX_DEFAULT;
/* maximum number of profile probes */
static uint32_t profile_total; /* current number of profile probes */
static struct cdevsw profile_cdevsw = {
.d_version = D_VERSION,
.d_open = profile_open,
.d_name = "profile",
};
static dtrace_pattr_t profile_attr = {
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
};
static dtrace_pops_t profile_pops = {
profile_provide,
NULL,
profile_enable,
profile_disable,
NULL,
NULL,
NULL,
NULL,
NULL,
profile_destroy
};
static struct cdev *profile_cdev;
static dtrace_provider_id_t profile_id;
static hrtime_t profile_interval_min = NANOSEC / 5000; /* 5000 hz */
static int profile_aframes = 0; /* override */
static sbintime_t
nsec_to_sbt(hrtime_t nsec)
{
time_t sec;
/*
* We need to calculate nsec * 2^32 / 10^9
* Seconds and nanoseconds are split to avoid overflow.
*/
sec = nsec / NANOSEC;
nsec = nsec % NANOSEC;
return (((sbintime_t)sec << 32) | ((sbintime_t)nsec << 32) / NANOSEC);
}
static hrtime_t
sbt_to_nsec(sbintime_t sbt)
{
return ((sbt >> 32) * NANOSEC +
(((uint32_t)sbt * (hrtime_t)NANOSEC) >> 32));
}
static void
profile_fire(void *arg)
{
profile_probe_percpu_t *pcpu = arg;
profile_probe_t *prof = pcpu->profc_probe;
hrtime_t late;
struct trapframe *frame;
uintfptr_t pc, upc;
#ifdef illumos
late = gethrtime() - pcpu->profc_expected;
#else
late = sbt_to_nsec(sbinuptime() - pcpu->profc_expected);
#endif
pc = 0;
upc = 0;
/*
* td_intr_frame can be unset if this is a catch up event
* after waking up from idle sleep.
* This can only happen on a CPU idle thread.
*/
frame = curthread->td_intr_frame;
if (frame != NULL) {
if (TRAPF_USERMODE(frame))
upc = TRAPF_PC(frame);
else
pc = TRAPF_PC(frame);
}
dtrace_probe(prof->prof_id, pc, upc, late, 0, 0);
pcpu->profc_expected += pcpu->profc_interval;
callout_schedule_sbt_curcpu(&pcpu->profc_cyclic,
pcpu->profc_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE);
}
static void
profile_tick(void *arg)
{
profile_probe_t *prof = arg;
struct trapframe *frame;
uintfptr_t pc, upc;
pc = 0;
upc = 0;
/*
* td_intr_frame can be unset if this is a catch up event
* after waking up from idle sleep.
* This can only happen on a CPU idle thread.
*/
frame = curthread->td_intr_frame;
if (frame != NULL) {
if (TRAPF_USERMODE(frame))
upc = TRAPF_PC(frame);
else
pc = TRAPF_PC(frame);
}
dtrace_probe(prof->prof_id, pc, upc, 0, 0, 0);
prof->prof_expected += prof->prof_interval;
callout_schedule_sbt(&prof->prof_cyclic,
prof->prof_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE);
}
static void
profile_create(hrtime_t interval, char *name, int kind)
{
profile_probe_t *prof;
if (interval < profile_interval_min)
return;
if (dtrace_probe_lookup(profile_id, NULL, NULL, name) != 0)
return;
atomic_add_32(&profile_total, 1);
if (profile_total > profile_max) {
atomic_add_32(&profile_total, -1);
return;
}
prof = kmem_zalloc(sizeof (profile_probe_t), KM_SLEEP);
(void) strcpy(prof->prof_name, name);
#ifdef illumos
prof->prof_interval = interval;
prof->prof_cyclic = CYCLIC_NONE;
#else
prof->prof_interval = nsec_to_sbt(interval);
- callout_init(&prof->prof_cyclic, CALLOUT_MPSAFE);
+ callout_init(&prof->prof_cyclic, 1);
#endif
prof->prof_kind = kind;
prof->prof_id = dtrace_probe_create(profile_id,
NULL, NULL, name,
profile_aframes ? profile_aframes : PROF_ARTIFICIAL_FRAMES, prof);
}
/*ARGSUSED*/
static void
profile_provide(void *arg, dtrace_probedesc_t *desc)
{
int i, j, rate, kind;
hrtime_t val = 0, mult = 1, len = 0;
char *name, *suffix = NULL;
const struct {
char *prefix;
int kind;
} types[] = {
{ PROF_PREFIX_PROFILE, PROF_PROFILE },
{ PROF_PREFIX_TICK, PROF_TICK },
{ 0, 0 }
};
const struct {
char *name;
hrtime_t mult;
} suffixes[] = {
{ "ns", NANOSEC / NANOSEC },
{ "nsec", NANOSEC / NANOSEC },
{ "us", NANOSEC / MICROSEC },
{ "usec", NANOSEC / MICROSEC },
{ "ms", NANOSEC / MILLISEC },
{ "msec", NANOSEC / MILLISEC },
{ "s", NANOSEC / SEC },
{ "sec", NANOSEC / SEC },
{ "m", NANOSEC * (hrtime_t)60 },
{ "min", NANOSEC * (hrtime_t)60 },
{ "h", NANOSEC * (hrtime_t)(60 * 60) },
{ "hour", NANOSEC * (hrtime_t)(60 * 60) },
{ "d", NANOSEC * (hrtime_t)(24 * 60 * 60) },
{ "day", NANOSEC * (hrtime_t)(24 * 60 * 60) },
{ "hz", 0 },
{ NULL }
};
if (desc == NULL) {
char n[PROF_NAMELEN];
/*
* If no description was provided, provide all of our probes.
*/
for (i = 0; i < sizeof (profile_rates) / sizeof (int); i++) {
if ((rate = profile_rates[i]) == 0)
continue;
(void) snprintf(n, PROF_NAMELEN, "%s%d",
PROF_PREFIX_PROFILE, rate);
profile_create(NANOSEC / rate, n, PROF_PROFILE);
}
for (i = 0; i < sizeof (profile_ticks) / sizeof (int); i++) {
if ((rate = profile_ticks[i]) == 0)
continue;
(void) snprintf(n, PROF_NAMELEN, "%s%d",
PROF_PREFIX_TICK, rate);
profile_create(NANOSEC / rate, n, PROF_TICK);
}
return;
}
name = desc->dtpd_name;
for (i = 0; types[i].prefix != NULL; i++) {
len = strlen(types[i].prefix);
if (strncmp(name, types[i].prefix, len) != 0)
continue;
break;
}
if (types[i].prefix == NULL)
return;
kind = types[i].kind;
j = strlen(name) - len;
/*
* We need to start before any time suffix.
*/
for (j = strlen(name); j >= len; j--) {
if (name[j] >= '0' && name[j] <= '9')
break;
suffix = &name[j];
}
ASSERT(suffix != NULL);
/*
* Now determine the numerical value present in the probe name.
*/
for (; j >= len; j--) {
if (name[j] < '0' || name[j] > '9')
return;
val += (name[j] - '0') * mult;
mult *= (hrtime_t)10;
}
if (val == 0)
return;
/*
* Look-up the suffix to determine the multiplier.
*/
for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
if (strcasecmp(suffixes[i].name, suffix) == 0) {
mult = suffixes[i].mult;
break;
}
}
if (suffixes[i].name == NULL && *suffix != '\0')
return;
if (mult == 0) {
/*
* The default is frequency-per-second.
*/
val = NANOSEC / val;
} else {
val *= mult;
}
profile_create(val, name, kind);
}
/* ARGSUSED */
static void
profile_destroy(void *arg, dtrace_id_t id, void *parg)
{
profile_probe_t *prof = parg;
#ifdef illumos
ASSERT(prof->prof_cyclic == CYCLIC_NONE);
#else
ASSERT(!callout_active(&prof->prof_cyclic) && prof->prof_pcpus == NULL);
#endif
kmem_free(prof, sizeof (profile_probe_t));
ASSERT(profile_total >= 1);
atomic_add_32(&profile_total, -1);
}
#ifdef illumos
/*ARGSUSED*/
static void
profile_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
{
profile_probe_t *prof = arg;
profile_probe_percpu_t *pcpu;
pcpu = kmem_zalloc(sizeof (profile_probe_percpu_t), KM_SLEEP);
pcpu->profc_probe = prof;
hdlr->cyh_func = profile_fire;
hdlr->cyh_arg = pcpu;
when->cyt_interval = prof->prof_interval;
when->cyt_when = gethrtime() + when->cyt_interval;
pcpu->profc_expected = when->cyt_when;
pcpu->profc_interval = when->cyt_interval;
}
/*ARGSUSED*/
static void
profile_offline(void *arg, cpu_t *cpu, void *oarg)
{
profile_probe_percpu_t *pcpu = oarg;
ASSERT(pcpu->profc_probe == arg);
kmem_free(pcpu, sizeof (profile_probe_percpu_t));
}
/* ARGSUSED */
static void
profile_enable(void *arg, dtrace_id_t id, void *parg)
{
profile_probe_t *prof = parg;
cyc_omni_handler_t omni;
cyc_handler_t hdlr;
cyc_time_t when;
ASSERT(prof->prof_interval != 0);
ASSERT(MUTEX_HELD(&cpu_lock));
if (prof->prof_kind == PROF_TICK) {
hdlr.cyh_func = profile_tick;
hdlr.cyh_arg = prof;
when.cyt_interval = prof->prof_interval;
when.cyt_when = gethrtime() + when.cyt_interval;
} else {
ASSERT(prof->prof_kind == PROF_PROFILE);
omni.cyo_online = profile_online;
omni.cyo_offline = profile_offline;
omni.cyo_arg = prof;
}
if (prof->prof_kind == PROF_TICK) {
prof->prof_cyclic = cyclic_add(&hdlr, &when);
} else {
prof->prof_cyclic = cyclic_add_omni(&omni);
}
}
/* ARGSUSED */
static void
profile_disable(void *arg, dtrace_id_t id, void *parg)
{
profile_probe_t *prof = parg;
ASSERT(prof->prof_cyclic != CYCLIC_NONE);
ASSERT(MUTEX_HELD(&cpu_lock));
cyclic_remove(prof->prof_cyclic);
prof->prof_cyclic = CYCLIC_NONE;
}
#else
static void
profile_enable_omni(profile_probe_t *prof)
{
profile_probe_percpu_t *pcpu;
int cpu;
prof->prof_pcpus = kmem_zalloc((mp_maxid + 1) * sizeof(pcpu), KM_SLEEP);
CPU_FOREACH(cpu) {
pcpu = kmem_zalloc(sizeof(profile_probe_percpu_t), KM_SLEEP);
prof->prof_pcpus[cpu] = pcpu;
pcpu->profc_probe = prof;
pcpu->profc_expected = sbinuptime() + prof->prof_interval;
pcpu->profc_interval = prof->prof_interval;
- callout_init(&pcpu->profc_cyclic, CALLOUT_MPSAFE);
+ callout_init(&pcpu->profc_cyclic, 1);
callout_reset_sbt_on(&pcpu->profc_cyclic,
pcpu->profc_expected, 0, profile_fire, pcpu,
cpu, C_DIRECT_EXEC | C_ABSOLUTE);
}
}
static void
profile_disable_omni(profile_probe_t *prof)
{
profile_probe_percpu_t *pcpu;
int cpu;
ASSERT(prof->prof_pcpus != NULL);
CPU_FOREACH(cpu) {
pcpu = prof->prof_pcpus[cpu];
ASSERT(pcpu->profc_probe == prof);
ASSERT(callout_active(&pcpu->profc_cyclic));
callout_stop(&pcpu->profc_cyclic);
callout_drain(&pcpu->profc_cyclic);
kmem_free(pcpu, sizeof(profile_probe_percpu_t));
}
kmem_free(prof->prof_pcpus, (mp_maxid + 1) * sizeof(pcpu));
prof->prof_pcpus = NULL;
}
/* ARGSUSED */
static void
profile_enable(void *arg, dtrace_id_t id, void *parg)
{
profile_probe_t *prof = parg;
if (prof->prof_kind == PROF_TICK) {
prof->prof_expected = sbinuptime() + prof->prof_interval;
callout_reset_sbt(&prof->prof_cyclic,
prof->prof_expected, 0, profile_tick, prof,
C_DIRECT_EXEC | C_ABSOLUTE);
} else {
ASSERT(prof->prof_kind == PROF_PROFILE);
profile_enable_omni(prof);
}
}
/* ARGSUSED */
static void
profile_disable(void *arg, dtrace_id_t id, void *parg)
{
profile_probe_t *prof = parg;
if (prof->prof_kind == PROF_TICK) {
ASSERT(callout_active(&prof->prof_cyclic));
callout_stop(&prof->prof_cyclic);
callout_drain(&prof->prof_cyclic);
} else {
ASSERT(prof->prof_kind == PROF_PROFILE);
profile_disable_omni(prof);
}
}
#endif
static void
profile_load(void *dummy)
{
/* Create the /dev/dtrace/profile entry. */
profile_cdev = make_dev(&profile_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
"dtrace/profile");
if (dtrace_register("profile", &profile_attr, DTRACE_PRIV_USER,
NULL, &profile_pops, NULL, &profile_id) != 0)
return;
}
static int
profile_unload()
{
int error = 0;
if ((error = dtrace_unregister(profile_id)) != 0)
return (error);
destroy_dev(profile_cdev);
return (error);
}
/* ARGSUSED */
static int
profile_modevent(module_t mod __unused, int type, void *data __unused)
{
int error = 0;
switch (type) {
case MOD_LOAD:
break;
case MOD_UNLOAD:
break;
case MOD_SHUTDOWN:
break;
default:
error = EOPNOTSUPP;
break;
}
return (error);
}
/* ARGSUSED */
static int
profile_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused)
{
return (0);
}
SYSINIT(profile_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_load, NULL);
SYSUNINIT(profile_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_unload, NULL);
DEV_MODULE(profile, profile_modevent, NULL);
MODULE_VERSION(profile, 1);
MODULE_DEPEND(profile, dtrace, 1, 1, 1);
MODULE_DEPEND(profile, opensolaris, 1, 1, 1);
Index: head/sys/compat/ndis/subr_ntoskrnl.c
===================================================================
--- head/sys/compat/ndis/subr_ntoskrnl.c (revision 283290)
+++ head/sys/compat/ndis/subr_ntoskrnl.c (revision 283291)
@@ -1,4457 +1,4457 @@
/*-
* Copyright (c) 2003
* Bill Paul <wpaul@windriver.com>. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by Bill Paul.
* 4. Neither the name of the author nor the names of any co-contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/ctype.h>
#include <sys/unistd.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/errno.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/callout.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/condvar.h>
#include <sys/kthread.h>
#include <sys/module.h>
#include <sys/smp.h>
#include <sys/sched.h>
#include <sys/sysctl.h>
#include <machine/atomic.h>
#include <machine/bus.h>
#include <machine/stdarg.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
#include <vm/uma.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <vm/vm_extern.h>
#include <compat/ndis/pe_var.h>
#include <compat/ndis/cfg_var.h>
#include <compat/ndis/resource_var.h>
#include <compat/ndis/ntoskrnl_var.h>
#include <compat/ndis/hal_var.h>
#include <compat/ndis/ndis_var.h>
#ifdef NTOSKRNL_DEBUG_TIMERS
static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers, CTLTYPE_INT | CTLFLAG_RW,
NULL, 0, sysctl_show_timers, "I",
"Show ntoskrnl timer stats");
#endif
struct kdpc_queue {
list_entry kq_disp;
struct thread *kq_td;
int kq_cpu;
int kq_exit;
int kq_running;
kspin_lock kq_lock;
nt_kevent kq_proc;
nt_kevent kq_done;
};
typedef struct kdpc_queue kdpc_queue;
struct wb_ext {
struct cv we_cv;
struct thread *we_td;
};
typedef struct wb_ext wb_ext;
#define NTOSKRNL_TIMEOUTS 256
#ifdef NTOSKRNL_DEBUG_TIMERS
static uint64_t ntoskrnl_timer_fires;
static uint64_t ntoskrnl_timer_sets;
static uint64_t ntoskrnl_timer_reloads;
static uint64_t ntoskrnl_timer_cancels;
#endif
struct callout_entry {
struct callout ce_callout;
list_entry ce_list;
};
typedef struct callout_entry callout_entry;
static struct list_entry ntoskrnl_calllist;
static struct mtx ntoskrnl_calllock;
struct kuser_shared_data kuser_shared_data;
static struct list_entry ntoskrnl_intlist;
static kspin_lock ntoskrnl_intlock;
static uint8_t RtlEqualUnicodeString(unicode_string *,
unicode_string *, uint8_t);
static void RtlCopyString(ansi_string *, const ansi_string *);
static void RtlCopyUnicodeString(unicode_string *,
unicode_string *);
static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
static irp *IoBuildAsynchronousFsdRequest(uint32_t,
device_object *, void *, uint32_t, uint64_t *, io_status_block *);
static irp *IoBuildDeviceIoControlRequest(uint32_t,
device_object *, void *, uint32_t, void *, uint32_t,
uint8_t, nt_kevent *, io_status_block *);
static irp *IoAllocateIrp(uint8_t, uint8_t);
static void IoReuseIrp(irp *, uint32_t);
static void IoFreeIrp(irp *);
static void IoInitializeIrp(irp *, uint16_t, uint8_t);
static irp *IoMakeAssociatedIrp(irp *, uint8_t);
static uint32_t KeWaitForMultipleObjects(uint32_t,
nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
int64_t *, wait_block *);
static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
static void ntoskrnl_satisfy_multiple_waits(wait_block *);
static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
static void ntoskrnl_insert_timer(ktimer *, int);
static void ntoskrnl_remove_timer(ktimer *);
#ifdef NTOSKRNL_DEBUG_TIMERS
static void ntoskrnl_show_timers(void);
#endif
static void ntoskrnl_timercall(void *);
static void ntoskrnl_dpc_thread(void *);
static void ntoskrnl_destroy_dpc_threads(void);
static void ntoskrnl_destroy_workitem_threads(void);
static void ntoskrnl_workitem_thread(void *);
static void ntoskrnl_workitem(device_object *, void *);
static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
static uint16_t READ_REGISTER_USHORT(uint16_t *);
static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
static uint32_t READ_REGISTER_ULONG(uint32_t *);
static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
static uint8_t READ_REGISTER_UCHAR(uint8_t *);
static int64_t _allmul(int64_t, int64_t);
static int64_t _alldiv(int64_t, int64_t);
static int64_t _allrem(int64_t, int64_t);
static int64_t _allshr(int64_t, uint8_t);
static int64_t _allshl(int64_t, uint8_t);
static uint64_t _aullmul(uint64_t, uint64_t);
static uint64_t _aulldiv(uint64_t, uint64_t);
static uint64_t _aullrem(uint64_t, uint64_t);
static uint64_t _aullshr(uint64_t, uint8_t);
static uint64_t _aullshl(uint64_t, uint8_t);
static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
static void InitializeSListHead(slist_header *);
static slist_entry *ntoskrnl_popsl(slist_header *);
static void ExFreePoolWithTag(void *, uint32_t);
static void ExInitializePagedLookasideList(paged_lookaside_list *,
lookaside_alloc_func *, lookaside_free_func *,
uint32_t, size_t, uint32_t, uint16_t);
static void ExDeletePagedLookasideList(paged_lookaside_list *);
static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
lookaside_alloc_func *, lookaside_free_func *,
uint32_t, size_t, uint32_t, uint16_t);
static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
static slist_entry
*ExInterlockedPushEntrySList(slist_header *,
slist_entry *, kspin_lock *);
static slist_entry
*ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
static uint32_t InterlockedIncrement(volatile uint32_t *);
static uint32_t InterlockedDecrement(volatile uint32_t *);
static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
uint64_t, uint64_t, uint64_t, enum nt_caching_type);
static void MmFreeContiguousMemory(void *);
static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
enum nt_caching_type);
static uint32_t MmSizeOfMdl(void *, size_t);
static void *MmMapLockedPages(mdl *, uint8_t);
static void *MmMapLockedPagesSpecifyCache(mdl *,
uint8_t, uint32_t, void *, uint32_t, uint32_t);
static void MmUnmapLockedPages(void *, mdl *);
static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
static void RtlZeroMemory(void *, size_t);
static void RtlSecureZeroMemory(void *, size_t);
static void RtlFillMemory(void *, size_t, uint8_t);
static void RtlMoveMemory(void *, const void *, size_t);
static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *);
static void RtlCopyMemory(void *, const void *, size_t);
static size_t RtlCompareMemory(const void *, const void *, size_t);
static ndis_status RtlUnicodeStringToInteger(unicode_string *,
uint32_t, uint32_t *);
static int atoi (const char *);
static long atol (const char *);
static int rand(void);
static void srand(unsigned int);
static void KeQuerySystemTime(uint64_t *);
static uint32_t KeTickCount(void);
static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t,
uint32_t, void **);
static void ntoskrnl_thrfunc(void *);
static ndis_status PsCreateSystemThread(ndis_handle *,
uint32_t, void *, ndis_handle, void *, void *, void *);
static ndis_status PsTerminateSystemThread(ndis_status);
static ndis_status IoGetDeviceObjectPointer(unicode_string *,
uint32_t, void *, device_object *);
static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
uint32_t, void *, uint32_t *);
static void KeInitializeMutex(kmutant *, uint32_t);
static uint32_t KeReleaseMutex(kmutant *, uint8_t);
static uint32_t KeReadStateMutex(kmutant *);
static ndis_status ObReferenceObjectByHandle(ndis_handle,
uint32_t, void *, uint8_t, void **, void **);
static void ObfDereferenceObject(void *);
static uint32_t ZwClose(ndis_handle);
static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
uint32_t, void *);
static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
static void *ntoskrnl_memset(void *, int, size_t);
static void *ntoskrnl_memmove(void *, void *, size_t);
static void *ntoskrnl_memchr(void *, unsigned char, size_t);
static char *ntoskrnl_strstr(char *, char *);
static char *ntoskrnl_strncat(char *, char *, size_t);
static int ntoskrnl_toupper(int);
static int ntoskrnl_tolower(int);
static funcptr ntoskrnl_findwrap(funcptr);
static uint32_t DbgPrint(char *, ...);
static void DbgBreakPoint(void);
static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
static int32_t KeSetPriorityThread(struct thread *, int32_t);
static void dummy(void);
static struct mtx ntoskrnl_dispatchlock;
static struct mtx ntoskrnl_interlock;
static kspin_lock ntoskrnl_cancellock;
static int ntoskrnl_kth = 0;
static struct nt_objref_head ntoskrnl_reflist;
static uma_zone_t mdl_zone;
static uma_zone_t iw_zone;
static struct kdpc_queue *kq_queues;
static struct kdpc_queue *wq_queues;
static int wq_idx = 0;
int
ntoskrnl_libinit()
{
image_patch_table *patch;
int error;
struct proc *p;
kdpc_queue *kq;
callout_entry *e;
int i;
mtx_init(&ntoskrnl_dispatchlock,
"ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
KeInitializeSpinLock(&ntoskrnl_cancellock);
KeInitializeSpinLock(&ntoskrnl_intlock);
TAILQ_INIT(&ntoskrnl_reflist);
InitializeListHead(&ntoskrnl_calllist);
InitializeListHead(&ntoskrnl_intlist);
mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
kq_queues = ExAllocatePoolWithTag(NonPagedPool,
#ifdef NTOSKRNL_MULTIPLE_DPCS
sizeof(kdpc_queue) * mp_ncpus, 0);
#else
sizeof(kdpc_queue), 0);
#endif
if (kq_queues == NULL)
return (ENOMEM);
wq_queues = ExAllocatePoolWithTag(NonPagedPool,
sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
if (wq_queues == NULL)
return (ENOMEM);
#ifdef NTOSKRNL_MULTIPLE_DPCS
bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
#else
bzero((char *)kq_queues, sizeof(kdpc_queue));
#endif
bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
/*
* Launch the DPC threads.
*/
#ifdef NTOSKRNL_MULTIPLE_DPCS
for (i = 0; i < mp_ncpus; i++) {
#else
for (i = 0; i < 1; i++) {
#endif
kq = kq_queues + i;
kq->kq_cpu = i;
error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
if (error)
panic("failed to launch DPC thread");
}
/*
* Launch the workitem threads.
*/
for (i = 0; i < WORKITEM_THREADS; i++) {
kq = wq_queues + i;
error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
if (error)
panic("failed to launch workitem thread");
}
patch = ntoskrnl_functbl;
while (patch->ipt_func != NULL) {
windrv_wrap((funcptr)patch->ipt_func,
(funcptr *)&patch->ipt_wrap,
patch->ipt_argcnt, patch->ipt_ftype);
patch++;
}
for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
e = ExAllocatePoolWithTag(NonPagedPool,
sizeof(callout_entry), 0);
if (e == NULL)
panic("failed to allocate timeouts");
mtx_lock_spin(&ntoskrnl_calllock);
InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
mtx_unlock_spin(&ntoskrnl_calllock);
}
/*
* MDLs are supposed to be variable size (they describe
* buffers containing some number of pages, but we don't
* know ahead of time how many pages that will be). But
* always allocating them off the heap is very slow. As
* a compromise, we create an MDL UMA zone big enough to
* handle any buffer requiring up to 16 pages, and we
* use those for any MDLs for buffers of 16 pages or less
* in size. For buffers larger than that (which we assume
* will be few and far between, we allocate the MDLs off
* the heap.
*/
mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
return (0);
}
int
ntoskrnl_libfini()
{
image_patch_table *patch;
callout_entry *e;
list_entry *l;
patch = ntoskrnl_functbl;
while (patch->ipt_func != NULL) {
windrv_unwrap(patch->ipt_wrap);
patch++;
}
/* Stop the workitem queues. */
ntoskrnl_destroy_workitem_threads();
/* Stop the DPC queues. */
ntoskrnl_destroy_dpc_threads();
ExFreePool(kq_queues);
ExFreePool(wq_queues);
uma_zdestroy(mdl_zone);
uma_zdestroy(iw_zone);
mtx_lock_spin(&ntoskrnl_calllock);
while(!IsListEmpty(&ntoskrnl_calllist)) {
l = RemoveHeadList(&ntoskrnl_calllist);
e = CONTAINING_RECORD(l, callout_entry, ce_list);
mtx_unlock_spin(&ntoskrnl_calllock);
ExFreePool(e);
mtx_lock_spin(&ntoskrnl_calllock);
}
mtx_unlock_spin(&ntoskrnl_calllock);
mtx_destroy(&ntoskrnl_dispatchlock);
mtx_destroy(&ntoskrnl_interlock);
mtx_destroy(&ntoskrnl_calllock);
return (0);
}
/*
* We need to be able to reference this externally from the wrapper;
* GCC only generates a local implementation of memset.
*/
static void *
ntoskrnl_memset(buf, ch, size)
void *buf;
int ch;
size_t size;
{
return (memset(buf, ch, size));
}
static void *
ntoskrnl_memmove(dst, src, size)
void *src;
void *dst;
size_t size;
{
bcopy(src, dst, size);
return (dst);
}
static void *
ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
{
if (len != 0) {
unsigned char *p = buf;
do {
if (*p++ == ch)
return (p - 1);
} while (--len != 0);
}
return (NULL);
}
static char *
ntoskrnl_strstr(s, find)
char *s, *find;
{
char c, sc;
size_t len;
if ((c = *find++) != 0) {
len = strlen(find);
do {
do {
if ((sc = *s++) == 0)
return (NULL);
} while (sc != c);
} while (strncmp(s, find, len) != 0);
s--;
}
return ((char *)s);
}
/* Taken from libc */
static char *
ntoskrnl_strncat(dst, src, n)
char *dst;
char *src;
size_t n;
{
if (n != 0) {
char *d = dst;
const char *s = src;
while (*d != 0)
d++;
do {
if ((*d = *s++) == 0)
break;
d++;
} while (--n != 0);
*d = 0;
}
return (dst);
}
static int
ntoskrnl_toupper(c)
int c;
{
return (toupper(c));
}
static int
ntoskrnl_tolower(c)
int c;
{
return (tolower(c));
}
static uint8_t
RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
uint8_t caseinsensitive)
{
int i;
if (str1->us_len != str2->us_len)
return (FALSE);
for (i = 0; i < str1->us_len; i++) {
if (caseinsensitive == TRUE) {
if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
toupper((char)(str2->us_buf[i] & 0xFF)))
return (FALSE);
} else {
if (str1->us_buf[i] != str2->us_buf[i])
return (FALSE);
}
}
return (TRUE);
}
static void
RtlCopyString(dst, src)
ansi_string *dst;
const ansi_string *src;
{
if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) {
dst->as_len = min(src->as_len, dst->as_maxlen);
memcpy(dst->as_buf, src->as_buf, dst->as_len);
if (dst->as_len < dst->as_maxlen)
dst->as_buf[dst->as_len] = 0;
} else
dst->as_len = 0;
}
static void
RtlCopyUnicodeString(dest, src)
unicode_string *dest;
unicode_string *src;
{
if (dest->us_maxlen >= src->us_len)
dest->us_len = src->us_len;
else
dest->us_len = dest->us_maxlen;
memcpy(dest->us_buf, src->us_buf, dest->us_len);
}
static void
ntoskrnl_ascii_to_unicode(ascii, unicode, len)
char *ascii;
uint16_t *unicode;
int len;
{
int i;
uint16_t *ustr;
ustr = unicode;
for (i = 0; i < len; i++) {
*ustr = (uint16_t)ascii[i];
ustr++;
}
}
static void
ntoskrnl_unicode_to_ascii(unicode, ascii, len)
uint16_t *unicode;
char *ascii;
int len;
{
int i;
uint8_t *astr;
astr = ascii;
for (i = 0; i < len / 2; i++) {
*astr = (uint8_t)unicode[i];
astr++;
}
}
uint32_t
RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
{
if (dest == NULL || src == NULL)
return (STATUS_INVALID_PARAMETER);
dest->as_len = src->us_len / 2;
if (dest->as_maxlen < dest->as_len)
dest->as_len = dest->as_maxlen;
if (allocate == TRUE) {
dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
(src->us_len / 2) + 1, 0);
if (dest->as_buf == NULL)
return (STATUS_INSUFFICIENT_RESOURCES);
dest->as_len = dest->as_maxlen = src->us_len / 2;
} else {
dest->as_len = src->us_len / 2; /* XXX */
if (dest->as_maxlen < dest->as_len)
dest->as_len = dest->as_maxlen;
}
ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
dest->as_len * 2);
return (STATUS_SUCCESS);
}
uint32_t
RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
uint8_t allocate)
{
if (dest == NULL || src == NULL)
return (STATUS_INVALID_PARAMETER);
if (allocate == TRUE) {
dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
src->as_len * 2, 0);
if (dest->us_buf == NULL)
return (STATUS_INSUFFICIENT_RESOURCES);
dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
} else {
dest->us_len = src->as_len * 2; /* XXX */
if (dest->us_maxlen < dest->us_len)
dest->us_len = dest->us_maxlen;
}
ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
dest->us_len / 2);
return (STATUS_SUCCESS);
}
void *
ExAllocatePoolWithTag(pooltype, len, tag)
uint32_t pooltype;
size_t len;
uint32_t tag;
{
void *buf;
buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
if (buf == NULL)
return (NULL);
return (buf);
}
static void
ExFreePoolWithTag(buf, tag)
void *buf;
uint32_t tag;
{
ExFreePool(buf);
}
void
ExFreePool(buf)
void *buf;
{
free(buf, M_DEVBUF);
}
uint32_t
IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
driver_object *drv;
void *clid;
uint32_t extlen;
void **ext;
{
custom_extension *ce;
ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
+ extlen, 0);
if (ce == NULL)
return (STATUS_INSUFFICIENT_RESOURCES);
ce->ce_clid = clid;
InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
*ext = (void *)(ce + 1);
return (STATUS_SUCCESS);
}
void *
IoGetDriverObjectExtension(drv, clid)
driver_object *drv;
void *clid;
{
list_entry *e;
custom_extension *ce;
/*
* Sanity check. Our dummy bus drivers don't have
* any driver extentions.
*/
if (drv->dro_driverext == NULL)
return (NULL);
e = drv->dro_driverext->dre_usrext.nle_flink;
while (e != &drv->dro_driverext->dre_usrext) {
ce = (custom_extension *)e;
if (ce->ce_clid == clid)
return ((void *)(ce + 1));
e = e->nle_flink;
}
return (NULL);
}
uint32_t
IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
uint32_t devtype, uint32_t devchars, uint8_t exclusive,
device_object **newdev)
{
device_object *dev;
dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
if (dev == NULL)
return (STATUS_INSUFFICIENT_RESOURCES);
dev->do_type = devtype;
dev->do_drvobj = drv;
dev->do_currirp = NULL;
dev->do_flags = 0;
if (devextlen) {
dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
devextlen, 0);
if (dev->do_devext == NULL) {
ExFreePool(dev);
return (STATUS_INSUFFICIENT_RESOURCES);
}
bzero(dev->do_devext, devextlen);
} else
dev->do_devext = NULL;
dev->do_size = sizeof(device_object) + devextlen;
dev->do_refcnt = 1;
dev->do_attacheddev = NULL;
dev->do_nextdev = NULL;
dev->do_devtype = devtype;
dev->do_stacksize = 1;
dev->do_alignreq = 1;
dev->do_characteristics = devchars;
dev->do_iotimer = NULL;
KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
/*
* Vpd is used for disk/tape devices,
* but we don't support those. (Yet.)
*/
dev->do_vpb = NULL;
dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
sizeof(devobj_extension), 0);
if (dev->do_devobj_ext == NULL) {
if (dev->do_devext != NULL)
ExFreePool(dev->do_devext);
ExFreePool(dev);
return (STATUS_INSUFFICIENT_RESOURCES);
}
dev->do_devobj_ext->dve_type = 0;
dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
dev->do_devobj_ext->dve_devobj = dev;
/*
* Attach this device to the driver object's list
* of devices. Note: this is not the same as attaching
* the device to the device stack. The driver's AddDevice
* routine must explicitly call IoAddDeviceToDeviceStack()
* to do that.
*/
if (drv->dro_devobj == NULL) {
drv->dro_devobj = dev;
dev->do_nextdev = NULL;
} else {
dev->do_nextdev = drv->dro_devobj;
drv->dro_devobj = dev;
}
*newdev = dev;
return (STATUS_SUCCESS);
}
void
IoDeleteDevice(dev)
device_object *dev;
{
device_object *prev;
if (dev == NULL)
return;
if (dev->do_devobj_ext != NULL)
ExFreePool(dev->do_devobj_ext);
if (dev->do_devext != NULL)
ExFreePool(dev->do_devext);
/* Unlink the device from the driver's device list. */
prev = dev->do_drvobj->dro_devobj;
if (prev == dev)
dev->do_drvobj->dro_devobj = dev->do_nextdev;
else {
while (prev->do_nextdev != dev)
prev = prev->do_nextdev;
prev->do_nextdev = dev->do_nextdev;
}
ExFreePool(dev);
}
device_object *
IoGetAttachedDevice(dev)
device_object *dev;
{
device_object *d;
if (dev == NULL)
return (NULL);
d = dev;
while (d->do_attacheddev != NULL)
d = d->do_attacheddev;
return (d);
}
static irp *
IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
uint32_t func;
device_object *dobj;
void *buf;
uint32_t len;
uint64_t *off;
nt_kevent *event;
io_status_block *status;
{
irp *ip;
ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
if (ip == NULL)
return (NULL);
ip->irp_usrevent = event;
return (ip);
}
static irp *
IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
uint32_t func;
device_object *dobj;
void *buf;
uint32_t len;
uint64_t *off;
io_status_block *status;
{
irp *ip;
io_stack_location *sl;
ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
if (ip == NULL)
return (NULL);
ip->irp_usriostat = status;
ip->irp_tail.irp_overlay.irp_thread = NULL;
sl = IoGetNextIrpStackLocation(ip);
sl->isl_major = func;
sl->isl_minor = 0;
sl->isl_flags = 0;
sl->isl_ctl = 0;
sl->isl_devobj = dobj;
sl->isl_fileobj = NULL;
sl->isl_completionfunc = NULL;
ip->irp_userbuf = buf;
if (dobj->do_flags & DO_BUFFERED_IO) {
ip->irp_assoc.irp_sysbuf =
ExAllocatePoolWithTag(NonPagedPool, len, 0);
if (ip->irp_assoc.irp_sysbuf == NULL) {
IoFreeIrp(ip);
return (NULL);
}
bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
}
if (dobj->do_flags & DO_DIRECT_IO) {
ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
if (ip->irp_mdl == NULL) {
if (ip->irp_assoc.irp_sysbuf != NULL)
ExFreePool(ip->irp_assoc.irp_sysbuf);
IoFreeIrp(ip);
return (NULL);
}
ip->irp_userbuf = NULL;
ip->irp_assoc.irp_sysbuf = NULL;
}
if (func == IRP_MJ_READ) {
sl->isl_parameters.isl_read.isl_len = len;
if (off != NULL)
sl->isl_parameters.isl_read.isl_byteoff = *off;
else
sl->isl_parameters.isl_read.isl_byteoff = 0;
}
if (func == IRP_MJ_WRITE) {
sl->isl_parameters.isl_write.isl_len = len;
if (off != NULL)
sl->isl_parameters.isl_write.isl_byteoff = *off;
else
sl->isl_parameters.isl_write.isl_byteoff = 0;
}
return (ip);
}
static irp *
IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
nt_kevent *event, io_status_block *status)
{
irp *ip;
io_stack_location *sl;
uint32_t buflen;
ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
if (ip == NULL)
return (NULL);
ip->irp_usrevent = event;
ip->irp_usriostat = status;
ip->irp_tail.irp_overlay.irp_thread = NULL;
sl = IoGetNextIrpStackLocation(ip);
sl->isl_major = isinternal == TRUE ?
IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
sl->isl_minor = 0;
sl->isl_flags = 0;
sl->isl_ctl = 0;
sl->isl_devobj = dobj;
sl->isl_fileobj = NULL;
sl->isl_completionfunc = NULL;
sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
switch(IO_METHOD(iocode)) {
case METHOD_BUFFERED:
if (ilen > olen)
buflen = ilen;
else
buflen = olen;
if (buflen) {
ip->irp_assoc.irp_sysbuf =
ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
if (ip->irp_assoc.irp_sysbuf == NULL) {
IoFreeIrp(ip);
return (NULL);
}
}
if (ilen && ibuf != NULL) {
bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
buflen - ilen);
} else
bzero(ip->irp_assoc.irp_sysbuf, ilen);
ip->irp_userbuf = obuf;
break;
case METHOD_IN_DIRECT:
case METHOD_OUT_DIRECT:
if (ilen && ibuf != NULL) {
ip->irp_assoc.irp_sysbuf =
ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
if (ip->irp_assoc.irp_sysbuf == NULL) {
IoFreeIrp(ip);
return (NULL);
}
bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
}
if (olen && obuf != NULL) {
ip->irp_mdl = IoAllocateMdl(obuf, olen,
FALSE, FALSE, ip);
/*
* Normally we would MmProbeAndLockPages()
* here, but we don't have to in our
* imlementation.
*/
}
break;
case METHOD_NEITHER:
ip->irp_userbuf = obuf;
sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
break;
default:
break;
}
/*
* Ideally, we should associate this IRP with the calling
* thread here.
*/
return (ip);
}
static irp *
IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
{
irp *i;
i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
if (i == NULL)
return (NULL);
IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
return (i);
}
static irp *
IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
{
irp *associrp;
associrp = IoAllocateIrp(stsize, FALSE);
if (associrp == NULL)
return (NULL);
mtx_lock(&ntoskrnl_dispatchlock);
associrp->irp_flags |= IRP_ASSOCIATED_IRP;
associrp->irp_tail.irp_overlay.irp_thread =
ip->irp_tail.irp_overlay.irp_thread;
associrp->irp_assoc.irp_master = ip;
mtx_unlock(&ntoskrnl_dispatchlock);
return (associrp);
}
static void
IoFreeIrp(ip)
irp *ip;
{
ExFreePool(ip);
}
static void
IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
{
bzero((char *)io, IoSizeOfIrp(ssize));
io->irp_size = psize;
io->irp_stackcnt = ssize;
io->irp_currentstackloc = ssize;
InitializeListHead(&io->irp_thlist);
io->irp_tail.irp_overlay.irp_csl =
(io_stack_location *)(io + 1) + ssize;
}
static void
IoReuseIrp(ip, status)
irp *ip;
uint32_t status;
{
uint8_t allocflags;
allocflags = ip->irp_allocflags;
IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
ip->irp_iostat.isb_status = status;
ip->irp_allocflags = allocflags;
}
void
IoAcquireCancelSpinLock(uint8_t *irql)
{
KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
}
void
IoReleaseCancelSpinLock(uint8_t irql)
{
KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
}
uint8_t
IoCancelIrp(irp *ip)
{
cancel_func cfunc;
uint8_t cancelirql;
IoAcquireCancelSpinLock(&cancelirql);
cfunc = IoSetCancelRoutine(ip, NULL);
ip->irp_cancel = TRUE;
if (cfunc == NULL) {
IoReleaseCancelSpinLock(cancelirql);
return (FALSE);
}
ip->irp_cancelirql = cancelirql;
MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
return (uint8_t)IoSetCancelValue(ip, TRUE);
}
uint32_t
IofCallDriver(dobj, ip)
device_object *dobj;
irp *ip;
{
driver_object *drvobj;
io_stack_location *sl;
uint32_t status;
driver_dispatch disp;
drvobj = dobj->do_drvobj;
if (ip->irp_currentstackloc <= 0)
panic("IoCallDriver(): out of stack locations");
IoSetNextIrpStackLocation(ip);
sl = IoGetCurrentIrpStackLocation(ip);
sl->isl_devobj = dobj;
disp = drvobj->dro_dispatch[sl->isl_major];
status = MSCALL2(disp, dobj, ip);
return (status);
}
void
IofCompleteRequest(irp *ip, uint8_t prioboost)
{
uint32_t status;
device_object *dobj;
io_stack_location *sl;
completion_func cf;
KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
("incorrect IRP(%p) status (STATUS_PENDING)", ip));
sl = IoGetCurrentIrpStackLocation(ip);
IoSkipCurrentIrpStackLocation(ip);
do {
if (sl->isl_ctl & SL_PENDING_RETURNED)
ip->irp_pendingreturned = TRUE;
if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
else
dobj = NULL;
if (sl->isl_completionfunc != NULL &&
((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
(ip->irp_iostat.isb_status != STATUS_SUCCESS &&
sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
(ip->irp_cancel == TRUE &&
sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
cf = sl->isl_completionfunc;
status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
if (status == STATUS_MORE_PROCESSING_REQUIRED)
return;
} else {
if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
(ip->irp_pendingreturned == TRUE))
IoMarkIrpPending(ip);
}
/* move to the next. */
IoSkipCurrentIrpStackLocation(ip);
sl++;
} while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
if (ip->irp_usriostat != NULL)
*ip->irp_usriostat = ip->irp_iostat;
if (ip->irp_usrevent != NULL)
KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
/* Handle any associated IRPs. */
if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
uint32_t masterirpcnt;
irp *masterirp;
mdl *m;
masterirp = ip->irp_assoc.irp_master;
masterirpcnt =
InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
while ((m = ip->irp_mdl) != NULL) {
ip->irp_mdl = m->mdl_next;
IoFreeMdl(m);
}
IoFreeIrp(ip);
if (masterirpcnt == 0)
IoCompleteRequest(masterirp, IO_NO_INCREMENT);
return;
}
/* With any luck, these conditions will never arise. */
if (ip->irp_flags & IRP_PAGING_IO) {
if (ip->irp_mdl != NULL)
IoFreeMdl(ip->irp_mdl);
IoFreeIrp(ip);
}
}
void
ntoskrnl_intr(arg)
void *arg;
{
kinterrupt *iobj;
uint8_t irql;
uint8_t claimed;
list_entry *l;
KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
l = ntoskrnl_intlist.nle_flink;
while (l != &ntoskrnl_intlist) {
iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
if (claimed == TRUE)
break;
l = l->nle_flink;
}
KeReleaseSpinLock(&ntoskrnl_intlock, irql);
}
uint8_t
KeAcquireInterruptSpinLock(iobj)
kinterrupt *iobj;
{
uint8_t irql;
KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
return (irql);
}
void
KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
{
KeReleaseSpinLock(&ntoskrnl_intlock, irql);
}
uint8_t
KeSynchronizeExecution(iobj, syncfunc, syncctx)
kinterrupt *iobj;
void *syncfunc;
void *syncctx;
{
uint8_t irql;
KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
MSCALL1(syncfunc, syncctx);
KeReleaseSpinLock(&ntoskrnl_intlock, irql);
return (TRUE);
}
/*
* IoConnectInterrupt() is passed only the interrupt vector and
* irql that a device wants to use, but no device-specific tag
* of any kind. This conflicts rather badly with FreeBSD's
* bus_setup_intr(), which needs the device_t for the device
* requesting interrupt delivery. In order to bypass this
* inconsistency, we implement a second level of interrupt
* dispatching on top of bus_setup_intr(). All devices use
* ntoskrnl_intr() as their ISR, and any device requesting
* interrupts will be registered with ntoskrnl_intr()'s interrupt
* dispatch list. When an interrupt arrives, we walk the list
* and invoke all the registered ISRs. This effectively makes all
* interrupts shared, but it's the only way to duplicate the
* semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
*/
uint32_t
IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
{
uint8_t curirql;
*iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
if (*iobj == NULL)
return (STATUS_INSUFFICIENT_RESOURCES);
(*iobj)->ki_svcfunc = svcfunc;
(*iobj)->ki_svcctx = svcctx;
if (lock == NULL) {
KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
(*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
} else
(*iobj)->ki_lock = lock;
KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
return (STATUS_SUCCESS);
}
void
IoDisconnectInterrupt(iobj)
kinterrupt *iobj;
{
uint8_t irql;
if (iobj == NULL)
return;
KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
RemoveEntryList((&iobj->ki_list));
KeReleaseSpinLock(&ntoskrnl_intlock, irql);
ExFreePool(iobj);
}
device_object *
IoAttachDeviceToDeviceStack(src, dst)
device_object *src;
device_object *dst;
{
device_object *attached;
mtx_lock(&ntoskrnl_dispatchlock);
attached = IoGetAttachedDevice(dst);
attached->do_attacheddev = src;
src->do_attacheddev = NULL;
src->do_stacksize = attached->do_stacksize + 1;
mtx_unlock(&ntoskrnl_dispatchlock);
return (attached);
}
void
IoDetachDevice(topdev)
device_object *topdev;
{
device_object *tail;
mtx_lock(&ntoskrnl_dispatchlock);
/* First, break the chain. */
tail = topdev->do_attacheddev;
if (tail == NULL) {
mtx_unlock(&ntoskrnl_dispatchlock);
return;
}
topdev->do_attacheddev = tail->do_attacheddev;
topdev->do_refcnt--;
/* Now reduce the stacksize count for the takm_il objects. */
tail = topdev->do_attacheddev;
while (tail != NULL) {
tail->do_stacksize--;
tail = tail->do_attacheddev;
}
mtx_unlock(&ntoskrnl_dispatchlock);
}
/*
* For the most part, an object is considered signalled if
* dh_sigstate == TRUE. The exception is for mutant objects
* (mutexes), where the logic works like this:
*
* - If the thread already owns the object and sigstate is
* less than or equal to 0, then the object is considered
* signalled (recursive acquisition).
* - If dh_sigstate == 1, the object is also considered
* signalled.
*/
static int
ntoskrnl_is_signalled(obj, td)
nt_dispatch_header *obj;
struct thread *td;
{
kmutant *km;
if (obj->dh_type == DISP_TYPE_MUTANT) {
km = (kmutant *)obj;
if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
obj->dh_sigstate == 1)
return (TRUE);
return (FALSE);
}
if (obj->dh_sigstate > 0)
return (TRUE);
return (FALSE);
}
static void
ntoskrnl_satisfy_wait(obj, td)
nt_dispatch_header *obj;
struct thread *td;
{
kmutant *km;
switch (obj->dh_type) {
case DISP_TYPE_MUTANT:
km = (struct kmutant *)obj;
obj->dh_sigstate--;
/*
* If sigstate reaches 0, the mutex is now
* non-signalled (the new thread owns it).
*/
if (obj->dh_sigstate == 0) {
km->km_ownerthread = td;
if (km->km_abandoned == TRUE)
km->km_abandoned = FALSE;
}
break;
/* Synchronization objects get reset to unsignalled. */
case DISP_TYPE_SYNCHRONIZATION_EVENT:
case DISP_TYPE_SYNCHRONIZATION_TIMER:
obj->dh_sigstate = 0;
break;
case DISP_TYPE_SEMAPHORE:
obj->dh_sigstate--;
break;
default:
break;
}
}
static void
ntoskrnl_satisfy_multiple_waits(wb)
wait_block *wb;
{
wait_block *cur;
struct thread *td;
cur = wb;
td = wb->wb_kthread;
do {
ntoskrnl_satisfy_wait(wb->wb_object, td);
cur->wb_awakened = TRUE;
cur = cur->wb_next;
} while (cur != wb);
}
/* Always called with dispatcher lock held. */
static void
ntoskrnl_waittest(obj, increment)
nt_dispatch_header *obj;
uint32_t increment;
{
wait_block *w, *next;
list_entry *e;
struct thread *td;
wb_ext *we;
int satisfied;
/*
* Once an object has been signalled, we walk its list of
* wait blocks. If a wait block can be awakened, then satisfy
* waits as necessary and wake the thread.
*
* The rules work like this:
*
* If a wait block is marked as WAITTYPE_ANY, then
* we can satisfy the wait conditions on the current
* object and wake the thread right away. Satisfying
* the wait also has the effect of breaking us out
* of the search loop.
*
* If the object is marked as WAITTYLE_ALL, then the
* wait block will be part of a circularly linked
* list of wait blocks belonging to a waiting thread
* that's sleeping in KeWaitForMultipleObjects(). In
* order to wake the thread, all the objects in the
* wait list must be in the signalled state. If they
* are, we then satisfy all of them and wake the
* thread.
*
*/
e = obj->dh_waitlisthead.nle_flink;
while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
we = w->wb_ext;
td = we->we_td;
satisfied = FALSE;
if (w->wb_waittype == WAITTYPE_ANY) {
/*
* Thread can be awakened if
* any wait is satisfied.
*/
ntoskrnl_satisfy_wait(obj, td);
satisfied = TRUE;
w->wb_awakened = TRUE;
} else {
/*
* Thread can only be woken up
* if all waits are satisfied.
* If the thread is waiting on multiple
* objects, they should all be linked
* through the wb_next pointers in the
* wait blocks.
*/
satisfied = TRUE;
next = w->wb_next;
while (next != w) {
if (ntoskrnl_is_signalled(obj, td) == FALSE) {
satisfied = FALSE;
break;
}
next = next->wb_next;
}
ntoskrnl_satisfy_multiple_waits(w);
}
if (satisfied == TRUE)
cv_broadcastpri(&we->we_cv,
(w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
e = e->nle_flink;
}
}
/*
* Return the number of 100 nanosecond intervals since
* January 1, 1601. (?!?!)
*/
void
ntoskrnl_time(tval)
uint64_t *tval;
{
struct timespec ts;
nanotime(&ts);
*tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
}
static void
KeQuerySystemTime(current_time)
uint64_t *current_time;
{
ntoskrnl_time(current_time);
}
static uint32_t
KeTickCount(void)
{
struct timeval tv;
getmicrouptime(&tv);
return tvtohz(&tv);
}
/*
* KeWaitForSingleObject() is a tricky beast, because it can be used
* with several different object types: semaphores, timers, events,
* mutexes and threads. Semaphores don't appear very often, but the
* other object types are quite common. KeWaitForSingleObject() is
* what's normally used to acquire a mutex, and it can be used to
* wait for a thread termination.
*
* The Windows NDIS API is implemented in terms of Windows kernel
* primitives, and some of the object manipulation is duplicated in
* NDIS. For example, NDIS has timers and events, which are actually
* Windows kevents and ktimers. Now, you're supposed to only use the
* NDIS variants of these objects within the confines of the NDIS API,
* but there are some naughty developers out there who will use
* KeWaitForSingleObject() on NDIS timer and event objects, so we
* have to support that as well. Conseqently, our NDIS timer and event
* code has to be closely tied into our ntoskrnl timer and event code,
* just as it is in Windows.
*
* KeWaitForSingleObject() may do different things for different kinds
* of objects:
*
* - For events, we check if the event has been signalled. If the
* event is already in the signalled state, we just return immediately,
* otherwise we wait for it to be set to the signalled state by someone
* else calling KeSetEvent(). Events can be either synchronization or
* notification events.
*
* - For timers, if the timer has already fired and the timer is in
* the signalled state, we just return, otherwise we wait on the
* timer. Unlike an event, timers get signalled automatically when
* they expire rather than someone having to trip them manually.
* Timers initialized with KeInitializeTimer() are always notification
* events: KeInitializeTimerEx() lets you initialize a timer as
* either a notification or synchronization event.
*
* - For mutexes, we try to acquire the mutex and if we can't, we wait
* on the mutex until it's available and then grab it. When a mutex is
* released, it enters the signalled state, which wakes up one of the
* threads waiting to acquire it. Mutexes are always synchronization
* events.
*
* - For threads, the only thing we do is wait until the thread object
* enters a signalled state, which occurs when the thread terminates.
* Threads are always notification events.
*
* A notification event wakes up all threads waiting on an object. A
* synchronization event wakes up just one. Also, a synchronization event
* is auto-clearing, which means we automatically set the event back to
* the non-signalled state once the wakeup is done.
*/
uint32_t
KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
uint8_t alertable, int64_t *duetime)
{
wait_block w;
struct thread *td = curthread;
struct timeval tv;
int error = 0;
uint64_t curtime;
wb_ext we;
nt_dispatch_header *obj;
obj = arg;
if (obj == NULL)
return (STATUS_INVALID_PARAMETER);
mtx_lock(&ntoskrnl_dispatchlock);
cv_init(&we.we_cv, "KeWFS");
we.we_td = td;
/*
* Check to see if this object is already signalled,
* and just return without waiting if it is.
*/
if (ntoskrnl_is_signalled(obj, td) == TRUE) {
/* Sanity check the signal state value. */
if (obj->dh_sigstate != INT32_MIN) {
ntoskrnl_satisfy_wait(obj, curthread);
mtx_unlock(&ntoskrnl_dispatchlock);
return (STATUS_SUCCESS);
} else {
/*
* There's a limit to how many times we can
* recursively acquire a mutant. If we hit
* the limit, something is very wrong.
*/
if (obj->dh_type == DISP_TYPE_MUTANT) {
mtx_unlock(&ntoskrnl_dispatchlock);
panic("mutant limit exceeded");
}
}
}
bzero((char *)&w, sizeof(wait_block));
w.wb_object = obj;
w.wb_ext = &we;
w.wb_waittype = WAITTYPE_ANY;
w.wb_next = &w;
w.wb_waitkey = 0;
w.wb_awakened = FALSE;
w.wb_oldpri = td->td_priority;
InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
/*
* The timeout value is specified in 100 nanosecond units
* and can be a positive or negative number. If it's positive,
* then the duetime is absolute, and we need to convert it
* to an absolute offset relative to now in order to use it.
* If it's negative, then the duetime is relative and we
* just have to convert the units.
*/
if (duetime != NULL) {
if (*duetime < 0) {
tv.tv_sec = - (*duetime) / 10000000;
tv.tv_usec = (- (*duetime) / 10) -
(tv.tv_sec * 1000000);
} else {
ntoskrnl_time(&curtime);
if (*duetime < curtime)
tv.tv_sec = tv.tv_usec = 0;
else {
tv.tv_sec = ((*duetime) - curtime) / 10000000;
tv.tv_usec = ((*duetime) - curtime) / 10 -
(tv.tv_sec * 1000000);
}
}
}
if (duetime == NULL)
cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
else
error = cv_timedwait(&we.we_cv,
&ntoskrnl_dispatchlock, tvtohz(&tv));
RemoveEntryList(&w.wb_waitlist);
cv_destroy(&we.we_cv);
/* We timed out. Leave the object alone and return status. */
if (error == EWOULDBLOCK) {
mtx_unlock(&ntoskrnl_dispatchlock);
return (STATUS_TIMEOUT);
}
mtx_unlock(&ntoskrnl_dispatchlock);
return (STATUS_SUCCESS);
/*
return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
mode, alertable, duetime, &w));
*/
}
static uint32_t
KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
wait_block *wb_array)
{
struct thread *td = curthread;
wait_block *whead, *w;
wait_block _wb_array[MAX_WAIT_OBJECTS];
nt_dispatch_header *cur;
struct timeval tv;
int i, wcnt = 0, error = 0;
uint64_t curtime;
struct timespec t1, t2;
uint32_t status = STATUS_SUCCESS;
wb_ext we;
if (cnt > MAX_WAIT_OBJECTS)
return (STATUS_INVALID_PARAMETER);
if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
return (STATUS_INVALID_PARAMETER);
mtx_lock(&ntoskrnl_dispatchlock);
cv_init(&we.we_cv, "KeWFM");
we.we_td = td;
if (wb_array == NULL)
whead = _wb_array;
else
whead = wb_array;
bzero((char *)whead, sizeof(wait_block) * cnt);
/* First pass: see if we can satisfy any waits immediately. */
wcnt = 0;
w = whead;
for (i = 0; i < cnt; i++) {
InsertTailList((&obj[i]->dh_waitlisthead),
(&w->wb_waitlist));
w->wb_ext = &we;
w->wb_object = obj[i];
w->wb_waittype = wtype;
w->wb_waitkey = i;
w->wb_awakened = FALSE;
w->wb_oldpri = td->td_priority;
w->wb_next = w + 1;
w++;
wcnt++;
if (ntoskrnl_is_signalled(obj[i], td)) {
/*
* There's a limit to how many times
* we can recursively acquire a mutant.
* If we hit the limit, something
* is very wrong.
*/
if (obj[i]->dh_sigstate == INT32_MIN &&
obj[i]->dh_type == DISP_TYPE_MUTANT) {
mtx_unlock(&ntoskrnl_dispatchlock);
panic("mutant limit exceeded");
}
/*
* If this is a WAITTYPE_ANY wait, then
* satisfy the waited object and exit
* right now.
*/
if (wtype == WAITTYPE_ANY) {
ntoskrnl_satisfy_wait(obj[i], td);
status = STATUS_WAIT_0 + i;
goto wait_done;
} else {
w--;
wcnt--;
w->wb_object = NULL;
RemoveEntryList(&w->wb_waitlist);
}
}
}
/*
* If this is a WAITTYPE_ALL wait and all objects are
* already signalled, satisfy the waits and exit now.
*/
if (wtype == WAITTYPE_ALL && wcnt == 0) {
for (i = 0; i < cnt; i++)
ntoskrnl_satisfy_wait(obj[i], td);
status = STATUS_SUCCESS;
goto wait_done;
}
/*
* Create a circular waitblock list. The waitcount
* must always be non-zero when we get here.
*/
(w - 1)->wb_next = whead;
/* Wait on any objects that aren't yet signalled. */
/* Calculate timeout, if any. */
if (duetime != NULL) {
if (*duetime < 0) {
tv.tv_sec = - (*duetime) / 10000000;
tv.tv_usec = (- (*duetime) / 10) -
(tv.tv_sec * 1000000);
} else {
ntoskrnl_time(&curtime);
if (*duetime < curtime)
tv.tv_sec = tv.tv_usec = 0;
else {
tv.tv_sec = ((*duetime) - curtime) / 10000000;
tv.tv_usec = ((*duetime) - curtime) / 10 -
(tv.tv_sec * 1000000);
}
}
}
while (wcnt) {
nanotime(&t1);
if (duetime == NULL)
cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
else
error = cv_timedwait(&we.we_cv,
&ntoskrnl_dispatchlock, tvtohz(&tv));
/* Wait with timeout expired. */
if (error) {
status = STATUS_TIMEOUT;
goto wait_done;
}
nanotime(&t2);
/* See what's been signalled. */
w = whead;
do {
cur = w->wb_object;
if (ntoskrnl_is_signalled(cur, td) == TRUE ||
w->wb_awakened == TRUE) {
/* Sanity check the signal state value. */
if (cur->dh_sigstate == INT32_MIN &&
cur->dh_type == DISP_TYPE_MUTANT) {
mtx_unlock(&ntoskrnl_dispatchlock);
panic("mutant limit exceeded");
}
wcnt--;
if (wtype == WAITTYPE_ANY) {
status = w->wb_waitkey &
STATUS_WAIT_0;
goto wait_done;
}
}
w = w->wb_next;
} while (w != whead);
/*
* If all objects have been signalled, or if this
* is a WAITTYPE_ANY wait and we were woke up by
* someone, we can bail.
*/
if (wcnt == 0) {
status = STATUS_SUCCESS;
goto wait_done;
}
/*
* If this is WAITTYPE_ALL wait, and there's still
* objects that haven't been signalled, deduct the
* time that's elapsed so far from the timeout and
* wait again (or continue waiting indefinitely if
* there's no timeout).
*/
if (duetime != NULL) {
tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
}
}
wait_done:
cv_destroy(&we.we_cv);
for (i = 0; i < cnt; i++) {
if (whead[i].wb_object != NULL)
RemoveEntryList(&whead[i].wb_waitlist);
}
mtx_unlock(&ntoskrnl_dispatchlock);
return (status);
}
static void
WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
{
bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
}
static uint16_t
READ_REGISTER_USHORT(reg)
uint16_t *reg;
{
return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
}
static void
WRITE_REGISTER_ULONG(reg, val)
uint32_t *reg;
uint32_t val;
{
bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
}
static uint32_t
READ_REGISTER_ULONG(reg)
uint32_t *reg;
{
return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
}
static uint8_t
READ_REGISTER_UCHAR(uint8_t *reg)
{
return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
}
static void
WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
{
bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
}
static int64_t
_allmul(a, b)
int64_t a;
int64_t b;
{
return (a * b);
}
static int64_t
_alldiv(a, b)
int64_t a;
int64_t b;
{
return (a / b);
}
static int64_t
_allrem(a, b)
int64_t a;
int64_t b;
{
return (a % b);
}
static uint64_t
_aullmul(a, b)
uint64_t a;
uint64_t b;
{
return (a * b);
}
static uint64_t
_aulldiv(a, b)
uint64_t a;
uint64_t b;
{
return (a / b);
}
static uint64_t
_aullrem(a, b)
uint64_t a;
uint64_t b;
{
return (a % b);
}
static int64_t
_allshl(int64_t a, uint8_t b)
{
return (a << b);
}
static uint64_t
_aullshl(uint64_t a, uint8_t b)
{
return (a << b);
}
static int64_t
_allshr(int64_t a, uint8_t b)
{
return (a >> b);
}
static uint64_t
_aullshr(uint64_t a, uint8_t b)
{
return (a >> b);
}
static slist_entry *
ntoskrnl_pushsl(head, entry)
slist_header *head;
slist_entry *entry;
{
slist_entry *oldhead;
oldhead = head->slh_list.slh_next;
entry->sl_next = head->slh_list.slh_next;
head->slh_list.slh_next = entry;
head->slh_list.slh_depth++;
head->slh_list.slh_seq++;
return (oldhead);
}
static void
InitializeSListHead(head)
slist_header *head;
{
memset(head, 0, sizeof(*head));
}
static slist_entry *
ntoskrnl_popsl(head)
slist_header *head;
{
slist_entry *first;
first = head->slh_list.slh_next;
if (first != NULL) {
head->slh_list.slh_next = first->sl_next;
head->slh_list.slh_depth--;
head->slh_list.slh_seq++;
}
return (first);
}
/*
* We need this to make lookaside lists work for amd64.
* We pass a pointer to ExAllocatePoolWithTag() the lookaside
* list structure. For amd64 to work right, this has to be a
* pointer to the wrapped version of the routine, not the
* original. Letting the Windows driver invoke the original
* function directly will result in a convention calling
* mismatch and a pretty crash. On x86, this effectively
* becomes a no-op since ipt_func and ipt_wrap are the same.
*/
static funcptr
ntoskrnl_findwrap(func)
funcptr func;
{
image_patch_table *patch;
patch = ntoskrnl_functbl;
while (patch->ipt_func != NULL) {
if ((funcptr)patch->ipt_func == func)
return ((funcptr)patch->ipt_wrap);
patch++;
}
return (NULL);
}
static void
ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
{
bzero((char *)lookaside, sizeof(paged_lookaside_list));
if (size < sizeof(slist_entry))
lookaside->nll_l.gl_size = sizeof(slist_entry);
else
lookaside->nll_l.gl_size = size;
lookaside->nll_l.gl_tag = tag;
if (allocfunc == NULL)
lookaside->nll_l.gl_allocfunc =
ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
else
lookaside->nll_l.gl_allocfunc = allocfunc;
if (freefunc == NULL)
lookaside->nll_l.gl_freefunc =
ntoskrnl_findwrap((funcptr)ExFreePool);
else
lookaside->nll_l.gl_freefunc = freefunc;
#ifdef __i386__
KeInitializeSpinLock(&lookaside->nll_obsoletelock);
#endif
lookaside->nll_l.gl_type = NonPagedPool;
lookaside->nll_l.gl_depth = depth;
lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
}
static void
ExDeletePagedLookasideList(lookaside)
paged_lookaside_list *lookaside;
{
void *buf;
void (*freefunc)(void *);
freefunc = lookaside->nll_l.gl_freefunc;
while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
MSCALL1(freefunc, buf);
}
static void
ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
{
bzero((char *)lookaside, sizeof(npaged_lookaside_list));
if (size < sizeof(slist_entry))
lookaside->nll_l.gl_size = sizeof(slist_entry);
else
lookaside->nll_l.gl_size = size;
lookaside->nll_l.gl_tag = tag;
if (allocfunc == NULL)
lookaside->nll_l.gl_allocfunc =
ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
else
lookaside->nll_l.gl_allocfunc = allocfunc;
if (freefunc == NULL)
lookaside->nll_l.gl_freefunc =
ntoskrnl_findwrap((funcptr)ExFreePool);
else
lookaside->nll_l.gl_freefunc = freefunc;
#ifdef __i386__
KeInitializeSpinLock(&lookaside->nll_obsoletelock);
#endif
lookaside->nll_l.gl_type = NonPagedPool;
lookaside->nll_l.gl_depth = depth;
lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
}
static void
ExDeleteNPagedLookasideList(lookaside)
npaged_lookaside_list *lookaside;
{
void *buf;
void (*freefunc)(void *);
freefunc = lookaside->nll_l.gl_freefunc;
while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
MSCALL1(freefunc, buf);
}
slist_entry *
InterlockedPushEntrySList(head, entry)
slist_header *head;
slist_entry *entry;
{
slist_entry *oldhead;
mtx_lock_spin(&ntoskrnl_interlock);
oldhead = ntoskrnl_pushsl(head, entry);
mtx_unlock_spin(&ntoskrnl_interlock);
return (oldhead);
}
slist_entry *
InterlockedPopEntrySList(head)
slist_header *head;
{
slist_entry *first;
mtx_lock_spin(&ntoskrnl_interlock);
first = ntoskrnl_popsl(head);
mtx_unlock_spin(&ntoskrnl_interlock);
return (first);
}
static slist_entry *
ExInterlockedPushEntrySList(head, entry, lock)
slist_header *head;
slist_entry *entry;
kspin_lock *lock;
{
return (InterlockedPushEntrySList(head, entry));
}
static slist_entry *
ExInterlockedPopEntrySList(head, lock)
slist_header *head;
kspin_lock *lock;
{
return (InterlockedPopEntrySList(head));
}
uint16_t
ExQueryDepthSList(head)
slist_header *head;
{
uint16_t depth;
mtx_lock_spin(&ntoskrnl_interlock);
depth = head->slh_list.slh_depth;
mtx_unlock_spin(&ntoskrnl_interlock);
return (depth);
}
void
KeInitializeSpinLock(lock)
kspin_lock *lock;
{
*lock = 0;
}
#ifdef __i386__
void
KefAcquireSpinLockAtDpcLevel(lock)
kspin_lock *lock;
{
#ifdef NTOSKRNL_DEBUG_SPINLOCKS
int i = 0;
#endif
while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
/* sit and spin */;
#ifdef NTOSKRNL_DEBUG_SPINLOCKS
i++;
if (i > 200000000)
panic("DEADLOCK!");
#endif
}
}
void
KefReleaseSpinLockFromDpcLevel(lock)
kspin_lock *lock;
{
atomic_store_rel_int((volatile u_int *)lock, 0);
}
uint8_t
KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
{
uint8_t oldirql;
if (KeGetCurrentIrql() > DISPATCH_LEVEL)
panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
KeAcquireSpinLockAtDpcLevel(lock);
return (oldirql);
}
#else
void
KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
{
while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
/* sit and spin */;
}
void
KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
{
atomic_store_rel_int((volatile u_int *)lock, 0);
}
#endif /* __i386__ */
uintptr_t
InterlockedExchange(dst, val)
volatile uint32_t *dst;
uintptr_t val;
{
uintptr_t r;
mtx_lock_spin(&ntoskrnl_interlock);
r = *dst;
*dst = val;
mtx_unlock_spin(&ntoskrnl_interlock);
return (r);
}
static uint32_t
InterlockedIncrement(addend)
volatile uint32_t *addend;
{
atomic_add_long((volatile u_long *)addend, 1);
return (*addend);
}
static uint32_t
InterlockedDecrement(addend)
volatile uint32_t *addend;
{
atomic_subtract_long((volatile u_long *)addend, 1);
return (*addend);
}
static void
ExInterlockedAddLargeStatistic(addend, inc)
uint64_t *addend;
uint32_t inc;
{
mtx_lock_spin(&ntoskrnl_interlock);
*addend += inc;
mtx_unlock_spin(&ntoskrnl_interlock);
};
mdl *
IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
uint8_t chargequota, irp *iopkt)
{
mdl *m;
int zone = 0;
if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
m = ExAllocatePoolWithTag(NonPagedPool,
MmSizeOfMdl(vaddr, len), 0);
else {
m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
zone++;
}
if (m == NULL)
return (NULL);
MmInitializeMdl(m, vaddr, len);
/*
* MmInitializMdl() clears the flags field, so we
* have to set this here. If the MDL came from the
* MDL UMA zone, tag it so we can release it to
* the right place later.
*/
if (zone)
m->mdl_flags = MDL_ZONE_ALLOCED;
if (iopkt != NULL) {
if (secondarybuf == TRUE) {
mdl *last;
last = iopkt->irp_mdl;
while (last->mdl_next != NULL)
last = last->mdl_next;
last->mdl_next = m;
} else {
if (iopkt->irp_mdl != NULL)
panic("leaking an MDL in IoAllocateMdl()");
iopkt->irp_mdl = m;
}
}
return (m);
}
void
IoFreeMdl(m)
mdl *m;
{
if (m == NULL)
return;
if (m->mdl_flags & MDL_ZONE_ALLOCED)
uma_zfree(mdl_zone, m);
else
ExFreePool(m);
}
static void *
MmAllocateContiguousMemory(size, highest)
uint32_t size;
uint64_t highest;
{
void *addr;
size_t pagelength = roundup(size, PAGE_SIZE);
addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
return (addr);
}
static void *
MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
boundary, cachetype)
uint32_t size;
uint64_t lowest;
uint64_t highest;
uint64_t boundary;
enum nt_caching_type cachetype;
{
vm_memattr_t memattr;
void *ret;
switch (cachetype) {
case MmNonCached:
memattr = VM_MEMATTR_UNCACHEABLE;
break;
case MmWriteCombined:
memattr = VM_MEMATTR_WRITE_COMBINING;
break;
case MmNonCachedUnordered:
memattr = VM_MEMATTR_UNCACHEABLE;
break;
case MmCached:
case MmHardwareCoherentCached:
case MmUSWCCached:
default:
memattr = VM_MEMATTR_DEFAULT;
break;
}
ret = (void *)kmem_alloc_contig(kernel_arena, size, M_ZERO | M_NOWAIT,
lowest, highest, PAGE_SIZE, boundary, memattr);
if (ret != NULL)
malloc_type_allocated(M_DEVBUF, round_page(size));
return (ret);
}
static void
MmFreeContiguousMemory(base)
void *base;
{
ExFreePool(base);
}
static void
MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
void *base;
uint32_t size;
enum nt_caching_type cachetype;
{
contigfree(base, size, M_DEVBUF);
}
static uint32_t
MmSizeOfMdl(vaddr, len)
void *vaddr;
size_t len;
{
uint32_t l;
l = sizeof(struct mdl) +
(sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
return (l);
}
/*
* The Microsoft documentation says this routine fills in the
* page array of an MDL with the _physical_ page addresses that
* comprise the buffer, but we don't really want to do that here.
* Instead, we just fill in the page array with the kernel virtual
* addresses of the buffers.
*/
void
MmBuildMdlForNonPagedPool(m)
mdl *m;
{
vm_offset_t *mdl_pages;
int pagecnt, i;
pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
panic("not enough pages in MDL to describe buffer");
mdl_pages = MmGetMdlPfnArray(m);
for (i = 0; i < pagecnt; i++)
*mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
}
static void *
MmMapLockedPages(mdl *buf, uint8_t accessmode)
{
buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
return (MmGetMdlVirtualAddress(buf));
}
static void *
MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
void *vaddr, uint32_t bugcheck, uint32_t prio)
{
return (MmMapLockedPages(buf, accessmode));
}
static void
MmUnmapLockedPages(vaddr, buf)
void *vaddr;
mdl *buf;
{
buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
}
/*
* This function has a problem in that it will break if you
* compile this module without PAE and try to use it on a PAE
* kernel. Unfortunately, there's no way around this at the
* moment. It's slightly less broken that using pmap_kextract().
* You'd think the virtual memory subsystem would help us out
* here, but it doesn't.
*/
static uint64_t
MmGetPhysicalAddress(void *base)
{
return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
}
void *
MmGetSystemRoutineAddress(ustr)
unicode_string *ustr;
{
ansi_string astr;
if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE))
return (NULL);
return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf));
}
uint8_t
MmIsAddressValid(vaddr)
void *vaddr;
{
if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
return (TRUE);
return (FALSE);
}
void *
MmMapIoSpace(paddr, len, cachetype)
uint64_t paddr;
uint32_t len;
uint32_t cachetype;
{
devclass_t nexus_class;
device_t *nexus_devs, devp;
int nexus_count = 0;
device_t matching_dev = NULL;
struct resource *res;
int i;
vm_offset_t v;
/* There will always be at least one nexus. */
nexus_class = devclass_find("nexus");
devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
for (i = 0; i < nexus_count; i++) {
devp = nexus_devs[i];
matching_dev = ntoskrnl_finddev(devp, paddr, &res);
if (matching_dev)
break;
}
free(nexus_devs, M_TEMP);
if (matching_dev == NULL)
return (NULL);
v = (vm_offset_t)rman_get_virtual(res);
if (paddr > rman_get_start(res))
v += paddr - rman_get_start(res);
return ((void *)v);
}
void
MmUnmapIoSpace(vaddr, len)
void *vaddr;
size_t len;
{
}
static device_t
ntoskrnl_finddev(dev, paddr, res)
device_t dev;
uint64_t paddr;
struct resource **res;
{
device_t *children = NULL;
device_t matching_dev;
int childcnt;
struct resource *r;
struct resource_list *rl;
struct resource_list_entry *rle;
uint32_t flags;
int i;
/* We only want devices that have been successfully probed. */
if (device_is_alive(dev) == FALSE)
return (NULL);
rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
if (rl != NULL) {
STAILQ_FOREACH(rle, rl, link) {
r = rle->res;
if (r == NULL)
continue;
flags = rman_get_flags(r);
if (rle->type == SYS_RES_MEMORY &&
paddr >= rman_get_start(r) &&
paddr <= rman_get_end(r)) {
if (!(flags & RF_ACTIVE))
bus_activate_resource(dev,
SYS_RES_MEMORY, 0, r);
*res = r;
return (dev);
}
}
}
/*
* If this device has children, do another
* level of recursion to inspect them.
*/
device_get_children(dev, &children, &childcnt);
for (i = 0; i < childcnt; i++) {
matching_dev = ntoskrnl_finddev(children[i], paddr, res);
if (matching_dev != NULL) {
free(children, M_TEMP);
return (matching_dev);
}
}
/* Won't somebody please think of the children! */
if (children != NULL)
free(children, M_TEMP);
return (NULL);
}
/*
* Workitems are unlike DPCs, in that they run in a user-mode thread
* context rather than at DISPATCH_LEVEL in kernel context. In our
* case we run them in kernel context anyway.
*/
static void
ntoskrnl_workitem_thread(arg)
void *arg;
{
kdpc_queue *kq;
list_entry *l;
io_workitem *iw;
uint8_t irql;
kq = arg;
InitializeListHead(&kq->kq_disp);
kq->kq_td = curthread;
kq->kq_exit = 0;
KeInitializeSpinLock(&kq->kq_lock);
KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
while (1) {
KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
KeAcquireSpinLock(&kq->kq_lock, &irql);
if (kq->kq_exit) {
kq->kq_exit = 0;
KeReleaseSpinLock(&kq->kq_lock, irql);
break;
}
while (!IsListEmpty(&kq->kq_disp)) {
l = RemoveHeadList(&kq->kq_disp);
iw = CONTAINING_RECORD(l,
io_workitem, iw_listentry);
InitializeListHead((&iw->iw_listentry));
if (iw->iw_func == NULL)
continue;
KeReleaseSpinLock(&kq->kq_lock, irql);
MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
KeAcquireSpinLock(&kq->kq_lock, &irql);
}
KeReleaseSpinLock(&kq->kq_lock, irql);
}
kproc_exit(0);
return; /* notreached */
}
static ndis_status
RtlCharToInteger(src, base, val)
const char *src;
uint32_t base;
uint32_t *val;
{
int negative = 0;
uint32_t res;
if (!src || !val)
return (STATUS_ACCESS_VIOLATION);
while (*src != '\0' && *src <= ' ')
src++;
if (*src == '+')
src++;
else if (*src == '-') {
src++;
negative = 1;
}
if (base == 0) {
base = 10;
if (*src == '0') {
src++;
if (*src == 'b') {
base = 2;
src++;
} else if (*src == 'o') {
base = 8;
src++;
} else if (*src == 'x') {
base = 16;
src++;
}
}
} else if (!(base == 2 || base == 8 || base == 10 || base == 16))
return (STATUS_INVALID_PARAMETER);
for (res = 0; *src; src++) {
int v;
if (isdigit(*src))
v = *src - '0';
else if (isxdigit(*src))
v = tolower(*src) - 'a' + 10;
else
v = base;
if (v >= base)
return (STATUS_INVALID_PARAMETER);
res = res * base + v;
}
*val = negative ? -res : res;
return (STATUS_SUCCESS);
}
static void
ntoskrnl_destroy_workitem_threads(void)
{
kdpc_queue *kq;
int i;
for (i = 0; i < WORKITEM_THREADS; i++) {
kq = wq_queues + i;
kq->kq_exit = 1;
KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
while (kq->kq_exit)
tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
}
}
io_workitem *
IoAllocateWorkItem(dobj)
device_object *dobj;
{
io_workitem *iw;
iw = uma_zalloc(iw_zone, M_NOWAIT);
if (iw == NULL)
return (NULL);
InitializeListHead(&iw->iw_listentry);
iw->iw_dobj = dobj;
mtx_lock(&ntoskrnl_dispatchlock);
iw->iw_idx = wq_idx;
WORKIDX_INC(wq_idx);
mtx_unlock(&ntoskrnl_dispatchlock);
return (iw);
}
void
IoFreeWorkItem(iw)
io_workitem *iw;
{
uma_zfree(iw_zone, iw);
}
void
IoQueueWorkItem(iw, iw_func, qtype, ctx)
io_workitem *iw;
io_workitem_func iw_func;
uint32_t qtype;
void *ctx;
{
kdpc_queue *kq;
list_entry *l;
io_workitem *cur;
uint8_t irql;
kq = wq_queues + iw->iw_idx;
KeAcquireSpinLock(&kq->kq_lock, &irql);
/*
* Traverse the list and make sure this workitem hasn't
* already been inserted. Queuing the same workitem
* twice will hose the list but good.
*/
l = kq->kq_disp.nle_flink;
while (l != &kq->kq_disp) {
cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
if (cur == iw) {
/* Already queued -- do nothing. */
KeReleaseSpinLock(&kq->kq_lock, irql);
return;
}
l = l->nle_flink;
}
iw->iw_func = iw_func;
iw->iw_ctx = ctx;
InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
KeReleaseSpinLock(&kq->kq_lock, irql);
KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
}
static void
ntoskrnl_workitem(dobj, arg)
device_object *dobj;
void *arg;
{
io_workitem *iw;
work_queue_item *w;
work_item_func f;
iw = arg;
w = (work_queue_item *)dobj;
f = (work_item_func)w->wqi_func;
uma_zfree(iw_zone, iw);
MSCALL2(f, w, w->wqi_ctx);
}
/*
* The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
* warns that it's unsafe and to use IoQueueWorkItem() instead. The
* problem with ExQueueWorkItem() is that it can't guard against
* the condition where a driver submits a job to the work queue and
* is then unloaded before the job is able to run. IoQueueWorkItem()
* acquires a reference to the device's device_object via the
* object manager and retains it until after the job has completed,
* which prevents the driver from being unloaded before the job
* runs. (We don't currently support this behavior, though hopefully
* that will change once the object manager API is fleshed out a bit.)
*
* Having said all that, the ExQueueWorkItem() API remains, because
* there are still other parts of Windows that use it, including
* NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
* We fake up the ExQueueWorkItem() API on top of our implementation
* of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
* for ExQueueWorkItem() jobs, and we pass a pointer to the work
* queue item (provided by the caller) in to IoAllocateWorkItem()
* instead of the device_object. We need to save this pointer so
* we can apply a sanity check: as with the DPC queue and other
* workitem queues, we can't allow the same work queue item to
* be queued twice. If it's already pending, we silently return
*/
void
ExQueueWorkItem(w, qtype)
work_queue_item *w;
uint32_t qtype;
{
io_workitem *iw;
io_workitem_func iwf;
kdpc_queue *kq;
list_entry *l;
io_workitem *cur;
uint8_t irql;
/*
* We need to do a special sanity test to make sure
* the ExQueueWorkItem() API isn't used to queue
* the same workitem twice. Rather than checking the
* io_workitem pointer itself, we test the attached
* device object, which is really a pointer to the
* legacy work queue item structure.
*/
kq = wq_queues + WORKITEM_LEGACY_THREAD;
KeAcquireSpinLock(&kq->kq_lock, &irql);
l = kq->kq_disp.nle_flink;
while (l != &kq->kq_disp) {
cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
if (cur->iw_dobj == (device_object *)w) {
/* Already queued -- do nothing. */
KeReleaseSpinLock(&kq->kq_lock, irql);
return;
}
l = l->nle_flink;
}
KeReleaseSpinLock(&kq->kq_lock, irql);
iw = IoAllocateWorkItem((device_object *)w);
if (iw == NULL)
return;
iw->iw_idx = WORKITEM_LEGACY_THREAD;
iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
IoQueueWorkItem(iw, iwf, qtype, iw);
}
static void
RtlZeroMemory(dst, len)
void *dst;
size_t len;
{
bzero(dst, len);
}
static void
RtlSecureZeroMemory(dst, len)
void *dst;
size_t len;
{
memset(dst, 0, len);
}
static void
RtlFillMemory(void *dst, size_t len, uint8_t c)
{
memset(dst, c, len);
}
static void
RtlMoveMemory(dst, src, len)
void *dst;
const void *src;
size_t len;
{
memmove(dst, src, len);
}
static void
RtlCopyMemory(dst, src, len)
void *dst;
const void *src;
size_t len;
{
bcopy(src, dst, len);
}
static size_t
RtlCompareMemory(s1, s2, len)
const void *s1;
const void *s2;
size_t len;
{
size_t i;
uint8_t *m1, *m2;
m1 = __DECONST(char *, s1);
m2 = __DECONST(char *, s2);
for (i = 0; i < len && m1[i] == m2[i]; i++);
return (i);
}
void
RtlInitAnsiString(dst, src)
ansi_string *dst;
char *src;
{
ansi_string *a;
a = dst;
if (a == NULL)
return;
if (src == NULL) {
a->as_len = a->as_maxlen = 0;
a->as_buf = NULL;
} else {
a->as_buf = src;
a->as_len = a->as_maxlen = strlen(src);
}
}
void
RtlInitUnicodeString(dst, src)
unicode_string *dst;
uint16_t *src;
{
unicode_string *u;
int i;
u = dst;
if (u == NULL)
return;
if (src == NULL) {
u->us_len = u->us_maxlen = 0;
u->us_buf = NULL;
} else {
i = 0;
while(src[i] != 0)
i++;
u->us_buf = src;
u->us_len = u->us_maxlen = i * 2;
}
}
ndis_status
RtlUnicodeStringToInteger(ustr, base, val)
unicode_string *ustr;
uint32_t base;
uint32_t *val;
{
uint16_t *uchr;
int len, neg = 0;
char abuf[64];
char *astr;
uchr = ustr->us_buf;
len = ustr->us_len;
bzero(abuf, sizeof(abuf));
if ((char)((*uchr) & 0xFF) == '-') {
neg = 1;
uchr++;
len -= 2;
} else if ((char)((*uchr) & 0xFF) == '+') {
neg = 0;
uchr++;
len -= 2;
}
if (base == 0) {
if ((char)((*uchr) & 0xFF) == 'b') {
base = 2;
uchr++;
len -= 2;
} else if ((char)((*uchr) & 0xFF) == 'o') {
base = 8;
uchr++;
len -= 2;
} else if ((char)((*uchr) & 0xFF) == 'x') {
base = 16;
uchr++;
len -= 2;
} else
base = 10;
}
astr = abuf;
if (neg) {
strcpy(astr, "-");
astr++;
}
ntoskrnl_unicode_to_ascii(uchr, astr, len);
*val = strtoul(abuf, NULL, base);
return (STATUS_SUCCESS);
}
void
RtlFreeUnicodeString(ustr)
unicode_string *ustr;
{
if (ustr->us_buf == NULL)
return;
ExFreePool(ustr->us_buf);
ustr->us_buf = NULL;
}
void
RtlFreeAnsiString(astr)
ansi_string *astr;
{
if (astr->as_buf == NULL)
return;
ExFreePool(astr->as_buf);
astr->as_buf = NULL;
}
static int
atoi(str)
const char *str;
{
return (int)strtol(str, (char **)NULL, 10);
}
static long
atol(str)
const char *str;
{
return strtol(str, (char **)NULL, 10);
}
static int
rand(void)
{
struct timeval tv;
microtime(&tv);
srandom(tv.tv_usec);
return ((int)random());
}
static void
srand(seed)
unsigned int seed;
{
srandom(seed);
}
static uint8_t
IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
{
if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
return (TRUE);
return (FALSE);
}
static int32_t
IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type,
uint32_t mask, void **key)
{
return (NDIS_STATUS_INVALID_DEVICE_REQUEST);
}
static ndis_status
IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
unicode_string *name;
uint32_t reqaccess;
void *fileobj;
device_object *devobj;
{
return (STATUS_SUCCESS);
}
static ndis_status
IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
device_object *devobj;
uint32_t regprop;
uint32_t buflen;
void *prop;
uint32_t *reslen;
{
driver_object *drv;
uint16_t **name;
drv = devobj->do_drvobj;
switch (regprop) {
case DEVPROP_DRIVER_KEYNAME:
name = prop;
*name = drv->dro_drivername.us_buf;
*reslen = drv->dro_drivername.us_len;
break;
default:
return (STATUS_INVALID_PARAMETER_2);
break;
}
return (STATUS_SUCCESS);
}
static void
KeInitializeMutex(kmutex, level)
kmutant *kmutex;
uint32_t level;
{
InitializeListHead((&kmutex->km_header.dh_waitlisthead));
kmutex->km_abandoned = FALSE;
kmutex->km_apcdisable = 1;
kmutex->km_header.dh_sigstate = 1;
kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
kmutex->km_ownerthread = NULL;
}
static uint32_t
KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
{
uint32_t prevstate;
mtx_lock(&ntoskrnl_dispatchlock);
prevstate = kmutex->km_header.dh_sigstate;
if (kmutex->km_ownerthread != curthread) {
mtx_unlock(&ntoskrnl_dispatchlock);
return (STATUS_MUTANT_NOT_OWNED);
}
kmutex->km_header.dh_sigstate++;
kmutex->km_abandoned = FALSE;
if (kmutex->km_header.dh_sigstate == 1) {
kmutex->km_ownerthread = NULL;
ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
}
mtx_unlock(&ntoskrnl_dispatchlock);
return (prevstate);
}
static uint32_t
KeReadStateMutex(kmutex)
kmutant *kmutex;
{
return (kmutex->km_header.dh_sigstate);
}
void
KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
{
InitializeListHead((&kevent->k_header.dh_waitlisthead));
kevent->k_header.dh_sigstate = state;
if (type == EVENT_TYPE_NOTIFY)
kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
else
kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
}
uint32_t
KeResetEvent(kevent)
nt_kevent *kevent;
{
uint32_t prevstate;
mtx_lock(&ntoskrnl_dispatchlock);
prevstate = kevent->k_header.dh_sigstate;
kevent->k_header.dh_sigstate = FALSE;
mtx_unlock(&ntoskrnl_dispatchlock);
return (prevstate);
}
uint32_t
KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
{
uint32_t prevstate;
wait_block *w;
nt_dispatch_header *dh;
struct thread *td;
wb_ext *we;
mtx_lock(&ntoskrnl_dispatchlock);
prevstate = kevent->k_header.dh_sigstate;
dh = &kevent->k_header;
if (IsListEmpty(&dh->dh_waitlisthead))
/*
* If there's nobody in the waitlist, just set
* the state to signalled.
*/
dh->dh_sigstate = 1;
else {
/*
* Get the first waiter. If this is a synchronization
* event, just wake up that one thread (don't bother
* setting the state to signalled since we're supposed
* to automatically clear synchronization events anyway).
*
* If it's a notification event, or the first
* waiter is doing a WAITTYPE_ALL wait, go through
* the full wait satisfaction process.
*/
w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
wait_block, wb_waitlist);
we = w->wb_ext;
td = we->we_td;
if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
w->wb_waittype == WAITTYPE_ALL) {
if (prevstate == 0) {
dh->dh_sigstate = 1;
ntoskrnl_waittest(dh, increment);
}
} else {
w->wb_awakened |= TRUE;
cv_broadcastpri(&we->we_cv,
(w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
}
}
mtx_unlock(&ntoskrnl_dispatchlock);
return (prevstate);
}
void
KeClearEvent(kevent)
nt_kevent *kevent;
{
kevent->k_header.dh_sigstate = FALSE;
}
uint32_t
KeReadStateEvent(kevent)
nt_kevent *kevent;
{
return (kevent->k_header.dh_sigstate);
}
/*
* The object manager in Windows is responsible for managing
* references and access to various types of objects, including
* device_objects, events, threads, timers and so on. However,
* there's a difference in the way objects are handled in user
* mode versus kernel mode.
*
* In user mode (i.e. Win32 applications), all objects are
* managed by the object manager. For example, when you create
* a timer or event object, you actually end up with an
* object_header (for the object manager's bookkeeping
* purposes) and an object body (which contains the actual object
* structure, e.g. ktimer, kevent, etc...). This allows Windows
* to manage resource quotas and to enforce access restrictions
* on basically every kind of system object handled by the kernel.
*
* However, in kernel mode, you only end up using the object
* manager some of the time. For example, in a driver, you create
* a timer object by simply allocating the memory for a ktimer
* structure and initializing it with KeInitializeTimer(). Hence,
* the timer has no object_header and no reference counting or
* security/resource checks are done on it. The assumption in
* this case is that if you're running in kernel mode, you know
* what you're doing, and you're already at an elevated privilege
* anyway.
*
* There are some exceptions to this. The two most important ones
* for our purposes are device_objects and threads. We need to use
* the object manager to do reference counting on device_objects,
* and for threads, you can only get a pointer to a thread's
* dispatch header by using ObReferenceObjectByHandle() on the
* handle returned by PsCreateSystemThread().
*/
static ndis_status
ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
uint8_t accessmode, void **object, void **handleinfo)
{
nt_objref *nr;
nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
if (nr == NULL)
return (STATUS_INSUFFICIENT_RESOURCES);
InitializeListHead((&nr->no_dh.dh_waitlisthead));
nr->no_obj = handle;
nr->no_dh.dh_type = DISP_TYPE_THREAD;
nr->no_dh.dh_sigstate = 0;
nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
sizeof(uint32_t));
TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
*object = nr;
return (STATUS_SUCCESS);
}
static void
ObfDereferenceObject(object)
void *object;
{
nt_objref *nr;
nr = object;
TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
free(nr, M_DEVBUF);
}
static uint32_t
ZwClose(handle)
ndis_handle handle;
{
return (STATUS_SUCCESS);
}
static uint32_t
WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
uint32_t traceclass;
void *traceinfo;
uint32_t infolen;
uint32_t reqlen;
void *buf;
{
return (STATUS_NOT_FOUND);
}
static uint32_t
WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
void *guid, uint16_t messagenum, ...)
{
return (STATUS_SUCCESS);
}
static uint32_t
IoWMIRegistrationControl(dobj, action)
device_object *dobj;
uint32_t action;
{
return (STATUS_SUCCESS);
}
/*
* This is here just in case the thread returns without calling
* PsTerminateSystemThread().
*/
static void
ntoskrnl_thrfunc(arg)
void *arg;
{
thread_context *thrctx;
uint32_t (*tfunc)(void *);
void *tctx;
uint32_t rval;
thrctx = arg;
tfunc = thrctx->tc_thrfunc;
tctx = thrctx->tc_thrctx;
free(thrctx, M_TEMP);
rval = MSCALL1(tfunc, tctx);
PsTerminateSystemThread(rval);
return; /* notreached */
}
static ndis_status
PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
clientid, thrfunc, thrctx)
ndis_handle *handle;
uint32_t reqaccess;
void *objattrs;
ndis_handle phandle;
void *clientid;
void *thrfunc;
void *thrctx;
{
int error;
thread_context *tc;
struct proc *p;
tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
if (tc == NULL)
return (STATUS_INSUFFICIENT_RESOURCES);
tc->tc_thrctx = thrctx;
tc->tc_thrfunc = thrfunc;
error = kproc_create(ntoskrnl_thrfunc, tc, &p,
RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
if (error) {
free(tc, M_TEMP);
return (STATUS_INSUFFICIENT_RESOURCES);
}
*handle = p;
ntoskrnl_kth++;
return (STATUS_SUCCESS);
}
/*
* In Windows, the exit of a thread is an event that you're allowed
* to wait on, assuming you've obtained a reference to the thread using
* ObReferenceObjectByHandle(). Unfortunately, the only way we can
* simulate this behavior is to register each thread we create in a
* reference list, and if someone holds a reference to us, we poke
* them.
*/
static ndis_status
PsTerminateSystemThread(status)
ndis_status status;
{
struct nt_objref *nr;
mtx_lock(&ntoskrnl_dispatchlock);
TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
if (nr->no_obj != curthread->td_proc)
continue;
nr->no_dh.dh_sigstate = 1;
ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
break;
}
mtx_unlock(&ntoskrnl_dispatchlock);
ntoskrnl_kth--;
kproc_exit(0);
return (0); /* notreached */
}
static uint32_t
DbgPrint(char *fmt, ...)
{
va_list ap;
if (bootverbose) {
va_start(ap, fmt);
vprintf(fmt, ap);
va_end(ap);
}
return (STATUS_SUCCESS);
}
static void
DbgBreakPoint(void)
{
kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
}
static void
KeBugCheckEx(code, param1, param2, param3, param4)
uint32_t code;
u_long param1;
u_long param2;
u_long param3;
u_long param4;
{
panic("KeBugCheckEx: STOP 0x%X", code);
}
static void
ntoskrnl_timercall(arg)
void *arg;
{
ktimer *timer;
struct timeval tv;
kdpc *dpc;
mtx_lock(&ntoskrnl_dispatchlock);
timer = arg;
#ifdef NTOSKRNL_DEBUG_TIMERS
ntoskrnl_timer_fires++;
#endif
ntoskrnl_remove_timer(timer);
/*
* This should never happen, but complain
* if it does.
*/
if (timer->k_header.dh_inserted == FALSE) {
mtx_unlock(&ntoskrnl_dispatchlock);
printf("NTOS: timer %p fired even though "
"it was canceled\n", timer);
return;
}
/* Mark the timer as no longer being on the timer queue. */
timer->k_header.dh_inserted = FALSE;
/* Now signal the object and satisfy any waits on it. */
timer->k_header.dh_sigstate = 1;
ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
/*
* If this is a periodic timer, re-arm it
* so it will fire again. We do this before
* calling any deferred procedure calls because
* it's possible the DPC might cancel the timer,
* in which case it would be wrong for us to
* re-arm it again afterwards.
*/
if (timer->k_period) {
tv.tv_sec = 0;
tv.tv_usec = timer->k_period * 1000;
timer->k_header.dh_inserted = TRUE;
ntoskrnl_insert_timer(timer, tvtohz(&tv));
#ifdef NTOSKRNL_DEBUG_TIMERS
ntoskrnl_timer_reloads++;
#endif
}
dpc = timer->k_dpc;
mtx_unlock(&ntoskrnl_dispatchlock);
/* If there's a DPC associated with the timer, queue it up. */
if (dpc != NULL)
KeInsertQueueDpc(dpc, NULL, NULL);
}
#ifdef NTOSKRNL_DEBUG_TIMERS
static int
sysctl_show_timers(SYSCTL_HANDLER_ARGS)
{
int ret;
ret = 0;
ntoskrnl_show_timers();
return (sysctl_handle_int(oidp, &ret, 0, req));
}
static void
ntoskrnl_show_timers()
{
int i = 0;
list_entry *l;
mtx_lock_spin(&ntoskrnl_calllock);
l = ntoskrnl_calllist.nle_flink;
while(l != &ntoskrnl_calllist) {
i++;
l = l->nle_flink;
}
mtx_unlock_spin(&ntoskrnl_calllock);
printf("\n");
printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
printf("timer sets: %qu\n", ntoskrnl_timer_sets);
printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
printf("timer fires: %qu\n", ntoskrnl_timer_fires);
printf("\n");
}
#endif
/*
* Must be called with dispatcher lock held.
*/
static void
ntoskrnl_insert_timer(timer, ticks)
ktimer *timer;
int ticks;
{
callout_entry *e;
list_entry *l;
struct callout *c;
/*
* Try and allocate a timer.
*/
mtx_lock_spin(&ntoskrnl_calllock);
if (IsListEmpty(&ntoskrnl_calllist)) {
mtx_unlock_spin(&ntoskrnl_calllock);
#ifdef NTOSKRNL_DEBUG_TIMERS
ntoskrnl_show_timers();
#endif
panic("out of timers!");
}
l = RemoveHeadList(&ntoskrnl_calllist);
mtx_unlock_spin(&ntoskrnl_calllock);
e = CONTAINING_RECORD(l, callout_entry, ce_list);
c = &e->ce_callout;
timer->k_callout = c;
- callout_init(c, CALLOUT_MPSAFE);
+ callout_init(c, 1);
callout_reset(c, ticks, ntoskrnl_timercall, timer);
}
static void
ntoskrnl_remove_timer(timer)
ktimer *timer;
{
callout_entry *e;
e = (callout_entry *)timer->k_callout;
callout_stop(timer->k_callout);
mtx_lock_spin(&ntoskrnl_calllock);
InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
mtx_unlock_spin(&ntoskrnl_calllock);
}
void
KeInitializeTimer(timer)
ktimer *timer;
{
if (timer == NULL)
return;
KeInitializeTimerEx(timer, EVENT_TYPE_NOTIFY);
}
void
KeInitializeTimerEx(timer, type)
ktimer *timer;
uint32_t type;
{
if (timer == NULL)
return;
bzero((char *)timer, sizeof(ktimer));
InitializeListHead((&timer->k_header.dh_waitlisthead));
timer->k_header.dh_sigstate = FALSE;
timer->k_header.dh_inserted = FALSE;
if (type == EVENT_TYPE_NOTIFY)
timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
else
timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
}
/*
* DPC subsystem. A Windows Defered Procedure Call has the following
* properties:
* - It runs at DISPATCH_LEVEL.
* - It can have one of 3 importance values that control when it
* runs relative to other DPCs in the queue.
* - On SMP systems, it can be set to run on a specific processor.
* In order to satisfy the last property, we create a DPC thread for
* each CPU in the system and bind it to that CPU. Each thread
* maintains three queues with different importance levels, which
* will be processed in order from lowest to highest.
*
* In Windows, interrupt handlers run as DPCs. (Not to be confused
* with ISRs, which run in interrupt context and can preempt DPCs.)
* ISRs are given the highest importance so that they'll take
* precedence over timers and other things.
*/
static void
ntoskrnl_dpc_thread(arg)
void *arg;
{
kdpc_queue *kq;
kdpc *d;
list_entry *l;
uint8_t irql;
kq = arg;
InitializeListHead(&kq->kq_disp);
kq->kq_td = curthread;
kq->kq_exit = 0;
kq->kq_running = FALSE;
KeInitializeSpinLock(&kq->kq_lock);
KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
/*
* Elevate our priority. DPCs are used to run interrupt
* handlers, and they should trigger as soon as possible
* once scheduled by an ISR.
*/
thread_lock(curthread);
#ifdef NTOSKRNL_MULTIPLE_DPCS
sched_bind(curthread, kq->kq_cpu);
#endif
sched_prio(curthread, PRI_MIN_KERN);
thread_unlock(curthread);
while (1) {
KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
KeAcquireSpinLock(&kq->kq_lock, &irql);
if (kq->kq_exit) {
kq->kq_exit = 0;
KeReleaseSpinLock(&kq->kq_lock, irql);
break;
}
kq->kq_running = TRUE;
while (!IsListEmpty(&kq->kq_disp)) {
l = RemoveHeadList((&kq->kq_disp));
d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
InitializeListHead((&d->k_dpclistentry));
KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
d->k_sysarg1, d->k_sysarg2);
KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
}
kq->kq_running = FALSE;
KeReleaseSpinLock(&kq->kq_lock, irql);
KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
}
kproc_exit(0);
return; /* notreached */
}
static void
ntoskrnl_destroy_dpc_threads(void)
{
kdpc_queue *kq;
kdpc dpc;
int i;
kq = kq_queues;
#ifdef NTOSKRNL_MULTIPLE_DPCS
for (i = 0; i < mp_ncpus; i++) {
#else
for (i = 0; i < 1; i++) {
#endif
kq += i;
kq->kq_exit = 1;
KeInitializeDpc(&dpc, NULL, NULL);
KeSetTargetProcessorDpc(&dpc, i);
KeInsertQueueDpc(&dpc, NULL, NULL);
while (kq->kq_exit)
tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
}
}
static uint8_t
ntoskrnl_insert_dpc(head, dpc)
list_entry *head;
kdpc *dpc;
{
list_entry *l;
kdpc *d;
l = head->nle_flink;
while (l != head) {
d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
if (d == dpc)
return (FALSE);
l = l->nle_flink;
}
if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
InsertTailList((head), (&dpc->k_dpclistentry));
else
InsertHeadList((head), (&dpc->k_dpclistentry));
return (TRUE);
}
void
KeInitializeDpc(dpc, dpcfunc, dpcctx)
kdpc *dpc;
void *dpcfunc;
void *dpcctx;
{
if (dpc == NULL)
return;
dpc->k_deferedfunc = dpcfunc;
dpc->k_deferredctx = dpcctx;
dpc->k_num = KDPC_CPU_DEFAULT;
dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
InitializeListHead((&dpc->k_dpclistentry));
}
uint8_t
KeInsertQueueDpc(dpc, sysarg1, sysarg2)
kdpc *dpc;
void *sysarg1;
void *sysarg2;
{
kdpc_queue *kq;
uint8_t r;
uint8_t irql;
if (dpc == NULL)
return (FALSE);
kq = kq_queues;
#ifdef NTOSKRNL_MULTIPLE_DPCS
KeRaiseIrql(DISPATCH_LEVEL, &irql);
/*
* By default, the DPC is queued to run on the same CPU
* that scheduled it.
*/
if (dpc->k_num == KDPC_CPU_DEFAULT)
kq += curthread->td_oncpu;
else
kq += dpc->k_num;
KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
#else
KeAcquireSpinLock(&kq->kq_lock, &irql);
#endif
r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
if (r == TRUE) {
dpc->k_sysarg1 = sysarg1;
dpc->k_sysarg2 = sysarg2;
}
KeReleaseSpinLock(&kq->kq_lock, irql);
if (r == FALSE)
return (r);
KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
return (r);
}
uint8_t
KeRemoveQueueDpc(dpc)
kdpc *dpc;
{
kdpc_queue *kq;
uint8_t irql;
if (dpc == NULL)
return (FALSE);
#ifdef NTOSKRNL_MULTIPLE_DPCS
KeRaiseIrql(DISPATCH_LEVEL, &irql);
kq = kq_queues + dpc->k_num;
KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
#else
kq = kq_queues;
KeAcquireSpinLock(&kq->kq_lock, &irql);
#endif
if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
KeLowerIrql(irql);
return (FALSE);
}
RemoveEntryList((&dpc->k_dpclistentry));
InitializeListHead((&dpc->k_dpclistentry));
KeReleaseSpinLock(&kq->kq_lock, irql);
return (TRUE);
}
void
KeSetImportanceDpc(dpc, imp)
kdpc *dpc;
uint32_t imp;
{
if (imp != KDPC_IMPORTANCE_LOW &&
imp != KDPC_IMPORTANCE_MEDIUM &&
imp != KDPC_IMPORTANCE_HIGH)
return;
dpc->k_importance = (uint8_t)imp;
}
void
KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
{
if (cpu > mp_ncpus)
return;
dpc->k_num = cpu;
}
void
KeFlushQueuedDpcs(void)
{
kdpc_queue *kq;
int i;
/*
* Poke each DPC queue and wait
* for them to drain.
*/
#ifdef NTOSKRNL_MULTIPLE_DPCS
for (i = 0; i < mp_ncpus; i++) {
#else
for (i = 0; i < 1; i++) {
#endif
kq = kq_queues + i;
KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
}
}
uint32_t
KeGetCurrentProcessorNumber(void)
{
return ((uint32_t)curthread->td_oncpu);
}
uint8_t
KeSetTimerEx(timer, duetime, period, dpc)
ktimer *timer;
int64_t duetime;
uint32_t period;
kdpc *dpc;
{
struct timeval tv;
uint64_t curtime;
uint8_t pending;
if (timer == NULL)
return (FALSE);
mtx_lock(&ntoskrnl_dispatchlock);
if (timer->k_header.dh_inserted == TRUE) {
ntoskrnl_remove_timer(timer);
#ifdef NTOSKRNL_DEBUG_TIMERS
ntoskrnl_timer_cancels++;
#endif
timer->k_header.dh_inserted = FALSE;
pending = TRUE;
} else
pending = FALSE;
timer->k_duetime = duetime;
timer->k_period = period;
timer->k_header.dh_sigstate = FALSE;
timer->k_dpc = dpc;
if (duetime < 0) {
tv.tv_sec = - (duetime) / 10000000;
tv.tv_usec = (- (duetime) / 10) -
(tv.tv_sec * 1000000);
} else {
ntoskrnl_time(&curtime);
if (duetime < curtime)
tv.tv_sec = tv.tv_usec = 0;
else {
tv.tv_sec = ((duetime) - curtime) / 10000000;
tv.tv_usec = ((duetime) - curtime) / 10 -
(tv.tv_sec * 1000000);
}
}
timer->k_header.dh_inserted = TRUE;
ntoskrnl_insert_timer(timer, tvtohz(&tv));
#ifdef NTOSKRNL_DEBUG_TIMERS
ntoskrnl_timer_sets++;
#endif
mtx_unlock(&ntoskrnl_dispatchlock);
return (pending);
}
uint8_t
KeSetTimer(timer, duetime, dpc)
ktimer *timer;
int64_t duetime;
kdpc *dpc;
{
return (KeSetTimerEx(timer, duetime, 0, dpc));
}
/*
* The Windows DDK documentation seems to say that cancelling
* a timer that has a DPC will result in the DPC also being
* cancelled, but this isn't really the case.
*/
uint8_t
KeCancelTimer(timer)
ktimer *timer;
{
uint8_t pending;
if (timer == NULL)
return (FALSE);
mtx_lock(&ntoskrnl_dispatchlock);
pending = timer->k_header.dh_inserted;
if (timer->k_header.dh_inserted == TRUE) {
timer->k_header.dh_inserted = FALSE;
ntoskrnl_remove_timer(timer);
#ifdef NTOSKRNL_DEBUG_TIMERS
ntoskrnl_timer_cancels++;
#endif
}
mtx_unlock(&ntoskrnl_dispatchlock);
return (pending);
}
uint8_t
KeReadStateTimer(timer)
ktimer *timer;
{
return (timer->k_header.dh_sigstate);
}
static int32_t
KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
{
ktimer timer;
if (wait_mode != 0)
panic("invalid wait_mode %d", wait_mode);
KeInitializeTimer(&timer);
KeSetTimer(&timer, *interval, NULL);
KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
return STATUS_SUCCESS;
}
static uint64_t
KeQueryInterruptTime(void)
{
int ticks;
struct timeval tv;
getmicrouptime(&tv);
ticks = tvtohz(&tv);
return ticks * ((10000000 + hz - 1) / hz);
}
static struct thread *
KeGetCurrentThread(void)
{
return curthread;
}
static int32_t
KeSetPriorityThread(td, pri)
struct thread *td;
int32_t pri;
{
int32_t old;
if (td == NULL)
return LOW_REALTIME_PRIORITY;
if (td->td_priority <= PRI_MIN_KERN)
old = HIGH_PRIORITY;
else if (td->td_priority >= PRI_MAX_KERN)
old = LOW_PRIORITY;
else
old = LOW_REALTIME_PRIORITY;
thread_lock(td);
if (pri == HIGH_PRIORITY)
sched_prio(td, PRI_MIN_KERN);
if (pri == LOW_REALTIME_PRIORITY)
sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
if (pri == LOW_PRIORITY)
sched_prio(td, PRI_MAX_KERN);
thread_unlock(td);
return old;
}
static void
dummy()
{
printf("ntoskrnl dummy called...\n");
}
image_patch_table ntoskrnl_functbl[] = {
IMPORT_SFUNC(RtlZeroMemory, 2),
IMPORT_SFUNC(RtlSecureZeroMemory, 2),
IMPORT_SFUNC(RtlFillMemory, 3),
IMPORT_SFUNC(RtlMoveMemory, 3),
IMPORT_SFUNC(RtlCharToInteger, 3),
IMPORT_SFUNC(RtlCopyMemory, 3),
IMPORT_SFUNC(RtlCopyString, 2),
IMPORT_SFUNC(RtlCompareMemory, 3),
IMPORT_SFUNC(RtlEqualUnicodeString, 3),
IMPORT_SFUNC(RtlCopyUnicodeString, 2),
IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
IMPORT_SFUNC(RtlInitAnsiString, 2),
IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
IMPORT_SFUNC(RtlInitUnicodeString, 2),
IMPORT_SFUNC(RtlFreeAnsiString, 1),
IMPORT_SFUNC(RtlFreeUnicodeString, 1),
IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
IMPORT_CFUNC(sprintf, 0),
IMPORT_CFUNC(vsprintf, 0),
IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
IMPORT_CFUNC(DbgPrint, 0),
IMPORT_SFUNC(DbgBreakPoint, 0),
IMPORT_SFUNC(KeBugCheckEx, 5),
IMPORT_CFUNC(strncmp, 0),
IMPORT_CFUNC(strcmp, 0),
IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
IMPORT_CFUNC(strncpy, 0),
IMPORT_CFUNC(strcpy, 0),
IMPORT_CFUNC(strlen, 0),
IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
IMPORT_CFUNC_MAP(strchr, index, 0),
IMPORT_CFUNC_MAP(strrchr, rindex, 0),
IMPORT_CFUNC(memcpy, 0),
IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
IMPORT_FFUNC(IofCallDriver, 2),
IMPORT_FFUNC(IofCompleteRequest, 2),
IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
IMPORT_SFUNC(IoCancelIrp, 1),
IMPORT_SFUNC(IoConnectInterrupt, 11),
IMPORT_SFUNC(IoDisconnectInterrupt, 1),
IMPORT_SFUNC(IoCreateDevice, 7),
IMPORT_SFUNC(IoDeleteDevice, 1),
IMPORT_SFUNC(IoGetAttachedDevice, 1),
IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
IMPORT_SFUNC(IoDetachDevice, 1),
IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
IMPORT_SFUNC(IoAllocateIrp, 2),
IMPORT_SFUNC(IoReuseIrp, 2),
IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
IMPORT_SFUNC(IoFreeIrp, 1),
IMPORT_SFUNC(IoInitializeIrp, 3),
IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
IMPORT_SFUNC(KeSynchronizeExecution, 3),
IMPORT_SFUNC(KeWaitForSingleObject, 5),
IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
IMPORT_SFUNC(_allmul, 4),
IMPORT_SFUNC(_alldiv, 4),
IMPORT_SFUNC(_allrem, 4),
IMPORT_RFUNC(_allshr, 0),
IMPORT_RFUNC(_allshl, 0),
IMPORT_SFUNC(_aullmul, 4),
IMPORT_SFUNC(_aulldiv, 4),
IMPORT_SFUNC(_aullrem, 4),
IMPORT_RFUNC(_aullshr, 0),
IMPORT_RFUNC(_aullshl, 0),
IMPORT_CFUNC(atoi, 0),
IMPORT_CFUNC(atol, 0),
IMPORT_CFUNC(rand, 0),
IMPORT_CFUNC(srand, 0),
IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
IMPORT_FFUNC(InterlockedPopEntrySList, 1),
IMPORT_FFUNC(InitializeSListHead, 1),
IMPORT_FFUNC(InterlockedPushEntrySList, 2),
IMPORT_SFUNC(ExQueryDepthSList, 1),
IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
InterlockedPopEntrySList, 1),
IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
InterlockedPushEntrySList, 2),
IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
IMPORT_SFUNC(ExFreePoolWithTag, 2),
IMPORT_SFUNC(ExFreePool, 1),
#ifdef __i386__
IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
#else
/*
* For AMD64, we can get away with just mapping
* KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
* because the calling conventions end up being the same.
* On i386, we have to be careful because KfAcquireSpinLock()
* is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
*/
IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
#endif
IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
IMPORT_FFUNC(InterlockedIncrement, 1),
IMPORT_FFUNC(InterlockedDecrement, 1),
IMPORT_FFUNC(InterlockedExchange, 2),
IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
IMPORT_SFUNC(IoAllocateMdl, 5),
IMPORT_SFUNC(IoFreeMdl, 1),
IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
IMPORT_SFUNC(MmFreeContiguousMemory, 1),
IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
IMPORT_SFUNC(MmSizeOfMdl, 1),
IMPORT_SFUNC(MmMapLockedPages, 2),
IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
IMPORT_SFUNC(MmUnmapLockedPages, 2),
IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
IMPORT_SFUNC(MmGetPhysicalAddress, 1),
IMPORT_SFUNC(MmGetSystemRoutineAddress, 1),
IMPORT_SFUNC(MmIsAddressValid, 1),
IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
IMPORT_SFUNC(MmUnmapIoSpace, 2),
IMPORT_SFUNC(KeInitializeSpinLock, 1),
IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4),
IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
IMPORT_SFUNC(IoGetDeviceProperty, 5),
IMPORT_SFUNC(IoAllocateWorkItem, 1),
IMPORT_SFUNC(IoFreeWorkItem, 1),
IMPORT_SFUNC(IoQueueWorkItem, 4),
IMPORT_SFUNC(ExQueueWorkItem, 2),
IMPORT_SFUNC(ntoskrnl_workitem, 2),
IMPORT_SFUNC(KeInitializeMutex, 2),
IMPORT_SFUNC(KeReleaseMutex, 2),
IMPORT_SFUNC(KeReadStateMutex, 1),
IMPORT_SFUNC(KeInitializeEvent, 3),
IMPORT_SFUNC(KeSetEvent, 3),
IMPORT_SFUNC(KeResetEvent, 1),
IMPORT_SFUNC(KeClearEvent, 1),
IMPORT_SFUNC(KeReadStateEvent, 1),
IMPORT_SFUNC(KeInitializeTimer, 1),
IMPORT_SFUNC(KeInitializeTimerEx, 2),
IMPORT_SFUNC(KeSetTimer, 3),
IMPORT_SFUNC(KeSetTimerEx, 4),
IMPORT_SFUNC(KeCancelTimer, 1),
IMPORT_SFUNC(KeReadStateTimer, 1),
IMPORT_SFUNC(KeInitializeDpc, 3),
IMPORT_SFUNC(KeInsertQueueDpc, 3),
IMPORT_SFUNC(KeRemoveQueueDpc, 1),
IMPORT_SFUNC(KeSetImportanceDpc, 2),
IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
IMPORT_FFUNC(ObfDereferenceObject, 1),
IMPORT_SFUNC(ZwClose, 1),
IMPORT_SFUNC(PsCreateSystemThread, 7),
IMPORT_SFUNC(PsTerminateSystemThread, 1),
IMPORT_SFUNC(IoWMIRegistrationControl, 2),
IMPORT_SFUNC(WmiQueryTraceInformation, 5),
IMPORT_CFUNC(WmiTraceMessage, 0),
IMPORT_SFUNC(KeQuerySystemTime, 1),
IMPORT_CFUNC(KeTickCount, 0),
IMPORT_SFUNC(KeDelayExecutionThread, 3),
IMPORT_SFUNC(KeQueryInterruptTime, 0),
IMPORT_SFUNC(KeGetCurrentThread, 0),
IMPORT_SFUNC(KeSetPriorityThread, 2),
/*
* This last entry is a catch-all for any function we haven't
* implemented yet. The PE import list patching routine will
* use it for any function that doesn't have an explicit match
* in this table.
*/
{ NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
/* End of list. */
{ NULL, NULL, NULL }
};
Index: head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c
===================================================================
--- head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c (revision 283290)
+++ head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c (revision 283291)
@@ -1,1448 +1,1448 @@
/* $FreeBSD$ */
/*
* Copyright (C) 2012 by Darren Reed.
*
* See the IPFILTER.LICENCE file for details on licencing.
*/
#if !defined(lint)
static const char sccsid[] = "@(#)ip_fil.c 2.41 6/5/96 (C) 1993-2000 Darren Reed";
static const char rcsid[] = "@(#)$Id$";
#endif
#if defined(KERNEL) || defined(_KERNEL)
# undef KERNEL
# undef _KERNEL
# define KERNEL 1
# define _KERNEL 1
#endif
#if defined(__FreeBSD_version) && (__FreeBSD_version >= 400000) && \
!defined(KLD_MODULE) && !defined(IPFILTER_LKM)
# include "opt_inet6.h"
#endif
#if defined(__FreeBSD_version) && (__FreeBSD_version >= 440000) && \
!defined(KLD_MODULE) && !defined(IPFILTER_LKM)
# include "opt_random_ip_id.h"
#endif
#include <sys/param.h>
#include <sys/errno.h>
#include <sys/types.h>
#include <sys/file.h>
# include <sys/fcntl.h>
# include <sys/filio.h>
#include <sys/time.h>
#include <sys/systm.h>
# include <sys/dirent.h>
#if defined(__FreeBSD_version) && (__FreeBSD_version >= 800000)
#include <sys/jail.h>
#endif
# include <sys/mbuf.h>
# include <sys/sockopt.h>
#if !defined(__hpux)
# include <sys/mbuf.h>
#endif
#include <sys/socket.h>
# include <sys/selinfo.h>
# include <netinet/tcp_var.h>
#include <net/if.h>
# include <net/if_var.h>
# include <net/netisr.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/in_var.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/ip_var.h>
#include <netinet/tcp.h>
#if defined(__FreeBSD_version) && (__FreeBSD_version >= 800000)
#include <net/vnet.h>
#else
#define CURVNET_SET(arg)
#define CURVNET_RESTORE()
#endif
#if defined(__osf__)
# include <netinet/tcp_timer.h>
#endif
#include <netinet/udp.h>
#include <netinet/tcpip.h>
#include <netinet/ip_icmp.h>
#include "netinet/ip_compat.h"
#ifdef USE_INET6
# include <netinet/icmp6.h>
#endif
#include "netinet/ip_fil.h"
#include "netinet/ip_nat.h"
#include "netinet/ip_frag.h"
#include "netinet/ip_state.h"
#include "netinet/ip_proxy.h"
#include "netinet/ip_auth.h"
#include "netinet/ip_sync.h"
#include "netinet/ip_lookup.h"
#include "netinet/ip_dstlist.h"
#ifdef IPFILTER_SCAN
#include "netinet/ip_scan.h"
#endif
#include "netinet/ip_pool.h"
# include <sys/malloc.h>
#include <sys/kernel.h>
#ifdef CSUM_DATA_VALID
#include <machine/in_cksum.h>
#endif
extern int ip_optcopy __P((struct ip *, struct ip *));
# ifdef IPFILTER_M_IPFILTER
MALLOC_DEFINE(M_IPFILTER, "ipfilter", "IP Filter packet filter data structures");
# endif
static int (*ipf_savep) __P((void *, ip_t *, int, void *, int, struct mbuf **));
static int ipf_send_ip __P((fr_info_t *, mb_t *));
static void ipf_timer_func __P((void *arg));
int ipf_locks_done = 0;
ipf_main_softc_t ipfmain;
# include <sys/conf.h>
# if defined(NETBSD_PF)
# include <net/pfil.h>
# endif /* NETBSD_PF */
/*
* We provide the ipf_checkp name just to minimize changes later.
*/
int (*ipf_checkp) __P((void *, ip_t *ip, int hlen, void *ifp, int out, mb_t **mp));
static eventhandler_tag ipf_arrivetag, ipf_departtag, ipf_clonetag;
static void ipf_ifevent(void *arg);
static void ipf_ifevent(arg)
void *arg;
{
ipf_sync(arg, NULL);
}
static int
ipf_check_wrapper(void *arg, struct mbuf **mp, struct ifnet *ifp, int dir)
{
struct ip *ip = mtod(*mp, struct ip *);
int rv;
/*
* IPFilter expects evreything in network byte order
*/
#if (__FreeBSD_version < 1000019)
ip->ip_len = htons(ip->ip_len);
ip->ip_off = htons(ip->ip_off);
#endif
rv = ipf_check(&ipfmain, ip, ip->ip_hl << 2, ifp, (dir == PFIL_OUT),
mp);
#if (__FreeBSD_version < 1000019)
if ((rv == 0) && (*mp != NULL)) {
ip = mtod(*mp, struct ip *);
ip->ip_len = ntohs(ip->ip_len);
ip->ip_off = ntohs(ip->ip_off);
}
#endif
return rv;
}
# ifdef USE_INET6
# include <netinet/ip6.h>
static int
ipf_check_wrapper6(void *arg, struct mbuf **mp, struct ifnet *ifp, int dir)
{
return (ipf_check(&ipfmain, mtod(*mp, struct ip *),
sizeof(struct ip6_hdr), ifp, (dir == PFIL_OUT), mp));
}
# endif
#if defined(IPFILTER_LKM)
int ipf_identify(s)
char *s;
{
if (strcmp(s, "ipl") == 0)
return 1;
return 0;
}
#endif /* IPFILTER_LKM */
static void
ipf_timer_func(arg)
void *arg;
{
ipf_main_softc_t *softc = arg;
SPL_INT(s);
SPL_NET(s);
READ_ENTER(&softc->ipf_global);
if (softc->ipf_running > 0)
ipf_slowtimer(softc);
if (softc->ipf_running == -1 || softc->ipf_running == 1) {
#if 0
softc->ipf_slow_ch = timeout(ipf_timer_func, softc, hz/2);
#endif
- callout_init(&softc->ipf_slow_ch, CALLOUT_MPSAFE);
+ callout_init(&softc->ipf_slow_ch, 1);
callout_reset(&softc->ipf_slow_ch,
(hz / IPF_HZ_DIVIDE) * IPF_HZ_MULT,
ipf_timer_func, softc);
}
RWLOCK_EXIT(&softc->ipf_global);
SPL_X(s);
}
int
ipfattach(softc)
ipf_main_softc_t *softc;
{
#ifdef USE_SPL
int s;
#endif
SPL_NET(s);
if (softc->ipf_running > 0) {
SPL_X(s);
return EBUSY;
}
if (ipf_init_all(softc) < 0) {
SPL_X(s);
return EIO;
}
if (ipf_checkp != ipf_check) {
ipf_savep = ipf_checkp;
ipf_checkp = ipf_check;
}
bzero((char *)ipfmain.ipf_selwait, sizeof(ipfmain.ipf_selwait));
softc->ipf_running = 1;
if (softc->ipf_control_forwarding & 1)
V_ipforwarding = 1;
SPL_X(s);
#if 0
softc->ipf_slow_ch = timeout(ipf_timer_func, softc,
(hz / IPF_HZ_DIVIDE) * IPF_HZ_MULT);
#endif
- callout_init(&softc->ipf_slow_ch, CALLOUT_MPSAFE);
+ callout_init(&softc->ipf_slow_ch, 1);
callout_reset(&softc->ipf_slow_ch, (hz / IPF_HZ_DIVIDE) * IPF_HZ_MULT,
ipf_timer_func, softc);
return 0;
}
/*
* Disable the filter by removing the hooks from the IP input/output
* stream.
*/
int
ipfdetach(softc)
ipf_main_softc_t *softc;
{
#ifdef USE_SPL
int s;
#endif
if (softc->ipf_control_forwarding & 2)
V_ipforwarding = 0;
SPL_NET(s);
#if 0
if (softc->ipf_slow_ch.callout != NULL)
untimeout(ipf_timer_func, softc, softc->ipf_slow_ch);
bzero(&softc->ipf_slow, sizeof(softc->ipf_slow));
#endif
callout_drain(&softc->ipf_slow_ch);
#ifndef NETBSD_PF
if (ipf_checkp != NULL)
ipf_checkp = ipf_savep;
ipf_savep = NULL;
#endif
ipf_fini_all(softc);
softc->ipf_running = -2;
SPL_X(s);
return 0;
}
/*
* Filter ioctl interface.
*/
int
ipfioctl(dev, cmd, data, mode
, p)
struct thread *p;
# define p_cred td_ucred
# define p_uid td_ucred->cr_ruid
struct cdev *dev;
ioctlcmd_t cmd;
caddr_t data;
int mode;
{
int error = 0, unit = 0;
SPL_INT(s);
#if (BSD >= 199306)
if (securelevel_ge(p->p_cred, 3) && (mode & FWRITE))
{
ipfmain.ipf_interror = 130001;
return EPERM;
}
#endif
unit = GET_MINOR(dev);
if ((IPL_LOGMAX < unit) || (unit < 0)) {
ipfmain.ipf_interror = 130002;
return ENXIO;
}
if (ipfmain.ipf_running <= 0) {
if (unit != IPL_LOGIPF && cmd != SIOCIPFINTERROR) {
ipfmain.ipf_interror = 130003;
return EIO;
}
if (cmd != SIOCIPFGETNEXT && cmd != SIOCIPFGET &&
cmd != SIOCIPFSET && cmd != SIOCFRENB &&
cmd != SIOCGETFS && cmd != SIOCGETFF &&
cmd != SIOCIPFINTERROR) {
ipfmain.ipf_interror = 130004;
return EIO;
}
}
SPL_NET(s);
CURVNET_SET(TD_TO_VNET(p));
error = ipf_ioctlswitch(&ipfmain, unit, data, cmd, mode, p->p_uid, p);
CURVNET_RESTORE();
if (error != -1) {
SPL_X(s);
return error;
}
SPL_X(s);
return error;
}
/*
* ipf_send_reset - this could conceivably be a call to tcp_respond(), but that
* requires a large amount of setting up and isn't any more efficient.
*/
int
ipf_send_reset(fin)
fr_info_t *fin;
{
struct tcphdr *tcp, *tcp2;
int tlen = 0, hlen;
struct mbuf *m;
#ifdef USE_INET6
ip6_t *ip6;
#endif
ip_t *ip;
tcp = fin->fin_dp;
if (tcp->th_flags & TH_RST)
return -1; /* feedback loop */
if (ipf_checkl4sum(fin) == -1)
return -1;
tlen = fin->fin_dlen - (TCP_OFF(tcp) << 2) +
((tcp->th_flags & TH_SYN) ? 1 : 0) +
((tcp->th_flags & TH_FIN) ? 1 : 0);
#ifdef USE_INET6
hlen = (fin->fin_v == 6) ? sizeof(ip6_t) : sizeof(ip_t);
#else
hlen = sizeof(ip_t);
#endif
#ifdef MGETHDR
MGETHDR(m, M_NOWAIT, MT_HEADER);
#else
MGET(m, M_NOWAIT, MT_HEADER);
#endif
if (m == NULL)
return -1;
if (sizeof(*tcp2) + hlen > MLEN) {
if (!(MCLGET(m, M_NOWAIT))) {
FREE_MB_T(m);
return -1;
}
}
m->m_len = sizeof(*tcp2) + hlen;
#if (BSD >= 199103)
m->m_data += max_linkhdr;
m->m_pkthdr.len = m->m_len;
m->m_pkthdr.rcvif = (struct ifnet *)0;
#endif
ip = mtod(m, struct ip *);
bzero((char *)ip, hlen);
#ifdef USE_INET6
ip6 = (ip6_t *)ip;
#endif
tcp2 = (struct tcphdr *)((char *)ip + hlen);
tcp2->th_sport = tcp->th_dport;
tcp2->th_dport = tcp->th_sport;
if (tcp->th_flags & TH_ACK) {
tcp2->th_seq = tcp->th_ack;
tcp2->th_flags = TH_RST;
tcp2->th_ack = 0;
} else {
tcp2->th_seq = 0;
tcp2->th_ack = ntohl(tcp->th_seq);
tcp2->th_ack += tlen;
tcp2->th_ack = htonl(tcp2->th_ack);
tcp2->th_flags = TH_RST|TH_ACK;
}
TCP_X2_A(tcp2, 0);
TCP_OFF_A(tcp2, sizeof(*tcp2) >> 2);
tcp2->th_win = tcp->th_win;
tcp2->th_sum = 0;
tcp2->th_urp = 0;
#ifdef USE_INET6
if (fin->fin_v == 6) {
ip6->ip6_flow = ((ip6_t *)fin->fin_ip)->ip6_flow;
ip6->ip6_plen = htons(sizeof(struct tcphdr));
ip6->ip6_nxt = IPPROTO_TCP;
ip6->ip6_hlim = 0;
ip6->ip6_src = fin->fin_dst6.in6;
ip6->ip6_dst = fin->fin_src6.in6;
tcp2->th_sum = in6_cksum(m, IPPROTO_TCP,
sizeof(*ip6), sizeof(*tcp2));
return ipf_send_ip(fin, m);
}
#endif
ip->ip_p = IPPROTO_TCP;
ip->ip_len = htons(sizeof(struct tcphdr));
ip->ip_src.s_addr = fin->fin_daddr;
ip->ip_dst.s_addr = fin->fin_saddr;
tcp2->th_sum = in_cksum(m, hlen + sizeof(*tcp2));
ip->ip_len = htons(hlen + sizeof(*tcp2));
return ipf_send_ip(fin, m);
}
/*
* ip_len must be in network byte order when called.
*/
static int
ipf_send_ip(fin, m)
fr_info_t *fin;
mb_t *m;
{
fr_info_t fnew;
ip_t *ip, *oip;
int hlen;
ip = mtod(m, ip_t *);
bzero((char *)&fnew, sizeof(fnew));
fnew.fin_main_soft = fin->fin_main_soft;
IP_V_A(ip, fin->fin_v);
switch (fin->fin_v)
{
case 4 :
oip = fin->fin_ip;
hlen = sizeof(*oip);
fnew.fin_v = 4;
fnew.fin_p = ip->ip_p;
fnew.fin_plen = ntohs(ip->ip_len);
IP_HL_A(ip, sizeof(*oip) >> 2);
ip->ip_tos = oip->ip_tos;
ip->ip_id = fin->fin_ip->ip_id;
#if defined(FreeBSD) && (__FreeBSD_version > 460000)
ip->ip_off = htons(path_mtu_discovery ? IP_DF : 0);
#else
ip->ip_off = 0;
#endif
ip->ip_ttl = V_ip_defttl;
ip->ip_sum = 0;
break;
#ifdef USE_INET6
case 6 :
{
ip6_t *ip6 = (ip6_t *)ip;
ip6->ip6_vfc = 0x60;
ip6->ip6_hlim = IPDEFTTL;
hlen = sizeof(*ip6);
fnew.fin_p = ip6->ip6_nxt;
fnew.fin_v = 6;
fnew.fin_plen = ntohs(ip6->ip6_plen) + hlen;
break;
}
#endif
default :
return EINVAL;
}
#ifdef IPSEC
m->m_pkthdr.rcvif = NULL;
#endif
fnew.fin_ifp = fin->fin_ifp;
fnew.fin_flx = FI_NOCKSUM;
fnew.fin_m = m;
fnew.fin_ip = ip;
fnew.fin_mp = &m;
fnew.fin_hlen = hlen;
fnew.fin_dp = (char *)ip + hlen;
(void) ipf_makefrip(hlen, ip, &fnew);
return ipf_fastroute(m, &m, &fnew, NULL);
}
int
ipf_send_icmp_err(type, fin, dst)
int type;
fr_info_t *fin;
int dst;
{
int err, hlen, xtra, iclen, ohlen, avail, code;
struct in_addr dst4;
struct icmp *icmp;
struct mbuf *m;
i6addr_t dst6;
void *ifp;
#ifdef USE_INET6
ip6_t *ip6;
#endif
ip_t *ip, *ip2;
if ((type < 0) || (type >= ICMP_MAXTYPE))
return -1;
code = fin->fin_icode;
#ifdef USE_INET6
#if 0
/* XXX Fix an off by one error: s/>/>=/
was:
if ((code < 0) || (code > sizeof(icmptoicmp6unreach)/sizeof(int)))
Fix obtained from NetBSD ip_fil_netbsd.c r1.4: */
#endif
if ((code < 0) || (code >= sizeof(icmptoicmp6unreach)/sizeof(int)))
return -1;
#endif
if (ipf_checkl4sum(fin) == -1)
return -1;
#ifdef MGETHDR
MGETHDR(m, M_NOWAIT, MT_HEADER);
#else
MGET(m, M_NOWAIT, MT_HEADER);
#endif
if (m == NULL)
return -1;
avail = MHLEN;
xtra = 0;
hlen = 0;
ohlen = 0;
dst4.s_addr = 0;
ifp = fin->fin_ifp;
if (fin->fin_v == 4) {
if ((fin->fin_p == IPPROTO_ICMP) && !(fin->fin_flx & FI_SHORT))
switch (ntohs(fin->fin_data[0]) >> 8)
{
case ICMP_ECHO :
case ICMP_TSTAMP :
case ICMP_IREQ :
case ICMP_MASKREQ :
break;
default :
FREE_MB_T(m);
return 0;
}
if (dst == 0) {
if (ipf_ifpaddr(&ipfmain, 4, FRI_NORMAL, ifp,
&dst6, NULL) == -1) {
FREE_MB_T(m);
return -1;
}
dst4 = dst6.in4;
} else
dst4.s_addr = fin->fin_daddr;
hlen = sizeof(ip_t);
ohlen = fin->fin_hlen;
iclen = hlen + offsetof(struct icmp, icmp_ip) + ohlen;
if (fin->fin_hlen < fin->fin_plen)
xtra = MIN(fin->fin_dlen, 8);
else
xtra = 0;
}
#ifdef USE_INET6
else if (fin->fin_v == 6) {
hlen = sizeof(ip6_t);
ohlen = sizeof(ip6_t);
iclen = hlen + offsetof(struct icmp, icmp_ip) + ohlen;
type = icmptoicmp6types[type];
if (type == ICMP6_DST_UNREACH)
code = icmptoicmp6unreach[code];
if (iclen + max_linkhdr + fin->fin_plen > avail) {
if (!(MCLGET(m, M_NOWAIT))) {
FREE_MB_T(m);
return -1;
}
avail = MCLBYTES;
}
xtra = MIN(fin->fin_plen, avail - iclen - max_linkhdr);
xtra = MIN(xtra, IPV6_MMTU - iclen);
if (dst == 0) {
if (ipf_ifpaddr(&ipfmain, 6, FRI_NORMAL, ifp,
&dst6, NULL) == -1) {
FREE_MB_T(m);
return -1;
}
} else
dst6 = fin->fin_dst6;
}
#endif
else {
FREE_MB_T(m);
return -1;
}
avail -= (max_linkhdr + iclen);
if (avail < 0) {
FREE_MB_T(m);
return -1;
}
if (xtra > avail)
xtra = avail;
iclen += xtra;
m->m_data += max_linkhdr;
m->m_pkthdr.rcvif = (struct ifnet *)0;
m->m_pkthdr.len = iclen;
m->m_len = iclen;
ip = mtod(m, ip_t *);
icmp = (struct icmp *)((char *)ip + hlen);
ip2 = (ip_t *)&icmp->icmp_ip;
icmp->icmp_type = type;
icmp->icmp_code = fin->fin_icode;
icmp->icmp_cksum = 0;
#ifdef icmp_nextmtu
if (type == ICMP_UNREACH && fin->fin_icode == ICMP_UNREACH_NEEDFRAG) {
if (fin->fin_mtu != 0) {
icmp->icmp_nextmtu = htons(fin->fin_mtu);
} else if (ifp != NULL) {
icmp->icmp_nextmtu = htons(GETIFMTU_4(ifp));
} else { /* make up a number... */
icmp->icmp_nextmtu = htons(fin->fin_plen - 20);
}
}
#endif
bcopy((char *)fin->fin_ip, (char *)ip2, ohlen);
#ifdef USE_INET6
ip6 = (ip6_t *)ip;
if (fin->fin_v == 6) {
ip6->ip6_flow = ((ip6_t *)fin->fin_ip)->ip6_flow;
ip6->ip6_plen = htons(iclen - hlen);
ip6->ip6_nxt = IPPROTO_ICMPV6;
ip6->ip6_hlim = 0;
ip6->ip6_src = dst6.in6;
ip6->ip6_dst = fin->fin_src6.in6;
if (xtra > 0)
bcopy((char *)fin->fin_ip + ohlen,
(char *)&icmp->icmp_ip + ohlen, xtra);
icmp->icmp_cksum = in6_cksum(m, IPPROTO_ICMPV6,
sizeof(*ip6), iclen - hlen);
} else
#endif
{
ip->ip_p = IPPROTO_ICMP;
ip->ip_src.s_addr = dst4.s_addr;
ip->ip_dst.s_addr = fin->fin_saddr;
if (xtra > 0)
bcopy((char *)fin->fin_ip + ohlen,
(char *)&icmp->icmp_ip + ohlen, xtra);
icmp->icmp_cksum = ipf_cksum((u_short *)icmp,
sizeof(*icmp) + 8);
ip->ip_len = htons(iclen);
ip->ip_p = IPPROTO_ICMP;
}
err = ipf_send_ip(fin, m);
return err;
}
/*
* m0 - pointer to mbuf where the IP packet starts
* mpp - pointer to the mbuf pointer that is the start of the mbuf chain
*/
int
ipf_fastroute(m0, mpp, fin, fdp)
mb_t *m0, **mpp;
fr_info_t *fin;
frdest_t *fdp;
{
register struct ip *ip, *mhip;
register struct mbuf *m = *mpp;
register struct route *ro;
int len, off, error = 0, hlen, code;
struct ifnet *ifp, *sifp;
struct sockaddr_in *dst;
struct route iproute;
u_short ip_off;
frdest_t node;
frentry_t *fr;
ro = NULL;
#ifdef M_WRITABLE
/*
* HOT FIX/KLUDGE:
*
* If the mbuf we're about to send is not writable (because of
* a cluster reference, for example) we'll need to make a copy
* of it since this routine modifies the contents.
*
* If you have non-crappy network hardware that can transmit data
* from the mbuf, rather than making a copy, this is gonna be a
* problem.
*/
if (M_WRITABLE(m) == 0) {
m0 = m_dup(m, M_NOWAIT);
if (m0 != 0) {
FREE_MB_T(m);
m = m0;
*mpp = m;
} else {
error = ENOBUFS;
FREE_MB_T(m);
goto done;
}
}
#endif
#ifdef USE_INET6
if (fin->fin_v == 6) {
/*
* currently "to <if>" and "to <if>:ip#" are not supported
* for IPv6
*/
return ip6_output(m, NULL, NULL, 0, NULL, NULL, NULL);
}
#endif
hlen = fin->fin_hlen;
ip = mtod(m0, struct ip *);
ifp = NULL;
/*
* Route packet.
*/
ro = &iproute;
bzero(ro, sizeof (*ro));
dst = (struct sockaddr_in *)&ro->ro_dst;
dst->sin_family = AF_INET;
dst->sin_addr = ip->ip_dst;
fr = fin->fin_fr;
if ((fr != NULL) && !(fr->fr_flags & FR_KEEPSTATE) && (fdp != NULL) &&
(fdp->fd_type == FRD_DSTLIST)) {
if (ipf_dstlist_select_node(fin, fdp->fd_ptr, NULL, &node) == 0)
fdp = &node;
}
if (fdp != NULL)
ifp = fdp->fd_ptr;
else
ifp = fin->fin_ifp;
if ((ifp == NULL) && ((fr == NULL) || !(fr->fr_flags & FR_FASTROUTE))) {
error = -2;
goto bad;
}
if ((fdp != NULL) && (fdp->fd_ip.s_addr != 0))
dst->sin_addr = fdp->fd_ip;
dst->sin_len = sizeof(*dst);
in_rtalloc(ro, M_GETFIB(m0));
if ((ifp == NULL) && (ro->ro_rt != NULL))
ifp = ro->ro_rt->rt_ifp;
if ((ro->ro_rt == NULL) || (ifp == NULL)) {
if (in_localaddr(ip->ip_dst))
error = EHOSTUNREACH;
else
error = ENETUNREACH;
goto bad;
}
if (ro->ro_rt->rt_flags & RTF_GATEWAY)
dst = (struct sockaddr_in *)ro->ro_rt->rt_gateway;
if (ro->ro_rt)
counter_u64_add(ro->ro_rt->rt_pksent, 1);
/*
* For input packets which are being "fastrouted", they won't
* go back through output filtering and miss their chance to get
* NAT'd and counted. Duplicated packets aren't considered to be
* part of the normal packet stream, so do not NAT them or pass
* them through stateful checking, etc.
*/
if ((fdp != &fr->fr_dif) && (fin->fin_out == 0)) {
sifp = fin->fin_ifp;
fin->fin_ifp = ifp;
fin->fin_out = 1;
(void) ipf_acctpkt(fin, NULL);
fin->fin_fr = NULL;
if (!fr || !(fr->fr_flags & FR_RETMASK)) {
u_32_t pass;
(void) ipf_state_check(fin, &pass);
}
switch (ipf_nat_checkout(fin, NULL))
{
case 0 :
break;
case 1 :
ip->ip_sum = 0;
break;
case -1 :
error = -1;
goto bad;
break;
}
fin->fin_ifp = sifp;
fin->fin_out = 0;
} else
ip->ip_sum = 0;
/*
* If small enough for interface, can just send directly.
*/
if (ntohs(ip->ip_len) <= ifp->if_mtu) {
if (!ip->ip_sum)
ip->ip_sum = in_cksum(m, hlen);
error = (*ifp->if_output)(ifp, m, (struct sockaddr *)dst,
ro
);
goto done;
}
/*
* Too large for interface; fragment if possible.
* Must be able to put at least 8 bytes per fragment.
*/
ip_off = ntohs(ip->ip_off);
if (ip_off & IP_DF) {
error = EMSGSIZE;
goto bad;
}
len = (ifp->if_mtu - hlen) &~ 7;
if (len < 8) {
error = EMSGSIZE;
goto bad;
}
{
int mhlen, firstlen = len;
struct mbuf **mnext = &m->m_act;
/*
* Loop through length of segment after first fragment,
* make new header and copy data of each part and link onto chain.
*/
m0 = m;
mhlen = sizeof (struct ip);
for (off = hlen + len; off < ntohs(ip->ip_len); off += len) {
#ifdef MGETHDR
MGETHDR(m, M_NOWAIT, MT_HEADER);
#else
MGET(m, M_NOWAIT, MT_HEADER);
#endif
if (m == 0) {
m = m0;
error = ENOBUFS;
goto bad;
}
m->m_data += max_linkhdr;
mhip = mtod(m, struct ip *);
bcopy((char *)ip, (char *)mhip, sizeof(*ip));
if (hlen > sizeof (struct ip)) {
mhlen = ip_optcopy(ip, mhip) + sizeof (struct ip);
IP_HL_A(mhip, mhlen >> 2);
}
m->m_len = mhlen;
mhip->ip_off = ((off - hlen) >> 3) + ip_off;
if (off + len >= ntohs(ip->ip_len))
len = ntohs(ip->ip_len) - off;
else
mhip->ip_off |= IP_MF;
mhip->ip_len = htons((u_short)(len + mhlen));
*mnext = m;
m->m_next = m_copy(m0, off, len);
if (m->m_next == 0) {
error = ENOBUFS; /* ??? */
goto sendorfree;
}
m->m_pkthdr.len = mhlen + len;
m->m_pkthdr.rcvif = NULL;
mhip->ip_off = htons((u_short)mhip->ip_off);
mhip->ip_sum = 0;
mhip->ip_sum = in_cksum(m, mhlen);
mnext = &m->m_act;
}
/*
* Update first fragment by trimming what's been copied out
* and updating header, then send each fragment (in order).
*/
m_adj(m0, hlen + firstlen - ip->ip_len);
ip->ip_len = htons((u_short)(hlen + firstlen));
ip->ip_off = htons((u_short)IP_MF);
ip->ip_sum = 0;
ip->ip_sum = in_cksum(m0, hlen);
sendorfree:
for (m = m0; m; m = m0) {
m0 = m->m_act;
m->m_act = 0;
if (error == 0)
error = (*ifp->if_output)(ifp, m,
(struct sockaddr *)dst,
ro
);
else
FREE_MB_T(m);
}
}
done:
if (!error)
ipfmain.ipf_frouteok[0]++;
else
ipfmain.ipf_frouteok[1]++;
if ((ro != NULL) && (ro->ro_rt != NULL)) {
RTFREE(ro->ro_rt);
}
return 0;
bad:
if (error == EMSGSIZE) {
sifp = fin->fin_ifp;
code = fin->fin_icode;
fin->fin_icode = ICMP_UNREACH_NEEDFRAG;
fin->fin_ifp = ifp;
(void) ipf_send_icmp_err(ICMP_UNREACH, fin, 1);
fin->fin_ifp = sifp;
fin->fin_icode = code;
}
FREE_MB_T(m);
goto done;
}
int
ipf_verifysrc(fin)
fr_info_t *fin;
{
struct sockaddr_in *dst;
struct route iproute;
bzero((char *)&iproute, sizeof(iproute));
dst = (struct sockaddr_in *)&iproute.ro_dst;
dst->sin_len = sizeof(*dst);
dst->sin_family = AF_INET;
dst->sin_addr = fin->fin_src;
in_rtalloc(&iproute, 0);
if (iproute.ro_rt == NULL)
return 0;
return (fin->fin_ifp == iproute.ro_rt->rt_ifp);
}
/*
* return the first IP Address associated with an interface
*/
int
ipf_ifpaddr(softc, v, atype, ifptr, inp, inpmask)
ipf_main_softc_t *softc;
int v, atype;
void *ifptr;
i6addr_t *inp, *inpmask;
{
#ifdef USE_INET6
struct in6_addr *inp6 = NULL;
#endif
struct sockaddr *sock, *mask;
struct sockaddr_in *sin;
struct ifaddr *ifa;
struct ifnet *ifp;
if ((ifptr == NULL) || (ifptr == (void *)-1))
return -1;
sin = NULL;
ifp = ifptr;
if (v == 4)
inp->in4.s_addr = 0;
#ifdef USE_INET6
else if (v == 6)
bzero((char *)inp, sizeof(*inp));
#endif
ifa = TAILQ_FIRST(&ifp->if_addrhead);
sock = ifa->ifa_addr;
while (sock != NULL && ifa != NULL) {
sin = (struct sockaddr_in *)sock;
if ((v == 4) && (sin->sin_family == AF_INET))
break;
#ifdef USE_INET6
if ((v == 6) && (sin->sin_family == AF_INET6)) {
inp6 = &((struct sockaddr_in6 *)sin)->sin6_addr;
if (!IN6_IS_ADDR_LINKLOCAL(inp6) &&
!IN6_IS_ADDR_LOOPBACK(inp6))
break;
}
#endif
ifa = TAILQ_NEXT(ifa, ifa_link);
if (ifa != NULL)
sock = ifa->ifa_addr;
}
if (ifa == NULL || sin == NULL)
return -1;
mask = ifa->ifa_netmask;
if (atype == FRI_BROADCAST)
sock = ifa->ifa_broadaddr;
else if (atype == FRI_PEERADDR)
sock = ifa->ifa_dstaddr;
if (sock == NULL)
return -1;
#ifdef USE_INET6
if (v == 6) {
return ipf_ifpfillv6addr(atype, (struct sockaddr_in6 *)sock,
(struct sockaddr_in6 *)mask,
inp, inpmask);
}
#endif
return ipf_ifpfillv4addr(atype, (struct sockaddr_in *)sock,
(struct sockaddr_in *)mask,
&inp->in4, &inpmask->in4);
}
u_32_t
ipf_newisn(fin)
fr_info_t *fin;
{
u_32_t newiss;
newiss = arc4random();
return newiss;
}
INLINE int
ipf_checkv4sum(fin)
fr_info_t *fin;
{
#ifdef CSUM_DATA_VALID
int manual = 0;
u_short sum;
ip_t *ip;
mb_t *m;
if ((fin->fin_flx & FI_NOCKSUM) != 0)
return 0;
if ((fin->fin_flx & FI_SHORT) != 0)
return 1;
if (fin->fin_cksum != FI_CK_NEEDED)
return (fin->fin_cksum > FI_CK_NEEDED) ? 0 : -1;
m = fin->fin_m;
if (m == NULL) {
manual = 1;
goto skipauto;
}
ip = fin->fin_ip;
if ((m->m_pkthdr.csum_flags & (CSUM_IP_CHECKED|CSUM_IP_VALID)) ==
CSUM_IP_CHECKED) {
fin->fin_cksum = FI_CK_BAD;
fin->fin_flx |= FI_BAD;
return -1;
}
if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR)
sum = m->m_pkthdr.csum_data;
else
sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
htonl(m->m_pkthdr.csum_data +
fin->fin_dlen + fin->fin_p));
sum ^= 0xffff;
if (sum != 0) {
fin->fin_cksum = FI_CK_BAD;
fin->fin_flx |= FI_BAD;
} else {
fin->fin_cksum = FI_CK_SUMOK;
return 0;
}
} else {
if (m->m_pkthdr.csum_flags == CSUM_DELAY_DATA) {
fin->fin_cksum = FI_CK_L4FULL;
return 0;
} else if (m->m_pkthdr.csum_flags == CSUM_TCP ||
m->m_pkthdr.csum_flags == CSUM_UDP) {
fin->fin_cksum = FI_CK_L4PART;
return 0;
} else if (m->m_pkthdr.csum_flags == CSUM_IP) {
fin->fin_cksum = FI_CK_L4PART;
return 0;
} else {
manual = 1;
}
}
skipauto:
if (manual != 0) {
if (ipf_checkl4sum(fin) == -1) {
fin->fin_flx |= FI_BAD;
return -1;
}
}
#else
if (ipf_checkl4sum(fin) == -1) {
fin->fin_flx |= FI_BAD;
return -1;
}
#endif
return 0;
}
#ifdef USE_INET6
INLINE int
ipf_checkv6sum(fin)
fr_info_t *fin;
{
if ((fin->fin_flx & FI_NOCKSUM) != 0)
return 0;
if ((fin->fin_flx & FI_SHORT) != 0)
return 1;
if (fin->fin_cksum != FI_CK_NEEDED)
return (fin->fin_cksum > FI_CK_NEEDED) ? 0 : -1;
if (ipf_checkl4sum(fin) == -1) {
fin->fin_flx |= FI_BAD;
return -1;
}
return 0;
}
#endif /* USE_INET6 */
size_t
mbufchainlen(m0)
struct mbuf *m0;
{
size_t len;
if ((m0->m_flags & M_PKTHDR) != 0) {
len = m0->m_pkthdr.len;
} else {
struct mbuf *m;
for (m = m0, len = 0; m != NULL; m = m->m_next)
len += m->m_len;
}
return len;
}
/* ------------------------------------------------------------------------ */
/* Function: ipf_pullup */
/* Returns: NULL == pullup failed, else pointer to protocol header */
/* Parameters: xmin(I)- pointer to buffer where data packet starts */
/* fin(I) - pointer to packet information */
/* len(I) - number of bytes to pullup */
/* */
/* Attempt to move at least len bytes (from the start of the buffer) into a */
/* single buffer for ease of access. Operating system native functions are */
/* used to manage buffers - if necessary. If the entire packet ends up in */
/* a single buffer, set the FI_COALESCE flag even though ipf_coalesce() has */
/* not been called. Both fin_ip and fin_dp are updated before exiting _IF_ */
/* and ONLY if the pullup succeeds. */
/* */
/* We assume that 'xmin' is a pointer to a buffer that is part of the chain */
/* of buffers that starts at *fin->fin_mp. */
/* ------------------------------------------------------------------------ */
void *
ipf_pullup(xmin, fin, len)
mb_t *xmin;
fr_info_t *fin;
int len;
{
int dpoff, ipoff;
mb_t *m = xmin;
char *ip;
if (m == NULL)
return NULL;
ip = (char *)fin->fin_ip;
if ((fin->fin_flx & FI_COALESCE) != 0)
return ip;
ipoff = fin->fin_ipoff;
if (fin->fin_dp != NULL)
dpoff = (char *)fin->fin_dp - (char *)ip;
else
dpoff = 0;
if (M_LEN(m) < len) {
mb_t *n = *fin->fin_mp;
/*
* Assume that M_PKTHDR is set and just work with what is left
* rather than check..
* Should not make any real difference, anyway.
*/
if (m != n) {
/*
* Record the mbuf that points to the mbuf that we're
* about to go to work on so that we can update the
* m_next appropriately later.
*/
for (; n->m_next != m; n = n->m_next)
;
} else {
n = NULL;
}
#ifdef MHLEN
if (len > MHLEN)
#else
if (len > MLEN)
#endif
{
#ifdef HAVE_M_PULLDOWN
if (m_pulldown(m, 0, len, NULL) == NULL)
m = NULL;
#else
FREE_MB_T(*fin->fin_mp);
m = NULL;
n = NULL;
#endif
} else
{
m = m_pullup(m, len);
}
if (n != NULL)
n->m_next = m;
if (m == NULL) {
/*
* When n is non-NULL, it indicates that m pointed to
* a sub-chain (tail) of the mbuf and that the head
* of this chain has not yet been free'd.
*/
if (n != NULL) {
FREE_MB_T(*fin->fin_mp);
}
*fin->fin_mp = NULL;
fin->fin_m = NULL;
return NULL;
}
if (n == NULL)
*fin->fin_mp = m;
while (M_LEN(m) == 0) {
m = m->m_next;
}
fin->fin_m = m;
ip = MTOD(m, char *) + ipoff;
fin->fin_ip = (ip_t *)ip;
if (fin->fin_dp != NULL)
fin->fin_dp = (char *)fin->fin_ip + dpoff;
if (fin->fin_fraghdr != NULL)
fin->fin_fraghdr = (char *)ip +
((char *)fin->fin_fraghdr -
(char *)fin->fin_ip);
}
if (len == fin->fin_plen)
fin->fin_flx |= FI_COALESCE;
return ip;
}
int
ipf_inject(fin, m)
fr_info_t *fin;
mb_t *m;
{
int error = 0;
if (fin->fin_out == 0) {
netisr_dispatch(NETISR_IP, m);
} else {
fin->fin_ip->ip_len = ntohs(fin->fin_ip->ip_len);
fin->fin_ip->ip_off = ntohs(fin->fin_ip->ip_off);
error = ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL);
}
return error;
}
int ipf_pfil_unhook(void) {
#if defined(NETBSD_PF) && (__FreeBSD_version >= 500011)
struct pfil_head *ph_inet;
# ifdef USE_INET6
struct pfil_head *ph_inet6;
# endif
#endif
#ifdef NETBSD_PF
ph_inet = pfil_head_get(PFIL_TYPE_AF, AF_INET);
if (ph_inet != NULL)
pfil_remove_hook((void *)ipf_check_wrapper, NULL,
PFIL_IN|PFIL_OUT|PFIL_WAITOK, ph_inet);
# ifdef USE_INET6
ph_inet6 = pfil_head_get(PFIL_TYPE_AF, AF_INET6);
if (ph_inet6 != NULL)
pfil_remove_hook((void *)ipf_check_wrapper6, NULL,
PFIL_IN|PFIL_OUT|PFIL_WAITOK, ph_inet6);
# endif
#endif
return (0);
}
int ipf_pfil_hook(void) {
#if defined(NETBSD_PF) && (__FreeBSD_version >= 500011)
struct pfil_head *ph_inet;
# ifdef USE_INET6
struct pfil_head *ph_inet6;
# endif
#endif
# ifdef NETBSD_PF
ph_inet = pfil_head_get(PFIL_TYPE_AF, AF_INET);
# ifdef USE_INET6
ph_inet6 = pfil_head_get(PFIL_TYPE_AF, AF_INET6);
# endif
if (ph_inet == NULL
# ifdef USE_INET6
&& ph_inet6 == NULL
# endif
) {
return ENODEV;
}
if (ph_inet != NULL)
pfil_add_hook((void *)ipf_check_wrapper, NULL,
PFIL_IN|PFIL_OUT|PFIL_WAITOK, ph_inet);
# ifdef USE_INET6
if (ph_inet6 != NULL)
pfil_add_hook((void *)ipf_check_wrapper6, NULL,
PFIL_IN|PFIL_OUT|PFIL_WAITOK, ph_inet6);
# endif
# endif
return (0);
}
void
ipf_event_reg(void)
{
ipf_arrivetag = EVENTHANDLER_REGISTER(ifnet_arrival_event, \
ipf_ifevent, &ipfmain, \
EVENTHANDLER_PRI_ANY);
ipf_departtag = EVENTHANDLER_REGISTER(ifnet_departure_event, \
ipf_ifevent, &ipfmain, \
EVENTHANDLER_PRI_ANY);
ipf_clonetag = EVENTHANDLER_REGISTER(if_clone_event, ipf_ifevent, \
&ipfmain, EVENTHANDLER_PRI_ANY);
}
void
ipf_event_dereg(void)
{
if (ipf_arrivetag != NULL) {
EVENTHANDLER_DEREGISTER(ifnet_arrival_event, ipf_arrivetag);
}
if (ipf_departtag != NULL) {
EVENTHANDLER_DEREGISTER(ifnet_departure_event, ipf_departtag);
}
if (ipf_clonetag != NULL) {
EVENTHANDLER_DEREGISTER(if_clone_event, ipf_clonetag);
}
}
u_32_t
ipf_random()
{
return arc4random();
}
u_int
ipf_pcksum(fin, hlen, sum)
fr_info_t *fin;
int hlen;
u_int sum;
{
struct mbuf *m;
u_int sum2;
int off;
m = fin->fin_m;
off = (char *)fin->fin_dp - (char *)fin->fin_ip;
m->m_data += hlen;
m->m_len -= hlen;
sum2 = in_cksum(fin->fin_m, fin->fin_plen - off);
m->m_len += hlen;
m->m_data -= hlen;
/*
* Both sum and sum2 are partial sums, so combine them together.
*/
sum += ~sum2 & 0xffff;
while (sum > 0xffff)
sum = (sum & 0xffff) + (sum >> 16);
sum2 = ~sum & 0xffff;
return sum2;
}
Index: head/sys/contrib/vchiq/interface/compat/vchi_bsd.c
===================================================================
--- head/sys/contrib/vchiq/interface/compat/vchi_bsd.c (revision 283290)
+++ head/sys/contrib/vchiq/interface/compat/vchi_bsd.c (revision 283291)
@@ -1,532 +1,532 @@
/*-
* Copyright (c) 2010 Max Khon <fjoe@freebsd.org>
* All rights reserved.
*
* This software was developed by Max Khon under sponsorship from
* the FreeBSD Foundation and Ethon Technologies GmbH.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $Id: bsd-compat.c 9253 2010-09-02 10:12:09Z fjoe $
*/
#include <sys/types.h>
#include <sys/limits.h>
#include <sys/bus.h>
#include <sys/callout.h>
#include <sys/firmware.h>
#include <sys/param.h>
#include <sys/proc.h>
#include <sys/syscallsubr.h>
#include <sys/systm.h>
#include <sys/taskqueue.h>
#include <machine/stdarg.h>
#include "mbox_if.h"
#include <interface/compat/vchi_bsd.h>
MALLOC_DEFINE(M_VCHI, "VCHI", "VCHI");
/*
* Timer API
*/
static void
run_timer(void *arg)
{
struct timer_list *t = (struct timer_list *) arg;
void (*function)(unsigned long);
mtx_lock_spin(&t->mtx);
if (callout_pending(&t->callout)) {
/* callout was reset */
mtx_unlock_spin(&t->mtx);
return;
}
if (!callout_active(&t->callout)) {
/* callout was stopped */
mtx_unlock_spin(&t->mtx);
return;
}
callout_deactivate(&t->callout);
function = t->function;
mtx_unlock_spin(&t->mtx);
function(t->data);
}
void
init_timer(struct timer_list *t)
{
mtx_init(&t->mtx, "dahdi timer lock", NULL, MTX_SPIN);
- callout_init(&t->callout, CALLOUT_MPSAFE);
+ callout_init(&t->callout, 1);
t->expires = 0;
/*
* function and data are not initialized intentionally:
* they are not initialized by Linux implementation too
*/
}
void
setup_timer(struct timer_list *t, void (*function)(unsigned long), unsigned long data)
{
t->function = function;
t->data = data;
init_timer(t);
}
void
mod_timer(struct timer_list *t, unsigned long expires)
{
mtx_lock_spin(&t->mtx);
callout_reset(&t->callout, expires - jiffies, run_timer, t);
mtx_unlock_spin(&t->mtx);
}
void
add_timer(struct timer_list *t)
{
mod_timer(t, t->expires);
}
int
del_timer_sync(struct timer_list *t)
{
mtx_lock_spin(&t->mtx);
callout_stop(&t->callout);
mtx_unlock_spin(&t->mtx);
mtx_destroy(&t->mtx);
return 0;
}
int
del_timer(struct timer_list *t)
{
del_timer_sync(t);
return 0;
}
/*
* Completion API
*/
void
init_completion(struct completion *c)
{
cv_init(&c->cv, "VCHI completion cv");
mtx_init(&c->lock, "VCHI completion lock", "condvar", MTX_DEF);
c->done = 0;
}
void
destroy_completion(struct completion *c)
{
cv_destroy(&c->cv);
mtx_destroy(&c->lock);
}
void
complete(struct completion *c)
{
mtx_lock(&c->lock);
if (c->done >= 0) {
KASSERT(c->done < INT_MAX, ("c->done overflow")); /* XXX check */
c->done++;
cv_signal(&c->cv);
} else {
KASSERT(c->done == -1, ("Invalid value of c->done: %d", c->done));
}
mtx_unlock(&c->lock);
}
void
complete_all(struct completion *c)
{
mtx_lock(&c->lock);
if (c->done >= 0) {
KASSERT(c->done < INT_MAX, ("c->done overflow")); /* XXX check */
c->done = -1;
cv_broadcast(&c->cv);
} else {
KASSERT(c->done == -1, ("Invalid value of c->done: %d", c->done));
}
mtx_unlock(&c->lock);
}
void
INIT_COMPLETION_locked(struct completion *c)
{
mtx_lock(&c->lock);
c->done = 0;
mtx_unlock(&c->lock);
}
static void
_completion_claim(struct completion *c)
{
KASSERT(mtx_owned(&c->lock),
("_completion_claim should be called with acquired lock"));
KASSERT(c->done != 0, ("_completion_claim on non-waited completion"));
if (c->done > 0)
c->done--;
else
KASSERT(c->done == -1, ("Invalid value of c->done: %d", c->done));
}
void
wait_for_completion(struct completion *c)
{
mtx_lock(&c->lock);
if (!c->done)
cv_wait(&c->cv, &c->lock);
c->done--;
mtx_unlock(&c->lock);
}
int
try_wait_for_completion(struct completion *c)
{
int res = 0;
mtx_lock(&c->lock);
if (!c->done)
res = 1;
else
c->done--;
mtx_unlock(&c->lock);
return res == 0;
}
int
wait_for_completion_interruptible_timeout(struct completion *c, unsigned long timeout)
{
int res = 0;
unsigned long start, now;
start = jiffies;
mtx_lock(&c->lock);
while (c->done == 0) {
res = cv_timedwait_sig(&c->cv, &c->lock, timeout);
if (res)
goto out;
now = jiffies;
if (timeout < (now - start)) {
res = EWOULDBLOCK;
goto out;
}
timeout -= (now - start);
start = now;
}
_completion_claim(c);
res = 0;
out:
mtx_unlock(&c->lock);
if (res == EWOULDBLOCK) {
return 0;
} else if ((res == EINTR) || (res == ERESTART)) {
return -ERESTART;
} else {
KASSERT((res == 0), ("res = %d", res));
return timeout;
}
}
int
wait_for_completion_interruptible(struct completion *c)
{
int res = 0;
mtx_lock(&c->lock);
while (c->done == 0) {
res = cv_wait_sig(&c->cv, &c->lock);
if (res)
goto out;
}
_completion_claim(c);
out:
mtx_unlock(&c->lock);
if ((res == EINTR) || (res == ERESTART))
res = -ERESTART;
return res;
}
int
wait_for_completion_killable(struct completion *c)
{
return wait_for_completion_interruptible(c);
}
/*
* Semaphore API
*/
void sema_sysinit(void *arg)
{
struct semaphore *s = arg;
printf("sema_sysinit\n");
_sema_init(s, 1);
}
void
_sema_init(struct semaphore *s, int value)
{
bzero(s, sizeof(*s));
mtx_init(&s->mtx, "sema lock", "VCHIQ sepmaphore backing lock",
MTX_DEF | MTX_NOWITNESS | MTX_QUIET);
cv_init(&s->cv, "sema cv");
s->value = value;
}
void
_sema_destroy(struct semaphore *s)
{
mtx_destroy(&s->mtx);
cv_destroy(&s->cv);
}
void
down(struct semaphore *s)
{
mtx_lock(&s->mtx);
while (s->value == 0) {
s->waiters++;
cv_wait(&s->cv, &s->mtx);
s->waiters--;
}
s->value--;
mtx_unlock(&s->mtx);
}
int
down_interruptible(struct semaphore *s)
{
int ret ;
ret = 0;
mtx_lock(&s->mtx);
while (s->value == 0) {
s->waiters++;
ret = cv_wait_sig(&s->cv, &s->mtx);
s->waiters--;
if (ret == EINTR) {
mtx_unlock(&s->mtx);
return (-EINTR);
}
if (ret == ERESTART)
continue;
}
s->value--;
mtx_unlock(&s->mtx);
return (0);
}
int
down_trylock(struct semaphore *s)
{
int ret;
ret = 0;
mtx_lock(&s->mtx);
if (s->value > 0) {
/* Success. */
s->value--;
ret = 0;
} else {
ret = -EAGAIN;
}
mtx_unlock(&s->mtx);
return (ret);
}
void
up(struct semaphore *s)
{
mtx_lock(&s->mtx);
s->value++;
if (s->waiters && s->value > 0)
cv_signal(&s->cv);
mtx_unlock(&s->mtx);
}
/*
* Logging API
*/
void
rlprintf(int pps, const char *fmt, ...)
{
va_list ap;
static struct timeval last_printf;
static int count;
if (ppsratecheck(&last_printf, &count, pps)) {
va_start(ap, fmt);
vprintf(fmt, ap);
va_end(ap);
}
}
void
device_rlprintf(int pps, device_t dev, const char *fmt, ...)
{
va_list ap;
static struct timeval last_printf;
static int count;
if (ppsratecheck(&last_printf, &count, pps)) {
va_start(ap, fmt);
device_print_prettyname(dev);
vprintf(fmt, ap);
va_end(ap);
}
}
/*
* Signals API
*/
void
flush_signals(VCHIQ_THREAD_T thr)
{
printf("Implement ME: %s\n", __func__);
}
int
fatal_signal_pending(VCHIQ_THREAD_T thr)
{
printf("Implement ME: %s\n", __func__);
return (0);
}
/*
* kthread API
*/
/*
* This is a hack to avoid memory leak
*/
#define MAX_THREAD_DATA_SLOTS 32
static int thread_data_slot = 0;
struct thread_data {
void *data;
int (*threadfn)(void *);
};
static struct thread_data thread_slots[MAX_THREAD_DATA_SLOTS];
static void
kthread_wrapper(void *data)
{
struct thread_data *slot;
slot = data;
slot->threadfn(slot->data);
}
VCHIQ_THREAD_T
vchiq_thread_create(int (*threadfn)(void *data),
void *data,
const char namefmt[], ...)
{
VCHIQ_THREAD_T newp;
va_list ap;
char name[MAXCOMLEN+1];
struct thread_data *slot;
if (thread_data_slot >= MAX_THREAD_DATA_SLOTS) {
printf("kthread_create: out of thread data slots\n");
return (NULL);
}
slot = &thread_slots[thread_data_slot];
slot->data = data;
slot->threadfn = threadfn;
va_start(ap, namefmt);
vsnprintf(name, sizeof(name), namefmt, ap);
va_end(ap);
newp = NULL;
if (kproc_create(kthread_wrapper, (void*)slot, &newp, 0, 0,
"%s", name) != 0) {
/* Just to be sure */
newp = NULL;
}
else
thread_data_slot++;
return newp;
}
void
set_user_nice(VCHIQ_THREAD_T thr, int nice)
{
/* NOOP */
}
void
wake_up_process(VCHIQ_THREAD_T thr)
{
/* NOOP */
}
void
bcm_mbox_write(int channel, uint32_t data)
{
device_t mbox;
mbox = devclass_get_device(devclass_find("mbox"), 0);
if (mbox)
MBOX_WRITE(mbox, channel, data);
}
Index: head/sys/dev/acpica/acpi.c
===================================================================
--- head/sys/dev/acpica/acpi.c (revision 283290)
+++ head/sys/dev/acpica/acpi.c (revision 283291)
@@ -1,4052 +1,4052 @@
/*-
* Copyright (c) 2000 Takanori Watanabe <takawata@jp.freebsd.org>
* Copyright (c) 2000 Mitsuru IWASAKI <iwasaki@jp.freebsd.org>
* Copyright (c) 2000, 2001 Michael Smith
* Copyright (c) 2000 BSDi
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_acpi.h"
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/fcntl.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/ioccom.h>
#include <sys/reboot.h>
#include <sys/sysctl.h>
#include <sys/ctype.h>
#include <sys/linker.h>
#include <sys/power.h>
#include <sys/sbuf.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/timetc.h>
#if defined(__i386__) || defined(__amd64__)
#include <machine/pci_cfgreg.h>
#endif
#include <machine/resource.h>
#include <machine/bus.h>
#include <sys/rman.h>
#include <isa/isavar.h>
#include <isa/pnpvar.h>
#include <contrib/dev/acpica/include/acpi.h>
#include <contrib/dev/acpica/include/accommon.h>
#include <contrib/dev/acpica/include/acnamesp.h>
#include <dev/acpica/acpivar.h>
#include <dev/acpica/acpiio.h>
#include <vm/vm_param.h>
static MALLOC_DEFINE(M_ACPIDEV, "acpidev", "ACPI devices");
/* Hooks for the ACPI CA debugging infrastructure */
#define _COMPONENT ACPI_BUS
ACPI_MODULE_NAME("ACPI")
static d_open_t acpiopen;
static d_close_t acpiclose;
static d_ioctl_t acpiioctl;
static struct cdevsw acpi_cdevsw = {
.d_version = D_VERSION,
.d_open = acpiopen,
.d_close = acpiclose,
.d_ioctl = acpiioctl,
.d_name = "acpi",
};
struct acpi_interface {
ACPI_STRING *data;
int num;
};
/* Global mutex for locking access to the ACPI subsystem. */
struct mtx acpi_mutex;
struct callout acpi_sleep_timer;
/* Bitmap of device quirks. */
int acpi_quirks;
/* Supported sleep states. */
static BOOLEAN acpi_sleep_states[ACPI_S_STATE_COUNT];
static void acpi_lookup(void *arg, const char *name, device_t *dev);
static int acpi_modevent(struct module *mod, int event, void *junk);
static int acpi_probe(device_t dev);
static int acpi_attach(device_t dev);
static int acpi_suspend(device_t dev);
static int acpi_resume(device_t dev);
static int acpi_shutdown(device_t dev);
static device_t acpi_add_child(device_t bus, u_int order, const char *name,
int unit);
static int acpi_print_child(device_t bus, device_t child);
static void acpi_probe_nomatch(device_t bus, device_t child);
static void acpi_driver_added(device_t dev, driver_t *driver);
static int acpi_read_ivar(device_t dev, device_t child, int index,
uintptr_t *result);
static int acpi_write_ivar(device_t dev, device_t child, int index,
uintptr_t value);
static struct resource_list *acpi_get_rlist(device_t dev, device_t child);
static void acpi_reserve_resources(device_t dev);
static int acpi_sysres_alloc(device_t dev);
static int acpi_set_resource(device_t dev, device_t child, int type,
int rid, u_long start, u_long count);
static struct resource *acpi_alloc_resource(device_t bus, device_t child,
int type, int *rid, u_long start, u_long end,
u_long count, u_int flags);
static int acpi_adjust_resource(device_t bus, device_t child, int type,
struct resource *r, u_long start, u_long end);
static int acpi_release_resource(device_t bus, device_t child, int type,
int rid, struct resource *r);
static void acpi_delete_resource(device_t bus, device_t child, int type,
int rid);
static uint32_t acpi_isa_get_logicalid(device_t dev);
static int acpi_isa_get_compatid(device_t dev, uint32_t *cids, int count);
static char *acpi_device_id_probe(device_t bus, device_t dev, char **ids);
static ACPI_STATUS acpi_device_eval_obj(device_t bus, device_t dev,
ACPI_STRING pathname, ACPI_OBJECT_LIST *parameters,
ACPI_BUFFER *ret);
static ACPI_STATUS acpi_device_scan_cb(ACPI_HANDLE h, UINT32 level,
void *context, void **retval);
static ACPI_STATUS acpi_device_scan_children(device_t bus, device_t dev,
int max_depth, acpi_scan_cb_t user_fn, void *arg);
static int acpi_set_powerstate(device_t child, int state);
static int acpi_isa_pnp_probe(device_t bus, device_t child,
struct isa_pnp_id *ids);
static void acpi_probe_children(device_t bus);
static void acpi_probe_order(ACPI_HANDLE handle, int *order);
static ACPI_STATUS acpi_probe_child(ACPI_HANDLE handle, UINT32 level,
void *context, void **status);
static void acpi_sleep_enable(void *arg);
static ACPI_STATUS acpi_sleep_disable(struct acpi_softc *sc);
static ACPI_STATUS acpi_EnterSleepState(struct acpi_softc *sc, int state);
static void acpi_shutdown_final(void *arg, int howto);
static void acpi_enable_fixed_events(struct acpi_softc *sc);
static BOOLEAN acpi_has_hid(ACPI_HANDLE handle);
static void acpi_resync_clock(struct acpi_softc *sc);
static int acpi_wake_sleep_prep(ACPI_HANDLE handle, int sstate);
static int acpi_wake_run_prep(ACPI_HANDLE handle, int sstate);
static int acpi_wake_prep_walk(int sstate);
static int acpi_wake_sysctl_walk(device_t dev);
static int acpi_wake_set_sysctl(SYSCTL_HANDLER_ARGS);
static void acpi_system_eventhandler_sleep(void *arg, int state);
static void acpi_system_eventhandler_wakeup(void *arg, int state);
static int acpi_sname2sstate(const char *sname);
static const char *acpi_sstate2sname(int sstate);
static int acpi_supported_sleep_state_sysctl(SYSCTL_HANDLER_ARGS);
static int acpi_sleep_state_sysctl(SYSCTL_HANDLER_ARGS);
static int acpi_debug_objects_sysctl(SYSCTL_HANDLER_ARGS);
static int acpi_pm_func(u_long cmd, void *arg, ...);
static int acpi_child_location_str_method(device_t acdev, device_t child,
char *buf, size_t buflen);
static int acpi_child_pnpinfo_str_method(device_t acdev, device_t child,
char *buf, size_t buflen);
#if defined(__i386__) || defined(__amd64__)
static void acpi_enable_pcie(void);
#endif
static void acpi_hint_device_unit(device_t acdev, device_t child,
const char *name, int *unitp);
static void acpi_reset_interfaces(device_t dev);
static device_method_t acpi_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, acpi_probe),
DEVMETHOD(device_attach, acpi_attach),
DEVMETHOD(device_shutdown, acpi_shutdown),
DEVMETHOD(device_detach, bus_generic_detach),
DEVMETHOD(device_suspend, acpi_suspend),
DEVMETHOD(device_resume, acpi_resume),
/* Bus interface */
DEVMETHOD(bus_add_child, acpi_add_child),
DEVMETHOD(bus_print_child, acpi_print_child),
DEVMETHOD(bus_probe_nomatch, acpi_probe_nomatch),
DEVMETHOD(bus_driver_added, acpi_driver_added),
DEVMETHOD(bus_read_ivar, acpi_read_ivar),
DEVMETHOD(bus_write_ivar, acpi_write_ivar),
DEVMETHOD(bus_get_resource_list, acpi_get_rlist),
DEVMETHOD(bus_set_resource, acpi_set_resource),
DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
DEVMETHOD(bus_alloc_resource, acpi_alloc_resource),
DEVMETHOD(bus_adjust_resource, acpi_adjust_resource),
DEVMETHOD(bus_release_resource, acpi_release_resource),
DEVMETHOD(bus_delete_resource, acpi_delete_resource),
DEVMETHOD(bus_child_pnpinfo_str, acpi_child_pnpinfo_str_method),
DEVMETHOD(bus_child_location_str, acpi_child_location_str_method),
DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
DEVMETHOD(bus_setup_intr, bus_generic_setup_intr),
DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr),
DEVMETHOD(bus_hint_device_unit, acpi_hint_device_unit),
DEVMETHOD(bus_get_domain, acpi_get_domain),
/* ACPI bus */
DEVMETHOD(acpi_id_probe, acpi_device_id_probe),
DEVMETHOD(acpi_evaluate_object, acpi_device_eval_obj),
DEVMETHOD(acpi_pwr_for_sleep, acpi_device_pwr_for_sleep),
DEVMETHOD(acpi_scan_children, acpi_device_scan_children),
/* ISA emulation */
DEVMETHOD(isa_pnp_probe, acpi_isa_pnp_probe),
DEVMETHOD_END
};
static driver_t acpi_driver = {
"acpi",
acpi_methods,
sizeof(struct acpi_softc),
};
static devclass_t acpi_devclass;
DRIVER_MODULE(acpi, nexus, acpi_driver, acpi_devclass, acpi_modevent, 0);
MODULE_VERSION(acpi, 1);
ACPI_SERIAL_DECL(acpi, "ACPI root bus");
/* Local pools for managing system resources for ACPI child devices. */
static struct rman acpi_rman_io, acpi_rman_mem;
#define ACPI_MINIMUM_AWAKETIME 5
/* Holds the description of the acpi0 device. */
static char acpi_desc[ACPI_OEM_ID_SIZE + ACPI_OEM_TABLE_ID_SIZE + 2];
SYSCTL_NODE(_debug, OID_AUTO, acpi, CTLFLAG_RD, NULL, "ACPI debugging");
static char acpi_ca_version[12];
SYSCTL_STRING(_debug_acpi, OID_AUTO, acpi_ca_version, CTLFLAG_RD,
acpi_ca_version, 0, "Version of Intel ACPI-CA");
/*
* Allow overriding _OSI methods.
*/
static char acpi_install_interface[256];
TUNABLE_STR("hw.acpi.install_interface", acpi_install_interface,
sizeof(acpi_install_interface));
static char acpi_remove_interface[256];
TUNABLE_STR("hw.acpi.remove_interface", acpi_remove_interface,
sizeof(acpi_remove_interface));
/* Allow users to dump Debug objects without ACPI debugger. */
static int acpi_debug_objects;
TUNABLE_INT("debug.acpi.enable_debug_objects", &acpi_debug_objects);
SYSCTL_PROC(_debug_acpi, OID_AUTO, enable_debug_objects,
CTLFLAG_RW | CTLTYPE_INT, NULL, 0, acpi_debug_objects_sysctl, "I",
"Enable Debug objects");
/* Allow the interpreter to ignore common mistakes in BIOS. */
static int acpi_interpreter_slack = 1;
TUNABLE_INT("debug.acpi.interpreter_slack", &acpi_interpreter_slack);
SYSCTL_INT(_debug_acpi, OID_AUTO, interpreter_slack, CTLFLAG_RDTUN,
&acpi_interpreter_slack, 1, "Turn on interpreter slack mode.");
/* Ignore register widths set by FADT and use default widths instead. */
static int acpi_ignore_reg_width = 1;
TUNABLE_INT("debug.acpi.default_register_width", &acpi_ignore_reg_width);
SYSCTL_INT(_debug_acpi, OID_AUTO, default_register_width, CTLFLAG_RDTUN,
&acpi_ignore_reg_width, 1, "Ignore register widths set by FADT");
#ifdef __amd64__
/* Reset system clock while resuming. XXX Remove once tested. */
static int acpi_reset_clock = 1;
TUNABLE_INT("debug.acpi.reset_clock", &acpi_reset_clock);
SYSCTL_INT(_debug_acpi, OID_AUTO, reset_clock, CTLFLAG_RW,
&acpi_reset_clock, 1, "Reset system clock while resuming.");
#endif
/* Allow users to override quirks. */
TUNABLE_INT("debug.acpi.quirks", &acpi_quirks);
static int acpi_susp_bounce;
SYSCTL_INT(_debug_acpi, OID_AUTO, suspend_bounce, CTLFLAG_RW,
&acpi_susp_bounce, 0, "Don't actually suspend, just test devices.");
/*
* ACPI can only be loaded as a module by the loader; activating it after
* system bootstrap time is not useful, and can be fatal to the system.
* It also cannot be unloaded, since the entire system bus hierarchy hangs
* off it.
*/
static int
acpi_modevent(struct module *mod, int event, void *junk)
{
switch (event) {
case MOD_LOAD:
if (!cold) {
printf("The ACPI driver cannot be loaded after boot.\n");
return (EPERM);
}
break;
case MOD_UNLOAD:
if (!cold && power_pm_get_type() == POWER_PM_TYPE_ACPI)
return (EBUSY);
break;
default:
break;
}
return (0);
}
/*
* Perform early initialization.
*/
ACPI_STATUS
acpi_Startup(void)
{
static int started = 0;
ACPI_STATUS status;
int val;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
/* Only run the startup code once. The MADT driver also calls this. */
if (started)
return_VALUE (AE_OK);
started = 1;
/*
* Pre-allocate space for RSDT/XSDT and DSDT tables and allow resizing
* if more tables exist.
*/
if (ACPI_FAILURE(status = AcpiInitializeTables(NULL, 2, TRUE))) {
printf("ACPI: Table initialisation failed: %s\n",
AcpiFormatException(status));
return_VALUE (status);
}
/* Set up any quirks we have for this system. */
if (acpi_quirks == ACPI_Q_OK)
acpi_table_quirks(&acpi_quirks);
/* If the user manually set the disabled hint to 0, force-enable ACPI. */
if (resource_int_value("acpi", 0, "disabled", &val) == 0 && val == 0)
acpi_quirks &= ~ACPI_Q_BROKEN;
if (acpi_quirks & ACPI_Q_BROKEN) {
printf("ACPI disabled by blacklist. Contact your BIOS vendor.\n");
status = AE_SUPPORT;
}
return_VALUE (status);
}
/*
* Detect ACPI and perform early initialisation.
*/
int
acpi_identify(void)
{
ACPI_TABLE_RSDP *rsdp;
ACPI_TABLE_HEADER *rsdt;
ACPI_PHYSICAL_ADDRESS paddr;
struct sbuf sb;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
if (!cold)
return (ENXIO);
/* Check that we haven't been disabled with a hint. */
if (resource_disabled("acpi", 0))
return (ENXIO);
/* Check for other PM systems. */
if (power_pm_get_type() != POWER_PM_TYPE_NONE &&
power_pm_get_type() != POWER_PM_TYPE_ACPI) {
printf("ACPI identify failed, other PM system enabled.\n");
return (ENXIO);
}
/* Initialize root tables. */
if (ACPI_FAILURE(acpi_Startup())) {
printf("ACPI: Try disabling either ACPI or apic support.\n");
return (ENXIO);
}
if ((paddr = AcpiOsGetRootPointer()) == 0 ||
(rsdp = AcpiOsMapMemory(paddr, sizeof(ACPI_TABLE_RSDP))) == NULL)
return (ENXIO);
if (rsdp->Revision > 1 && rsdp->XsdtPhysicalAddress != 0)
paddr = (ACPI_PHYSICAL_ADDRESS)rsdp->XsdtPhysicalAddress;
else
paddr = (ACPI_PHYSICAL_ADDRESS)rsdp->RsdtPhysicalAddress;
AcpiOsUnmapMemory(rsdp, sizeof(ACPI_TABLE_RSDP));
if ((rsdt = AcpiOsMapMemory(paddr, sizeof(ACPI_TABLE_HEADER))) == NULL)
return (ENXIO);
sbuf_new(&sb, acpi_desc, sizeof(acpi_desc), SBUF_FIXEDLEN);
sbuf_bcat(&sb, rsdt->OemId, ACPI_OEM_ID_SIZE);
sbuf_trim(&sb);
sbuf_putc(&sb, ' ');
sbuf_bcat(&sb, rsdt->OemTableId, ACPI_OEM_TABLE_ID_SIZE);
sbuf_trim(&sb);
sbuf_finish(&sb);
sbuf_delete(&sb);
AcpiOsUnmapMemory(rsdt, sizeof(ACPI_TABLE_HEADER));
snprintf(acpi_ca_version, sizeof(acpi_ca_version), "%x", ACPI_CA_VERSION);
return (0);
}
/*
* Fetch some descriptive data from ACPI to put in our attach message.
*/
static int
acpi_probe(device_t dev)
{
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
device_set_desc(dev, acpi_desc);
return_VALUE (BUS_PROBE_NOWILDCARD);
}
static int
acpi_attach(device_t dev)
{
struct acpi_softc *sc;
ACPI_STATUS status;
int error, state;
UINT32 flags;
UINT8 TypeA, TypeB;
char *env;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
sc = device_get_softc(dev);
sc->acpi_dev = dev;
- callout_init(&sc->susp_force_to, TRUE);
+ callout_init(&sc->susp_force_to, 1);
error = ENXIO;
/* Initialize resource manager. */
acpi_rman_io.rm_type = RMAN_ARRAY;
acpi_rman_io.rm_start = 0;
acpi_rman_io.rm_end = 0xffff;
acpi_rman_io.rm_descr = "ACPI I/O ports";
if (rman_init(&acpi_rman_io) != 0)
panic("acpi rman_init IO ports failed");
acpi_rman_mem.rm_type = RMAN_ARRAY;
acpi_rman_mem.rm_start = 0;
acpi_rman_mem.rm_end = ~0ul;
acpi_rman_mem.rm_descr = "ACPI I/O memory addresses";
if (rman_init(&acpi_rman_mem) != 0)
panic("acpi rman_init memory failed");
/* Initialise the ACPI mutex */
mtx_init(&acpi_mutex, "ACPI global lock", NULL, MTX_DEF);
/*
* Set the globals from our tunables. This is needed because ACPI-CA
* uses UINT8 for some values and we have no tunable_byte.
*/
AcpiGbl_EnableInterpreterSlack = acpi_interpreter_slack ? TRUE : FALSE;
AcpiGbl_EnableAmlDebugObject = acpi_debug_objects ? TRUE : FALSE;
AcpiGbl_UseDefaultRegisterWidths = acpi_ignore_reg_width ? TRUE : FALSE;
#ifndef ACPI_DEBUG
/*
* Disable all debugging layers and levels.
*/
AcpiDbgLayer = 0;
AcpiDbgLevel = 0;
#endif
/* Start up the ACPI CA subsystem. */
status = AcpiInitializeSubsystem();
if (ACPI_FAILURE(status)) {
device_printf(dev, "Could not initialize Subsystem: %s\n",
AcpiFormatException(status));
goto out;
}
/* Override OS interfaces if the user requested. */
acpi_reset_interfaces(dev);
/* Load ACPI name space. */
status = AcpiLoadTables();
if (ACPI_FAILURE(status)) {
device_printf(dev, "Could not load Namespace: %s\n",
AcpiFormatException(status));
goto out;
}
#if defined(__i386__) || defined(__amd64__)
/* Handle MCFG table if present. */
acpi_enable_pcie();
#endif
/*
* Note that some systems (specifically, those with namespace evaluation
* issues that require the avoidance of parts of the namespace) must
* avoid running _INI and _STA on everything, as well as dodging the final
* object init pass.
*
* For these devices, we set ACPI_NO_DEVICE_INIT and ACPI_NO_OBJECT_INIT).
*
* XXX We should arrange for the object init pass after we have attached
* all our child devices, but on many systems it works here.
*/
flags = 0;
if (testenv("debug.acpi.avoid"))
flags = ACPI_NO_DEVICE_INIT | ACPI_NO_OBJECT_INIT;
/* Bring the hardware and basic handlers online. */
if (ACPI_FAILURE(status = AcpiEnableSubsystem(flags))) {
device_printf(dev, "Could not enable ACPI: %s\n",
AcpiFormatException(status));
goto out;
}
/*
* Call the ECDT probe function to provide EC functionality before
* the namespace has been evaluated.
*
* XXX This happens before the sysresource devices have been probed and
* attached so its resources come from nexus0. In practice, this isn't
* a problem but should be addressed eventually.
*/
acpi_ec_ecdt_probe(dev);
/* Bring device objects and regions online. */
if (ACPI_FAILURE(status = AcpiInitializeObjects(flags))) {
device_printf(dev, "Could not initialize ACPI objects: %s\n",
AcpiFormatException(status));
goto out;
}
/*
* Setup our sysctl tree.
*
* XXX: This doesn't check to make sure that none of these fail.
*/
sysctl_ctx_init(&sc->acpi_sysctl_ctx);
sc->acpi_sysctl_tree = SYSCTL_ADD_NODE(&sc->acpi_sysctl_ctx,
SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
device_get_name(dev), CTLFLAG_RD, 0, "");
SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "supported_sleep_state", CTLTYPE_STRING | CTLFLAG_RD,
0, 0, acpi_supported_sleep_state_sysctl, "A", "");
SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "power_button_state", CTLTYPE_STRING | CTLFLAG_RW,
&sc->acpi_power_button_sx, 0, acpi_sleep_state_sysctl, "A", "");
SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "sleep_button_state", CTLTYPE_STRING | CTLFLAG_RW,
&sc->acpi_sleep_button_sx, 0, acpi_sleep_state_sysctl, "A", "");
SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "lid_switch_state", CTLTYPE_STRING | CTLFLAG_RW,
&sc->acpi_lid_switch_sx, 0, acpi_sleep_state_sysctl, "A", "");
SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "standby_state", CTLTYPE_STRING | CTLFLAG_RW,
&sc->acpi_standby_sx, 0, acpi_sleep_state_sysctl, "A", "");
SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "suspend_state", CTLTYPE_STRING | CTLFLAG_RW,
&sc->acpi_suspend_sx, 0, acpi_sleep_state_sysctl, "A", "");
SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "sleep_delay", CTLFLAG_RW, &sc->acpi_sleep_delay, 0,
"sleep delay in seconds");
SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "s4bios", CTLFLAG_RW, &sc->acpi_s4bios, 0, "S4BIOS mode");
SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "verbose", CTLFLAG_RW, &sc->acpi_verbose, 0, "verbose mode");
SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "disable_on_reboot", CTLFLAG_RW,
&sc->acpi_do_disable, 0, "Disable ACPI when rebooting/halting system");
SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
OID_AUTO, "handle_reboot", CTLFLAG_RW,
&sc->acpi_handle_reboot, 0, "Use ACPI Reset Register to reboot");
/*
* Default to 1 second before sleeping to give some machines time to
* stabilize.
*/
sc->acpi_sleep_delay = 1;
if (bootverbose)
sc->acpi_verbose = 1;
if ((env = kern_getenv("hw.acpi.verbose")) != NULL) {
if (strcmp(env, "0") != 0)
sc->acpi_verbose = 1;
freeenv(env);
}
/* Only enable reboot by default if the FADT says it is available. */
if (AcpiGbl_FADT.Flags & ACPI_FADT_RESET_REGISTER)
sc->acpi_handle_reboot = 1;
#if !ACPI_REDUCED_HARDWARE
/* Only enable S4BIOS by default if the FACS says it is available. */
if (AcpiGbl_FACS != NULL && AcpiGbl_FACS->Flags & ACPI_FACS_S4_BIOS_PRESENT)
sc->acpi_s4bios = 1;
#endif
/* Probe all supported sleep states. */
acpi_sleep_states[ACPI_STATE_S0] = TRUE;
for (state = ACPI_STATE_S1; state < ACPI_S_STATE_COUNT; state++)
if (ACPI_SUCCESS(AcpiEvaluateObject(ACPI_ROOT_OBJECT,
__DECONST(char *, AcpiGbl_SleepStateNames[state]), NULL, NULL)) &&
ACPI_SUCCESS(AcpiGetSleepTypeData(state, &TypeA, &TypeB)))
acpi_sleep_states[state] = TRUE;
/*
* Dispatch the default sleep state to devices. The lid switch is set
* to UNKNOWN by default to avoid surprising users.
*/
sc->acpi_power_button_sx = acpi_sleep_states[ACPI_STATE_S5] ?
ACPI_STATE_S5 : ACPI_STATE_UNKNOWN;
sc->acpi_lid_switch_sx = ACPI_STATE_UNKNOWN;
sc->acpi_standby_sx = acpi_sleep_states[ACPI_STATE_S1] ?
ACPI_STATE_S1 : ACPI_STATE_UNKNOWN;
sc->acpi_suspend_sx = acpi_sleep_states[ACPI_STATE_S3] ?
ACPI_STATE_S3 : ACPI_STATE_UNKNOWN;
/* Pick the first valid sleep state for the sleep button default. */
sc->acpi_sleep_button_sx = ACPI_STATE_UNKNOWN;
for (state = ACPI_STATE_S1; state <= ACPI_STATE_S4; state++)
if (acpi_sleep_states[state]) {
sc->acpi_sleep_button_sx = state;
break;
}
acpi_enable_fixed_events(sc);
/*
* Scan the namespace and attach/initialise children.
*/
/* Register our shutdown handler. */
EVENTHANDLER_REGISTER(shutdown_final, acpi_shutdown_final, sc,
SHUTDOWN_PRI_LAST);
/*
* Register our acpi event handlers.
* XXX should be configurable eg. via userland policy manager.
*/
EVENTHANDLER_REGISTER(acpi_sleep_event, acpi_system_eventhandler_sleep,
sc, ACPI_EVENT_PRI_LAST);
EVENTHANDLER_REGISTER(acpi_wakeup_event, acpi_system_eventhandler_wakeup,
sc, ACPI_EVENT_PRI_LAST);
/* Flag our initial states. */
sc->acpi_enabled = TRUE;
sc->acpi_sstate = ACPI_STATE_S0;
sc->acpi_sleep_disabled = TRUE;
/* Create the control device */
sc->acpi_dev_t = make_dev(&acpi_cdevsw, 0, UID_ROOT, GID_WHEEL, 0644,
"acpi");
sc->acpi_dev_t->si_drv1 = sc;
if ((error = acpi_machdep_init(dev)))
goto out;
/* Register ACPI again to pass the correct argument of pm_func. */
power_pm_register(POWER_PM_TYPE_ACPI, acpi_pm_func, sc);
if (!acpi_disabled("bus")) {
EVENTHANDLER_REGISTER(dev_lookup, acpi_lookup, NULL, 1000);
acpi_probe_children(dev);
}
/* Update all GPEs and enable runtime GPEs. */
status = AcpiUpdateAllGpes();
if (ACPI_FAILURE(status))
device_printf(dev, "Could not update all GPEs: %s\n",
AcpiFormatException(status));
/* Allow sleep request after a while. */
callout_init_mtx(&acpi_sleep_timer, &acpi_mutex, 0);
callout_reset(&acpi_sleep_timer, hz * ACPI_MINIMUM_AWAKETIME,
acpi_sleep_enable, sc);
error = 0;
out:
return_VALUE (error);
}
static void
acpi_set_power_children(device_t dev, int state)
{
device_t child;
device_t *devlist;
int dstate, i, numdevs;
if (device_get_children(dev, &devlist, &numdevs) != 0)
return;
/*
* Retrieve and set D-state for the sleep state if _SxD is present.
* Skip children who aren't attached since they are handled separately.
*/
for (i = 0; i < numdevs; i++) {
child = devlist[i];
dstate = state;
if (device_is_attached(child) &&
acpi_device_pwr_for_sleep(dev, child, &dstate) == 0)
acpi_set_powerstate(child, dstate);
}
free(devlist, M_TEMP);
}
static int
acpi_suspend(device_t dev)
{
int error;
GIANT_REQUIRED;
error = bus_generic_suspend(dev);
if (error == 0)
acpi_set_power_children(dev, ACPI_STATE_D3);
return (error);
}
static int
acpi_resume(device_t dev)
{
GIANT_REQUIRED;
acpi_set_power_children(dev, ACPI_STATE_D0);
return (bus_generic_resume(dev));
}
static int
acpi_shutdown(device_t dev)
{
GIANT_REQUIRED;
/* Allow children to shutdown first. */
bus_generic_shutdown(dev);
/*
* Enable any GPEs that are able to power-on the system (i.e., RTC).
* Also, disable any that are not valid for this state (most).
*/
acpi_wake_prep_walk(ACPI_STATE_S5);
return (0);
}
/*
* Handle a new device being added
*/
static device_t
acpi_add_child(device_t bus, u_int order, const char *name, int unit)
{
struct acpi_device *ad;
device_t child;
if ((ad = malloc(sizeof(*ad), M_ACPIDEV, M_NOWAIT | M_ZERO)) == NULL)
return (NULL);
resource_list_init(&ad->ad_rl);
child = device_add_child_ordered(bus, order, name, unit);
if (child != NULL)
device_set_ivars(child, ad);
else
free(ad, M_ACPIDEV);
return (child);
}
static int
acpi_print_child(device_t bus, device_t child)
{
struct acpi_device *adev = device_get_ivars(child);
struct resource_list *rl = &adev->ad_rl;
int retval = 0;
retval += bus_print_child_header(bus, child);
retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#lx");
retval += resource_list_print_type(rl, "iomem", SYS_RES_MEMORY, "%#lx");
retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%ld");
retval += resource_list_print_type(rl, "drq", SYS_RES_DRQ, "%ld");
if (device_get_flags(child))
retval += printf(" flags %#x", device_get_flags(child));
retval += bus_print_child_domain(bus, child);
retval += bus_print_child_footer(bus, child);
return (retval);
}
/*
* If this device is an ACPI child but no one claimed it, attempt
* to power it off. We'll power it back up when a driver is added.
*
* XXX Disabled for now since many necessary devices (like fdc and
* ATA) don't claim the devices we created for them but still expect
* them to be powered up.
*/
static void
acpi_probe_nomatch(device_t bus, device_t child)
{
#ifdef ACPI_ENABLE_POWERDOWN_NODRIVER
acpi_set_powerstate(child, ACPI_STATE_D3);
#endif
}
/*
* If a new driver has a chance to probe a child, first power it up.
*
* XXX Disabled for now (see acpi_probe_nomatch for details).
*/
static void
acpi_driver_added(device_t dev, driver_t *driver)
{
device_t child, *devlist;
int i, numdevs;
DEVICE_IDENTIFY(driver, dev);
if (device_get_children(dev, &devlist, &numdevs))
return;
for (i = 0; i < numdevs; i++) {
child = devlist[i];
if (device_get_state(child) == DS_NOTPRESENT) {
#ifdef ACPI_ENABLE_POWERDOWN_NODRIVER
acpi_set_powerstate(child, ACPI_STATE_D0);
if (device_probe_and_attach(child) != 0)
acpi_set_powerstate(child, ACPI_STATE_D3);
#else
device_probe_and_attach(child);
#endif
}
}
free(devlist, M_TEMP);
}
/* Location hint for devctl(8) */
static int
acpi_child_location_str_method(device_t cbdev, device_t child, char *buf,
size_t buflen)
{
struct acpi_device *dinfo = device_get_ivars(child);
char buf2[32];
int pxm;
if (dinfo->ad_handle) {
snprintf(buf, buflen, "handle=%s", acpi_name(dinfo->ad_handle));
if (ACPI_SUCCESS(acpi_GetInteger(dinfo->ad_handle, "_PXM", &pxm))) {
snprintf(buf2, 32, " _PXM=%d", pxm);
strlcat(buf, buf2, buflen);
}
} else {
snprintf(buf, buflen, "unknown");
}
return (0);
}
/* PnP information for devctl(8) */
static int
acpi_child_pnpinfo_str_method(device_t cbdev, device_t child, char *buf,
size_t buflen)
{
struct acpi_device *dinfo = device_get_ivars(child);
ACPI_DEVICE_INFO *adinfo;
if (ACPI_FAILURE(AcpiGetObjectInfo(dinfo->ad_handle, &adinfo))) {
snprintf(buf, buflen, "unknown");
return (0);
}
snprintf(buf, buflen, "_HID=%s _UID=%lu",
(adinfo->Valid & ACPI_VALID_HID) ?
adinfo->HardwareId.String : "none",
(adinfo->Valid & ACPI_VALID_UID) ?
strtoul(adinfo->UniqueId.String, NULL, 10) : 0UL);
AcpiOsFree(adinfo);
return (0);
}
/*
* Handle per-device ivars
*/
static int
acpi_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
{
struct acpi_device *ad;
if ((ad = device_get_ivars(child)) == NULL) {
device_printf(child, "device has no ivars\n");
return (ENOENT);
}
/* ACPI and ISA compatibility ivars */
switch(index) {
case ACPI_IVAR_HANDLE:
*(ACPI_HANDLE *)result = ad->ad_handle;
break;
case ACPI_IVAR_PRIVATE:
*(void **)result = ad->ad_private;
break;
case ACPI_IVAR_FLAGS:
*(int *)result = ad->ad_flags;
break;
case ISA_IVAR_VENDORID:
case ISA_IVAR_SERIAL:
case ISA_IVAR_COMPATID:
*(int *)result = -1;
break;
case ISA_IVAR_LOGICALID:
*(int *)result = acpi_isa_get_logicalid(child);
break;
default:
return (ENOENT);
}
return (0);
}
static int
acpi_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
{
struct acpi_device *ad;
if ((ad = device_get_ivars(child)) == NULL) {
device_printf(child, "device has no ivars\n");
return (ENOENT);
}
switch(index) {
case ACPI_IVAR_HANDLE:
ad->ad_handle = (ACPI_HANDLE)value;
break;
case ACPI_IVAR_PRIVATE:
ad->ad_private = (void *)value;
break;
case ACPI_IVAR_FLAGS:
ad->ad_flags = (int)value;
break;
default:
panic("bad ivar write request (%d)", index);
return (ENOENT);
}
return (0);
}
/*
* Handle child resource allocation/removal
*/
static struct resource_list *
acpi_get_rlist(device_t dev, device_t child)
{
struct acpi_device *ad;
ad = device_get_ivars(child);
return (&ad->ad_rl);
}
static int
acpi_match_resource_hint(device_t dev, int type, long value)
{
struct acpi_device *ad = device_get_ivars(dev);
struct resource_list *rl = &ad->ad_rl;
struct resource_list_entry *rle;
STAILQ_FOREACH(rle, rl, link) {
if (rle->type != type)
continue;
if (rle->start <= value && rle->end >= value)
return (1);
}
return (0);
}
/*
* Wire device unit numbers based on resource matches in hints.
*/
static void
acpi_hint_device_unit(device_t acdev, device_t child, const char *name,
int *unitp)
{
const char *s;
long value;
int line, matches, unit;
/*
* Iterate over all the hints for the devices with the specified
* name to see if one's resources are a subset of this device.
*/
line = 0;
for (;;) {
if (resource_find_dev(&line, name, &unit, "at", NULL) != 0)
break;
/* Must have an "at" for acpi or isa. */
resource_string_value(name, unit, "at", &s);
if (!(strcmp(s, "acpi0") == 0 || strcmp(s, "acpi") == 0 ||
strcmp(s, "isa0") == 0 || strcmp(s, "isa") == 0))
continue;
/*
* Check for matching resources. We must have at least one match.
* Since I/O and memory resources cannot be shared, if we get a
* match on either of those, ignore any mismatches in IRQs or DRQs.
*
* XXX: We may want to revisit this to be more lenient and wire
* as long as it gets one match.
*/
matches = 0;
if (resource_long_value(name, unit, "port", &value) == 0) {
/*
* Floppy drive controllers are notorious for having a
* wide variety of resources not all of which include the
* first port that is specified by the hint (typically
* 0x3f0) (see the comment above fdc_isa_alloc_resources()
* in fdc_isa.c). However, they do all seem to include
* port + 2 (e.g. 0x3f2) so for a floppy device, look for
* 'value + 2' in the port resources instead of the hint
* value.
*/
if (strcmp(name, "fdc") == 0)
value += 2;
if (acpi_match_resource_hint(child, SYS_RES_IOPORT, value))
matches++;
else
continue;
}
if (resource_long_value(name, unit, "maddr", &value) == 0) {
if (acpi_match_resource_hint(child, SYS_RES_MEMORY, value))
matches++;
else
continue;
}
if (matches > 0)
goto matched;
if (resource_long_value(name, unit, "irq", &value) == 0) {
if (acpi_match_resource_hint(child, SYS_RES_IRQ, value))
matches++;
else
continue;
}
if (resource_long_value(name, unit, "drq", &value) == 0) {
if (acpi_match_resource_hint(child, SYS_RES_DRQ, value))
matches++;
else
continue;
}
matched:
if (matches > 0) {
/* We have a winner! */
*unitp = unit;
break;
}
}
}
/*
* Fetch the VM domain for the given device 'dev'.
*
* Return 1 + domain if there's a domain, 0 if not found;
* -1 upon an error.
*/
int
acpi_parse_pxm(device_t dev, int *domain)
{
#if MAXMEMDOM > 1
ACPI_HANDLE h;
int d, pxm;
h = acpi_get_handle(dev);
if ((h != NULL) &&
ACPI_SUCCESS(acpi_GetInteger(h, "_PXM", &pxm))) {
d = acpi_map_pxm_to_vm_domainid(pxm);
if (d < 0)
return (-1);
*domain = d;
return (1);
}
#endif
return (0);
}
/*
* Fetch the NUMA domain for the given device.
*
* If a device has a _PXM method, map that to a NUMA domain.
*
* If none is found, then it'll call the parent method.
* If there's no domain, return ENOENT.
*/
int
acpi_get_domain(device_t dev, device_t child, int *domain)
{
int ret;
ret = acpi_parse_pxm(child, domain);
/* Error */
if (ret == -1)
return (ENOENT);
/* Found */
if (ret == 1)
return (0);
/* No _PXM node; go up a level */
return (bus_generic_get_domain(dev, child, domain));
}
/*
* Pre-allocate/manage all memory and IO resources. Since rman can't handle
* duplicates, we merge any in the sysresource attach routine.
*/
static int
acpi_sysres_alloc(device_t dev)
{
struct resource *res;
struct resource_list *rl;
struct resource_list_entry *rle;
struct rman *rm;
char *sysres_ids[] = { "PNP0C01", "PNP0C02", NULL };
device_t *children;
int child_count, i;
/*
* Probe/attach any sysresource devices. This would be unnecessary if we
* had multi-pass probe/attach.
*/
if (device_get_children(dev, &children, &child_count) != 0)
return (ENXIO);
for (i = 0; i < child_count; i++) {
if (ACPI_ID_PROBE(dev, children[i], sysres_ids) != NULL)
device_probe_and_attach(children[i]);
}
free(children, M_TEMP);
rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
STAILQ_FOREACH(rle, rl, link) {
if (rle->res != NULL) {
device_printf(dev, "duplicate resource for %lx\n", rle->start);
continue;
}
/* Only memory and IO resources are valid here. */
switch (rle->type) {
case SYS_RES_IOPORT:
rm = &acpi_rman_io;
break;
case SYS_RES_MEMORY:
rm = &acpi_rman_mem;
break;
default:
continue;
}
/* Pre-allocate resource and add to our rman pool. */
res = BUS_ALLOC_RESOURCE(device_get_parent(dev), dev, rle->type,
&rle->rid, rle->start, rle->start + rle->count - 1, rle->count, 0);
if (res != NULL) {
rman_manage_region(rm, rman_get_start(res), rman_get_end(res));
rle->res = res;
} else if (bootverbose)
device_printf(dev, "reservation of %lx, %lx (%d) failed\n",
rle->start, rle->count, rle->type);
}
return (0);
}
static char *pcilink_ids[] = { "PNP0C0F", NULL };
static char *sysres_ids[] = { "PNP0C01", "PNP0C02", NULL };
/*
* Reserve declared resources for devices found during attach once system
* resources have been allocated.
*/
static void
acpi_reserve_resources(device_t dev)
{
struct resource_list_entry *rle;
struct resource_list *rl;
struct acpi_device *ad;
struct acpi_softc *sc;
device_t *children;
int child_count, i;
sc = device_get_softc(dev);
if (device_get_children(dev, &children, &child_count) != 0)
return;
for (i = 0; i < child_count; i++) {
ad = device_get_ivars(children[i]);
rl = &ad->ad_rl;
/* Don't reserve system resources. */
if (ACPI_ID_PROBE(dev, children[i], sysres_ids) != NULL)
continue;
STAILQ_FOREACH(rle, rl, link) {
/*
* Don't reserve IRQ resources. There are many sticky things
* to get right otherwise (e.g. IRQs for psm, atkbd, and HPET
* when using legacy routing).
*/
if (rle->type == SYS_RES_IRQ)
continue;
/*
* Don't reserve the resource if it is already allocated.
* The acpi_ec(4) driver can allocate its resources early
* if ECDT is present.
*/
if (rle->res != NULL)
continue;
/*
* Try to reserve the resource from our parent. If this
* fails because the resource is a system resource, just
* let it be. The resource range is already reserved so
* that other devices will not use it. If the driver
* needs to allocate the resource, then
* acpi_alloc_resource() will sub-alloc from the system
* resource.
*/
resource_list_reserve(rl, dev, children[i], rle->type, &rle->rid,
rle->start, rle->end, rle->count, 0);
}
}
free(children, M_TEMP);
sc->acpi_resources_reserved = 1;
}
static int
acpi_set_resource(device_t dev, device_t child, int type, int rid,
u_long start, u_long count)
{
struct acpi_softc *sc = device_get_softc(dev);
struct acpi_device *ad = device_get_ivars(child);
struct resource_list *rl = &ad->ad_rl;
ACPI_DEVICE_INFO *devinfo;
u_long end;
/* Ignore IRQ resources for PCI link devices. */
if (type == SYS_RES_IRQ && ACPI_ID_PROBE(dev, child, pcilink_ids) != NULL)
return (0);
/*
* Ignore most resources for PCI root bridges. Some BIOSes
* incorrectly enumerate the memory ranges they decode as plain
* memory resources instead of as ResourceProducer ranges. Other
* BIOSes incorrectly list system resource entries for I/O ranges
* under the PCI bridge. Do allow the one known-correct case on
* x86 of a PCI bridge claiming the I/O ports used for PCI config
* access.
*/
if (type == SYS_RES_MEMORY || type == SYS_RES_IOPORT) {
if (ACPI_SUCCESS(AcpiGetObjectInfo(ad->ad_handle, &devinfo))) {
if ((devinfo->Flags & ACPI_PCI_ROOT_BRIDGE) != 0) {
#if defined(__i386__) || defined(__amd64__)
if (!(type == SYS_RES_IOPORT && start == CONF1_ADDR_PORT))
#endif
{
AcpiOsFree(devinfo);
return (0);
}
}
AcpiOsFree(devinfo);
}
}
/* If the resource is already allocated, fail. */
if (resource_list_busy(rl, type, rid))
return (EBUSY);
/* If the resource is already reserved, release it. */
if (resource_list_reserved(rl, type, rid))
resource_list_unreserve(rl, dev, child, type, rid);
/* Add the resource. */
end = (start + count - 1);
resource_list_add(rl, type, rid, start, end, count);
/* Don't reserve resources until the system resources are allocated. */
if (!sc->acpi_resources_reserved)
return (0);
/* Don't reserve system resources. */
if (ACPI_ID_PROBE(dev, child, sysres_ids) != NULL)
return (0);
/*
* Don't reserve IRQ resources. There are many sticky things to
* get right otherwise (e.g. IRQs for psm, atkbd, and HPET when
* using legacy routing).
*/
if (type == SYS_RES_IRQ)
return (0);
/*
* Reserve the resource.
*
* XXX: Ignores failure for now. Failure here is probably a
* BIOS/firmware bug?
*/
resource_list_reserve(rl, dev, child, type, &rid, start, end, count, 0);
return (0);
}
static struct resource *
acpi_alloc_resource(device_t bus, device_t child, int type, int *rid,
u_long start, u_long end, u_long count, u_int flags)
{
ACPI_RESOURCE ares;
struct acpi_device *ad;
struct resource_list_entry *rle;
struct resource_list *rl;
struct resource *res;
int isdefault = (start == 0UL && end == ~0UL);
/*
* First attempt at allocating the resource. For direct children,
* use resource_list_alloc() to handle reserved resources. For
* other devices, pass the request up to our parent.
*/
if (bus == device_get_parent(child)) {
ad = device_get_ivars(child);
rl = &ad->ad_rl;
/*
* Simulate the behavior of the ISA bus for direct children
* devices. That is, if a non-default range is specified for
* a resource that doesn't exist, use bus_set_resource() to
* add the resource before allocating it. Note that these
* resources will not be reserved.
*/
if (!isdefault && resource_list_find(rl, type, *rid) == NULL)
resource_list_add(rl, type, *rid, start, end, count);
res = resource_list_alloc(rl, bus, child, type, rid, start, end, count,
flags);
if (res != NULL && type == SYS_RES_IRQ) {
/*
* Since bus_config_intr() takes immediate effect, we cannot
* configure the interrupt associated with a device when we
* parse the resources but have to defer it until a driver
* actually allocates the interrupt via bus_alloc_resource().
*
* XXX: Should we handle the lookup failing?
*/
if (ACPI_SUCCESS(acpi_lookup_irq_resource(child, *rid, res, &ares)))
acpi_config_intr(child, &ares);
}
/*
* If this is an allocation of the "default" range for a given
* RID, fetch the exact bounds for this resource from the
* resource list entry to try to allocate the range from the
* system resource regions.
*/
if (res == NULL && isdefault) {
rle = resource_list_find(rl, type, *rid);
if (rle != NULL) {
start = rle->start;
end = rle->end;
count = rle->count;
}
}
} else
res = BUS_ALLOC_RESOURCE(device_get_parent(bus), child, type, rid,
start, end, count, flags);
/*
* If the first attempt failed and this is an allocation of a
* specific range, try to satisfy the request via a suballocation
* from our system resource regions.
*/
if (res == NULL && start + count - 1 == end)
res = acpi_alloc_sysres(child, type, rid, start, end, count, flags);
return (res);
}
/*
* Attempt to allocate a specific resource range from the system
* resource ranges. Note that we only handle memory and I/O port
* system resources.
*/
struct resource *
acpi_alloc_sysres(device_t child, int type, int *rid, u_long start, u_long end,
u_long count, u_int flags)
{
struct rman *rm;
struct resource *res;
switch (type) {
case SYS_RES_IOPORT:
rm = &acpi_rman_io;
break;
case SYS_RES_MEMORY:
rm = &acpi_rman_mem;
break;
default:
return (NULL);
}
KASSERT(start + count - 1 == end, ("wildcard resource range"));
res = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
child);
if (res == NULL)
return (NULL);
rman_set_rid(res, *rid);
/* If requested, activate the resource using the parent's method. */
if (flags & RF_ACTIVE)
if (bus_activate_resource(child, type, *rid, res) != 0) {
rman_release_resource(res);
return (NULL);
}
return (res);
}
static int
acpi_is_resource_managed(int type, struct resource *r)
{
/* We only handle memory and IO resources through rman. */
switch (type) {
case SYS_RES_IOPORT:
return (rman_is_region_manager(r, &acpi_rman_io));
case SYS_RES_MEMORY:
return (rman_is_region_manager(r, &acpi_rman_mem));
}
return (0);
}
static int
acpi_adjust_resource(device_t bus, device_t child, int type, struct resource *r,
u_long start, u_long end)
{
if (acpi_is_resource_managed(type, r))
return (rman_adjust_resource(r, start, end));
return (bus_generic_adjust_resource(bus, child, type, r, start, end));
}
static int
acpi_release_resource(device_t bus, device_t child, int type, int rid,
struct resource *r)
{
int ret;
/*
* If this resource belongs to one of our internal managers,
* deactivate it and release it to the local pool.
*/
if (acpi_is_resource_managed(type, r)) {
if (rman_get_flags(r) & RF_ACTIVE) {
ret = bus_deactivate_resource(child, type, rid, r);
if (ret != 0)
return (ret);
}
return (rman_release_resource(r));
}
return (bus_generic_rl_release_resource(bus, child, type, rid, r));
}
static void
acpi_delete_resource(device_t bus, device_t child, int type, int rid)
{
struct resource_list *rl;
rl = acpi_get_rlist(bus, child);
if (resource_list_busy(rl, type, rid)) {
device_printf(bus, "delete_resource: Resource still owned by child"
" (type=%d, rid=%d)\n", type, rid);
return;
}
resource_list_unreserve(rl, bus, child, type, rid);
resource_list_delete(rl, type, rid);
}
/* Allocate an IO port or memory resource, given its GAS. */
int
acpi_bus_alloc_gas(device_t dev, int *type, int *rid, ACPI_GENERIC_ADDRESS *gas,
struct resource **res, u_int flags)
{
int error, res_type;
error = ENOMEM;
if (type == NULL || rid == NULL || gas == NULL || res == NULL)
return (EINVAL);
/* We only support memory and IO spaces. */
switch (gas->SpaceId) {
case ACPI_ADR_SPACE_SYSTEM_MEMORY:
res_type = SYS_RES_MEMORY;
break;
case ACPI_ADR_SPACE_SYSTEM_IO:
res_type = SYS_RES_IOPORT;
break;
default:
return (EOPNOTSUPP);
}
/*
* If the register width is less than 8, assume the BIOS author means
* it is a bit field and just allocate a byte.
*/
if (gas->BitWidth && gas->BitWidth < 8)
gas->BitWidth = 8;
/* Validate the address after we're sure we support the space. */
if (gas->Address == 0 || gas->BitWidth == 0)
return (EINVAL);
bus_set_resource(dev, res_type, *rid, gas->Address,
gas->BitWidth / 8);
*res = bus_alloc_resource_any(dev, res_type, rid, RF_ACTIVE | flags);
if (*res != NULL) {
*type = res_type;
error = 0;
} else
bus_delete_resource(dev, res_type, *rid);
return (error);
}
/* Probe _HID and _CID for compatible ISA PNP ids. */
static uint32_t
acpi_isa_get_logicalid(device_t dev)
{
ACPI_DEVICE_INFO *devinfo;
ACPI_HANDLE h;
uint32_t pnpid;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
/* Fetch and validate the HID. */
if ((h = acpi_get_handle(dev)) == NULL ||
ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
return_VALUE (0);
pnpid = (devinfo->Valid & ACPI_VALID_HID) != 0 &&
devinfo->HardwareId.Length >= ACPI_EISAID_STRING_SIZE ?
PNP_EISAID(devinfo->HardwareId.String) : 0;
AcpiOsFree(devinfo);
return_VALUE (pnpid);
}
static int
acpi_isa_get_compatid(device_t dev, uint32_t *cids, int count)
{
ACPI_DEVICE_INFO *devinfo;
ACPI_PNP_DEVICE_ID *ids;
ACPI_HANDLE h;
uint32_t *pnpid;
int i, valid;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
pnpid = cids;
/* Fetch and validate the CID */
if ((h = acpi_get_handle(dev)) == NULL ||
ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
return_VALUE (0);
if ((devinfo->Valid & ACPI_VALID_CID) == 0) {
AcpiOsFree(devinfo);
return_VALUE (0);
}
if (devinfo->CompatibleIdList.Count < count)
count = devinfo->CompatibleIdList.Count;
ids = devinfo->CompatibleIdList.Ids;
for (i = 0, valid = 0; i < count; i++)
if (ids[i].Length >= ACPI_EISAID_STRING_SIZE &&
strncmp(ids[i].String, "PNP", 3) == 0) {
*pnpid++ = PNP_EISAID(ids[i].String);
valid++;
}
AcpiOsFree(devinfo);
return_VALUE (valid);
}
static char *
acpi_device_id_probe(device_t bus, device_t dev, char **ids)
{
ACPI_HANDLE h;
ACPI_OBJECT_TYPE t;
int i;
h = acpi_get_handle(dev);
if (ids == NULL || h == NULL)
return (NULL);
t = acpi_get_type(dev);
if (t != ACPI_TYPE_DEVICE && t != ACPI_TYPE_PROCESSOR)
return (NULL);
/* Try to match one of the array of IDs with a HID or CID. */
for (i = 0; ids[i] != NULL; i++) {
if (acpi_MatchHid(h, ids[i]))
return (ids[i]);
}
return (NULL);
}
static ACPI_STATUS
acpi_device_eval_obj(device_t bus, device_t dev, ACPI_STRING pathname,
ACPI_OBJECT_LIST *parameters, ACPI_BUFFER *ret)
{
ACPI_HANDLE h;
if (dev == NULL)
h = ACPI_ROOT_OBJECT;
else if ((h = acpi_get_handle(dev)) == NULL)
return (AE_BAD_PARAMETER);
return (AcpiEvaluateObject(h, pathname, parameters, ret));
}
int
acpi_device_pwr_for_sleep(device_t bus, device_t dev, int *dstate)
{
struct acpi_softc *sc;
ACPI_HANDLE handle;
ACPI_STATUS status;
char sxd[8];
handle = acpi_get_handle(dev);
/*
* XXX If we find these devices, don't try to power them down.
* The serial and IRDA ports on my T23 hang the system when
* set to D3 and it appears that such legacy devices may
* need special handling in their drivers.
*/
if (dstate == NULL || handle == NULL ||
acpi_MatchHid(handle, "PNP0500") ||
acpi_MatchHid(handle, "PNP0501") ||
acpi_MatchHid(handle, "PNP0502") ||
acpi_MatchHid(handle, "PNP0510") ||
acpi_MatchHid(handle, "PNP0511"))
return (ENXIO);
/*
* Override next state with the value from _SxD, if present.
* Note illegal _S0D is evaluated because some systems expect this.
*/
sc = device_get_softc(bus);
snprintf(sxd, sizeof(sxd), "_S%dD", sc->acpi_sstate);
status = acpi_GetInteger(handle, sxd, dstate);
if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
device_printf(dev, "failed to get %s on %s: %s\n", sxd,
acpi_name(handle), AcpiFormatException(status));
return (ENXIO);
}
return (0);
}
/* Callback arg for our implementation of walking the namespace. */
struct acpi_device_scan_ctx {
acpi_scan_cb_t user_fn;
void *arg;
ACPI_HANDLE parent;
};
static ACPI_STATUS
acpi_device_scan_cb(ACPI_HANDLE h, UINT32 level, void *arg, void **retval)
{
struct acpi_device_scan_ctx *ctx;
device_t dev, old_dev;
ACPI_STATUS status;
ACPI_OBJECT_TYPE type;
/*
* Skip this device if we think we'll have trouble with it or it is
* the parent where the scan began.
*/
ctx = (struct acpi_device_scan_ctx *)arg;
if (acpi_avoid(h) || h == ctx->parent)
return (AE_OK);
/* If this is not a valid device type (e.g., a method), skip it. */
if (ACPI_FAILURE(AcpiGetType(h, &type)))
return (AE_OK);
if (type != ACPI_TYPE_DEVICE && type != ACPI_TYPE_PROCESSOR &&
type != ACPI_TYPE_THERMAL && type != ACPI_TYPE_POWER)
return (AE_OK);
/*
* Call the user function with the current device. If it is unchanged
* afterwards, return. Otherwise, we update the handle to the new dev.
*/
old_dev = acpi_get_device(h);
dev = old_dev;
status = ctx->user_fn(h, &dev, level, ctx->arg);
if (ACPI_FAILURE(status) || old_dev == dev)
return (status);
/* Remove the old child and its connection to the handle. */
if (old_dev != NULL) {
device_delete_child(device_get_parent(old_dev), old_dev);
AcpiDetachData(h, acpi_fake_objhandler);
}
/* Recreate the handle association if the user created a device. */
if (dev != NULL)
AcpiAttachData(h, acpi_fake_objhandler, dev);
return (AE_OK);
}
static ACPI_STATUS
acpi_device_scan_children(device_t bus, device_t dev, int max_depth,
acpi_scan_cb_t user_fn, void *arg)
{
ACPI_HANDLE h;
struct acpi_device_scan_ctx ctx;
if (acpi_disabled("children"))
return (AE_OK);
if (dev == NULL)
h = ACPI_ROOT_OBJECT;
else if ((h = acpi_get_handle(dev)) == NULL)
return (AE_BAD_PARAMETER);
ctx.user_fn = user_fn;
ctx.arg = arg;
ctx.parent = h;
return (AcpiWalkNamespace(ACPI_TYPE_ANY, h, max_depth,
acpi_device_scan_cb, NULL, &ctx, NULL));
}
/*
* Even though ACPI devices are not PCI, we use the PCI approach for setting
* device power states since it's close enough to ACPI.
*/
static int
acpi_set_powerstate(device_t child, int state)
{
ACPI_HANDLE h;
ACPI_STATUS status;
h = acpi_get_handle(child);
if (state < ACPI_STATE_D0 || state > ACPI_D_STATES_MAX)
return (EINVAL);
if (h == NULL)
return (0);
/* Ignore errors if the power methods aren't present. */
status = acpi_pwr_switch_consumer(h, state);
if (ACPI_SUCCESS(status)) {
if (bootverbose)
device_printf(child, "set ACPI power state D%d on %s\n",
state, acpi_name(h));
} else if (status != AE_NOT_FOUND)
device_printf(child,
"failed to set ACPI power state D%d on %s: %s\n", state,
acpi_name(h), AcpiFormatException(status));
return (0);
}
static int
acpi_isa_pnp_probe(device_t bus, device_t child, struct isa_pnp_id *ids)
{
int result, cid_count, i;
uint32_t lid, cids[8];
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
/*
* ISA-style drivers attached to ACPI may persist and
* probe manually if we return ENOENT. We never want
* that to happen, so don't ever return it.
*/
result = ENXIO;
/* Scan the supplied IDs for a match */
lid = acpi_isa_get_logicalid(child);
cid_count = acpi_isa_get_compatid(child, cids, 8);
while (ids && ids->ip_id) {
if (lid == ids->ip_id) {
result = 0;
goto out;
}
for (i = 0; i < cid_count; i++) {
if (cids[i] == ids->ip_id) {
result = 0;
goto out;
}
}
ids++;
}
out:
if (result == 0 && ids->ip_desc)
device_set_desc(child, ids->ip_desc);
return_VALUE (result);
}
#if defined(__i386__) || defined(__amd64__)
/*
* Look for a MCFG table. If it is present, use the settings for
* domain (segment) 0 to setup PCI config space access via the memory
* map.
*/
static void
acpi_enable_pcie(void)
{
ACPI_TABLE_HEADER *hdr;
ACPI_MCFG_ALLOCATION *alloc, *end;
ACPI_STATUS status;
status = AcpiGetTable(ACPI_SIG_MCFG, 1, &hdr);
if (ACPI_FAILURE(status))
return;
end = (ACPI_MCFG_ALLOCATION *)((char *)hdr + hdr->Length);
alloc = (ACPI_MCFG_ALLOCATION *)((ACPI_TABLE_MCFG *)hdr + 1);
while (alloc < end) {
if (alloc->PciSegment == 0) {
pcie_cfgregopen(alloc->Address, alloc->StartBusNumber,
alloc->EndBusNumber);
return;
}
alloc++;
}
}
#endif
/*
* Scan all of the ACPI namespace and attach child devices.
*
* We should only expect to find devices in the \_PR, \_TZ, \_SI, and
* \_SB scopes, and \_PR and \_TZ became obsolete in the ACPI 2.0 spec.
* However, in violation of the spec, some systems place their PCI link
* devices in \, so we have to walk the whole namespace. We check the
* type of namespace nodes, so this should be ok.
*/
static void
acpi_probe_children(device_t bus)
{
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
/*
* Scan the namespace and insert placeholders for all the devices that
* we find. We also probe/attach any early devices.
*
* Note that we use AcpiWalkNamespace rather than AcpiGetDevices because
* we want to create nodes for all devices, not just those that are
* currently present. (This assumes that we don't want to create/remove
* devices as they appear, which might be smarter.)
*/
ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "namespace scan\n"));
AcpiWalkNamespace(ACPI_TYPE_ANY, ACPI_ROOT_OBJECT, 100, acpi_probe_child,
NULL, bus, NULL);
/* Pre-allocate resources for our rman from any sysresource devices. */
acpi_sysres_alloc(bus);
/* Reserve resources already allocated to children. */
acpi_reserve_resources(bus);
/* Create any static children by calling device identify methods. */
ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "device identify routines\n"));
bus_generic_probe(bus);
/* Probe/attach all children, created statically and from the namespace. */
ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "acpi bus_generic_attach\n"));
bus_generic_attach(bus);
/* Attach wake sysctls. */
acpi_wake_sysctl_walk(bus);
ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "done attaching children\n"));
return_VOID;
}
/*
* Determine the probe order for a given device.
*/
static void
acpi_probe_order(ACPI_HANDLE handle, int *order)
{
ACPI_OBJECT_TYPE type;
/*
* 0. CPUs
* 1. I/O port and memory system resource holders
* 2. Clocks and timers (to handle early accesses)
* 3. Embedded controllers (to handle early accesses)
* 4. PCI Link Devices
*/
AcpiGetType(handle, &type);
if (type == ACPI_TYPE_PROCESSOR)
*order = 0;
else if (acpi_MatchHid(handle, "PNP0C01") ||
acpi_MatchHid(handle, "PNP0C02"))
*order = 1;
else if (acpi_MatchHid(handle, "PNP0100") ||
acpi_MatchHid(handle, "PNP0103") ||
acpi_MatchHid(handle, "PNP0B00"))
*order = 2;
else if (acpi_MatchHid(handle, "PNP0C09"))
*order = 3;
else if (acpi_MatchHid(handle, "PNP0C0F"))
*order = 4;
}
/*
* Evaluate a child device and determine whether we might attach a device to
* it.
*/
static ACPI_STATUS
acpi_probe_child(ACPI_HANDLE handle, UINT32 level, void *context, void **status)
{
struct acpi_prw_data prw;
ACPI_OBJECT_TYPE type;
ACPI_HANDLE h;
device_t bus, child;
char *handle_str;
int order;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
if (acpi_disabled("children"))
return_ACPI_STATUS (AE_OK);
/* Skip this device if we think we'll have trouble with it. */
if (acpi_avoid(handle))
return_ACPI_STATUS (AE_OK);
bus = (device_t)context;
if (ACPI_SUCCESS(AcpiGetType(handle, &type))) {
handle_str = acpi_name(handle);
switch (type) {
case ACPI_TYPE_DEVICE:
/*
* Since we scan from \, be sure to skip system scope objects.
* \_SB_ and \_TZ_ are defined in ACPICA as devices to work around
* BIOS bugs. For example, \_SB_ is to allow \_SB_._INI to be run
* during the intialization and \_TZ_ is to support Notify() on it.
*/
if (strcmp(handle_str, "\\_SB_") == 0 ||
strcmp(handle_str, "\\_TZ_") == 0)
break;
if (acpi_parse_prw(handle, &prw) == 0)
AcpiSetupGpeForWake(handle, prw.gpe_handle, prw.gpe_bit);
/*
* Ignore devices that do not have a _HID or _CID. They should
* be discovered by other buses (e.g. the PCI bus driver).
*/
if (!acpi_has_hid(handle))
break;
/* FALLTHROUGH */
case ACPI_TYPE_PROCESSOR:
case ACPI_TYPE_THERMAL:
case ACPI_TYPE_POWER:
/*
* Create a placeholder device for this node. Sort the
* placeholder so that the probe/attach passes will run
* breadth-first. Orders less than ACPI_DEV_BASE_ORDER
* are reserved for special objects (i.e., system
* resources).
*/
ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "scanning '%s'\n", handle_str));
order = level * 10 + ACPI_DEV_BASE_ORDER;
acpi_probe_order(handle, &order);
child = BUS_ADD_CHILD(bus, order, NULL, -1);
if (child == NULL)
break;
/* Associate the handle with the device_t and vice versa. */
acpi_set_handle(child, handle);
AcpiAttachData(handle, acpi_fake_objhandler, child);
/*
* Check that the device is present. If it's not present,
* leave it disabled (so that we have a device_t attached to
* the handle, but we don't probe it).
*
* XXX PCI link devices sometimes report "present" but not
* "functional" (i.e. if disabled). Go ahead and probe them
* anyway since we may enable them later.
*/
if (type == ACPI_TYPE_DEVICE && !acpi_DeviceIsPresent(child)) {
/* Never disable PCI link devices. */
if (acpi_MatchHid(handle, "PNP0C0F"))
break;
/*
* Docking stations should remain enabled since the system
* may be undocked at boot.
*/
if (ACPI_SUCCESS(AcpiGetHandle(handle, "_DCK", &h)))
break;
device_disable(child);
break;
}
/*
* Get the device's resource settings and attach them.
* Note that if the device has _PRS but no _CRS, we need
* to decide when it's appropriate to try to configure the
* device. Ignore the return value here; it's OK for the
* device not to have any resources.
*/
acpi_parse_resources(child, handle, &acpi_res_parse_set, NULL);
break;
}
}
return_ACPI_STATUS (AE_OK);
}
/*
* AcpiAttachData() requires an object handler but never uses it. This is a
* placeholder object handler so we can store a device_t in an ACPI_HANDLE.
*/
void
acpi_fake_objhandler(ACPI_HANDLE h, void *data)
{
}
static void
acpi_shutdown_final(void *arg, int howto)
{
struct acpi_softc *sc = (struct acpi_softc *)arg;
register_t intr;
ACPI_STATUS status;
/*
* XXX Shutdown code should only run on the BSP (cpuid 0).
* Some chipsets do not power off the system correctly if called from
* an AP.
*/
if ((howto & RB_POWEROFF) != 0) {
status = AcpiEnterSleepStatePrep(ACPI_STATE_S5);
if (ACPI_FAILURE(status)) {
device_printf(sc->acpi_dev, "AcpiEnterSleepStatePrep failed - %s\n",
AcpiFormatException(status));
return;
}
device_printf(sc->acpi_dev, "Powering system off\n");
intr = intr_disable();
status = AcpiEnterSleepState(ACPI_STATE_S5);
if (ACPI_FAILURE(status)) {
intr_restore(intr);
device_printf(sc->acpi_dev, "power-off failed - %s\n",
AcpiFormatException(status));
} else {
DELAY(1000000);
intr_restore(intr);
device_printf(sc->acpi_dev, "power-off failed - timeout\n");
}
} else if ((howto & RB_HALT) == 0 && sc->acpi_handle_reboot) {
/* Reboot using the reset register. */
status = AcpiReset();
if (ACPI_SUCCESS(status)) {
DELAY(1000000);
device_printf(sc->acpi_dev, "reset failed - timeout\n");
} else if (status != AE_NOT_EXIST)
device_printf(sc->acpi_dev, "reset failed - %s\n",
AcpiFormatException(status));
} else if (sc->acpi_do_disable && panicstr == NULL) {
/*
* Only disable ACPI if the user requested. On some systems, writing
* the disable value to SMI_CMD hangs the system.
*/
device_printf(sc->acpi_dev, "Shutting down\n");
AcpiTerminate();
}
}
static void
acpi_enable_fixed_events(struct acpi_softc *sc)
{
static int first_time = 1;
/* Enable and clear fixed events and install handlers. */
if ((AcpiGbl_FADT.Flags & ACPI_FADT_POWER_BUTTON) == 0) {
AcpiClearEvent(ACPI_EVENT_POWER_BUTTON);
AcpiInstallFixedEventHandler(ACPI_EVENT_POWER_BUTTON,
acpi_event_power_button_sleep, sc);
if (first_time)
device_printf(sc->acpi_dev, "Power Button (fixed)\n");
}
if ((AcpiGbl_FADT.Flags & ACPI_FADT_SLEEP_BUTTON) == 0) {
AcpiClearEvent(ACPI_EVENT_SLEEP_BUTTON);
AcpiInstallFixedEventHandler(ACPI_EVENT_SLEEP_BUTTON,
acpi_event_sleep_button_sleep, sc);
if (first_time)
device_printf(sc->acpi_dev, "Sleep Button (fixed)\n");
}
first_time = 0;
}
/*
* Returns true if the device is actually present and should
* be attached to. This requires the present, enabled, UI-visible
* and diagnostics-passed bits to be set.
*/
BOOLEAN
acpi_DeviceIsPresent(device_t dev)
{
ACPI_DEVICE_INFO *devinfo;
ACPI_HANDLE h;
BOOLEAN present;
if ((h = acpi_get_handle(dev)) == NULL ||
ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
return (FALSE);
/* If no _STA method, must be present */
present = (devinfo->Valid & ACPI_VALID_STA) == 0 ||
ACPI_DEVICE_PRESENT(devinfo->CurrentStatus) ? TRUE : FALSE;
AcpiOsFree(devinfo);
return (present);
}
/*
* Returns true if the battery is actually present and inserted.
*/
BOOLEAN
acpi_BatteryIsPresent(device_t dev)
{
ACPI_DEVICE_INFO *devinfo;
ACPI_HANDLE h;
BOOLEAN present;
if ((h = acpi_get_handle(dev)) == NULL ||
ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
return (FALSE);
/* If no _STA method, must be present */
present = (devinfo->Valid & ACPI_VALID_STA) == 0 ||
ACPI_BATTERY_PRESENT(devinfo->CurrentStatus) ? TRUE : FALSE;
AcpiOsFree(devinfo);
return (present);
}
/*
* Returns true if a device has at least one valid device ID.
*/
static BOOLEAN
acpi_has_hid(ACPI_HANDLE h)
{
ACPI_DEVICE_INFO *devinfo;
BOOLEAN ret;
if (h == NULL ||
ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
return (FALSE);
ret = FALSE;
if ((devinfo->Valid & ACPI_VALID_HID) != 0)
ret = TRUE;
else if ((devinfo->Valid & ACPI_VALID_CID) != 0)
if (devinfo->CompatibleIdList.Count > 0)
ret = TRUE;
AcpiOsFree(devinfo);
return (ret);
}
/*
* Match a HID string against a handle
*/
BOOLEAN
acpi_MatchHid(ACPI_HANDLE h, const char *hid)
{
ACPI_DEVICE_INFO *devinfo;
BOOLEAN ret;
int i;
if (hid == NULL || h == NULL ||
ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
return (FALSE);
ret = FALSE;
if ((devinfo->Valid & ACPI_VALID_HID) != 0 &&
strcmp(hid, devinfo->HardwareId.String) == 0)
ret = TRUE;
else if ((devinfo->Valid & ACPI_VALID_CID) != 0)
for (i = 0; i < devinfo->CompatibleIdList.Count; i++) {
if (strcmp(hid, devinfo->CompatibleIdList.Ids[i].String) == 0) {
ret = TRUE;
break;
}
}
AcpiOsFree(devinfo);
return (ret);
}
/*
* Return the handle of a named object within our scope, ie. that of (parent)
* or one if its parents.
*/
ACPI_STATUS
acpi_GetHandleInScope(ACPI_HANDLE parent, char *path, ACPI_HANDLE *result)
{
ACPI_HANDLE r;
ACPI_STATUS status;
/* Walk back up the tree to the root */
for (;;) {
status = AcpiGetHandle(parent, path, &r);
if (ACPI_SUCCESS(status)) {
*result = r;
return (AE_OK);
}
/* XXX Return error here? */
if (status != AE_NOT_FOUND)
return (AE_OK);
if (ACPI_FAILURE(AcpiGetParent(parent, &r)))
return (AE_NOT_FOUND);
parent = r;
}
}
/*
* Allocate a buffer with a preset data size.
*/
ACPI_BUFFER *
acpi_AllocBuffer(int size)
{
ACPI_BUFFER *buf;
if ((buf = malloc(size + sizeof(*buf), M_ACPIDEV, M_NOWAIT)) == NULL)
return (NULL);
buf->Length = size;
buf->Pointer = (void *)(buf + 1);
return (buf);
}
ACPI_STATUS
acpi_SetInteger(ACPI_HANDLE handle, char *path, UINT32 number)
{
ACPI_OBJECT arg1;
ACPI_OBJECT_LIST args;
arg1.Type = ACPI_TYPE_INTEGER;
arg1.Integer.Value = number;
args.Count = 1;
args.Pointer = &arg1;
return (AcpiEvaluateObject(handle, path, &args, NULL));
}
/*
* Evaluate a path that should return an integer.
*/
ACPI_STATUS
acpi_GetInteger(ACPI_HANDLE handle, char *path, UINT32 *number)
{
ACPI_STATUS status;
ACPI_BUFFER buf;
ACPI_OBJECT param;
if (handle == NULL)
handle = ACPI_ROOT_OBJECT;
/*
* Assume that what we've been pointed at is an Integer object, or
* a method that will return an Integer.
*/
buf.Pointer = &param;
buf.Length = sizeof(param);
status = AcpiEvaluateObject(handle, path, NULL, &buf);
if (ACPI_SUCCESS(status)) {
if (param.Type == ACPI_TYPE_INTEGER)
*number = param.Integer.Value;
else
status = AE_TYPE;
}
/*
* In some applications, a method that's expected to return an Integer
* may instead return a Buffer (probably to simplify some internal
* arithmetic). We'll try to fetch whatever it is, and if it's a Buffer,
* convert it into an Integer as best we can.
*
* This is a hack.
*/
if (status == AE_BUFFER_OVERFLOW) {
if ((buf.Pointer = AcpiOsAllocate(buf.Length)) == NULL) {
status = AE_NO_MEMORY;
} else {
status = AcpiEvaluateObject(handle, path, NULL, &buf);
if (ACPI_SUCCESS(status))
status = acpi_ConvertBufferToInteger(&buf, number);
AcpiOsFree(buf.Pointer);
}
}
return (status);
}
ACPI_STATUS
acpi_ConvertBufferToInteger(ACPI_BUFFER *bufp, UINT32 *number)
{
ACPI_OBJECT *p;
UINT8 *val;
int i;
p = (ACPI_OBJECT *)bufp->Pointer;
if (p->Type == ACPI_TYPE_INTEGER) {
*number = p->Integer.Value;
return (AE_OK);
}
if (p->Type != ACPI_TYPE_BUFFER)
return (AE_TYPE);
if (p->Buffer.Length > sizeof(int))
return (AE_BAD_DATA);
*number = 0;
val = p->Buffer.Pointer;
for (i = 0; i < p->Buffer.Length; i++)
*number += val[i] << (i * 8);
return (AE_OK);
}
/*
* Iterate over the elements of an a package object, calling the supplied
* function for each element.
*
* XXX possible enhancement might be to abort traversal on error.
*/
ACPI_STATUS
acpi_ForeachPackageObject(ACPI_OBJECT *pkg,
void (*func)(ACPI_OBJECT *comp, void *arg), void *arg)
{
ACPI_OBJECT *comp;
int i;
if (pkg == NULL || pkg->Type != ACPI_TYPE_PACKAGE)
return (AE_BAD_PARAMETER);
/* Iterate over components */
i = 0;
comp = pkg->Package.Elements;
for (; i < pkg->Package.Count; i++, comp++)
func(comp, arg);
return (AE_OK);
}
/*
* Find the (index)th resource object in a set.
*/
ACPI_STATUS
acpi_FindIndexedResource(ACPI_BUFFER *buf, int index, ACPI_RESOURCE **resp)
{
ACPI_RESOURCE *rp;
int i;
rp = (ACPI_RESOURCE *)buf->Pointer;
i = index;
while (i-- > 0) {
/* Range check */
if (rp > (ACPI_RESOURCE *)((u_int8_t *)buf->Pointer + buf->Length))
return (AE_BAD_PARAMETER);
/* Check for terminator */
if (rp->Type == ACPI_RESOURCE_TYPE_END_TAG || rp->Length == 0)
return (AE_NOT_FOUND);
rp = ACPI_NEXT_RESOURCE(rp);
}
if (resp != NULL)
*resp = rp;
return (AE_OK);
}
/*
* Append an ACPI_RESOURCE to an ACPI_BUFFER.
*
* Given a pointer to an ACPI_RESOURCE structure, expand the ACPI_BUFFER
* provided to contain it. If the ACPI_BUFFER is empty, allocate a sensible
* backing block. If the ACPI_RESOURCE is NULL, return an empty set of
* resources.
*/
#define ACPI_INITIAL_RESOURCE_BUFFER_SIZE 512
ACPI_STATUS
acpi_AppendBufferResource(ACPI_BUFFER *buf, ACPI_RESOURCE *res)
{
ACPI_RESOURCE *rp;
void *newp;
/* Initialise the buffer if necessary. */
if (buf->Pointer == NULL) {
buf->Length = ACPI_INITIAL_RESOURCE_BUFFER_SIZE;
if ((buf->Pointer = AcpiOsAllocate(buf->Length)) == NULL)
return (AE_NO_MEMORY);
rp = (ACPI_RESOURCE *)buf->Pointer;
rp->Type = ACPI_RESOURCE_TYPE_END_TAG;
rp->Length = ACPI_RS_SIZE_MIN;
}
if (res == NULL)
return (AE_OK);
/*
* Scan the current buffer looking for the terminator.
* This will either find the terminator or hit the end
* of the buffer and return an error.
*/
rp = (ACPI_RESOURCE *)buf->Pointer;
for (;;) {
/* Range check, don't go outside the buffer */
if (rp >= (ACPI_RESOURCE *)((u_int8_t *)buf->Pointer + buf->Length))
return (AE_BAD_PARAMETER);
if (rp->Type == ACPI_RESOURCE_TYPE_END_TAG || rp->Length == 0)
break;
rp = ACPI_NEXT_RESOURCE(rp);
}
/*
* Check the size of the buffer and expand if required.
*
* Required size is:
* size of existing resources before terminator +
* size of new resource and header +
* size of terminator.
*
* Note that this loop should really only run once, unless
* for some reason we are stuffing a *really* huge resource.
*/
while ((((u_int8_t *)rp - (u_int8_t *)buf->Pointer) +
res->Length + ACPI_RS_SIZE_NO_DATA +
ACPI_RS_SIZE_MIN) >= buf->Length) {
if ((newp = AcpiOsAllocate(buf->Length * 2)) == NULL)
return (AE_NO_MEMORY);
bcopy(buf->Pointer, newp, buf->Length);
rp = (ACPI_RESOURCE *)((u_int8_t *)newp +
((u_int8_t *)rp - (u_int8_t *)buf->Pointer));
AcpiOsFree(buf->Pointer);
buf->Pointer = newp;
buf->Length += buf->Length;
}
/* Insert the new resource. */
bcopy(res, rp, res->Length + ACPI_RS_SIZE_NO_DATA);
/* And add the terminator. */
rp = ACPI_NEXT_RESOURCE(rp);
rp->Type = ACPI_RESOURCE_TYPE_END_TAG;
rp->Length = ACPI_RS_SIZE_MIN;
return (AE_OK);
}
/*
* Set interrupt model.
*/
ACPI_STATUS
acpi_SetIntrModel(int model)
{
return (acpi_SetInteger(ACPI_ROOT_OBJECT, "_PIC", model));
}
/*
* Walk subtables of a table and call a callback routine for each
* subtable. The caller should provide the first subtable and a
* pointer to the end of the table. This can be used to walk tables
* such as MADT and SRAT that use subtable entries.
*/
void
acpi_walk_subtables(void *first, void *end, acpi_subtable_handler *handler,
void *arg)
{
ACPI_SUBTABLE_HEADER *entry;
for (entry = first; (void *)entry < end; ) {
/* Avoid an infinite loop if we hit a bogus entry. */
if (entry->Length < sizeof(ACPI_SUBTABLE_HEADER))
return;
handler(entry, arg);
entry = ACPI_ADD_PTR(ACPI_SUBTABLE_HEADER, entry, entry->Length);
}
}
/*
* DEPRECATED. This interface has serious deficiencies and will be
* removed.
*
* Immediately enter the sleep state. In the old model, acpiconf(8) ran
* rc.suspend and rc.resume so we don't have to notify devd(8) to do this.
*/
ACPI_STATUS
acpi_SetSleepState(struct acpi_softc *sc, int state)
{
static int once;
if (!once) {
device_printf(sc->acpi_dev,
"warning: acpi_SetSleepState() deprecated, need to update your software\n");
once = 1;
}
return (acpi_EnterSleepState(sc, state));
}
#if defined(__amd64__) || defined(__i386__)
static void
acpi_sleep_force_task(void *context)
{
struct acpi_softc *sc = (struct acpi_softc *)context;
if (ACPI_FAILURE(acpi_EnterSleepState(sc, sc->acpi_next_sstate)))
device_printf(sc->acpi_dev, "force sleep state S%d failed\n",
sc->acpi_next_sstate);
}
static void
acpi_sleep_force(void *arg)
{
struct acpi_softc *sc = (struct acpi_softc *)arg;
device_printf(sc->acpi_dev,
"suspend request timed out, forcing sleep now\n");
/*
* XXX Suspending from callout causes freezes in DEVICE_SUSPEND().
* Suspend from acpi_task thread instead.
*/
if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
acpi_sleep_force_task, sc)))
device_printf(sc->acpi_dev, "AcpiOsExecute() for sleeping failed\n");
}
#endif
/*
* Request that the system enter the given suspend state. All /dev/apm
* devices and devd(8) will be notified. Userland then has a chance to
* save state and acknowledge the request. The system sleeps once all
* acks are in.
*/
int
acpi_ReqSleepState(struct acpi_softc *sc, int state)
{
#if defined(__amd64__) || defined(__i386__)
struct apm_clone_data *clone;
ACPI_STATUS status;
if (state < ACPI_STATE_S1 || state > ACPI_S_STATES_MAX)
return (EINVAL);
if (!acpi_sleep_states[state])
return (EOPNOTSUPP);
/* If a suspend request is already in progress, just return. */
if (sc->acpi_next_sstate != 0) {
return (0);
}
/* Wait until sleep is enabled. */
while (sc->acpi_sleep_disabled) {
AcpiOsSleep(1000);
}
ACPI_LOCK(acpi);
sc->acpi_next_sstate = state;
/* S5 (soft-off) should be entered directly with no waiting. */
if (state == ACPI_STATE_S5) {
ACPI_UNLOCK(acpi);
status = acpi_EnterSleepState(sc, state);
return (ACPI_SUCCESS(status) ? 0 : ENXIO);
}
/* Record the pending state and notify all apm devices. */
STAILQ_FOREACH(clone, &sc->apm_cdevs, entries) {
clone->notify_status = APM_EV_NONE;
if ((clone->flags & ACPI_EVF_DEVD) == 0) {
selwakeuppri(&clone->sel_read, PZERO);
KNOTE_LOCKED(&clone->sel_read.si_note, 0);
}
}
/* If devd(8) is not running, immediately enter the sleep state. */
if (!devctl_process_running()) {
ACPI_UNLOCK(acpi);
status = acpi_EnterSleepState(sc, state);
return (ACPI_SUCCESS(status) ? 0 : ENXIO);
}
/*
* Set a timeout to fire if userland doesn't ack the suspend request
* in time. This way we still eventually go to sleep if we were
* overheating or running low on battery, even if userland is hung.
* We cancel this timeout once all userland acks are in or the
* suspend request is aborted.
*/
callout_reset(&sc->susp_force_to, 10 * hz, acpi_sleep_force, sc);
ACPI_UNLOCK(acpi);
/* Now notify devd(8) also. */
acpi_UserNotify("Suspend", ACPI_ROOT_OBJECT, state);
return (0);
#else
/* This platform does not support acpi suspend/resume. */
return (EOPNOTSUPP);
#endif
}
/*
* Acknowledge (or reject) a pending sleep state. The caller has
* prepared for suspend and is now ready for it to proceed. If the
* error argument is non-zero, it indicates suspend should be cancelled
* and gives an errno value describing why. Once all votes are in,
* we suspend the system.
*/
int
acpi_AckSleepState(struct apm_clone_data *clone, int error)
{
#if defined(__amd64__) || defined(__i386__)
struct acpi_softc *sc;
int ret, sleeping;
/* If no pending sleep state, return an error. */
ACPI_LOCK(acpi);
sc = clone->acpi_sc;
if (sc->acpi_next_sstate == 0) {
ACPI_UNLOCK(acpi);
return (ENXIO);
}
/* Caller wants to abort suspend process. */
if (error) {
sc->acpi_next_sstate = 0;
callout_stop(&sc->susp_force_to);
device_printf(sc->acpi_dev,
"listener on %s cancelled the pending suspend\n",
devtoname(clone->cdev));
ACPI_UNLOCK(acpi);
return (0);
}
/*
* Mark this device as acking the suspend request. Then, walk through
* all devices, seeing if they agree yet. We only count devices that
* are writable since read-only devices couldn't ack the request.
*/
sleeping = TRUE;
clone->notify_status = APM_EV_ACKED;
STAILQ_FOREACH(clone, &sc->apm_cdevs, entries) {
if ((clone->flags & ACPI_EVF_WRITE) != 0 &&
clone->notify_status != APM_EV_ACKED) {
sleeping = FALSE;
break;
}
}
/* If all devices have voted "yes", we will suspend now. */
if (sleeping)
callout_stop(&sc->susp_force_to);
ACPI_UNLOCK(acpi);
ret = 0;
if (sleeping) {
if (ACPI_FAILURE(acpi_EnterSleepState(sc, sc->acpi_next_sstate)))
ret = ENODEV;
}
return (ret);
#else
/* This platform does not support acpi suspend/resume. */
return (EOPNOTSUPP);
#endif
}
static void
acpi_sleep_enable(void *arg)
{
struct acpi_softc *sc = (struct acpi_softc *)arg;
ACPI_LOCK_ASSERT(acpi);
/* Reschedule if the system is not fully up and running. */
if (!AcpiGbl_SystemAwakeAndRunning) {
callout_schedule(&acpi_sleep_timer, hz * ACPI_MINIMUM_AWAKETIME);
return;
}
sc->acpi_sleep_disabled = FALSE;
}
static ACPI_STATUS
acpi_sleep_disable(struct acpi_softc *sc)
{
ACPI_STATUS status;
/* Fail if the system is not fully up and running. */
if (!AcpiGbl_SystemAwakeAndRunning)
return (AE_ERROR);
ACPI_LOCK(acpi);
status = sc->acpi_sleep_disabled ? AE_ERROR : AE_OK;
sc->acpi_sleep_disabled = TRUE;
ACPI_UNLOCK(acpi);
return (status);
}
enum acpi_sleep_state {
ACPI_SS_NONE,
ACPI_SS_GPE_SET,
ACPI_SS_DEV_SUSPEND,
ACPI_SS_SLP_PREP,
ACPI_SS_SLEPT,
};
/*
* Enter the desired system sleep state.
*
* Currently we support S1-S5 but S4 is only S4BIOS
*/
static ACPI_STATUS
acpi_EnterSleepState(struct acpi_softc *sc, int state)
{
register_t intr;
ACPI_STATUS status;
ACPI_EVENT_STATUS power_button_status;
enum acpi_sleep_state slp_state;
int sleep_result;
ACPI_FUNCTION_TRACE_U32((char *)(uintptr_t)__func__, state);
if (state < ACPI_STATE_S1 || state > ACPI_S_STATES_MAX)
return_ACPI_STATUS (AE_BAD_PARAMETER);
if (!acpi_sleep_states[state]) {
device_printf(sc->acpi_dev, "Sleep state S%d not supported by BIOS\n",
state);
return (AE_SUPPORT);
}
/* Re-entry once we're suspending is not allowed. */
status = acpi_sleep_disable(sc);
if (ACPI_FAILURE(status)) {
device_printf(sc->acpi_dev,
"suspend request ignored (not ready yet)\n");
return (status);
}
if (state == ACPI_STATE_S5) {
/*
* Shut down cleanly and power off. This will call us back through the
* shutdown handlers.
*/
shutdown_nice(RB_POWEROFF);
return_ACPI_STATUS (AE_OK);
}
EVENTHANDLER_INVOKE(power_suspend_early);
stop_all_proc();
EVENTHANDLER_INVOKE(power_suspend);
if (smp_started) {
thread_lock(curthread);
sched_bind(curthread, 0);
thread_unlock(curthread);
}
/*
* Be sure to hold Giant across DEVICE_SUSPEND/RESUME since non-MPSAFE
* drivers need this.
*/
mtx_lock(&Giant);
slp_state = ACPI_SS_NONE;
sc->acpi_sstate = state;
/* Enable any GPEs as appropriate and requested by the user. */
acpi_wake_prep_walk(state);
slp_state = ACPI_SS_GPE_SET;
/*
* Inform all devices that we are going to sleep. If at least one
* device fails, DEVICE_SUSPEND() automatically resumes the tree.
*
* XXX Note that a better two-pass approach with a 'veto' pass
* followed by a "real thing" pass would be better, but the current
* bus interface does not provide for this.
*/
if (DEVICE_SUSPEND(root_bus) != 0) {
device_printf(sc->acpi_dev, "device_suspend failed\n");
goto backout;
}
slp_state = ACPI_SS_DEV_SUSPEND;
/* If testing device suspend only, back out of everything here. */
if (acpi_susp_bounce)
goto backout;
status = AcpiEnterSleepStatePrep(state);
if (ACPI_FAILURE(status)) {
device_printf(sc->acpi_dev, "AcpiEnterSleepStatePrep failed - %s\n",
AcpiFormatException(status));
goto backout;
}
slp_state = ACPI_SS_SLP_PREP;
if (sc->acpi_sleep_delay > 0)
DELAY(sc->acpi_sleep_delay * 1000000);
intr = intr_disable();
if (state != ACPI_STATE_S1) {
sleep_result = acpi_sleep_machdep(sc, state);
acpi_wakeup_machdep(sc, state, sleep_result, 0);
/*
* XXX According to ACPI specification SCI_EN bit should be restored
* by ACPI platform (BIOS, firmware) to its pre-sleep state.
* Unfortunately some BIOSes fail to do that and that leads to
* unexpected and serious consequences during wake up like a system
* getting stuck in SMI handlers.
* This hack is picked up from Linux, which claims that it follows
* Windows behavior.
*/
if (sleep_result == 1 && state != ACPI_STATE_S4)
AcpiWriteBitRegister(ACPI_BITREG_SCI_ENABLE, ACPI_ENABLE_EVENT);
AcpiLeaveSleepStatePrep(state);
if (sleep_result == 1 && state == ACPI_STATE_S3) {
/*
* Prevent mis-interpretation of the wakeup by power button
* as a request for power off.
* Ideally we should post an appropriate wakeup event,
* perhaps using acpi_event_power_button_wake or alike.
*
* Clearing of power button status after wakeup is mandated
* by ACPI specification in section "Fixed Power Button".
*
* XXX As of ACPICA 20121114 AcpiGetEventStatus provides
* status as 0/1 corressponding to inactive/active despite
* its type being ACPI_EVENT_STATUS. In other words,
* we should not test for ACPI_EVENT_FLAG_SET for time being.
*/
if (ACPI_SUCCESS(AcpiGetEventStatus(ACPI_EVENT_POWER_BUTTON,
&power_button_status)) && power_button_status != 0) {
AcpiClearEvent(ACPI_EVENT_POWER_BUTTON);
device_printf(sc->acpi_dev,
"cleared fixed power button status\n");
}
}
intr_restore(intr);
/* call acpi_wakeup_machdep() again with interrupt enabled */
acpi_wakeup_machdep(sc, state, sleep_result, 1);
if (sleep_result == -1)
goto backout;
/* Re-enable ACPI hardware on wakeup from sleep state 4. */
if (state == ACPI_STATE_S4)
AcpiEnable();
} else {
status = AcpiEnterSleepState(state);
AcpiLeaveSleepStatePrep(state);
intr_restore(intr);
if (ACPI_FAILURE(status)) {
device_printf(sc->acpi_dev, "AcpiEnterSleepState failed - %s\n",
AcpiFormatException(status));
goto backout;
}
}
slp_state = ACPI_SS_SLEPT;
/*
* Back out state according to how far along we got in the suspend
* process. This handles both the error and success cases.
*/
backout:
if (slp_state >= ACPI_SS_GPE_SET) {
acpi_wake_prep_walk(state);
sc->acpi_sstate = ACPI_STATE_S0;
}
if (slp_state >= ACPI_SS_DEV_SUSPEND)
DEVICE_RESUME(root_bus);
if (slp_state >= ACPI_SS_SLP_PREP)
AcpiLeaveSleepState(state);
if (slp_state >= ACPI_SS_SLEPT) {
acpi_resync_clock(sc);
acpi_enable_fixed_events(sc);
}
sc->acpi_next_sstate = 0;
mtx_unlock(&Giant);
if (smp_started) {
thread_lock(curthread);
sched_unbind(curthread);
thread_unlock(curthread);
}
resume_all_proc();
EVENTHANDLER_INVOKE(power_resume);
/* Allow another sleep request after a while. */
callout_schedule(&acpi_sleep_timer, hz * ACPI_MINIMUM_AWAKETIME);
/* Run /etc/rc.resume after we are back. */
if (devctl_process_running())
acpi_UserNotify("Resume", ACPI_ROOT_OBJECT, state);
return_ACPI_STATUS (status);
}
static void
acpi_resync_clock(struct acpi_softc *sc)
{
#ifdef __amd64__
if (!acpi_reset_clock)
return;
/*
* Warm up timecounter again and reset system clock.
*/
(void)timecounter->tc_get_timecount(timecounter);
(void)timecounter->tc_get_timecount(timecounter);
inittodr(time_second + sc->acpi_sleep_delay);
#endif
}
/* Enable or disable the device's wake GPE. */
int
acpi_wake_set_enable(device_t dev, int enable)
{
struct acpi_prw_data prw;
ACPI_STATUS status;
int flags;
/* Make sure the device supports waking the system and get the GPE. */
if (acpi_parse_prw(acpi_get_handle(dev), &prw) != 0)
return (ENXIO);
flags = acpi_get_flags(dev);
if (enable) {
status = AcpiSetGpeWakeMask(prw.gpe_handle, prw.gpe_bit,
ACPI_GPE_ENABLE);
if (ACPI_FAILURE(status)) {
device_printf(dev, "enable wake failed\n");
return (ENXIO);
}
acpi_set_flags(dev, flags | ACPI_FLAG_WAKE_ENABLED);
} else {
status = AcpiSetGpeWakeMask(prw.gpe_handle, prw.gpe_bit,
ACPI_GPE_DISABLE);
if (ACPI_FAILURE(status)) {
device_printf(dev, "disable wake failed\n");
return (ENXIO);
}
acpi_set_flags(dev, flags & ~ACPI_FLAG_WAKE_ENABLED);
}
return (0);
}
static int
acpi_wake_sleep_prep(ACPI_HANDLE handle, int sstate)
{
struct acpi_prw_data prw;
device_t dev;
/* Check that this is a wake-capable device and get its GPE. */
if (acpi_parse_prw(handle, &prw) != 0)
return (ENXIO);
dev = acpi_get_device(handle);
/*
* The destination sleep state must be less than (i.e., higher power)
* or equal to the value specified by _PRW. If this GPE cannot be
* enabled for the next sleep state, then disable it. If it can and
* the user requested it be enabled, turn on any required power resources
* and set _PSW.
*/
if (sstate > prw.lowest_wake) {
AcpiSetGpeWakeMask(prw.gpe_handle, prw.gpe_bit, ACPI_GPE_DISABLE);
if (bootverbose)
device_printf(dev, "wake_prep disabled wake for %s (S%d)\n",
acpi_name(handle), sstate);
} else if (dev && (acpi_get_flags(dev) & ACPI_FLAG_WAKE_ENABLED) != 0) {
acpi_pwr_wake_enable(handle, 1);
acpi_SetInteger(handle, "_PSW", 1);
if (bootverbose)
device_printf(dev, "wake_prep enabled for %s (S%d)\n",
acpi_name(handle), sstate);
}
return (0);
}
static int
acpi_wake_run_prep(ACPI_HANDLE handle, int sstate)
{
struct acpi_prw_data prw;
device_t dev;
/*
* Check that this is a wake-capable device and get its GPE. Return
* now if the user didn't enable this device for wake.
*/
if (acpi_parse_prw(handle, &prw) != 0)
return (ENXIO);
dev = acpi_get_device(handle);
if (dev == NULL || (acpi_get_flags(dev) & ACPI_FLAG_WAKE_ENABLED) == 0)
return (0);
/*
* If this GPE couldn't be enabled for the previous sleep state, it was
* disabled before going to sleep so re-enable it. If it was enabled,
* clear _PSW and turn off any power resources it used.
*/
if (sstate > prw.lowest_wake) {
AcpiSetGpeWakeMask(prw.gpe_handle, prw.gpe_bit, ACPI_GPE_ENABLE);
if (bootverbose)
device_printf(dev, "run_prep re-enabled %s\n", acpi_name(handle));
} else {
acpi_SetInteger(handle, "_PSW", 0);
acpi_pwr_wake_enable(handle, 0);
if (bootverbose)
device_printf(dev, "run_prep cleaned up for %s\n",
acpi_name(handle));
}
return (0);
}
static ACPI_STATUS
acpi_wake_prep(ACPI_HANDLE handle, UINT32 level, void *context, void **status)
{
int sstate;
/* If suspending, run the sleep prep function, otherwise wake. */
sstate = *(int *)context;
if (AcpiGbl_SystemAwakeAndRunning)
acpi_wake_sleep_prep(handle, sstate);
else
acpi_wake_run_prep(handle, sstate);
return (AE_OK);
}
/* Walk the tree rooted at acpi0 to prep devices for suspend/resume. */
static int
acpi_wake_prep_walk(int sstate)
{
ACPI_HANDLE sb_handle;
if (ACPI_SUCCESS(AcpiGetHandle(ACPI_ROOT_OBJECT, "\\_SB_", &sb_handle)))
AcpiWalkNamespace(ACPI_TYPE_DEVICE, sb_handle, 100,
acpi_wake_prep, NULL, &sstate, NULL);
return (0);
}
/* Walk the tree rooted at acpi0 to attach per-device wake sysctls. */
static int
acpi_wake_sysctl_walk(device_t dev)
{
int error, i, numdevs;
device_t *devlist;
device_t child;
ACPI_STATUS status;
error = device_get_children(dev, &devlist, &numdevs);
if (error != 0 || numdevs == 0) {
if (numdevs == 0)
free(devlist, M_TEMP);
return (error);
}
for (i = 0; i < numdevs; i++) {
child = devlist[i];
acpi_wake_sysctl_walk(child);
if (!device_is_attached(child))
continue;
status = AcpiEvaluateObject(acpi_get_handle(child), "_PRW", NULL, NULL);
if (ACPI_SUCCESS(status)) {
SYSCTL_ADD_PROC(device_get_sysctl_ctx(child),
SYSCTL_CHILDREN(device_get_sysctl_tree(child)), OID_AUTO,
"wake", CTLTYPE_INT | CTLFLAG_RW, child, 0,
acpi_wake_set_sysctl, "I", "Device set to wake the system");
}
}
free(devlist, M_TEMP);
return (0);
}
/* Enable or disable wake from userland. */
static int
acpi_wake_set_sysctl(SYSCTL_HANDLER_ARGS)
{
int enable, error;
device_t dev;
dev = (device_t)arg1;
enable = (acpi_get_flags(dev) & ACPI_FLAG_WAKE_ENABLED) ? 1 : 0;
error = sysctl_handle_int(oidp, &enable, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
if (enable != 0 && enable != 1)
return (EINVAL);
return (acpi_wake_set_enable(dev, enable));
}
/* Parse a device's _PRW into a structure. */
int
acpi_parse_prw(ACPI_HANDLE h, struct acpi_prw_data *prw)
{
ACPI_STATUS status;
ACPI_BUFFER prw_buffer;
ACPI_OBJECT *res, *res2;
int error, i, power_count;
if (h == NULL || prw == NULL)
return (EINVAL);
/*
* The _PRW object (7.2.9) is only required for devices that have the
* ability to wake the system from a sleeping state.
*/
error = EINVAL;
prw_buffer.Pointer = NULL;
prw_buffer.Length = ACPI_ALLOCATE_BUFFER;
status = AcpiEvaluateObject(h, "_PRW", NULL, &prw_buffer);
if (ACPI_FAILURE(status))
return (ENOENT);
res = (ACPI_OBJECT *)prw_buffer.Pointer;
if (res == NULL)
return (ENOENT);
if (!ACPI_PKG_VALID(res, 2))
goto out;
/*
* Element 1 of the _PRW object:
* The lowest power system sleeping state that can be entered while still
* providing wake functionality. The sleeping state being entered must
* be less than (i.e., higher power) or equal to this value.
*/
if (acpi_PkgInt32(res, 1, &prw->lowest_wake) != 0)
goto out;
/*
* Element 0 of the _PRW object:
*/
switch (res->Package.Elements[0].Type) {
case ACPI_TYPE_INTEGER:
/*
* If the data type of this package element is numeric, then this
* _PRW package element is the bit index in the GPEx_EN, in the
* GPE blocks described in the FADT, of the enable bit that is
* enabled for the wake event.
*/
prw->gpe_handle = NULL;
prw->gpe_bit = res->Package.Elements[0].Integer.Value;
error = 0;
break;
case ACPI_TYPE_PACKAGE:
/*
* If the data type of this package element is a package, then this
* _PRW package element is itself a package containing two
* elements. The first is an object reference to the GPE Block
* device that contains the GPE that will be triggered by the wake
* event. The second element is numeric and it contains the bit
* index in the GPEx_EN, in the GPE Block referenced by the
* first element in the package, of the enable bit that is enabled for
* the wake event.
*
* For example, if this field is a package then it is of the form:
* Package() {\_SB.PCI0.ISA.GPE, 2}
*/
res2 = &res->Package.Elements[0];
if (!ACPI_PKG_VALID(res2, 2))
goto out;
prw->gpe_handle = acpi_GetReference(NULL, &res2->Package.Elements[0]);
if (prw->gpe_handle == NULL)
goto out;
if (acpi_PkgInt32(res2, 1, &prw->gpe_bit) != 0)
goto out;
error = 0;
break;
default:
goto out;
}
/* Elements 2 to N of the _PRW object are power resources. */
power_count = res->Package.Count - 2;
if (power_count > ACPI_PRW_MAX_POWERRES) {
printf("ACPI device %s has too many power resources\n", acpi_name(h));
power_count = 0;
}
prw->power_res_count = power_count;
for (i = 0; i < power_count; i++)
prw->power_res[i] = res->Package.Elements[i];
out:
if (prw_buffer.Pointer != NULL)
AcpiOsFree(prw_buffer.Pointer);
return (error);
}
/*
* ACPI Event Handlers
*/
/* System Event Handlers (registered by EVENTHANDLER_REGISTER) */
static void
acpi_system_eventhandler_sleep(void *arg, int state)
{
struct acpi_softc *sc = (struct acpi_softc *)arg;
int ret;
ACPI_FUNCTION_TRACE_U32((char *)(uintptr_t)__func__, state);
/* Check if button action is disabled or unknown. */
if (state == ACPI_STATE_UNKNOWN)
return;
/* Request that the system prepare to enter the given suspend state. */
ret = acpi_ReqSleepState(sc, state);
if (ret != 0)
device_printf(sc->acpi_dev,
"request to enter state S%d failed (err %d)\n", state, ret);
return_VOID;
}
static void
acpi_system_eventhandler_wakeup(void *arg, int state)
{
ACPI_FUNCTION_TRACE_U32((char *)(uintptr_t)__func__, state);
/* Currently, nothing to do for wakeup. */
return_VOID;
}
/*
* ACPICA Event Handlers (FixedEvent, also called from button notify handler)
*/
static void
acpi_invoke_sleep_eventhandler(void *context)
{
EVENTHANDLER_INVOKE(acpi_sleep_event, *(int *)context);
}
static void
acpi_invoke_wake_eventhandler(void *context)
{
EVENTHANDLER_INVOKE(acpi_wakeup_event, *(int *)context);
}
UINT32
acpi_event_power_button_sleep(void *context)
{
struct acpi_softc *sc = (struct acpi_softc *)context;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
acpi_invoke_sleep_eventhandler, &sc->acpi_power_button_sx)))
return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
return_VALUE (ACPI_INTERRUPT_HANDLED);
}
UINT32
acpi_event_power_button_wake(void *context)
{
struct acpi_softc *sc = (struct acpi_softc *)context;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
acpi_invoke_wake_eventhandler, &sc->acpi_power_button_sx)))
return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
return_VALUE (ACPI_INTERRUPT_HANDLED);
}
UINT32
acpi_event_sleep_button_sleep(void *context)
{
struct acpi_softc *sc = (struct acpi_softc *)context;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
acpi_invoke_sleep_eventhandler, &sc->acpi_sleep_button_sx)))
return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
return_VALUE (ACPI_INTERRUPT_HANDLED);
}
UINT32
acpi_event_sleep_button_wake(void *context)
{
struct acpi_softc *sc = (struct acpi_softc *)context;
ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
acpi_invoke_wake_eventhandler, &sc->acpi_sleep_button_sx)))
return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
return_VALUE (ACPI_INTERRUPT_HANDLED);
}
/*
* XXX This static buffer is suboptimal. There is no locking so only
* use this for single-threaded callers.
*/
char *
acpi_name(ACPI_HANDLE handle)
{
ACPI_BUFFER buf;
static char data[256];
buf.Length = sizeof(data);
buf.Pointer = data;
if (handle && ACPI_SUCCESS(AcpiGetName(handle, ACPI_FULL_PATHNAME, &buf)))
return (data);
return ("(unknown)");
}
/*
* Debugging/bug-avoidance. Avoid trying to fetch info on various
* parts of the namespace.
*/
int
acpi_avoid(ACPI_HANDLE handle)
{
char *cp, *env, *np;
int len;
np = acpi_name(handle);
if (*np == '\\')
np++;
if ((env = kern_getenv("debug.acpi.avoid")) == NULL)
return (0);
/* Scan the avoid list checking for a match */
cp = env;
for (;;) {
while (*cp != 0 && isspace(*cp))
cp++;
if (*cp == 0)
break;
len = 0;
while (cp[len] != 0 && !isspace(cp[len]))
len++;
if (!strncmp(cp, np, len)) {
freeenv(env);
return(1);
}
cp += len;
}
freeenv(env);
return (0);
}
/*
* Debugging/bug-avoidance. Disable ACPI subsystem components.
*/
int
acpi_disabled(char *subsys)
{
char *cp, *env;
int len;
if ((env = kern_getenv("debug.acpi.disabled")) == NULL)
return (0);
if (strcmp(env, "all") == 0) {
freeenv(env);
return (1);
}
/* Scan the disable list, checking for a match. */
cp = env;
for (;;) {
while (*cp != '\0' && isspace(*cp))
cp++;
if (*cp == '\0')
break;
len = 0;
while (cp[len] != '\0' && !isspace(cp[len]))
len++;
if (strncmp(cp, subsys, len) == 0) {
freeenv(env);
return (1);
}
cp += len;
}
freeenv(env);
return (0);
}
static void
acpi_lookup(void *arg, const char *name, device_t *dev)
{
ACPI_HANDLE handle;
if (*dev != NULL)
return;
/*
* Allow any handle name that is specified as an absolute path and
* starts with '\'. We could restrict this to \_SB and friends,
* but see acpi_probe_children() for notes on why we scan the entire
* namespace for devices.
*
* XXX: The pathname argument to AcpiGetHandle() should be fixed to
* be const.
*/
if (name[0] != '\\')
return;
if (ACPI_FAILURE(AcpiGetHandle(ACPI_ROOT_OBJECT, __DECONST(char *, name),
&handle)))
return;
*dev = acpi_get_device(handle);
}
/*
* Control interface.
*
* We multiplex ioctls for all participating ACPI devices here. Individual
* drivers wanting to be accessible via /dev/acpi should use the
* register/deregister interface to make their handlers visible.
*/
struct acpi_ioctl_hook
{
TAILQ_ENTRY(acpi_ioctl_hook) link;
u_long cmd;
acpi_ioctl_fn fn;
void *arg;
};
static TAILQ_HEAD(,acpi_ioctl_hook) acpi_ioctl_hooks;
static int acpi_ioctl_hooks_initted;
int
acpi_register_ioctl(u_long cmd, acpi_ioctl_fn fn, void *arg)
{
struct acpi_ioctl_hook *hp;
if ((hp = malloc(sizeof(*hp), M_ACPIDEV, M_NOWAIT)) == NULL)
return (ENOMEM);
hp->cmd = cmd;
hp->fn = fn;
hp->arg = arg;
ACPI_LOCK(acpi);
if (acpi_ioctl_hooks_initted == 0) {
TAILQ_INIT(&acpi_ioctl_hooks);
acpi_ioctl_hooks_initted = 1;
}
TAILQ_INSERT_TAIL(&acpi_ioctl_hooks, hp, link);
ACPI_UNLOCK(acpi);
return (0);
}
void
acpi_deregister_ioctl(u_long cmd, acpi_ioctl_fn fn)
{
struct acpi_ioctl_hook *hp;
ACPI_LOCK(acpi);
TAILQ_FOREACH(hp, &acpi_ioctl_hooks, link)
if (hp->cmd == cmd && hp->fn == fn)
break;
if (hp != NULL) {
TAILQ_REMOVE(&acpi_ioctl_hooks, hp, link);
free(hp, M_ACPIDEV);
}
ACPI_UNLOCK(acpi);
}
static int
acpiopen(struct cdev *dev, int flag, int fmt, struct thread *td)
{
return (0);
}
static int
acpiclose(struct cdev *dev, int flag, int fmt, struct thread *td)
{
return (0);
}
static int
acpiioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
{
struct acpi_softc *sc;
struct acpi_ioctl_hook *hp;
int error, state;
error = 0;
hp = NULL;
sc = dev->si_drv1;
/*
* Scan the list of registered ioctls, looking for handlers.
*/
ACPI_LOCK(acpi);
if (acpi_ioctl_hooks_initted)
TAILQ_FOREACH(hp, &acpi_ioctl_hooks, link) {
if (hp->cmd == cmd)
break;
}
ACPI_UNLOCK(acpi);
if (hp)
return (hp->fn(cmd, addr, hp->arg));
/*
* Core ioctls are not permitted for non-writable user.
* Currently, other ioctls just fetch information.
* Not changing system behavior.
*/
if ((flag & FWRITE) == 0)
return (EPERM);
/* Core system ioctls. */
switch (cmd) {
case ACPIIO_REQSLPSTATE:
state = *(int *)addr;
if (state != ACPI_STATE_S5)
return (acpi_ReqSleepState(sc, state));
device_printf(sc->acpi_dev, "power off via acpi ioctl not supported\n");
error = EOPNOTSUPP;
break;
case ACPIIO_ACKSLPSTATE:
error = *(int *)addr;
error = acpi_AckSleepState(sc->acpi_clone, error);
break;
case ACPIIO_SETSLPSTATE: /* DEPRECATED */
state = *(int *)addr;
if (state < ACPI_STATE_S0 || state > ACPI_S_STATES_MAX)
return (EINVAL);
if (!acpi_sleep_states[state])
return (EOPNOTSUPP);
if (ACPI_FAILURE(acpi_SetSleepState(sc, state)))
error = ENXIO;
break;
default:
error = ENXIO;
break;
}
return (error);
}
static int
acpi_sname2sstate(const char *sname)
{
int sstate;
if (toupper(sname[0]) == 'S') {
sstate = sname[1] - '0';
if (sstate >= ACPI_STATE_S0 && sstate <= ACPI_STATE_S5 &&
sname[2] == '\0')
return (sstate);
} else if (strcasecmp(sname, "NONE") == 0)
return (ACPI_STATE_UNKNOWN);
return (-1);
}
static const char *
acpi_sstate2sname(int sstate)
{
static const char *snames[] = { "S0", "S1", "S2", "S3", "S4", "S5" };
if (sstate >= ACPI_STATE_S0 && sstate <= ACPI_STATE_S5)
return (snames[sstate]);
else if (sstate == ACPI_STATE_UNKNOWN)
return ("NONE");
return (NULL);
}
static int
acpi_supported_sleep_state_sysctl(SYSCTL_HANDLER_ARGS)
{
int error;
struct sbuf sb;
UINT8 state;
sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
for (state = ACPI_STATE_S1; state < ACPI_S_STATE_COUNT; state++)
if (acpi_sleep_states[state])
sbuf_printf(&sb, "%s ", acpi_sstate2sname(state));
sbuf_trim(&sb);
sbuf_finish(&sb);
error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
sbuf_delete(&sb);
return (error);
}
static int
acpi_sleep_state_sysctl(SYSCTL_HANDLER_ARGS)
{
char sleep_state[10];
int error, new_state, old_state;
old_state = *(int *)oidp->oid_arg1;
strlcpy(sleep_state, acpi_sstate2sname(old_state), sizeof(sleep_state));
error = sysctl_handle_string(oidp, sleep_state, sizeof(sleep_state), req);
if (error == 0 && req->newptr != NULL) {
new_state = acpi_sname2sstate(sleep_state);
if (new_state < ACPI_STATE_S1)
return (EINVAL);
if (new_state < ACPI_S_STATE_COUNT && !acpi_sleep_states[new_state])
return (EOPNOTSUPP);
if (new_state != old_state)
*(int *)oidp->oid_arg1 = new_state;
}
return (error);
}
/* Inform devctl(4) when we receive a Notify. */
void
acpi_UserNotify(const char *subsystem, ACPI_HANDLE h, uint8_t notify)
{
char notify_buf[16];
ACPI_BUFFER handle_buf;
ACPI_STATUS status;
if (subsystem == NULL)
return;
handle_buf.Pointer = NULL;
handle_buf.Length = ACPI_ALLOCATE_BUFFER;
status = AcpiNsHandleToPathname(h, &handle_buf);
if (ACPI_FAILURE(status))
return;
snprintf(notify_buf, sizeof(notify_buf), "notify=0x%02x", notify);
devctl_notify("ACPI", subsystem, handle_buf.Pointer, notify_buf);
AcpiOsFree(handle_buf.Pointer);
}
#ifdef ACPI_DEBUG
/*
* Support for parsing debug options from the kernel environment.
*
* Bits may be set in the AcpiDbgLayer and AcpiDbgLevel debug registers
* by specifying the names of the bits in the debug.acpi.layer and
* debug.acpi.level environment variables. Bits may be unset by
* prefixing the bit name with !.
*/
struct debugtag
{
char *name;
UINT32 value;
};
static struct debugtag dbg_layer[] = {
{"ACPI_UTILITIES", ACPI_UTILITIES},
{"ACPI_HARDWARE", ACPI_HARDWARE},
{"ACPI_EVENTS", ACPI_EVENTS},
{"ACPI_TABLES", ACPI_TABLES},
{"ACPI_NAMESPACE", ACPI_NAMESPACE},
{"ACPI_PARSER", ACPI_PARSER},
{"ACPI_DISPATCHER", ACPI_DISPATCHER},
{"ACPI_EXECUTER", ACPI_EXECUTER},
{"ACPI_RESOURCES", ACPI_RESOURCES},
{"ACPI_CA_DEBUGGER", ACPI_CA_DEBUGGER},
{"ACPI_OS_SERVICES", ACPI_OS_SERVICES},
{"ACPI_CA_DISASSEMBLER", ACPI_CA_DISASSEMBLER},
{"ACPI_ALL_COMPONENTS", ACPI_ALL_COMPONENTS},
{"ACPI_AC_ADAPTER", ACPI_AC_ADAPTER},
{"ACPI_BATTERY", ACPI_BATTERY},
{"ACPI_BUS", ACPI_BUS},
{"ACPI_BUTTON", ACPI_BUTTON},
{"ACPI_EC", ACPI_EC},
{"ACPI_FAN", ACPI_FAN},
{"ACPI_POWERRES", ACPI_POWERRES},
{"ACPI_PROCESSOR", ACPI_PROCESSOR},
{"ACPI_THERMAL", ACPI_THERMAL},
{"ACPI_TIMER", ACPI_TIMER},
{"ACPI_ALL_DRIVERS", ACPI_ALL_DRIVERS},
{NULL, 0}
};
static struct debugtag dbg_level[] = {
{"ACPI_LV_INIT", ACPI_LV_INIT},
{"ACPI_LV_DEBUG_OBJECT", ACPI_LV_DEBUG_OBJECT},
{"ACPI_LV_INFO", ACPI_LV_INFO},
{"ACPI_LV_REPAIR", ACPI_LV_REPAIR},
{"ACPI_LV_ALL_EXCEPTIONS", ACPI_LV_ALL_EXCEPTIONS},
/* Trace verbosity level 1 [Standard Trace Level] */
{"ACPI_LV_INIT_NAMES", ACPI_LV_INIT_NAMES},
{"ACPI_LV_PARSE", ACPI_LV_PARSE},
{"ACPI_LV_LOAD", ACPI_LV_LOAD},
{"ACPI_LV_DISPATCH", ACPI_LV_DISPATCH},
{"ACPI_LV_EXEC", ACPI_LV_EXEC},
{"ACPI_LV_NAMES", ACPI_LV_NAMES},
{"ACPI_LV_OPREGION", ACPI_LV_OPREGION},
{"ACPI_LV_BFIELD", ACPI_LV_BFIELD},
{"ACPI_LV_TABLES", ACPI_LV_TABLES},
{"ACPI_LV_VALUES", ACPI_LV_VALUES},
{"ACPI_LV_OBJECTS", ACPI_LV_OBJECTS},
{"ACPI_LV_RESOURCES", ACPI_LV_RESOURCES},
{"ACPI_LV_USER_REQUESTS", ACPI_LV_USER_REQUESTS},
{"ACPI_LV_PACKAGE", ACPI_LV_PACKAGE},
{"ACPI_LV_VERBOSITY1", ACPI_LV_VERBOSITY1},
/* Trace verbosity level 2 [Function tracing and memory allocation] */
{"ACPI_LV_ALLOCATIONS", ACPI_LV_ALLOCATIONS},
{"ACPI_LV_FUNCTIONS", ACPI_LV_FUNCTIONS},
{"ACPI_LV_OPTIMIZATIONS", ACPI_LV_OPTIMIZATIONS},
{"ACPI_LV_VERBOSITY2", ACPI_LV_VERBOSITY2},
{"ACPI_LV_ALL", ACPI_LV_ALL},
/* Trace verbosity level 3 [Threading, I/O, and Interrupts] */
{"ACPI_LV_MUTEX", ACPI_LV_MUTEX},
{"ACPI_LV_THREADS", ACPI_LV_THREADS},
{"ACPI_LV_IO", ACPI_LV_IO},
{"ACPI_LV_INTERRUPTS", ACPI_LV_INTERRUPTS},
{"ACPI_LV_VERBOSITY3", ACPI_LV_VERBOSITY3},
/* Exceptionally verbose output -- also used in the global "DebugLevel" */
{"ACPI_LV_AML_DISASSEMBLE", ACPI_LV_AML_DISASSEMBLE},
{"ACPI_LV_VERBOSE_INFO", ACPI_LV_VERBOSE_INFO},
{"ACPI_LV_FULL_TABLES", ACPI_LV_FULL_TABLES},
{"ACPI_LV_EVENTS", ACPI_LV_EVENTS},
{"ACPI_LV_VERBOSE", ACPI_LV_VERBOSE},
{NULL, 0}
};
static void
acpi_parse_debug(char *cp, struct debugtag *tag, UINT32 *flag)
{
char *ep;
int i, l;
int set;
while (*cp) {
if (isspace(*cp)) {
cp++;
continue;
}
ep = cp;
while (*ep && !isspace(*ep))
ep++;
if (*cp == '!') {
set = 0;
cp++;
if (cp == ep)
continue;
} else {
set = 1;
}
l = ep - cp;
for (i = 0; tag[i].name != NULL; i++) {
if (!strncmp(cp, tag[i].name, l)) {
if (set)
*flag |= tag[i].value;
else
*flag &= ~tag[i].value;
}
}
cp = ep;
}
}
static void
acpi_set_debugging(void *junk)
{
char *layer, *level;
if (cold) {
AcpiDbgLayer = 0;
AcpiDbgLevel = 0;
}
layer = kern_getenv("debug.acpi.layer");
level = kern_getenv("debug.acpi.level");
if (layer == NULL && level == NULL)
return;
printf("ACPI set debug");
if (layer != NULL) {
if (strcmp("NONE", layer) != 0)
printf(" layer '%s'", layer);
acpi_parse_debug(layer, &dbg_layer[0], &AcpiDbgLayer);
freeenv(layer);
}
if (level != NULL) {
if (strcmp("NONE", level) != 0)
printf(" level '%s'", level);
acpi_parse_debug(level, &dbg_level[0], &AcpiDbgLevel);
freeenv(level);
}
printf("\n");
}
SYSINIT(acpi_debugging, SI_SUB_TUNABLES, SI_ORDER_ANY, acpi_set_debugging,
NULL);
static int
acpi_debug_sysctl(SYSCTL_HANDLER_ARGS)
{
int error, *dbg;
struct debugtag *tag;
struct sbuf sb;
char temp[128];
if (sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND) == NULL)
return (ENOMEM);
if (strcmp(oidp->oid_arg1, "debug.acpi.layer") == 0) {
tag = &dbg_layer[0];
dbg = &AcpiDbgLayer;
} else {
tag = &dbg_level[0];
dbg = &AcpiDbgLevel;
}
/* Get old values if this is a get request. */
ACPI_SERIAL_BEGIN(acpi);
if (*dbg == 0) {
sbuf_cpy(&sb, "NONE");
} else if (req->newptr == NULL) {
for (; tag->name != NULL; tag++) {
if ((*dbg & tag->value) == tag->value)
sbuf_printf(&sb, "%s ", tag->name);
}
}
sbuf_trim(&sb);
sbuf_finish(&sb);
strlcpy(temp, sbuf_data(&sb), sizeof(temp));
sbuf_delete(&sb);
error = sysctl_handle_string(oidp, temp, sizeof(temp), req);
/* Check for error or no change */
if (error == 0 && req->newptr != NULL) {
*dbg = 0;
kern_setenv((char *)oidp->oid_arg1, temp);
acpi_set_debugging(NULL);
}
ACPI_SERIAL_END(acpi);
return (error);
}
SYSCTL_PROC(_debug_acpi, OID_AUTO, layer, CTLFLAG_RW | CTLTYPE_STRING,
"debug.acpi.layer", 0, acpi_debug_sysctl, "A", "");
SYSCTL_PROC(_debug_acpi, OID_AUTO, level, CTLFLAG_RW | CTLTYPE_STRING,
"debug.acpi.level", 0, acpi_debug_sysctl, "A", "");
#endif /* ACPI_DEBUG */
static int
acpi_debug_objects_sysctl(SYSCTL_HANDLER_ARGS)
{
int error;
int old;
old = acpi_debug_objects;
error = sysctl_handle_int(oidp, &acpi_debug_objects, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
if (old == acpi_debug_objects || (old && acpi_debug_objects))
return (0);
ACPI_SERIAL_BEGIN(acpi);
AcpiGbl_EnableAmlDebugObject = acpi_debug_objects ? TRUE : FALSE;
ACPI_SERIAL_END(acpi);
return (0);
}
static int
acpi_parse_interfaces(char *str, struct acpi_interface *iface)
{
char *p;
size_t len;
int i, j;
p = str;
while (isspace(*p) || *p == ',')
p++;
len = strlen(p);
if (len == 0)
return (0);
p = strdup(p, M_TEMP);
for (i = 0; i < len; i++)
if (p[i] == ',')
p[i] = '\0';
i = j = 0;
while (i < len)
if (isspace(p[i]) || p[i] == '\0')
i++;
else {
i += strlen(p + i) + 1;
j++;
}
if (j == 0) {
free(p, M_TEMP);
return (0);
}
iface->data = malloc(sizeof(*iface->data) * j, M_TEMP, M_WAITOK);
iface->num = j;
i = j = 0;
while (i < len)
if (isspace(p[i]) || p[i] == '\0')
i++;
else {
iface->data[j] = p + i;
i += strlen(p + i) + 1;
j++;
}
return (j);
}
static void
acpi_free_interfaces(struct acpi_interface *iface)
{
free(iface->data[0], M_TEMP);
free(iface->data, M_TEMP);
}
static void
acpi_reset_interfaces(device_t dev)
{
struct acpi_interface list;
ACPI_STATUS status;
int i;
if (acpi_parse_interfaces(acpi_install_interface, &list) > 0) {
for (i = 0; i < list.num; i++) {
status = AcpiInstallInterface(list.data[i]);
if (ACPI_FAILURE(status))
device_printf(dev,
"failed to install _OSI(\"%s\"): %s\n",
list.data[i], AcpiFormatException(status));
else if (bootverbose)
device_printf(dev, "installed _OSI(\"%s\")\n",
list.data[i]);
}
acpi_free_interfaces(&list);
}
if (acpi_parse_interfaces(acpi_remove_interface, &list) > 0) {
for (i = 0; i < list.num; i++) {
status = AcpiRemoveInterface(list.data[i]);
if (ACPI_FAILURE(status))
device_printf(dev,
"failed to remove _OSI(\"%s\"): %s\n",
list.data[i], AcpiFormatException(status));
else if (bootverbose)
device_printf(dev, "removed _OSI(\"%s\")\n",
list.data[i]);
}
acpi_free_interfaces(&list);
}
}
static int
acpi_pm_func(u_long cmd, void *arg, ...)
{
int state, acpi_state;
int error;
struct acpi_softc *sc;
va_list ap;
error = 0;
switch (cmd) {
case POWER_CMD_SUSPEND:
sc = (struct acpi_softc *)arg;
if (sc == NULL) {
error = EINVAL;
goto out;
}
va_start(ap, arg);
state = va_arg(ap, int);
va_end(ap);
switch (state) {
case POWER_SLEEP_STATE_STANDBY:
acpi_state = sc->acpi_standby_sx;
break;
case POWER_SLEEP_STATE_SUSPEND:
acpi_state = sc->acpi_suspend_sx;
break;
case POWER_SLEEP_STATE_HIBERNATE:
acpi_state = ACPI_STATE_S4;
break;
default:
error = EINVAL;
goto out;
}
if (ACPI_FAILURE(acpi_EnterSleepState(sc, acpi_state)))
error = ENXIO;
break;
default:
error = EINVAL;
goto out;
}
out:
return (error);
}
static void
acpi_pm_register(void *arg)
{
if (!cold || resource_disabled("acpi", 0))
return;
power_pm_register(POWER_PM_TYPE_ACPI, acpi_pm_func, NULL);
}
SYSINIT(power, SI_SUB_KLD, SI_ORDER_ANY, acpi_pm_register, 0);
Index: head/sys/dev/altera/jtag_uart/altera_jtag_uart_tty.c
===================================================================
--- head/sys/dev/altera/jtag_uart/altera_jtag_uart_tty.c (revision 283290)
+++ head/sys/dev/altera/jtag_uart/altera_jtag_uart_tty.c (revision 283291)
@@ -1,490 +1,490 @@
/*-
* Copyright (c) 2011-2012 Robert N. M. Watson
* All rights reserved.
*
* This software was developed by SRI International and the University of
* Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
* ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/cons.h>
#include <sys/endian.h>
#include <sys/kdb.h>
#include <sys/rman.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/reboot.h>
#include <sys/tty.h>
#include <ddb/ddb.h>
#include <machine/bus.h>
#include <dev/altera/jtag_uart/altera_jtag_uart.h>
/*
* If one of the Altera JTAG UARTs is currently the system console, register
* it here.
*/
static struct altera_jtag_uart_softc *aju_cons_sc;
static tsw_outwakeup_t aju_outwakeup;
static void aju_ac_callout(void *);
static void aju_io_callout(void *);
static struct ttydevsw aju_ttydevsw = {
.tsw_flags = TF_NOPREFIX,
.tsw_outwakeup = aju_outwakeup,
};
/*
* When polling for the AC bit, the number of times we have to not see it
* before assuming JTAG has disappeared on us. By default, two seconds.
*/
#define AJU_JTAG_MAXMISS 10
/*
* Polling intervals for input/output and JTAG connection events.
*/
#define AJU_IO_POLLINTERVAL (hz/100)
#define AJU_AC_POLLINTERVAL (hz/5)
/*
* Low-level read and write register routines; the Altera UART is little
* endian, so we byte swap 32-bit reads and writes.
*/
static inline uint32_t
aju_data_read(struct altera_jtag_uart_softc *sc)
{
return (le32toh(bus_read_4(sc->ajus_mem_res,
ALTERA_JTAG_UART_DATA_OFF)));
}
static inline void
aju_data_write(struct altera_jtag_uart_softc *sc, uint32_t v)
{
bus_write_4(sc->ajus_mem_res, ALTERA_JTAG_UART_DATA_OFF, htole32(v));
}
static inline uint32_t
aju_control_read(struct altera_jtag_uart_softc *sc)
{
return (le32toh(bus_read_4(sc->ajus_mem_res,
ALTERA_JTAG_UART_CONTROL_OFF)));
}
static inline void
aju_control_write(struct altera_jtag_uart_softc *sc, uint32_t v)
{
bus_write_4(sc->ajus_mem_res, ALTERA_JTAG_UART_CONTROL_OFF,
htole32(v));
}
/*
* Slightly higher-level routines aware of buffering and flow control.
*/
static inline int
aju_writable(struct altera_jtag_uart_softc *sc)
{
return ((aju_control_read(sc) &
ALTERA_JTAG_UART_CONTROL_WSPACE) != 0);
}
static inline int
aju_readable(struct altera_jtag_uart_softc *sc)
{
uint32_t v;
AJU_LOCK_ASSERT(sc);
if (*sc->ajus_buffer_validp)
return (1);
v = aju_data_read(sc);
if ((v & ALTERA_JTAG_UART_DATA_RVALID) != 0) {
*sc->ajus_buffer_validp = 1;
*sc->ajus_buffer_datap = (v & ALTERA_JTAG_UART_DATA_DATA);
return (1);
}
return (0);
}
static char
aju_read(struct altera_jtag_uart_softc *sc)
{
AJU_LOCK_ASSERT(sc);
while (!aju_readable(sc));
*sc->ajus_buffer_validp = 0;
return (*sc->ajus_buffer_datap);
}
/*
* Routines for enabling and disabling interrupts for read and write.
*/
static void
aju_intr_readable_enable(struct altera_jtag_uart_softc *sc)
{
uint32_t v;
AJU_LOCK_ASSERT(sc);
v = aju_control_read(sc);
v |= ALTERA_JTAG_UART_CONTROL_RE;
aju_control_write(sc, v);
}
static void
aju_intr_writable_enable(struct altera_jtag_uart_softc *sc)
{
uint32_t v;
AJU_LOCK_ASSERT(sc);
v = aju_control_read(sc);
v |= ALTERA_JTAG_UART_CONTROL_WE;
aju_control_write(sc, v);
}
static void
aju_intr_writable_disable(struct altera_jtag_uart_softc *sc)
{
uint32_t v;
AJU_LOCK_ASSERT(sc);
v = aju_control_read(sc);
v &= ~ALTERA_JTAG_UART_CONTROL_WE;
aju_control_write(sc, v);
}
static void
aju_intr_disable(struct altera_jtag_uart_softc *sc)
{
uint32_t v;
AJU_LOCK_ASSERT(sc);
v = aju_control_read(sc);
v &= ~(ALTERA_JTAG_UART_CONTROL_RE | ALTERA_JTAG_UART_CONTROL_WE);
aju_control_write(sc, v);
}
/*
* The actual work of checking for, and handling, available reads. This is
* used in both polled and interrupt-driven modes, as JTAG UARTs may be hooked
* up with, or without, IRQs allocated.
*/
static void
aju_handle_input(struct altera_jtag_uart_softc *sc, struct tty *tp)
{
int c;
tty_lock_assert(tp, MA_OWNED);
AJU_LOCK_ASSERT(sc);
while (aju_readable(sc)) {
c = aju_read(sc);
AJU_UNLOCK(sc);
#ifdef KDB
if (sc->ajus_flags & ALTERA_JTAG_UART_FLAG_CONSOLE)
kdb_alt_break(c, &sc->ajus_alt_break_state);
#endif
ttydisc_rint(tp, c, 0);
AJU_LOCK(sc);
}
AJU_UNLOCK(sc);
ttydisc_rint_done(tp);
AJU_LOCK(sc);
}
/*
* Send output to the UART until either there's none left to send, or we run
* out of room and need to await an interrupt so that we can start sending
* again.
*
* XXXRW: It would be nice to query WSPACE at the beginning and write to the
* FIFO in bugger chunks.
*/
static void
aju_handle_output(struct altera_jtag_uart_softc *sc, struct tty *tp)
{
uint32_t v;
uint8_t ch;
tty_lock_assert(tp, MA_OWNED);
AJU_LOCK_ASSERT(sc);
AJU_UNLOCK(sc);
while (ttydisc_getc_poll(tp) != 0) {
AJU_LOCK(sc);
v = aju_control_read(sc);
if ((v & ALTERA_JTAG_UART_CONTROL_WSPACE) != 0) {
AJU_UNLOCK(sc);
if (ttydisc_getc(tp, &ch, sizeof(ch)) != sizeof(ch))
panic("%s: ttydisc_getc", __func__);
AJU_LOCK(sc);
/*
* XXXRW: There is a slight race here in which we test
* for writability, drop the lock, get the character
* from the tty layer, re-acquire the lock, and then
* write. It's possible for other code --
* specifically, the low-level console -- to have
* written in the mean time, which might mean that
* there is no longer space. The BERI memory bus will
* cause this write to block, wedging the processor
* until space is available -- which could be a while
* if JTAG is not attached!
*
* The 'easy' fix is to drop the character if WSPACE
* has become unset. Not sure what the 'hard' fix is.
*/
aju_data_write(sc, ch);
} else {
/*
* If JTAG is not present, then we will drop this
* character instead of perhaps scheduling an
* interrupt to let us know when there is buffer
* space. Otherwise we might get a write interrupt
* later even though we aren't interested in sending
* anymore. Loop to drain TTY-layer buffer.
*/
if (*sc->ajus_jtag_presentp == 0) {
if (ttydisc_getc(tp, &ch, sizeof(ch)) !=
sizeof(ch))
panic("%s: ttydisc_getc 2", __func__);
AJU_UNLOCK(sc);
continue;
}
if (sc->ajus_irq_res != NULL)
aju_intr_writable_enable(sc);
return;
}
AJU_UNLOCK(sc);
}
AJU_LOCK(sc);
aju_intr_writable_disable(sc);
}
static void
aju_outwakeup(struct tty *tp)
{
struct altera_jtag_uart_softc *sc = tty_softc(tp);
tty_lock_assert(tp, MA_OWNED);
AJU_LOCK(sc);
aju_handle_output(sc, tp);
AJU_UNLOCK(sc);
}
static void
aju_io_callout(void *arg)
{
struct altera_jtag_uart_softc *sc = arg;
struct tty *tp = sc->ajus_ttyp;
tty_lock(tp);
AJU_LOCK(sc);
/*
* It would be convenient if we could share code with aju_intr() here
* by testing the control register for ALTERA_JTAG_UART_CONTROL_RI and
* ALTERA_JTAG_UART_CONTROL_WI. Unfortunately, it's not clear that
* this is supported, so do all the work to poll for both input and
* output.
*/
aju_handle_input(sc, tp);
aju_handle_output(sc, tp);
/*
* Reschedule next poll attempt. There's some argument that we should
* do adaptive polling based on the expectation of I/O: is something
* pending in the output buffer, or have we recently had input, but we
* don't.
*/
callout_reset(&sc->ajus_io_callout, AJU_IO_POLLINTERVAL,
aju_io_callout, sc);
AJU_UNLOCK(sc);
tty_unlock(tp);
}
static void
aju_ac_callout(void *arg)
{
struct altera_jtag_uart_softc *sc = arg;
struct tty *tp = sc->ajus_ttyp;
uint32_t v;
tty_lock(tp);
AJU_LOCK(sc);
v = aju_control_read(sc);
if (v & ALTERA_JTAG_UART_CONTROL_AC) {
v &= ~ALTERA_JTAG_UART_CONTROL_AC;
aju_control_write(sc, v);
if (*sc->ajus_jtag_presentp == 0) {
*sc->ajus_jtag_missedp = 0;
*sc->ajus_jtag_presentp = 1;
aju_handle_output(sc, tp);
}
} else if (*sc->ajus_jtag_presentp != 0) {
(*sc->ajus_jtag_missedp)++;
if (*sc->ajus_jtag_missedp >= AJU_JTAG_MAXMISS) {
*sc->ajus_jtag_presentp = 0;
aju_handle_output(sc, tp);
}
}
callout_reset(&sc->ajus_ac_callout, AJU_AC_POLLINTERVAL,
aju_ac_callout, sc);
AJU_UNLOCK(sc);
tty_unlock(tp);
}
static void
aju_intr(void *arg)
{
struct altera_jtag_uart_softc *sc = arg;
struct tty *tp = sc->ajus_ttyp;
uint32_t v;
tty_lock(tp);
AJU_LOCK(sc);
v = aju_control_read(sc);
if (v & ALTERA_JTAG_UART_CONTROL_RI)
aju_handle_input(sc, tp);
if (v & ALTERA_JTAG_UART_CONTROL_WI)
aju_handle_output(sc, tp);
AJU_UNLOCK(sc);
tty_unlock(tp);
}
int
altera_jtag_uart_attach(struct altera_jtag_uart_softc *sc)
{
struct tty *tp;
int error;
AJU_LOCK_INIT(sc);
/*
* XXXRW: Currently, we detect the console solely based on it using a
* reserved address, and borrow console-level locks and buffer if so.
* Is there a better way?
*/
if (rman_get_start(sc->ajus_mem_res) == BERI_UART_BASE) {
sc->ajus_lockp = &aju_cons_lock;
sc->ajus_buffer_validp = &aju_cons_buffer_valid;
sc->ajus_buffer_datap = &aju_cons_buffer_data;
sc->ajus_jtag_presentp = &aju_cons_jtag_present;
sc->ajus_jtag_missedp = &aju_cons_jtag_missed;
sc->ajus_flags |= ALTERA_JTAG_UART_FLAG_CONSOLE;
} else {
sc->ajus_lockp = &sc->ajus_lock;
sc->ajus_buffer_validp = &sc->ajus_buffer_valid;
sc->ajus_buffer_datap = &sc->ajus_buffer_data;
sc->ajus_jtag_presentp = &sc->ajus_jtag_present;
sc->ajus_jtag_missedp = &sc->ajus_jtag_missed;
}
/*
* Disable interrupts regardless of whether or not we plan to use
* them. We will register an interrupt handler now if they will be
* used, but not re-enable intil later once the remainder of the tty
* layer is properly initialised, as we're not ready for input yet.
*/
AJU_LOCK(sc);
aju_intr_disable(sc);
AJU_UNLOCK(sc);
if (sc->ajus_irq_res != NULL) {
error = bus_setup_intr(sc->ajus_dev, sc->ajus_irq_res,
INTR_ENTROPY | INTR_TYPE_TTY | INTR_MPSAFE, NULL,
aju_intr, sc, &sc->ajus_irq_cookie);
if (error) {
device_printf(sc->ajus_dev,
"could not activate interrupt\n");
AJU_LOCK_DESTROY(sc);
return (error);
}
}
tp = sc->ajus_ttyp = tty_alloc(&aju_ttydevsw, sc);
if (sc->ajus_flags & ALTERA_JTAG_UART_FLAG_CONSOLE) {
aju_cons_sc = sc;
tty_init_console(tp, 0);
}
tty_makedev(tp, NULL, "%s%d", AJU_TTYNAME, sc->ajus_unit);
/*
* If we will be using interrupts, enable them now; otherwise, start
* polling. From this point onwards, input can arrive.
*/
if (sc->ajus_irq_res != NULL) {
AJU_LOCK(sc);
aju_intr_readable_enable(sc);
AJU_UNLOCK(sc);
} else {
- callout_init(&sc->ajus_io_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->ajus_io_callout, 1);
callout_reset(&sc->ajus_io_callout, AJU_IO_POLLINTERVAL,
aju_io_callout, sc);
}
- callout_init(&sc->ajus_ac_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->ajus_ac_callout, 1);
callout_reset(&sc->ajus_ac_callout, AJU_AC_POLLINTERVAL,
aju_ac_callout, sc);
return (0);
}
void
altera_jtag_uart_detach(struct altera_jtag_uart_softc *sc)
{
struct tty *tp = sc->ajus_ttyp;
/*
* If we're using interrupts, disable and release the interrupt
* handler now. Otherwise drain the polling timeout.
*/
if (sc->ajus_irq_res != NULL) {
AJU_LOCK(sc);
aju_intr_disable(sc);
AJU_UNLOCK(sc);
bus_teardown_intr(sc->ajus_dev, sc->ajus_irq_res,
sc->ajus_irq_cookie);
} else
callout_drain(&sc->ajus_io_callout);
callout_drain(&sc->ajus_ac_callout);
if (sc->ajus_flags & ALTERA_JTAG_UART_FLAG_CONSOLE)
aju_cons_sc = NULL;
tty_lock(tp);
tty_rel_gone(tp);
AJU_LOCK_DESTROY(sc);
}
Index: head/sys/dev/ath/if_ath.c
===================================================================
--- head/sys/dev/ath/if_ath.c (revision 283290)
+++ head/sys/dev/ath/if_ath.c (revision 283291)
@@ -1,7259 +1,7259 @@
/*-
* Copyright (c) 2002-2009 Sam Leffler, Errno Consulting
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer,
* without modification.
* 2. Redistributions in binary form must reproduce at minimum a disclaimer
* similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
* redistribution must be conditioned upon including a substantially
* similar Disclaimer requirement for further binary redistribution.
*
* NO WARRANTY
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
* THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGES.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* Driver for the Atheros Wireless LAN controller.
*
* This software is derived from work of Atsushi Onoe; his contribution
* is greatly appreciated.
*/
#include "opt_inet.h"
#include "opt_ath.h"
/*
* This is needed for register operations which are performed
* by the driver - eg, calls to ath_hal_gettsf32().
*
* It's also required for any AH_DEBUG checks in here, eg the
* module dependencies.
*/
#include "opt_ah.h"
#include "opt_wlan.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/errno.h>
#include <sys/callout.h>
#include <sys/bus.h>
#include <sys/endian.h>
#include <sys/kthread.h>
#include <sys/taskqueue.h>
#include <sys/priv.h>
#include <sys/module.h>
#include <sys/ktr.h>
#include <sys/smp.h> /* for mp_ncpus */
#include <machine/bus.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/if_types.h>
#include <net/if_arp.h>
#include <net/ethernet.h>
#include <net/if_llc.h>
#include <net80211/ieee80211_var.h>
#include <net80211/ieee80211_regdomain.h>
#ifdef IEEE80211_SUPPORT_SUPERG
#include <net80211/ieee80211_superg.h>
#endif
#ifdef IEEE80211_SUPPORT_TDMA
#include <net80211/ieee80211_tdma.h>
#endif
#include <net/bpf.h>
#ifdef INET
#include <netinet/in.h>
#include <netinet/if_ether.h>
#endif
#include <dev/ath/if_athvar.h>
#include <dev/ath/ath_hal/ah_devid.h> /* XXX for softled */
#include <dev/ath/ath_hal/ah_diagcodes.h>
#include <dev/ath/if_ath_debug.h>
#include <dev/ath/if_ath_misc.h>
#include <dev/ath/if_ath_tsf.h>
#include <dev/ath/if_ath_tx.h>
#include <dev/ath/if_ath_sysctl.h>
#include <dev/ath/if_ath_led.h>
#include <dev/ath/if_ath_keycache.h>
#include <dev/ath/if_ath_rx.h>
#include <dev/ath/if_ath_rx_edma.h>
#include <dev/ath/if_ath_tx_edma.h>
#include <dev/ath/if_ath_beacon.h>
#include <dev/ath/if_ath_btcoex.h>
#include <dev/ath/if_ath_spectral.h>
#include <dev/ath/if_ath_lna_div.h>
#include <dev/ath/if_athdfs.h>
#ifdef ATH_TX99_DIAG
#include <dev/ath/ath_tx99/ath_tx99.h>
#endif
#ifdef ATH_DEBUG_ALQ
#include <dev/ath/if_ath_alq.h>
#endif
/*
* Only enable this if you're working on PS-POLL support.
*/
#define ATH_SW_PSQ
/*
* ATH_BCBUF determines the number of vap's that can transmit
* beacons and also (currently) the number of vap's that can
* have unique mac addresses/bssid. When staggering beacons
* 4 is probably a good max as otherwise the beacons become
* very closely spaced and there is limited time for cab q traffic
* to go out. You can burst beacons instead but that is not good
* for stations in power save and at some point you really want
* another radio (and channel).
*
* The limit on the number of mac addresses is tied to our use of
* the U/L bit and tracking addresses in a byte; it would be
* worthwhile to allow more for applications like proxy sta.
*/
CTASSERT(ATH_BCBUF <= 8);
static struct ieee80211vap *ath_vap_create(struct ieee80211com *,
const char [IFNAMSIZ], int, enum ieee80211_opmode, int,
const uint8_t [IEEE80211_ADDR_LEN],
const uint8_t [IEEE80211_ADDR_LEN]);
static void ath_vap_delete(struct ieee80211vap *);
static void ath_init(void *);
static void ath_stop_locked(struct ifnet *);
static void ath_stop(struct ifnet *);
static int ath_reset_vap(struct ieee80211vap *, u_long);
static int ath_transmit(struct ifnet *ifp, struct mbuf *m);
static void ath_qflush(struct ifnet *ifp);
static int ath_media_change(struct ifnet *);
static void ath_watchdog(void *);
static int ath_ioctl(struct ifnet *, u_long, caddr_t);
static void ath_fatal_proc(void *, int);
static void ath_bmiss_vap(struct ieee80211vap *);
static void ath_bmiss_proc(void *, int);
static void ath_key_update_begin(struct ieee80211vap *);
static void ath_key_update_end(struct ieee80211vap *);
static void ath_update_mcast_hw(struct ath_softc *);
static void ath_update_mcast(struct ifnet *);
static void ath_update_promisc(struct ifnet *);
static void ath_updateslot(struct ifnet *);
static void ath_bstuck_proc(void *, int);
static void ath_reset_proc(void *, int);
static int ath_desc_alloc(struct ath_softc *);
static void ath_desc_free(struct ath_softc *);
static struct ieee80211_node *ath_node_alloc(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN]);
static void ath_node_cleanup(struct ieee80211_node *);
static void ath_node_free(struct ieee80211_node *);
static void ath_node_getsignal(const struct ieee80211_node *,
int8_t *, int8_t *);
static void ath_txq_init(struct ath_softc *sc, struct ath_txq *, int);
static struct ath_txq *ath_txq_setup(struct ath_softc*, int qtype, int subtype);
static int ath_tx_setup(struct ath_softc *, int, int);
static void ath_tx_cleanupq(struct ath_softc *, struct ath_txq *);
static void ath_tx_cleanup(struct ath_softc *);
static int ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq,
int dosched);
static void ath_tx_proc_q0(void *, int);
static void ath_tx_proc_q0123(void *, int);
static void ath_tx_proc(void *, int);
static void ath_txq_sched_tasklet(void *, int);
static int ath_chan_set(struct ath_softc *, struct ieee80211_channel *);
static void ath_chan_change(struct ath_softc *, struct ieee80211_channel *);
static void ath_scan_start(struct ieee80211com *);
static void ath_scan_end(struct ieee80211com *);
static void ath_set_channel(struct ieee80211com *);
#ifdef ATH_ENABLE_11N
static void ath_update_chw(struct ieee80211com *);
#endif /* ATH_ENABLE_11N */
static void ath_calibrate(void *);
static int ath_newstate(struct ieee80211vap *, enum ieee80211_state, int);
static void ath_setup_stationkey(struct ieee80211_node *);
static void ath_newassoc(struct ieee80211_node *, int);
static int ath_setregdomain(struct ieee80211com *,
struct ieee80211_regdomain *, int,
struct ieee80211_channel []);
static void ath_getradiocaps(struct ieee80211com *, int, int *,
struct ieee80211_channel []);
static int ath_getchannels(struct ath_softc *);
static int ath_rate_setup(struct ath_softc *, u_int mode);
static void ath_setcurmode(struct ath_softc *, enum ieee80211_phymode);
static void ath_announce(struct ath_softc *);
static void ath_dfs_tasklet(void *, int);
static void ath_node_powersave(struct ieee80211_node *, int);
static int ath_node_set_tim(struct ieee80211_node *, int);
static void ath_node_recv_pspoll(struct ieee80211_node *, struct mbuf *);
#ifdef IEEE80211_SUPPORT_TDMA
#include <dev/ath/if_ath_tdma.h>
#endif
SYSCTL_DECL(_hw_ath);
/* XXX validate sysctl values */
static int ath_longcalinterval = 30; /* long cals every 30 secs */
SYSCTL_INT(_hw_ath, OID_AUTO, longcal, CTLFLAG_RW, &ath_longcalinterval,
0, "long chip calibration interval (secs)");
static int ath_shortcalinterval = 100; /* short cals every 100 ms */
SYSCTL_INT(_hw_ath, OID_AUTO, shortcal, CTLFLAG_RW, &ath_shortcalinterval,
0, "short chip calibration interval (msecs)");
static int ath_resetcalinterval = 20*60; /* reset cal state 20 mins */
SYSCTL_INT(_hw_ath, OID_AUTO, resetcal, CTLFLAG_RW, &ath_resetcalinterval,
0, "reset chip calibration results (secs)");
static int ath_anicalinterval = 100; /* ANI calibration - 100 msec */
SYSCTL_INT(_hw_ath, OID_AUTO, anical, CTLFLAG_RW, &ath_anicalinterval,
0, "ANI calibration (msecs)");
int ath_rxbuf = ATH_RXBUF; /* # rx buffers to allocate */
SYSCTL_INT(_hw_ath, OID_AUTO, rxbuf, CTLFLAG_RWTUN, &ath_rxbuf,
0, "rx buffers allocated");
int ath_txbuf = ATH_TXBUF; /* # tx buffers to allocate */
SYSCTL_INT(_hw_ath, OID_AUTO, txbuf, CTLFLAG_RWTUN, &ath_txbuf,
0, "tx buffers allocated");
int ath_txbuf_mgmt = ATH_MGMT_TXBUF; /* # mgmt tx buffers to allocate */
SYSCTL_INT(_hw_ath, OID_AUTO, txbuf_mgmt, CTLFLAG_RWTUN, &ath_txbuf_mgmt,
0, "tx (mgmt) buffers allocated");
int ath_bstuck_threshold = 4; /* max missed beacons */
SYSCTL_INT(_hw_ath, OID_AUTO, bstuck, CTLFLAG_RW, &ath_bstuck_threshold,
0, "max missed beacon xmits before chip reset");
MALLOC_DEFINE(M_ATHDEV, "athdev", "ath driver dma buffers");
void
ath_legacy_attach_comp_func(struct ath_softc *sc)
{
/*
* Special case certain configurations. Note the
* CAB queue is handled by these specially so don't
* include them when checking the txq setup mask.
*/
switch (sc->sc_txqsetup &~ (1<<sc->sc_cabq->axq_qnum)) {
case 0x01:
TASK_INIT(&sc->sc_txtask, 0, ath_tx_proc_q0, sc);
break;
case 0x0f:
TASK_INIT(&sc->sc_txtask, 0, ath_tx_proc_q0123, sc);
break;
default:
TASK_INIT(&sc->sc_txtask, 0, ath_tx_proc, sc);
break;
}
}
/*
* Set the target power mode.
*
* If this is called during a point in time where
* the hardware is being programmed elsewhere, it will
* simply store it away and update it when all current
* uses of the hardware are completed.
*/
void
_ath_power_setpower(struct ath_softc *sc, int power_state, const char *file, int line)
{
ATH_LOCK_ASSERT(sc);
sc->sc_target_powerstate = power_state;
DPRINTF(sc, ATH_DEBUG_PWRSAVE, "%s: (%s:%d) state=%d, refcnt=%d\n",
__func__,
file,
line,
power_state,
sc->sc_powersave_refcnt);
if (sc->sc_powersave_refcnt == 0 &&
power_state != sc->sc_cur_powerstate) {
sc->sc_cur_powerstate = power_state;
ath_hal_setpower(sc->sc_ah, power_state);
/*
* If the NIC is force-awake, then set the
* self-gen frame state appropriately.
*
* If the nic is in network sleep or full-sleep,
* we let the above call leave the self-gen
* state as "sleep".
*/
if (sc->sc_cur_powerstate == HAL_PM_AWAKE &&
sc->sc_target_selfgen_state != HAL_PM_AWAKE) {
ath_hal_setselfgenpower(sc->sc_ah,
sc->sc_target_selfgen_state);
}
}
}
/*
* Set the current self-generated frames state.
*
* This is separate from the target power mode. The chip may be
* awake but the desired state is "sleep", so frames sent to the
* destination has PWRMGT=1 in the 802.11 header. The NIC also
* needs to know to set PWRMGT=1 in self-generated frames.
*/
void
_ath_power_set_selfgen(struct ath_softc *sc, int power_state, const char *file, int line)
{
ATH_LOCK_ASSERT(sc);
DPRINTF(sc, ATH_DEBUG_PWRSAVE, "%s: (%s:%d) state=%d, refcnt=%d\n",
__func__,
file,
line,
power_state,
sc->sc_target_selfgen_state);
sc->sc_target_selfgen_state = power_state;
/*
* If the NIC is force-awake, then set the power state.
* Network-state and full-sleep will already transition it to
* mark self-gen frames as sleeping - and we can't
* guarantee the NIC is awake to program the self-gen frame
* setting anyway.
*/
if (sc->sc_cur_powerstate == HAL_PM_AWAKE) {
ath_hal_setselfgenpower(sc->sc_ah, power_state);
}
}
/*
* Set the hardware power mode and take a reference.
*
* This doesn't update the target power mode in the driver;
* it just updates the hardware power state.
*
* XXX it should only ever force the hardware awake; it should
* never be called to set it asleep.
*/
void
_ath_power_set_power_state(struct ath_softc *sc, int power_state, const char *file, int line)
{
ATH_LOCK_ASSERT(sc);
DPRINTF(sc, ATH_DEBUG_PWRSAVE, "%s: (%s:%d) state=%d, refcnt=%d\n",
__func__,
file,
line,
power_state,
sc->sc_powersave_refcnt);
sc->sc_powersave_refcnt++;
if (power_state != sc->sc_cur_powerstate) {
ath_hal_setpower(sc->sc_ah, power_state);
sc->sc_cur_powerstate = power_state;
/*
* Adjust the self-gen powerstate if appropriate.
*/
if (sc->sc_cur_powerstate == HAL_PM_AWAKE &&
sc->sc_target_selfgen_state != HAL_PM_AWAKE) {
ath_hal_setselfgenpower(sc->sc_ah,
sc->sc_target_selfgen_state);
}
}
}
/*
* Restore the power save mode to what it once was.
*
* This will decrement the reference counter and once it hits
* zero, it'll restore the powersave state.
*/
void
_ath_power_restore_power_state(struct ath_softc *sc, const char *file, int line)
{
ATH_LOCK_ASSERT(sc);
DPRINTF(sc, ATH_DEBUG_PWRSAVE, "%s: (%s:%d) refcnt=%d, target state=%d\n",
__func__,
file,
line,
sc->sc_powersave_refcnt,
sc->sc_target_powerstate);
if (sc->sc_powersave_refcnt == 0)
device_printf(sc->sc_dev, "%s: refcnt=0?\n", __func__);
else
sc->sc_powersave_refcnt--;
if (sc->sc_powersave_refcnt == 0 &&
sc->sc_target_powerstate != sc->sc_cur_powerstate) {
sc->sc_cur_powerstate = sc->sc_target_powerstate;
ath_hal_setpower(sc->sc_ah, sc->sc_target_powerstate);
}
/*
* Adjust the self-gen powerstate if appropriate.
*/
if (sc->sc_cur_powerstate == HAL_PM_AWAKE &&
sc->sc_target_selfgen_state != HAL_PM_AWAKE) {
ath_hal_setselfgenpower(sc->sc_ah,
sc->sc_target_selfgen_state);
}
}
/*
* Configure the initial HAL configuration values based on bus
* specific parameters.
*
* Some PCI IDs and other information may need tweaking.
*
* XXX TODO: ath9k and the Atheros HAL only program comm2g_switch_enable
* if BT antenna diversity isn't enabled.
*
* So, let's also figure out how to enable BT diversity for AR9485.
*/
static void
ath_setup_hal_config(struct ath_softc *sc, HAL_OPS_CONFIG *ah_config)
{
/* XXX TODO: only for PCI devices? */
if (sc->sc_pci_devinfo & (ATH_PCI_CUS198 | ATH_PCI_CUS230)) {
ah_config->ath_hal_ext_lna_ctl_gpio = 0x200; /* bit 9 */
ah_config->ath_hal_ext_atten_margin_cfg = AH_TRUE;
ah_config->ath_hal_min_gainidx = AH_TRUE;
ah_config->ath_hal_ant_ctrl_comm2g_switch_enable = 0x000bbb88;
/* XXX low_rssi_thresh */
/* XXX fast_div_bias */
device_printf(sc->sc_dev, "configuring for %s\n",
(sc->sc_pci_devinfo & ATH_PCI_CUS198) ?
"CUS198" : "CUS230");
}
if (sc->sc_pci_devinfo & ATH_PCI_CUS217)
device_printf(sc->sc_dev, "CUS217 card detected\n");
if (sc->sc_pci_devinfo & ATH_PCI_CUS252)
device_printf(sc->sc_dev, "CUS252 card detected\n");
if (sc->sc_pci_devinfo & ATH_PCI_AR9565_1ANT)
device_printf(sc->sc_dev, "WB335 1-ANT card detected\n");
if (sc->sc_pci_devinfo & ATH_PCI_AR9565_2ANT)
device_printf(sc->sc_dev, "WB335 2-ANT card detected\n");
if (sc->sc_pci_devinfo & ATH_PCI_KILLER)
device_printf(sc->sc_dev, "Killer Wireless card detected\n");
#if 0
/*
* Some WB335 cards do not support antenna diversity. Since
* we use a hardcoded value for AR9565 instead of using the
* EEPROM/OTP data, remove the combining feature from
* the HW capabilities bitmap.
*/
if (sc->sc_pci_devinfo & (ATH9K_PCI_AR9565_1ANT | ATH9K_PCI_AR9565_2ANT)) {
if (!(sc->sc_pci_devinfo & ATH9K_PCI_BT_ANT_DIV))
pCap->hw_caps &= ~ATH9K_HW_CAP_ANT_DIV_COMB;
}
if (sc->sc_pci_devinfo & ATH9K_PCI_BT_ANT_DIV) {
pCap->hw_caps |= ATH9K_HW_CAP_BT_ANT_DIV;
device_printf(sc->sc_dev, "Set BT/WLAN RX diversity capability\n");
}
#endif
if (sc->sc_pci_devinfo & ATH_PCI_D3_L1_WAR) {
ah_config->ath_hal_pcie_waen = 0x0040473b;
device_printf(sc->sc_dev, "Enable WAR for ASPM D3/L1\n");
}
#if 0
if (sc->sc_pci_devinfo & ATH9K_PCI_NO_PLL_PWRSAVE) {
ah->config.no_pll_pwrsave = true;
device_printf(sc->sc_dev, "Disable PLL PowerSave\n");
}
#endif
}
/*
* Attempt to fetch the MAC address from the kernel environment.
*
* Returns 0, macaddr in macaddr if successful; -1 otherwise.
*/
static int
ath_fetch_mac_kenv(struct ath_softc *sc, uint8_t *macaddr)
{
char devid_str[32];
int local_mac = 0;
char *local_macstr;
/*
* Fetch from the kenv rather than using hints.
*
* Hints would be nice but the transition to dynamic
* hints/kenv doesn't happen early enough for this
* to work reliably (eg on anything embedded.)
*/
snprintf(devid_str, 32, "hint.%s.%d.macaddr",
device_get_name(sc->sc_dev),
device_get_unit(sc->sc_dev));
if ((local_macstr = kern_getenv(devid_str)) != NULL) {
uint32_t tmpmac[ETHER_ADDR_LEN];
int count;
int i;
/* Have a MAC address; should use it */
device_printf(sc->sc_dev,
"Overriding MAC address from environment: '%s'\n",
local_macstr);
/* Extract out the MAC address */
count = sscanf(local_macstr, "%x%*c%x%*c%x%*c%x%*c%x%*c%x",
&tmpmac[0], &tmpmac[1],
&tmpmac[2], &tmpmac[3],
&tmpmac[4], &tmpmac[5]);
if (count == 6) {
/* Valid! */
local_mac = 1;
for (i = 0; i < ETHER_ADDR_LEN; i++)
macaddr[i] = tmpmac[i];
}
/* Done! */
freeenv(local_macstr);
local_macstr = NULL;
}
if (local_mac)
return (0);
return (-1);
}
#define HAL_MODE_HT20 (HAL_MODE_11NG_HT20 | HAL_MODE_11NA_HT20)
#define HAL_MODE_HT40 \
(HAL_MODE_11NG_HT40PLUS | HAL_MODE_11NG_HT40MINUS | \
HAL_MODE_11NA_HT40PLUS | HAL_MODE_11NA_HT40MINUS)
int
ath_attach(u_int16_t devid, struct ath_softc *sc)
{
struct ifnet *ifp;
struct ieee80211com *ic;
struct ath_hal *ah = NULL;
HAL_STATUS status;
int error = 0, i;
u_int wmodes;
uint8_t macaddr[IEEE80211_ADDR_LEN];
int rx_chainmask, tx_chainmask;
HAL_OPS_CONFIG ah_config;
DPRINTF(sc, ATH_DEBUG_ANY, "%s: devid 0x%x\n", __func__, devid);
CURVNET_SET(vnet0);
ifp = sc->sc_ifp = if_alloc(IFT_IEEE80211);
if (ifp == NULL) {
device_printf(sc->sc_dev, "can not if_alloc()\n");
error = ENOSPC;
CURVNET_RESTORE();
goto bad;
}
ic = ifp->if_l2com;
/* set these up early for if_printf use */
if_initname(ifp, device_get_name(sc->sc_dev),
device_get_unit(sc->sc_dev));
CURVNET_RESTORE();
/*
* Configure the initial configuration data.
*
* This is stuff that may be needed early during attach
* rather than done via configuration calls later.
*/
bzero(&ah_config, sizeof(ah_config));
ath_setup_hal_config(sc, &ah_config);
ah = ath_hal_attach(devid, sc, sc->sc_st, sc->sc_sh,
sc->sc_eepromdata, &ah_config, &status);
if (ah == NULL) {
if_printf(ifp, "unable to attach hardware; HAL status %u\n",
status);
error = ENXIO;
goto bad;
}
sc->sc_ah = ah;
sc->sc_invalid = 0; /* ready to go, enable interrupt handling */
#ifdef ATH_DEBUG
sc->sc_debug = ath_debug;
#endif
/*
* Setup the DMA/EDMA functions based on the current
* hardware support.
*
* This is required before the descriptors are allocated.
*/
if (ath_hal_hasedma(sc->sc_ah)) {
sc->sc_isedma = 1;
ath_recv_setup_edma(sc);
ath_xmit_setup_edma(sc);
} else {
ath_recv_setup_legacy(sc);
ath_xmit_setup_legacy(sc);
}
if (ath_hal_hasmybeacon(sc->sc_ah)) {
sc->sc_do_mybeacon = 1;
}
/*
* Check if the MAC has multi-rate retry support.
* We do this by trying to setup a fake extended
* descriptor. MAC's that don't have support will
* return false w/o doing anything. MAC's that do
* support it will return true w/o doing anything.
*/
sc->sc_mrretry = ath_hal_setupxtxdesc(ah, NULL, 0,0, 0,0, 0,0);
/*
* Check if the device has hardware counters for PHY
* errors. If so we need to enable the MIB interrupt
* so we can act on stat triggers.
*/
if (ath_hal_hwphycounters(ah))
sc->sc_needmib = 1;
/*
* Get the hardware key cache size.
*/
sc->sc_keymax = ath_hal_keycachesize(ah);
if (sc->sc_keymax > ATH_KEYMAX) {
if_printf(ifp, "Warning, using only %u of %u key cache slots\n",
ATH_KEYMAX, sc->sc_keymax);
sc->sc_keymax = ATH_KEYMAX;
}
/*
* Reset the key cache since some parts do not
* reset the contents on initial power up.
*/
for (i = 0; i < sc->sc_keymax; i++)
ath_hal_keyreset(ah, i);
/*
* Collect the default channel list.
*/
error = ath_getchannels(sc);
if (error != 0)
goto bad;
/*
* Setup rate tables for all potential media types.
*/
ath_rate_setup(sc, IEEE80211_MODE_11A);
ath_rate_setup(sc, IEEE80211_MODE_11B);
ath_rate_setup(sc, IEEE80211_MODE_11G);
ath_rate_setup(sc, IEEE80211_MODE_TURBO_A);
ath_rate_setup(sc, IEEE80211_MODE_TURBO_G);
ath_rate_setup(sc, IEEE80211_MODE_STURBO_A);
ath_rate_setup(sc, IEEE80211_MODE_11NA);
ath_rate_setup(sc, IEEE80211_MODE_11NG);
ath_rate_setup(sc, IEEE80211_MODE_HALF);
ath_rate_setup(sc, IEEE80211_MODE_QUARTER);
/* NB: setup here so ath_rate_update is happy */
ath_setcurmode(sc, IEEE80211_MODE_11A);
/*
* Allocate TX descriptors and populate the lists.
*/
error = ath_desc_alloc(sc);
if (error != 0) {
if_printf(ifp, "failed to allocate TX descriptors: %d\n",
error);
goto bad;
}
error = ath_txdma_setup(sc);
if (error != 0) {
if_printf(ifp, "failed to allocate TX descriptors: %d\n",
error);
goto bad;
}
/*
* Allocate RX descriptors and populate the lists.
*/
error = ath_rxdma_setup(sc);
if (error != 0) {
if_printf(ifp, "failed to allocate RX descriptors: %d\n",
error);
goto bad;
}
callout_init_mtx(&sc->sc_cal_ch, &sc->sc_mtx, 0);
callout_init_mtx(&sc->sc_wd_ch, &sc->sc_mtx, 0);
ATH_TXBUF_LOCK_INIT(sc);
sc->sc_tq = taskqueue_create("ath_taskq", M_NOWAIT,
taskqueue_thread_enqueue, &sc->sc_tq);
taskqueue_start_threads(&sc->sc_tq, 1, PI_NET,
"%s taskq", ifp->if_xname);
TASK_INIT(&sc->sc_rxtask, 0, sc->sc_rx.recv_tasklet, sc);
TASK_INIT(&sc->sc_bmisstask, 0, ath_bmiss_proc, sc);
TASK_INIT(&sc->sc_bstucktask,0, ath_bstuck_proc, sc);
TASK_INIT(&sc->sc_resettask,0, ath_reset_proc, sc);
TASK_INIT(&sc->sc_txqtask, 0, ath_txq_sched_tasklet, sc);
TASK_INIT(&sc->sc_fataltask, 0, ath_fatal_proc, sc);
/*
* Allocate hardware transmit queues: one queue for
* beacon frames and one data queue for each QoS
* priority. Note that the hal handles resetting
* these queues at the needed time.
*
* XXX PS-Poll
*/
sc->sc_bhalq = ath_beaconq_setup(sc);
if (sc->sc_bhalq == (u_int) -1) {
if_printf(ifp, "unable to setup a beacon xmit queue!\n");
error = EIO;
goto bad2;
}
sc->sc_cabq = ath_txq_setup(sc, HAL_TX_QUEUE_CAB, 0);
if (sc->sc_cabq == NULL) {
if_printf(ifp, "unable to setup CAB xmit queue!\n");
error = EIO;
goto bad2;
}
/* NB: insure BK queue is the lowest priority h/w queue */
if (!ath_tx_setup(sc, WME_AC_BK, HAL_WME_AC_BK)) {
if_printf(ifp, "unable to setup xmit queue for %s traffic!\n",
ieee80211_wme_acnames[WME_AC_BK]);
error = EIO;
goto bad2;
}
if (!ath_tx_setup(sc, WME_AC_BE, HAL_WME_AC_BE) ||
!ath_tx_setup(sc, WME_AC_VI, HAL_WME_AC_VI) ||
!ath_tx_setup(sc, WME_AC_VO, HAL_WME_AC_VO)) {
/*
* Not enough hardware tx queues to properly do WME;
* just punt and assign them all to the same h/w queue.
* We could do a better job of this if, for example,
* we allocate queues when we switch from station to
* AP mode.
*/
if (sc->sc_ac2q[WME_AC_VI] != NULL)
ath_tx_cleanupq(sc, sc->sc_ac2q[WME_AC_VI]);
if (sc->sc_ac2q[WME_AC_BE] != NULL)
ath_tx_cleanupq(sc, sc->sc_ac2q[WME_AC_BE]);
sc->sc_ac2q[WME_AC_BE] = sc->sc_ac2q[WME_AC_BK];
sc->sc_ac2q[WME_AC_VI] = sc->sc_ac2q[WME_AC_BK];
sc->sc_ac2q[WME_AC_VO] = sc->sc_ac2q[WME_AC_BK];
}
/*
* Attach the TX completion function.
*
* The non-EDMA chips may have some special case optimisations;
* this method gives everyone a chance to attach cleanly.
*/
sc->sc_tx.xmit_attach_comp_func(sc);
/*
* Setup rate control. Some rate control modules
* call back to change the anntena state so expose
* the necessary entry points.
* XXX maybe belongs in struct ath_ratectrl?
*/
sc->sc_setdefantenna = ath_setdefantenna;
sc->sc_rc = ath_rate_attach(sc);
if (sc->sc_rc == NULL) {
error = EIO;
goto bad2;
}
/* Attach DFS module */
if (! ath_dfs_attach(sc)) {
device_printf(sc->sc_dev,
"%s: unable to attach DFS\n", __func__);
error = EIO;
goto bad2;
}
/* Attach spectral module */
if (ath_spectral_attach(sc) < 0) {
device_printf(sc->sc_dev,
"%s: unable to attach spectral\n", __func__);
error = EIO;
goto bad2;
}
/* Attach bluetooth coexistence module */
if (ath_btcoex_attach(sc) < 0) {
device_printf(sc->sc_dev,
"%s: unable to attach bluetooth coexistence\n", __func__);
error = EIO;
goto bad2;
}
/* Attach LNA diversity module */
if (ath_lna_div_attach(sc) < 0) {
device_printf(sc->sc_dev,
"%s: unable to attach LNA diversity\n", __func__);
error = EIO;
goto bad2;
}
/* Start DFS processing tasklet */
TASK_INIT(&sc->sc_dfstask, 0, ath_dfs_tasklet, sc);
/* Configure LED state */
sc->sc_blinking = 0;
sc->sc_ledstate = 1;
sc->sc_ledon = 0; /* low true */
sc->sc_ledidle = (2700*hz)/1000; /* 2.7sec */
- callout_init(&sc->sc_ledtimer, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_ledtimer, 1);
/*
* Don't setup hardware-based blinking.
*
* Although some NICs may have this configured in the
* default reset register values, the user may wish
* to alter which pins have which function.
*
* The reference driver attaches the MAC network LED to GPIO1 and
* the MAC power LED to GPIO2. However, the DWA-552 cardbus
* NIC has these reversed.
*/
sc->sc_hardled = (1 == 0);
sc->sc_led_net_pin = -1;
sc->sc_led_pwr_pin = -1;
/*
* Auto-enable soft led processing for IBM cards and for
* 5211 minipci cards. Users can also manually enable/disable
* support with a sysctl.
*/
sc->sc_softled = (devid == AR5212_DEVID_IBM || devid == AR5211_DEVID);
ath_led_config(sc);
ath_hal_setledstate(ah, HAL_LED_INIT);
ifp->if_softc = sc;
ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST;
ifp->if_transmit = ath_transmit;
ifp->if_qflush = ath_qflush;
ifp->if_ioctl = ath_ioctl;
ifp->if_init = ath_init;
IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
IFQ_SET_READY(&ifp->if_snd);
ic->ic_ifp = ifp;
/* XXX not right but it's not used anywhere important */
ic->ic_phytype = IEEE80211_T_OFDM;
ic->ic_opmode = IEEE80211_M_STA;
ic->ic_caps =
IEEE80211_C_STA /* station mode */
| IEEE80211_C_IBSS /* ibss, nee adhoc, mode */
| IEEE80211_C_HOSTAP /* hostap mode */
| IEEE80211_C_MONITOR /* monitor mode */
| IEEE80211_C_AHDEMO /* adhoc demo mode */
| IEEE80211_C_WDS /* 4-address traffic works */
| IEEE80211_C_MBSS /* mesh point link mode */
| IEEE80211_C_SHPREAMBLE /* short preamble supported */
| IEEE80211_C_SHSLOT /* short slot time supported */
| IEEE80211_C_WPA /* capable of WPA1+WPA2 */
#ifndef ATH_ENABLE_11N
| IEEE80211_C_BGSCAN /* capable of bg scanning */
#endif
| IEEE80211_C_TXFRAG /* handle tx frags */
#ifdef ATH_ENABLE_DFS
| IEEE80211_C_DFS /* Enable radar detection */
#endif
| IEEE80211_C_PMGT /* Station side power mgmt */
| IEEE80211_C_SWSLEEP
;
/*
* Query the hal to figure out h/w crypto support.
*/
if (ath_hal_ciphersupported(ah, HAL_CIPHER_WEP))
ic->ic_cryptocaps |= IEEE80211_CRYPTO_WEP;
if (ath_hal_ciphersupported(ah, HAL_CIPHER_AES_OCB))
ic->ic_cryptocaps |= IEEE80211_CRYPTO_AES_OCB;
if (ath_hal_ciphersupported(ah, HAL_CIPHER_AES_CCM))
ic->ic_cryptocaps |= IEEE80211_CRYPTO_AES_CCM;
if (ath_hal_ciphersupported(ah, HAL_CIPHER_CKIP))
ic->ic_cryptocaps |= IEEE80211_CRYPTO_CKIP;
if (ath_hal_ciphersupported(ah, HAL_CIPHER_TKIP)) {
ic->ic_cryptocaps |= IEEE80211_CRYPTO_TKIP;
/*
* Check if h/w does the MIC and/or whether the
* separate key cache entries are required to
* handle both tx+rx MIC keys.
*/
if (ath_hal_ciphersupported(ah, HAL_CIPHER_MIC))
ic->ic_cryptocaps |= IEEE80211_CRYPTO_TKIPMIC;
/*
* If the h/w supports storing tx+rx MIC keys
* in one cache slot automatically enable use.
*/
if (ath_hal_hastkipsplit(ah) ||
!ath_hal_settkipsplit(ah, AH_FALSE))
sc->sc_splitmic = 1;
/*
* If the h/w can do TKIP MIC together with WME then
* we use it; otherwise we force the MIC to be done
* in software by the net80211 layer.
*/
if (ath_hal_haswmetkipmic(ah))
sc->sc_wmetkipmic = 1;
}
sc->sc_hasclrkey = ath_hal_ciphersupported(ah, HAL_CIPHER_CLR);
/*
* Check for multicast key search support.
*/
if (ath_hal_hasmcastkeysearch(sc->sc_ah) &&
!ath_hal_getmcastkeysearch(sc->sc_ah)) {
ath_hal_setmcastkeysearch(sc->sc_ah, 1);
}
sc->sc_mcastkey = ath_hal_getmcastkeysearch(ah);
/*
* Mark key cache slots associated with global keys
* as in use. If we knew TKIP was not to be used we
* could leave the +32, +64, and +32+64 slots free.
*/
for (i = 0; i < IEEE80211_WEP_NKID; i++) {
setbit(sc->sc_keymap, i);
setbit(sc->sc_keymap, i+64);
if (sc->sc_splitmic) {
setbit(sc->sc_keymap, i+32);
setbit(sc->sc_keymap, i+32+64);
}
}
/*
* TPC support can be done either with a global cap or
* per-packet support. The latter is not available on
* all parts. We're a bit pedantic here as all parts
* support a global cap.
*/
if (ath_hal_hastpc(ah) || ath_hal_hastxpowlimit(ah))
ic->ic_caps |= IEEE80211_C_TXPMGT;
/*
* Mark WME capability only if we have sufficient
* hardware queues to do proper priority scheduling.
*/
if (sc->sc_ac2q[WME_AC_BE] != sc->sc_ac2q[WME_AC_BK])
ic->ic_caps |= IEEE80211_C_WME;
/*
* Check for misc other capabilities.
*/
if (ath_hal_hasbursting(ah))
ic->ic_caps |= IEEE80211_C_BURST;
sc->sc_hasbmask = ath_hal_hasbssidmask(ah);
sc->sc_hasbmatch = ath_hal_hasbssidmatch(ah);
sc->sc_hastsfadd = ath_hal_hastsfadjust(ah);
sc->sc_rxslink = ath_hal_self_linked_final_rxdesc(ah);
sc->sc_rxtsf32 = ath_hal_has_long_rxdesc_tsf(ah);
sc->sc_hasenforcetxop = ath_hal_hasenforcetxop(ah);
sc->sc_rx_lnamixer = ath_hal_hasrxlnamixer(ah);
sc->sc_hasdivcomb = ath_hal_hasdivantcomb(ah);
if (ath_hal_hasfastframes(ah))
ic->ic_caps |= IEEE80211_C_FF;
wmodes = ath_hal_getwirelessmodes(ah);
if (wmodes & (HAL_MODE_108G|HAL_MODE_TURBO))
ic->ic_caps |= IEEE80211_C_TURBOP;
#ifdef IEEE80211_SUPPORT_TDMA
if (ath_hal_macversion(ah) > 0x78) {
ic->ic_caps |= IEEE80211_C_TDMA; /* capable of TDMA */
ic->ic_tdma_update = ath_tdma_update;
}
#endif
/*
* TODO: enforce that at least this many frames are available
* in the txbuf list before allowing data frames (raw or
* otherwise) to be transmitted.
*/
sc->sc_txq_data_minfree = 10;
/*
* Leave this as default to maintain legacy behaviour.
* Shortening the cabq/mcastq may end up causing some
* undesirable behaviour.
*/
sc->sc_txq_mcastq_maxdepth = ath_txbuf;
/*
* How deep can the node software TX queue get whilst it's asleep.
*/
sc->sc_txq_node_psq_maxdepth = 16;
/*
* Default the maximum queue depth for a given node
* to 1/4'th the TX buffers, or 64, whichever
* is larger.
*/
sc->sc_txq_node_maxdepth = MAX(64, ath_txbuf / 4);
/* Enable CABQ by default */
sc->sc_cabq_enable = 1;
/*
* Allow the TX and RX chainmasks to be overridden by
* environment variables and/or device.hints.
*
* This must be done early - before the hardware is
* calibrated or before the 802.11n stream calculation
* is done.
*/
if (resource_int_value(device_get_name(sc->sc_dev),
device_get_unit(sc->sc_dev), "rx_chainmask",
&rx_chainmask) == 0) {
device_printf(sc->sc_dev, "Setting RX chainmask to 0x%x\n",
rx_chainmask);
(void) ath_hal_setrxchainmask(sc->sc_ah, rx_chainmask);
}
if (resource_int_value(device_get_name(sc->sc_dev),
device_get_unit(sc->sc_dev), "tx_chainmask",
&tx_chainmask) == 0) {
device_printf(sc->sc_dev, "Setting TX chainmask to 0x%x\n",
tx_chainmask);
(void) ath_hal_settxchainmask(sc->sc_ah, tx_chainmask);
}
/*
* Query the TX/RX chainmask configuration.
*
* This is only relevant for 11n devices.
*/
ath_hal_getrxchainmask(ah, &sc->sc_rxchainmask);
ath_hal_gettxchainmask(ah, &sc->sc_txchainmask);
/*
* Disable MRR with protected frames by default.
* Only 802.11n series NICs can handle this.
*/
sc->sc_mrrprot = 0; /* XXX should be a capability */
/*
* Query the enterprise mode information the HAL.
*/
if (ath_hal_getcapability(ah, HAL_CAP_ENTERPRISE_MODE, 0,
&sc->sc_ent_cfg) == HAL_OK)
sc->sc_use_ent = 1;
#ifdef ATH_ENABLE_11N
/*
* Query HT capabilities
*/
if (ath_hal_getcapability(ah, HAL_CAP_HT, 0, NULL) == HAL_OK &&
(wmodes & (HAL_MODE_HT20 | HAL_MODE_HT40))) {
uint32_t rxs, txs;
device_printf(sc->sc_dev, "[HT] enabling HT modes\n");
sc->sc_mrrprot = 1; /* XXX should be a capability */
ic->ic_htcaps = IEEE80211_HTC_HT /* HT operation */
| IEEE80211_HTC_AMPDU /* A-MPDU tx/rx */
| IEEE80211_HTC_AMSDU /* A-MSDU tx/rx */
| IEEE80211_HTCAP_MAXAMSDU_3839
/* max A-MSDU length */
| IEEE80211_HTCAP_SMPS_OFF; /* SM power save off */
;
/*
* Enable short-GI for HT20 only if the hardware
* advertises support.
* Notably, anything earlier than the AR9287 doesn't.
*/
if ((ath_hal_getcapability(ah,
HAL_CAP_HT20_SGI, 0, NULL) == HAL_OK) &&
(wmodes & HAL_MODE_HT20)) {
device_printf(sc->sc_dev,
"[HT] enabling short-GI in 20MHz mode\n");
ic->ic_htcaps |= IEEE80211_HTCAP_SHORTGI20;
}
if (wmodes & HAL_MODE_HT40)
ic->ic_htcaps |= IEEE80211_HTCAP_CHWIDTH40
| IEEE80211_HTCAP_SHORTGI40;
/*
* TX/RX streams need to be taken into account when
* negotiating which MCS rates it'll receive and
* what MCS rates are available for TX.
*/
(void) ath_hal_getcapability(ah, HAL_CAP_STREAMS, 0, &txs);
(void) ath_hal_getcapability(ah, HAL_CAP_STREAMS, 1, &rxs);
ic->ic_txstream = txs;
ic->ic_rxstream = rxs;
/*
* Setup TX and RX STBC based on what the HAL allows and
* the currently configured chainmask set.
* Ie - don't enable STBC TX if only one chain is enabled.
* STBC RX is fine on a single RX chain; it just won't
* provide any real benefit.
*/
if (ath_hal_getcapability(ah, HAL_CAP_RX_STBC, 0,
NULL) == HAL_OK) {
sc->sc_rx_stbc = 1;
device_printf(sc->sc_dev,
"[HT] 1 stream STBC receive enabled\n");
ic->ic_htcaps |= IEEE80211_HTCAP_RXSTBC_1STREAM;
}
if (txs > 1 && ath_hal_getcapability(ah, HAL_CAP_TX_STBC, 0,
NULL) == HAL_OK) {
sc->sc_tx_stbc = 1;
device_printf(sc->sc_dev,
"[HT] 1 stream STBC transmit enabled\n");
ic->ic_htcaps |= IEEE80211_HTCAP_TXSTBC;
}
(void) ath_hal_getcapability(ah, HAL_CAP_RTS_AGGR_LIMIT, 1,
&sc->sc_rts_aggr_limit);
if (sc->sc_rts_aggr_limit != (64 * 1024))
device_printf(sc->sc_dev,
"[HT] RTS aggregates limited to %d KiB\n",
sc->sc_rts_aggr_limit / 1024);
device_printf(sc->sc_dev,
"[HT] %d RX streams; %d TX streams\n", rxs, txs);
}
#endif
/*
* Initial aggregation settings.
*/
sc->sc_hwq_limit_aggr = ATH_AGGR_MIN_QDEPTH;
sc->sc_hwq_limit_nonaggr = ATH_NONAGGR_MIN_QDEPTH;
sc->sc_tid_hwq_lo = ATH_AGGR_SCHED_LOW;
sc->sc_tid_hwq_hi = ATH_AGGR_SCHED_HIGH;
sc->sc_aggr_limit = ATH_AGGR_MAXSIZE;
sc->sc_delim_min_pad = 0;
/*
* Check if the hardware requires PCI register serialisation.
* Some of the Owl based MACs require this.
*/
if (mp_ncpus > 1 &&
ath_hal_getcapability(ah, HAL_CAP_SERIALISE_WAR,
0, NULL) == HAL_OK) {
sc->sc_ah->ah_config.ah_serialise_reg_war = 1;
device_printf(sc->sc_dev,
"Enabling register serialisation\n");
}
/*
* Initialise the deferred completed RX buffer list.
*/
TAILQ_INIT(&sc->sc_rx_rxlist[HAL_RX_QUEUE_HP]);
TAILQ_INIT(&sc->sc_rx_rxlist[HAL_RX_QUEUE_LP]);
/*
* Indicate we need the 802.11 header padded to a
* 32-bit boundary for 4-address and QoS frames.
*/
ic->ic_flags |= IEEE80211_F_DATAPAD;
/*
* Query the hal about antenna support.
*/
sc->sc_defant = ath_hal_getdefantenna(ah);
/*
* Not all chips have the VEOL support we want to
* use with IBSS beacons; check here for it.
*/
sc->sc_hasveol = ath_hal_hasveol(ah);
/* get mac address from kenv first, then hardware */
if (ath_fetch_mac_kenv(sc, macaddr) == 0) {
/* Tell the HAL now about the new MAC */
ath_hal_setmac(ah, macaddr);
} else {
ath_hal_getmac(ah, macaddr);
}
if (sc->sc_hasbmask)
ath_hal_getbssidmask(ah, sc->sc_hwbssidmask);
/* NB: used to size node table key mapping array */
ic->ic_max_keyix = sc->sc_keymax;
/* call MI attach routine. */
ieee80211_ifattach(ic, macaddr);
ic->ic_setregdomain = ath_setregdomain;
ic->ic_getradiocaps = ath_getradiocaps;
sc->sc_opmode = HAL_M_STA;
/* override default methods */
ic->ic_newassoc = ath_newassoc;
ic->ic_updateslot = ath_updateslot;
ic->ic_wme.wme_update = ath_wme_update;
ic->ic_vap_create = ath_vap_create;
ic->ic_vap_delete = ath_vap_delete;
ic->ic_raw_xmit = ath_raw_xmit;
ic->ic_update_mcast = ath_update_mcast;
ic->ic_update_promisc = ath_update_promisc;
ic->ic_node_alloc = ath_node_alloc;
sc->sc_node_free = ic->ic_node_free;
ic->ic_node_free = ath_node_free;
sc->sc_node_cleanup = ic->ic_node_cleanup;
ic->ic_node_cleanup = ath_node_cleanup;
ic->ic_node_getsignal = ath_node_getsignal;
ic->ic_scan_start = ath_scan_start;
ic->ic_scan_end = ath_scan_end;
ic->ic_set_channel = ath_set_channel;
#ifdef ATH_ENABLE_11N
/* 802.11n specific - but just override anyway */
sc->sc_addba_request = ic->ic_addba_request;
sc->sc_addba_response = ic->ic_addba_response;
sc->sc_addba_stop = ic->ic_addba_stop;
sc->sc_bar_response = ic->ic_bar_response;
sc->sc_addba_response_timeout = ic->ic_addba_response_timeout;
ic->ic_addba_request = ath_addba_request;
ic->ic_addba_response = ath_addba_response;
ic->ic_addba_response_timeout = ath_addba_response_timeout;
ic->ic_addba_stop = ath_addba_stop;
ic->ic_bar_response = ath_bar_response;
ic->ic_update_chw = ath_update_chw;
#endif /* ATH_ENABLE_11N */
#ifdef ATH_ENABLE_RADIOTAP_VENDOR_EXT
/*
* There's one vendor bitmap entry in the RX radiotap
* header; make sure that's taken into account.
*/
ieee80211_radiotap_attachv(ic,
&sc->sc_tx_th.wt_ihdr, sizeof(sc->sc_tx_th), 0,
ATH_TX_RADIOTAP_PRESENT,
&sc->sc_rx_th.wr_ihdr, sizeof(sc->sc_rx_th), 1,
ATH_RX_RADIOTAP_PRESENT);
#else
/*
* No vendor bitmap/extensions are present.
*/
ieee80211_radiotap_attach(ic,
&sc->sc_tx_th.wt_ihdr, sizeof(sc->sc_tx_th),
ATH_TX_RADIOTAP_PRESENT,
&sc->sc_rx_th.wr_ihdr, sizeof(sc->sc_rx_th),
ATH_RX_RADIOTAP_PRESENT);
#endif /* ATH_ENABLE_RADIOTAP_VENDOR_EXT */
/*
* Setup the ALQ logging if required
*/
#ifdef ATH_DEBUG_ALQ
if_ath_alq_init(&sc->sc_alq, device_get_nameunit(sc->sc_dev));
if_ath_alq_setcfg(&sc->sc_alq,
sc->sc_ah->ah_macVersion,
sc->sc_ah->ah_macRev,
sc->sc_ah->ah_phyRev,
sc->sc_ah->ah_magic);
#endif
/*
* Setup dynamic sysctl's now that country code and
* regdomain are available from the hal.
*/
ath_sysctlattach(sc);
ath_sysctl_stats_attach(sc);
ath_sysctl_hal_attach(sc);
if (bootverbose)
ieee80211_announce(ic);
ath_announce(sc);
/*
* Put it to sleep for now.
*/
ATH_LOCK(sc);
ath_power_setpower(sc, HAL_PM_FULL_SLEEP);
ATH_UNLOCK(sc);
return 0;
bad2:
ath_tx_cleanup(sc);
ath_desc_free(sc);
ath_txdma_teardown(sc);
ath_rxdma_teardown(sc);
bad:
if (ah)
ath_hal_detach(ah);
/*
* To work around scoping issues with CURVNET_SET/CURVNET_RESTORE..
*/
if (ifp != NULL && ifp->if_vnet) {
CURVNET_SET(ifp->if_vnet);
if_free(ifp);
CURVNET_RESTORE();
} else if (ifp != NULL)
if_free(ifp);
sc->sc_invalid = 1;
return error;
}
int
ath_detach(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
DPRINTF(sc, ATH_DEBUG_ANY, "%s: if_flags %x\n",
__func__, ifp->if_flags);
/*
* NB: the order of these is important:
* o stop the chip so no more interrupts will fire
* o call the 802.11 layer before detaching the hal to
* insure callbacks into the driver to delete global
* key cache entries can be handled
* o free the taskqueue which drains any pending tasks
* o reclaim the tx queue data structures after calling
* the 802.11 layer as we'll get called back to reclaim
* node state and potentially want to use them
* o to cleanup the tx queues the hal is called, so detach
* it last
* Other than that, it's straightforward...
*/
/*
* XXX Wake the hardware up first. ath_stop() will still
* wake it up first, but I'd rather do it here just to
* ensure it's awake.
*/
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ath_power_setpower(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
/*
* Stop things cleanly.
*/
ath_stop(ifp);
ieee80211_ifdetach(ifp->if_l2com);
taskqueue_free(sc->sc_tq);
#ifdef ATH_TX99_DIAG
if (sc->sc_tx99 != NULL)
sc->sc_tx99->detach(sc->sc_tx99);
#endif
ath_rate_detach(sc->sc_rc);
#ifdef ATH_DEBUG_ALQ
if_ath_alq_tidyup(&sc->sc_alq);
#endif
ath_lna_div_detach(sc);
ath_btcoex_detach(sc);
ath_spectral_detach(sc);
ath_dfs_detach(sc);
ath_desc_free(sc);
ath_txdma_teardown(sc);
ath_rxdma_teardown(sc);
ath_tx_cleanup(sc);
ath_hal_detach(sc->sc_ah); /* NB: sets chip in full sleep */
CURVNET_SET(ifp->if_vnet);
if_free(ifp);
CURVNET_RESTORE();
return 0;
}
/*
* MAC address handling for multiple BSS on the same radio.
* The first vap uses the MAC address from the EEPROM. For
* subsequent vap's we set the U/L bit (bit 1) in the MAC
* address and use the next six bits as an index.
*/
static void
assign_address(struct ath_softc *sc, uint8_t mac[IEEE80211_ADDR_LEN], int clone)
{
int i;
if (clone && sc->sc_hasbmask) {
/* NB: we only do this if h/w supports multiple bssid */
for (i = 0; i < 8; i++)
if ((sc->sc_bssidmask & (1<<i)) == 0)
break;
if (i != 0)
mac[0] |= (i << 2)|0x2;
} else
i = 0;
sc->sc_bssidmask |= 1<<i;
sc->sc_hwbssidmask[0] &= ~mac[0];
if (i == 0)
sc->sc_nbssid0++;
}
static void
reclaim_address(struct ath_softc *sc, const uint8_t mac[IEEE80211_ADDR_LEN])
{
int i = mac[0] >> 2;
uint8_t mask;
if (i != 0 || --sc->sc_nbssid0 == 0) {
sc->sc_bssidmask &= ~(1<<i);
/* recalculate bssid mask from remaining addresses */
mask = 0xff;
for (i = 1; i < 8; i++)
if (sc->sc_bssidmask & (1<<i))
mask &= ~((i<<2)|0x2);
sc->sc_hwbssidmask[0] |= mask;
}
}
/*
* Assign a beacon xmit slot. We try to space out
* assignments so when beacons are staggered the
* traffic coming out of the cab q has maximal time
* to go out before the next beacon is scheduled.
*/
static int
assign_bslot(struct ath_softc *sc)
{
u_int slot, free;
free = 0;
for (slot = 0; slot < ATH_BCBUF; slot++)
if (sc->sc_bslot[slot] == NULL) {
if (sc->sc_bslot[(slot+1)%ATH_BCBUF] == NULL &&
sc->sc_bslot[(slot-1)%ATH_BCBUF] == NULL)
return slot;
free = slot;
/* NB: keep looking for a double slot */
}
return free;
}
static struct ieee80211vap *
ath_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
enum ieee80211_opmode opmode, int flags,
const uint8_t bssid[IEEE80211_ADDR_LEN],
const uint8_t mac0[IEEE80211_ADDR_LEN])
{
struct ath_softc *sc = ic->ic_ifp->if_softc;
struct ath_vap *avp;
struct ieee80211vap *vap;
uint8_t mac[IEEE80211_ADDR_LEN];
int needbeacon, error;
enum ieee80211_opmode ic_opmode;
avp = (struct ath_vap *) malloc(sizeof(struct ath_vap),
M_80211_VAP, M_WAITOK | M_ZERO);
needbeacon = 0;
IEEE80211_ADDR_COPY(mac, mac0);
ATH_LOCK(sc);
ic_opmode = opmode; /* default to opmode of new vap */
switch (opmode) {
case IEEE80211_M_STA:
if (sc->sc_nstavaps != 0) { /* XXX only 1 for now */
device_printf(sc->sc_dev, "only 1 sta vap supported\n");
goto bad;
}
if (sc->sc_nvaps) {
/*
* With multiple vaps we must fall back
* to s/w beacon miss handling.
*/
flags |= IEEE80211_CLONE_NOBEACONS;
}
if (flags & IEEE80211_CLONE_NOBEACONS) {
/*
* Station mode w/o beacons are implemented w/ AP mode.
*/
ic_opmode = IEEE80211_M_HOSTAP;
}
break;
case IEEE80211_M_IBSS:
if (sc->sc_nvaps != 0) { /* XXX only 1 for now */
device_printf(sc->sc_dev,
"only 1 ibss vap supported\n");
goto bad;
}
needbeacon = 1;
break;
case IEEE80211_M_AHDEMO:
#ifdef IEEE80211_SUPPORT_TDMA
if (flags & IEEE80211_CLONE_TDMA) {
if (sc->sc_nvaps != 0) {
device_printf(sc->sc_dev,
"only 1 tdma vap supported\n");
goto bad;
}
needbeacon = 1;
flags |= IEEE80211_CLONE_NOBEACONS;
}
/* fall thru... */
#endif
case IEEE80211_M_MONITOR:
if (sc->sc_nvaps != 0 && ic->ic_opmode != opmode) {
/*
* Adopt existing mode. Adding a monitor or ahdemo
* vap to an existing configuration is of dubious
* value but should be ok.
*/
/* XXX not right for monitor mode */
ic_opmode = ic->ic_opmode;
}
break;
case IEEE80211_M_HOSTAP:
case IEEE80211_M_MBSS:
needbeacon = 1;
break;
case IEEE80211_M_WDS:
if (sc->sc_nvaps != 0 && ic->ic_opmode == IEEE80211_M_STA) {
device_printf(sc->sc_dev,
"wds not supported in sta mode\n");
goto bad;
}
/*
* Silently remove any request for a unique
* bssid; WDS vap's always share the local
* mac address.
*/
flags &= ~IEEE80211_CLONE_BSSID;
if (sc->sc_nvaps == 0)
ic_opmode = IEEE80211_M_HOSTAP;
else
ic_opmode = ic->ic_opmode;
break;
default:
device_printf(sc->sc_dev, "unknown opmode %d\n", opmode);
goto bad;
}
/*
* Check that a beacon buffer is available; the code below assumes it.
*/
if (needbeacon & TAILQ_EMPTY(&sc->sc_bbuf)) {
device_printf(sc->sc_dev, "no beacon buffer available\n");
goto bad;
}
/* STA, AHDEMO? */
if (opmode == IEEE80211_M_HOSTAP || opmode == IEEE80211_M_MBSS) {
assign_address(sc, mac, flags & IEEE80211_CLONE_BSSID);
ath_hal_setbssidmask(sc->sc_ah, sc->sc_hwbssidmask);
}
vap = &avp->av_vap;
/* XXX can't hold mutex across if_alloc */
ATH_UNLOCK(sc);
error = ieee80211_vap_setup(ic, vap, name, unit, opmode, flags,
bssid, mac);
ATH_LOCK(sc);
if (error != 0) {
device_printf(sc->sc_dev, "%s: error %d creating vap\n",
__func__, error);
goto bad2;
}
/* h/w crypto support */
vap->iv_key_alloc = ath_key_alloc;
vap->iv_key_delete = ath_key_delete;
vap->iv_key_set = ath_key_set;
vap->iv_key_update_begin = ath_key_update_begin;
vap->iv_key_update_end = ath_key_update_end;
/* override various methods */
avp->av_recv_mgmt = vap->iv_recv_mgmt;
vap->iv_recv_mgmt = ath_recv_mgmt;
vap->iv_reset = ath_reset_vap;
vap->iv_update_beacon = ath_beacon_update;
avp->av_newstate = vap->iv_newstate;
vap->iv_newstate = ath_newstate;
avp->av_bmiss = vap->iv_bmiss;
vap->iv_bmiss = ath_bmiss_vap;
avp->av_node_ps = vap->iv_node_ps;
vap->iv_node_ps = ath_node_powersave;
avp->av_set_tim = vap->iv_set_tim;
vap->iv_set_tim = ath_node_set_tim;
avp->av_recv_pspoll = vap->iv_recv_pspoll;
vap->iv_recv_pspoll = ath_node_recv_pspoll;
/* Set default parameters */
/*
* Anything earlier than some AR9300 series MACs don't
* support a smaller MPDU density.
*/
vap->iv_ampdu_density = IEEE80211_HTCAP_MPDUDENSITY_8;
/*
* All NICs can handle the maximum size, however
* AR5416 based MACs can only TX aggregates w/ RTS
* protection when the total aggregate size is <= 8k.
* However, for now that's enforced by the TX path.
*/
vap->iv_ampdu_rxmax = IEEE80211_HTCAP_MAXRXAMPDU_64K;
avp->av_bslot = -1;
if (needbeacon) {
/*
* Allocate beacon state and setup the q for buffered
* multicast frames. We know a beacon buffer is
* available because we checked above.
*/
avp->av_bcbuf = TAILQ_FIRST(&sc->sc_bbuf);
TAILQ_REMOVE(&sc->sc_bbuf, avp->av_bcbuf, bf_list);
if (opmode != IEEE80211_M_IBSS || !sc->sc_hasveol) {
/*
* Assign the vap to a beacon xmit slot. As above
* this cannot fail to find a free one.
*/
avp->av_bslot = assign_bslot(sc);
KASSERT(sc->sc_bslot[avp->av_bslot] == NULL,
("beacon slot %u not empty", avp->av_bslot));
sc->sc_bslot[avp->av_bslot] = vap;
sc->sc_nbcnvaps++;
}
if (sc->sc_hastsfadd && sc->sc_nbcnvaps > 0) {
/*
* Multple vaps are to transmit beacons and we
* have h/w support for TSF adjusting; enable
* use of staggered beacons.
*/
sc->sc_stagbeacons = 1;
}
ath_txq_init(sc, &avp->av_mcastq, ATH_TXQ_SWQ);
}
ic->ic_opmode = ic_opmode;
if (opmode != IEEE80211_M_WDS) {
sc->sc_nvaps++;
if (opmode == IEEE80211_M_STA)
sc->sc_nstavaps++;
if (opmode == IEEE80211_M_MBSS)
sc->sc_nmeshvaps++;
}
switch (ic_opmode) {
case IEEE80211_M_IBSS:
sc->sc_opmode = HAL_M_IBSS;
break;
case IEEE80211_M_STA:
sc->sc_opmode = HAL_M_STA;
break;
case IEEE80211_M_AHDEMO:
#ifdef IEEE80211_SUPPORT_TDMA
if (vap->iv_caps & IEEE80211_C_TDMA) {
sc->sc_tdma = 1;
/* NB: disable tsf adjust */
sc->sc_stagbeacons = 0;
}
/*
* NB: adhoc demo mode is a pseudo mode; to the hal it's
* just ap mode.
*/
/* fall thru... */
#endif
case IEEE80211_M_HOSTAP:
case IEEE80211_M_MBSS:
sc->sc_opmode = HAL_M_HOSTAP;
break;
case IEEE80211_M_MONITOR:
sc->sc_opmode = HAL_M_MONITOR;
break;
default:
/* XXX should not happen */
break;
}
if (sc->sc_hastsfadd) {
/*
* Configure whether or not TSF adjust should be done.
*/
ath_hal_settsfadjust(sc->sc_ah, sc->sc_stagbeacons);
}
if (flags & IEEE80211_CLONE_NOBEACONS) {
/*
* Enable s/w beacon miss handling.
*/
sc->sc_swbmiss = 1;
}
ATH_UNLOCK(sc);
/* complete setup */
ieee80211_vap_attach(vap, ath_media_change, ieee80211_media_status);
return vap;
bad2:
reclaim_address(sc, mac);
ath_hal_setbssidmask(sc->sc_ah, sc->sc_hwbssidmask);
bad:
free(avp, M_80211_VAP);
ATH_UNLOCK(sc);
return NULL;
}
static void
ath_vap_delete(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
struct ifnet *ifp = ic->ic_ifp;
struct ath_softc *sc = ifp->if_softc;
struct ath_hal *ah = sc->sc_ah;
struct ath_vap *avp = ATH_VAP(vap);
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
DPRINTF(sc, ATH_DEBUG_RESET, "%s: called\n", __func__);
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
/*
* Quiesce the hardware while we remove the vap. In
* particular we need to reclaim all references to
* the vap state by any frames pending on the tx queues.
*/
ath_hal_intrset(ah, 0); /* disable interrupts */
/* XXX Do all frames from all vaps/nodes need draining here? */
ath_stoprecv(sc, 1); /* stop recv side */
ath_draintxq(sc, ATH_RESET_DEFAULT); /* stop hw xmit side */
}
/* .. leave the hardware awake for now. */
ieee80211_vap_detach(vap);
/*
* XXX Danger Will Robinson! Danger!
*
* Because ieee80211_vap_detach() can queue a frame (the station
* diassociate message?) after we've drained the TXQ and
* flushed the software TXQ, we will end up with a frame queued
* to a node whose vap is about to be freed.
*
* To work around this, flush the hardware/software again.
* This may be racy - the ath task may be running and the packet
* may be being scheduled between sw->hw txq. Tsk.
*
* TODO: figure out why a new node gets allocated somewhere around
* here (after the ath_tx_swq() call; and after an ath_stop_locked()
* call!)
*/
ath_draintxq(sc, ATH_RESET_DEFAULT);
ATH_LOCK(sc);
/*
* Reclaim beacon state. Note this must be done before
* the vap instance is reclaimed as we may have a reference
* to it in the buffer for the beacon frame.
*/
if (avp->av_bcbuf != NULL) {
if (avp->av_bslot != -1) {
sc->sc_bslot[avp->av_bslot] = NULL;
sc->sc_nbcnvaps--;
}
ath_beacon_return(sc, avp->av_bcbuf);
avp->av_bcbuf = NULL;
if (sc->sc_nbcnvaps == 0) {
sc->sc_stagbeacons = 0;
if (sc->sc_hastsfadd)
ath_hal_settsfadjust(sc->sc_ah, 0);
}
/*
* Reclaim any pending mcast frames for the vap.
*/
ath_tx_draintxq(sc, &avp->av_mcastq);
}
/*
* Update bookkeeping.
*/
if (vap->iv_opmode == IEEE80211_M_STA) {
sc->sc_nstavaps--;
if (sc->sc_nstavaps == 0 && sc->sc_swbmiss)
sc->sc_swbmiss = 0;
} else if (vap->iv_opmode == IEEE80211_M_HOSTAP ||
vap->iv_opmode == IEEE80211_M_MBSS) {
reclaim_address(sc, vap->iv_myaddr);
ath_hal_setbssidmask(ah, sc->sc_hwbssidmask);
if (vap->iv_opmode == IEEE80211_M_MBSS)
sc->sc_nmeshvaps--;
}
if (vap->iv_opmode != IEEE80211_M_WDS)
sc->sc_nvaps--;
#ifdef IEEE80211_SUPPORT_TDMA
/* TDMA operation ceases when the last vap is destroyed */
if (sc->sc_tdma && sc->sc_nvaps == 0) {
sc->sc_tdma = 0;
sc->sc_swbmiss = 0;
}
#endif
free(avp, M_80211_VAP);
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
/*
* Restart rx+tx machines if still running (RUNNING will
* be reset if we just destroyed the last vap).
*/
if (ath_startrecv(sc) != 0)
if_printf(ifp, "%s: unable to restart recv logic\n",
__func__);
if (sc->sc_beacons) { /* restart beacons */
#ifdef IEEE80211_SUPPORT_TDMA
if (sc->sc_tdma)
ath_tdma_config(sc, NULL);
else
#endif
ath_beacon_config(sc, NULL);
}
ath_hal_intrset(ah, sc->sc_imask);
}
/* Ok, let the hardware asleep. */
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
}
void
ath_suspend(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
DPRINTF(sc, ATH_DEBUG_ANY, "%s: if_flags %x\n",
__func__, ifp->if_flags);
sc->sc_resume_up = (ifp->if_flags & IFF_UP) != 0;
ieee80211_suspend_all(ic);
/*
* NB: don't worry about putting the chip in low power
* mode; pci will power off our socket on suspend and
* CardBus detaches the device.
*
* XXX TODO: well, that's great, except for non-cardbus
* devices!
*/
/*
* XXX This doesn't wait until all pending taskqueue
* items and parallel transmit/receive/other threads
* are running!
*/
ath_hal_intrset(sc->sc_ah, 0);
taskqueue_block(sc->sc_tq);
ATH_LOCK(sc);
callout_stop(&sc->sc_cal_ch);
ATH_UNLOCK(sc);
/*
* XXX ensure sc_invalid is 1
*/
/* Disable the PCIe PHY, complete with workarounds */
ath_hal_enablepcie(sc->sc_ah, 1, 1);
}
/*
* Reset the key cache since some parts do not reset the
* contents on resume. First we clear all entries, then
* re-load keys that the 802.11 layer assumes are setup
* in h/w.
*/
static void
ath_reset_keycache(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct ath_hal *ah = sc->sc_ah;
int i;
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
for (i = 0; i < sc->sc_keymax; i++)
ath_hal_keyreset(ah, i);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
ieee80211_crypto_reload_keys(ic);
}
/*
* Fetch the current chainmask configuration based on the current
* operating channel and options.
*/
static void
ath_update_chainmasks(struct ath_softc *sc, struct ieee80211_channel *chan)
{
/*
* Set TX chainmask to the currently configured chainmask;
* the TX chainmask depends upon the current operating mode.
*/
sc->sc_cur_rxchainmask = sc->sc_rxchainmask;
if (IEEE80211_IS_CHAN_HT(chan)) {
sc->sc_cur_txchainmask = sc->sc_txchainmask;
} else {
sc->sc_cur_txchainmask = 1;
}
DPRINTF(sc, ATH_DEBUG_RESET,
"%s: TX chainmask is now 0x%x, RX is now 0x%x\n",
__func__,
sc->sc_cur_txchainmask,
sc->sc_cur_rxchainmask);
}
void
ath_resume(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct ath_hal *ah = sc->sc_ah;
HAL_STATUS status;
DPRINTF(sc, ATH_DEBUG_ANY, "%s: if_flags %x\n",
__func__, ifp->if_flags);
/* Re-enable PCIe, re-enable the PCIe bus */
ath_hal_enablepcie(ah, 0, 0);
/*
* Must reset the chip before we reload the
* keycache as we were powered down on suspend.
*/
ath_update_chainmasks(sc,
sc->sc_curchan != NULL ? sc->sc_curchan : ic->ic_curchan);
ath_hal_setchainmasks(sc->sc_ah, sc->sc_cur_txchainmask,
sc->sc_cur_rxchainmask);
/* Ensure we set the current power state to on */
ATH_LOCK(sc);
ath_power_setselfgen(sc, HAL_PM_AWAKE);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ath_power_setpower(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ath_hal_reset(ah, sc->sc_opmode,
sc->sc_curchan != NULL ? sc->sc_curchan : ic->ic_curchan,
AH_FALSE, &status);
ath_reset_keycache(sc);
ATH_RX_LOCK(sc);
sc->sc_rx_stopped = 1;
sc->sc_rx_resetted = 1;
ATH_RX_UNLOCK(sc);
/* Let DFS at it in case it's a DFS channel */
ath_dfs_radar_enable(sc, ic->ic_curchan);
/* Let spectral at in case spectral is enabled */
ath_spectral_enable(sc, ic->ic_curchan);
/*
* Let bluetooth coexistence at in case it's needed for this channel
*/
ath_btcoex_enable(sc, ic->ic_curchan);
/*
* If we're doing TDMA, enforce the TXOP limitation for chips that
* support it.
*/
if (sc->sc_hasenforcetxop && sc->sc_tdma)
ath_hal_setenforcetxop(sc->sc_ah, 1);
else
ath_hal_setenforcetxop(sc->sc_ah, 0);
/* Restore the LED configuration */
ath_led_config(sc);
ath_hal_setledstate(ah, HAL_LED_INIT);
if (sc->sc_resume_up)
ieee80211_resume_all(ic);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
/* XXX beacons ? */
}
void
ath_shutdown(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
DPRINTF(sc, ATH_DEBUG_ANY, "%s: if_flags %x\n",
__func__, ifp->if_flags);
ath_stop(ifp);
/* NB: no point powering down chip as we're about to reboot */
}
/*
* Interrupt handler. Most of the actual processing is deferred.
*/
void
ath_intr(void *arg)
{
struct ath_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
struct ath_hal *ah = sc->sc_ah;
HAL_INT status = 0;
uint32_t txqs;
/*
* If we're inside a reset path, just print a warning and
* clear the ISR. The reset routine will finish it for us.
*/
ATH_PCU_LOCK(sc);
if (sc->sc_inreset_cnt) {
HAL_INT status;
ath_hal_getisr(ah, &status); /* clear ISR */
ath_hal_intrset(ah, 0); /* disable further intr's */
DPRINTF(sc, ATH_DEBUG_ANY,
"%s: in reset, ignoring: status=0x%x\n",
__func__, status);
ATH_PCU_UNLOCK(sc);
return;
}
if (sc->sc_invalid) {
/*
* The hardware is not ready/present, don't touch anything.
* Note this can happen early on if the IRQ is shared.
*/
DPRINTF(sc, ATH_DEBUG_ANY, "%s: invalid; ignored\n", __func__);
ATH_PCU_UNLOCK(sc);
return;
}
if (!ath_hal_intrpend(ah)) { /* shared irq, not for us */
ATH_PCU_UNLOCK(sc);
return;
}
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
if ((ifp->if_flags & IFF_UP) == 0 ||
(ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
HAL_INT status;
DPRINTF(sc, ATH_DEBUG_ANY, "%s: if_flags 0x%x\n",
__func__, ifp->if_flags);
ath_hal_getisr(ah, &status); /* clear ISR */
ath_hal_intrset(ah, 0); /* disable further intr's */
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
return;
}
/*
* Figure out the reason(s) for the interrupt. Note
* that the hal returns a pseudo-ISR that may include
* bits we haven't explicitly enabled so we mask the
* value to insure we only process bits we requested.
*/
ath_hal_getisr(ah, &status); /* NB: clears ISR too */
DPRINTF(sc, ATH_DEBUG_INTR, "%s: status 0x%x\n", __func__, status);
ATH_KTR(sc, ATH_KTR_INTERRUPTS, 1, "ath_intr: mask=0x%.8x", status);
#ifdef ATH_DEBUG_ALQ
if_ath_alq_post_intr(&sc->sc_alq, status, ah->ah_intrstate,
ah->ah_syncstate);
#endif /* ATH_DEBUG_ALQ */
#ifdef ATH_KTR_INTR_DEBUG
ATH_KTR(sc, ATH_KTR_INTERRUPTS, 5,
"ath_intr: ISR=0x%.8x, ISR_S0=0x%.8x, ISR_S1=0x%.8x, ISR_S2=0x%.8x, ISR_S5=0x%.8x",
ah->ah_intrstate[0],
ah->ah_intrstate[1],
ah->ah_intrstate[2],
ah->ah_intrstate[3],
ah->ah_intrstate[6]);
#endif
/* Squirrel away SYNC interrupt debugging */
if (ah->ah_syncstate != 0) {
int i;
for (i = 0; i < 32; i++)
if (ah->ah_syncstate & (i << i))
sc->sc_intr_stats.sync_intr[i]++;
}
status &= sc->sc_imask; /* discard unasked for bits */
/* Short-circuit un-handled interrupts */
if (status == 0x0) {
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
return;
}
/*
* Take a note that we're inside the interrupt handler, so
* the reset routines know to wait.
*/
sc->sc_intr_cnt++;
ATH_PCU_UNLOCK(sc);
/*
* Handle the interrupt. We won't run concurrent with the reset
* or channel change routines as they'll wait for sc_intr_cnt
* to be 0 before continuing.
*/
if (status & HAL_INT_FATAL) {
sc->sc_stats.ast_hardware++;
ath_hal_intrset(ah, 0); /* disable intr's until reset */
taskqueue_enqueue(sc->sc_tq, &sc->sc_fataltask);
} else {
if (status & HAL_INT_SWBA) {
/*
* Software beacon alert--time to send a beacon.
* Handle beacon transmission directly; deferring
* this is too slow to meet timing constraints
* under load.
*/
#ifdef IEEE80211_SUPPORT_TDMA
if (sc->sc_tdma) {
if (sc->sc_tdmaswba == 0) {
struct ieee80211com *ic = ifp->if_l2com;
struct ieee80211vap *vap =
TAILQ_FIRST(&ic->ic_vaps);
ath_tdma_beacon_send(sc, vap);
sc->sc_tdmaswba =
vap->iv_tdma->tdma_bintval;
} else
sc->sc_tdmaswba--;
} else
#endif
{
ath_beacon_proc(sc, 0);
#ifdef IEEE80211_SUPPORT_SUPERG
/*
* Schedule the rx taskq in case there's no
* traffic so any frames held on the staging
* queue are aged and potentially flushed.
*/
sc->sc_rx.recv_sched(sc, 1);
#endif
}
}
if (status & HAL_INT_RXEOL) {
int imask;
ATH_KTR(sc, ATH_KTR_ERROR, 0, "ath_intr: RXEOL");
if (! sc->sc_isedma) {
ATH_PCU_LOCK(sc);
/*
* NB: the hardware should re-read the link when
* RXE bit is written, but it doesn't work at
* least on older hardware revs.
*/
sc->sc_stats.ast_rxeol++;
/*
* Disable RXEOL/RXORN - prevent an interrupt
* storm until the PCU logic can be reset.
* In case the interface is reset some other
* way before "sc_kickpcu" is called, don't
* modify sc_imask - that way if it is reset
* by a call to ath_reset() somehow, the
* interrupt mask will be correctly reprogrammed.
*/
imask = sc->sc_imask;
imask &= ~(HAL_INT_RXEOL | HAL_INT_RXORN);
ath_hal_intrset(ah, imask);
/*
* Only blank sc_rxlink if we've not yet kicked
* the PCU.
*
* This isn't entirely correct - the correct solution
* would be to have a PCU lock and engage that for
* the duration of the PCU fiddling; which would include
* running the RX process. Otherwise we could end up
* messing up the RX descriptor chain and making the
* RX desc list much shorter.
*/
if (! sc->sc_kickpcu)
sc->sc_rxlink = NULL;
sc->sc_kickpcu = 1;
ATH_PCU_UNLOCK(sc);
}
/*
* Enqueue an RX proc to handle whatever
* is in the RX queue.
* This will then kick the PCU if required.
*/
sc->sc_rx.recv_sched(sc, 1);
}
if (status & HAL_INT_TXURN) {
sc->sc_stats.ast_txurn++;
/* bump tx trigger level */
ath_hal_updatetxtriglevel(ah, AH_TRUE);
}
/*
* Handle both the legacy and RX EDMA interrupt bits.
* Note that HAL_INT_RXLP is also HAL_INT_RXDESC.
*/
if (status & (HAL_INT_RX | HAL_INT_RXHP | HAL_INT_RXLP)) {
sc->sc_stats.ast_rx_intr++;
sc->sc_rx.recv_sched(sc, 1);
}
if (status & HAL_INT_TX) {
sc->sc_stats.ast_tx_intr++;
/*
* Grab all the currently set bits in the HAL txq bitmap
* and blank them. This is the only place we should be
* doing this.
*/
if (! sc->sc_isedma) {
ATH_PCU_LOCK(sc);
txqs = 0xffffffff;
ath_hal_gettxintrtxqs(sc->sc_ah, &txqs);
ATH_KTR(sc, ATH_KTR_INTERRUPTS, 3,
"ath_intr: TX; txqs=0x%08x, txq_active was 0x%08x, now 0x%08x",
txqs,
sc->sc_txq_active,
sc->sc_txq_active | txqs);
sc->sc_txq_active |= txqs;
ATH_PCU_UNLOCK(sc);
}
taskqueue_enqueue(sc->sc_tq, &sc->sc_txtask);
}
if (status & HAL_INT_BMISS) {
sc->sc_stats.ast_bmiss++;
taskqueue_enqueue(sc->sc_tq, &sc->sc_bmisstask);
}
if (status & HAL_INT_GTT)
sc->sc_stats.ast_tx_timeout++;
if (status & HAL_INT_CST)
sc->sc_stats.ast_tx_cst++;
if (status & HAL_INT_MIB) {
sc->sc_stats.ast_mib++;
ATH_PCU_LOCK(sc);
/*
* Disable interrupts until we service the MIB
* interrupt; otherwise it will continue to fire.
*/
ath_hal_intrset(ah, 0);
/*
* Let the hal handle the event. We assume it will
* clear whatever condition caused the interrupt.
*/
ath_hal_mibevent(ah, &sc->sc_halstats);
/*
* Don't reset the interrupt if we've just
* kicked the PCU, or we may get a nested
* RXEOL before the rxproc has had a chance
* to run.
*/
if (sc->sc_kickpcu == 0)
ath_hal_intrset(ah, sc->sc_imask);
ATH_PCU_UNLOCK(sc);
}
if (status & HAL_INT_RXORN) {
/* NB: hal marks HAL_INT_FATAL when RXORN is fatal */
ATH_KTR(sc, ATH_KTR_ERROR, 0, "ath_intr: RXORN");
sc->sc_stats.ast_rxorn++;
}
if (status & HAL_INT_TSFOOR) {
device_printf(sc->sc_dev, "%s: TSFOOR\n", __func__);
sc->sc_syncbeacon = 1;
}
}
ATH_PCU_LOCK(sc);
sc->sc_intr_cnt--;
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
}
static void
ath_fatal_proc(void *arg, int pending)
{
struct ath_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
u_int32_t *state;
u_int32_t len;
void *sp;
if_printf(ifp, "hardware error; resetting\n");
/*
* Fatal errors are unrecoverable. Typically these
* are caused by DMA errors. Collect h/w state from
* the hal so we can diagnose what's going on.
*/
if (ath_hal_getfatalstate(sc->sc_ah, &sp, &len)) {
KASSERT(len >= 6*sizeof(u_int32_t), ("len %u bytes", len));
state = sp;
if_printf(ifp, "0x%08x 0x%08x 0x%08x, 0x%08x 0x%08x 0x%08x\n",
state[0], state[1] , state[2], state[3],
state[4], state[5]);
}
ath_reset(ifp, ATH_RESET_NOLOSS);
}
static void
ath_bmiss_vap(struct ieee80211vap *vap)
{
struct ath_softc *sc = vap->iv_ic->ic_ifp->if_softc;
/*
* Workaround phantom bmiss interrupts by sanity-checking
* the time of our last rx'd frame. If it is within the
* beacon miss interval then ignore the interrupt. If it's
* truly a bmiss we'll get another interrupt soon and that'll
* be dispatched up for processing. Note this applies only
* for h/w beacon miss events.
*/
/*
* XXX TODO: Just read the TSF during the interrupt path;
* that way we don't have to wake up again just to read it
* again.
*/
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
if ((vap->iv_flags_ext & IEEE80211_FEXT_SWBMISS) == 0) {
struct ifnet *ifp = vap->iv_ic->ic_ifp;
struct ath_softc *sc = ifp->if_softc;
u_int64_t lastrx = sc->sc_lastrx;
u_int64_t tsf = ath_hal_gettsf64(sc->sc_ah);
/* XXX should take a locked ref to iv_bss */
u_int bmisstimeout =
vap->iv_bmissthreshold * vap->iv_bss->ni_intval * 1024;
DPRINTF(sc, ATH_DEBUG_BEACON,
"%s: tsf %llu lastrx %lld (%llu) bmiss %u\n",
__func__, (unsigned long long) tsf,
(unsigned long long)(tsf - lastrx),
(unsigned long long) lastrx, bmisstimeout);
if (tsf - lastrx <= bmisstimeout) {
sc->sc_stats.ast_bmiss_phantom++;
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
return;
}
}
/*
* There's no need to keep the hardware awake during the call
* to av_bmiss().
*/
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
/*
* Attempt to force a beacon resync.
*/
sc->sc_syncbeacon = 1;
ATH_VAP(vap)->av_bmiss(vap);
}
/* XXX this needs a force wakeup! */
int
ath_hal_gethangstate(struct ath_hal *ah, uint32_t mask, uint32_t *hangs)
{
uint32_t rsize;
void *sp;
if (!ath_hal_getdiagstate(ah, HAL_DIAG_CHECK_HANGS, &mask, sizeof(mask), &sp, &rsize))
return 0;
KASSERT(rsize == sizeof(uint32_t), ("resultsize %u", rsize));
*hangs = *(uint32_t *)sp;
return 1;
}
static void
ath_bmiss_proc(void *arg, int pending)
{
struct ath_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
uint32_t hangs;
DPRINTF(sc, ATH_DEBUG_ANY, "%s: pending %u\n", __func__, pending);
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ath_beacon_miss(sc);
/*
* Do a reset upon any becaon miss event.
*
* It may be a non-recognised RX clear hang which needs a reset
* to clear.
*/
if (ath_hal_gethangstate(sc->sc_ah, 0xff, &hangs) && hangs != 0) {
ath_reset(ifp, ATH_RESET_NOLOSS);
if_printf(ifp, "bb hang detected (0x%x), resetting\n", hangs);
} else {
ath_reset(ifp, ATH_RESET_NOLOSS);
ieee80211_beacon_miss(ifp->if_l2com);
}
/* Force a beacon resync, in case they've drifted */
sc->sc_syncbeacon = 1;
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
}
/*
* Handle TKIP MIC setup to deal hardware that doesn't do MIC
* calcs together with WME. If necessary disable the crypto
* hardware and mark the 802.11 state so keys will be setup
* with the MIC work done in software.
*/
static void
ath_settkipmic(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
if ((ic->ic_cryptocaps & IEEE80211_CRYPTO_TKIP) && !sc->sc_wmetkipmic) {
if (ic->ic_flags & IEEE80211_F_WME) {
ath_hal_settkipmic(sc->sc_ah, AH_FALSE);
ic->ic_cryptocaps &= ~IEEE80211_CRYPTO_TKIPMIC;
} else {
ath_hal_settkipmic(sc->sc_ah, AH_TRUE);
ic->ic_cryptocaps |= IEEE80211_CRYPTO_TKIPMIC;
}
}
}
static void
ath_init(void *arg)
{
struct ath_softc *sc = (struct ath_softc *) arg;
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct ath_hal *ah = sc->sc_ah;
HAL_STATUS status;
DPRINTF(sc, ATH_DEBUG_ANY, "%s: if_flags 0x%x\n",
__func__, ifp->if_flags);
ATH_LOCK(sc);
/*
* Force the sleep state awake.
*/
ath_power_setselfgen(sc, HAL_PM_AWAKE);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ath_power_setpower(sc, HAL_PM_AWAKE);
/*
* Stop anything previously setup. This is safe
* whether this is the first time through or not.
*/
ath_stop_locked(ifp);
/*
* The basic interface to setting the hardware in a good
* state is ``reset''. On return the hardware is known to
* be powered up and with interrupts disabled. This must
* be followed by initialization of the appropriate bits
* and then setup of the interrupt mask.
*/
ath_settkipmic(sc);
ath_update_chainmasks(sc, ic->ic_curchan);
ath_hal_setchainmasks(sc->sc_ah, sc->sc_cur_txchainmask,
sc->sc_cur_rxchainmask);
if (!ath_hal_reset(ah, sc->sc_opmode, ic->ic_curchan, AH_FALSE, &status)) {
if_printf(ifp, "unable to reset hardware; hal status %u\n",
status);
ATH_UNLOCK(sc);
return;
}
ATH_RX_LOCK(sc);
sc->sc_rx_stopped = 1;
sc->sc_rx_resetted = 1;
ATH_RX_UNLOCK(sc);
ath_chan_change(sc, ic->ic_curchan);
/* Let DFS at it in case it's a DFS channel */
ath_dfs_radar_enable(sc, ic->ic_curchan);
/* Let spectral at in case spectral is enabled */
ath_spectral_enable(sc, ic->ic_curchan);
/*
* Let bluetooth coexistence at in case it's needed for this channel
*/
ath_btcoex_enable(sc, ic->ic_curchan);
/*
* If we're doing TDMA, enforce the TXOP limitation for chips that
* support it.
*/
if (sc->sc_hasenforcetxop && sc->sc_tdma)
ath_hal_setenforcetxop(sc->sc_ah, 1);
else
ath_hal_setenforcetxop(sc->sc_ah, 0);
/*
* Likewise this is set during reset so update
* state cached in the driver.
*/
sc->sc_diversity = ath_hal_getdiversity(ah);
sc->sc_lastlongcal = ticks;
sc->sc_resetcal = 1;
sc->sc_lastcalreset = 0;
sc->sc_lastani = ticks;
sc->sc_lastshortcal = ticks;
sc->sc_doresetcal = AH_FALSE;
/*
* Beacon timers were cleared here; give ath_newstate()
* a hint that the beacon timers should be poked when
* things transition to the RUN state.
*/
sc->sc_beacons = 0;
/*
* Setup the hardware after reset: the key cache
* is filled as needed and the receive engine is
* set going. Frame transmit is handled entirely
* in the frame output path; there's nothing to do
* here except setup the interrupt mask.
*/
if (ath_startrecv(sc) != 0) {
if_printf(ifp, "unable to start recv logic\n");
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
return;
}
/*
* Enable interrupts.
*/
sc->sc_imask = HAL_INT_RX | HAL_INT_TX
| HAL_INT_RXORN | HAL_INT_TXURN
| HAL_INT_FATAL | HAL_INT_GLOBAL;
/*
* Enable RX EDMA bits. Note these overlap with
* HAL_INT_RX and HAL_INT_RXDESC respectively.
*/
if (sc->sc_isedma)
sc->sc_imask |= (HAL_INT_RXHP | HAL_INT_RXLP);
/*
* If we're an EDMA NIC, we don't care about RXEOL.
* Writing a new descriptor in will simply restart
* RX DMA.
*/
if (! sc->sc_isedma)
sc->sc_imask |= HAL_INT_RXEOL;
/*
* Enable MIB interrupts when there are hardware phy counters.
* Note we only do this (at the moment) for station mode.
*/
if (sc->sc_needmib && ic->ic_opmode == IEEE80211_M_STA)
sc->sc_imask |= HAL_INT_MIB;
/*
* XXX add capability for this.
*
* If we're in STA mode (and maybe IBSS?) then register for
* TSFOOR interrupts.
*/
if (ic->ic_opmode == IEEE80211_M_STA)
sc->sc_imask |= HAL_INT_TSFOOR;
/* Enable global TX timeout and carrier sense timeout if available */
if (ath_hal_gtxto_supported(ah))
sc->sc_imask |= HAL_INT_GTT;
DPRINTF(sc, ATH_DEBUG_RESET, "%s: imask=0x%x\n",
__func__, sc->sc_imask);
ifp->if_drv_flags |= IFF_DRV_RUNNING;
callout_reset(&sc->sc_wd_ch, hz, ath_watchdog, sc);
ath_hal_intrset(ah, sc->sc_imask);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
#ifdef ATH_TX99_DIAG
if (sc->sc_tx99 != NULL)
sc->sc_tx99->start(sc->sc_tx99);
else
#endif
ieee80211_start_all(ic); /* start all vap's */
}
static void
ath_stop_locked(struct ifnet *ifp)
{
struct ath_softc *sc = ifp->if_softc;
struct ath_hal *ah = sc->sc_ah;
DPRINTF(sc, ATH_DEBUG_ANY, "%s: invalid %u if_flags 0x%x\n",
__func__, sc->sc_invalid, ifp->if_flags);
ATH_LOCK_ASSERT(sc);
/*
* Wake the hardware up before fiddling with it.
*/
ath_power_set_power_state(sc, HAL_PM_AWAKE);
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
/*
* Shutdown the hardware and driver:
* reset 802.11 state machine
* turn off timers
* disable interrupts
* turn off the radio
* clear transmit machinery
* clear receive machinery
* drain and release tx queues
* reclaim beacon resources
* power down hardware
*
* Note that some of this work is not possible if the
* hardware is gone (invalid).
*/
#ifdef ATH_TX99_DIAG
if (sc->sc_tx99 != NULL)
sc->sc_tx99->stop(sc->sc_tx99);
#endif
callout_stop(&sc->sc_wd_ch);
sc->sc_wd_timer = 0;
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
if (!sc->sc_invalid) {
if (sc->sc_softled) {
callout_stop(&sc->sc_ledtimer);
ath_hal_gpioset(ah, sc->sc_ledpin,
!sc->sc_ledon);
sc->sc_blinking = 0;
}
ath_hal_intrset(ah, 0);
}
/* XXX we should stop RX regardless of whether it's valid */
if (!sc->sc_invalid) {
ath_stoprecv(sc, 1);
ath_hal_phydisable(ah);
} else
sc->sc_rxlink = NULL;
ath_draintxq(sc, ATH_RESET_DEFAULT);
ath_beacon_free(sc); /* XXX not needed */
}
/* And now, restore the current power state */
ath_power_restore_power_state(sc);
}
/*
* Wait until all pending TX/RX has completed.
*
* This waits until all existing transmit, receive and interrupts
* have completed. It's assumed that the caller has first
* grabbed the reset lock so it doesn't try to do overlapping
* chip resets.
*/
#define MAX_TXRX_ITERATIONS 100
static void
ath_txrx_stop_locked(struct ath_softc *sc)
{
int i = MAX_TXRX_ITERATIONS;
ATH_UNLOCK_ASSERT(sc);
ATH_PCU_LOCK_ASSERT(sc);
/*
* Sleep until all the pending operations have completed.
*
* The caller must ensure that reset has been incremented
* or the pending operations may continue being queued.
*/
while (sc->sc_rxproc_cnt || sc->sc_txproc_cnt ||
sc->sc_txstart_cnt || sc->sc_intr_cnt) {
if (i <= 0)
break;
msleep(sc, &sc->sc_pcu_mtx, 0, "ath_txrx_stop",
msecs_to_ticks(10));
i--;
}
if (i <= 0)
device_printf(sc->sc_dev,
"%s: didn't finish after %d iterations\n",
__func__, MAX_TXRX_ITERATIONS);
}
#undef MAX_TXRX_ITERATIONS
#if 0
static void
ath_txrx_stop(struct ath_softc *sc)
{
ATH_UNLOCK_ASSERT(sc);
ATH_PCU_UNLOCK_ASSERT(sc);
ATH_PCU_LOCK(sc);
ath_txrx_stop_locked(sc);
ATH_PCU_UNLOCK(sc);
}
#endif
static void
ath_txrx_start(struct ath_softc *sc)
{
taskqueue_unblock(sc->sc_tq);
}
/*
* Grab the reset lock, and wait around until noone else
* is trying to do anything with it.
*
* This is totally horrible but we can't hold this lock for
* long enough to do TX/RX or we end up with net80211/ip stack
* LORs and eventual deadlock.
*
* "dowait" signals whether to spin, waiting for the reset
* lock count to reach 0. This should (for now) only be used
* during the reset path, as the rest of the code may not
* be locking-reentrant enough to behave correctly.
*
* Another, cleaner way should be found to serialise all of
* these operations.
*/
#define MAX_RESET_ITERATIONS 25
static int
ath_reset_grablock(struct ath_softc *sc, int dowait)
{
int w = 0;
int i = MAX_RESET_ITERATIONS;
ATH_PCU_LOCK_ASSERT(sc);
do {
if (sc->sc_inreset_cnt == 0) {
w = 1;
break;
}
if (dowait == 0) {
w = 0;
break;
}
ATH_PCU_UNLOCK(sc);
/*
* 1 tick is likely not enough time for long calibrations
* to complete. So we should wait quite a while.
*/
pause("ath_reset_grablock", msecs_to_ticks(100));
i--;
ATH_PCU_LOCK(sc);
} while (i > 0);
/*
* We always increment the refcounter, regardless
* of whether we succeeded to get it in an exclusive
* way.
*/
sc->sc_inreset_cnt++;
if (i <= 0)
device_printf(sc->sc_dev,
"%s: didn't finish after %d iterations\n",
__func__, MAX_RESET_ITERATIONS);
if (w == 0)
device_printf(sc->sc_dev,
"%s: warning, recursive reset path!\n",
__func__);
return w;
}
#undef MAX_RESET_ITERATIONS
/*
* XXX TODO: write ath_reset_releaselock
*/
static void
ath_stop(struct ifnet *ifp)
{
struct ath_softc *sc = ifp->if_softc;
ATH_LOCK(sc);
ath_stop_locked(ifp);
ATH_UNLOCK(sc);
}
/*
* Reset the hardware w/o losing operational state. This is
* basically a more efficient way of doing ath_stop, ath_init,
* followed by state transitions to the current 802.11
* operational state. Used to recover from various errors and
* to reset or reload hardware state.
*/
int
ath_reset(struct ifnet *ifp, ATH_RESET_TYPE reset_type)
{
struct ath_softc *sc = ifp->if_softc;
struct ieee80211com *ic = ifp->if_l2com;
struct ath_hal *ah = sc->sc_ah;
HAL_STATUS status;
int i;
DPRINTF(sc, ATH_DEBUG_RESET, "%s: called\n", __func__);
/* Ensure ATH_LOCK isn't held; ath_rx_proc can't be locked */
ATH_PCU_UNLOCK_ASSERT(sc);
ATH_UNLOCK_ASSERT(sc);
/* Try to (stop any further TX/RX from occuring */
taskqueue_block(sc->sc_tq);
/*
* Wake the hardware up.
*/
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ATH_PCU_LOCK(sc);
/*
* Grab the reset lock before TX/RX is stopped.
*
* This is needed to ensure that when the TX/RX actually does finish,
* no further TX/RX/reset runs in parallel with this.
*/
if (ath_reset_grablock(sc, 1) == 0) {
device_printf(sc->sc_dev, "%s: concurrent reset! Danger!\n",
__func__);
}
/* disable interrupts */
ath_hal_intrset(ah, 0);
/*
* Now, ensure that any in progress TX/RX completes before we
* continue.
*/
ath_txrx_stop_locked(sc);
ATH_PCU_UNLOCK(sc);
/*
* Regardless of whether we're doing a no-loss flush or
* not, stop the PCU and handle what's in the RX queue.
* That way frames aren't dropped which shouldn't be.
*/
ath_stoprecv(sc, (reset_type != ATH_RESET_NOLOSS));
ath_rx_flush(sc);
/*
* Should now wait for pending TX/RX to complete
* and block future ones from occuring. This needs to be
* done before the TX queue is drained.
*/
ath_draintxq(sc, reset_type); /* stop xmit side */
ath_settkipmic(sc); /* configure TKIP MIC handling */
/* NB: indicate channel change so we do a full reset */
ath_update_chainmasks(sc, ic->ic_curchan);
ath_hal_setchainmasks(sc->sc_ah, sc->sc_cur_txchainmask,
sc->sc_cur_rxchainmask);
if (!ath_hal_reset(ah, sc->sc_opmode, ic->ic_curchan, AH_TRUE, &status))
if_printf(ifp, "%s: unable to reset hardware; hal status %u\n",
__func__, status);
sc->sc_diversity = ath_hal_getdiversity(ah);
ATH_RX_LOCK(sc);
sc->sc_rx_stopped = 1;
sc->sc_rx_resetted = 1;
ATH_RX_UNLOCK(sc);
/* Let DFS at it in case it's a DFS channel */
ath_dfs_radar_enable(sc, ic->ic_curchan);
/* Let spectral at in case spectral is enabled */
ath_spectral_enable(sc, ic->ic_curchan);
/*
* Let bluetooth coexistence at in case it's needed for this channel
*/
ath_btcoex_enable(sc, ic->ic_curchan);
/*
* If we're doing TDMA, enforce the TXOP limitation for chips that
* support it.
*/
if (sc->sc_hasenforcetxop && sc->sc_tdma)
ath_hal_setenforcetxop(sc->sc_ah, 1);
else
ath_hal_setenforcetxop(sc->sc_ah, 0);
if (ath_startrecv(sc) != 0) /* restart recv */
if_printf(ifp, "%s: unable to start recv logic\n", __func__);
/*
* We may be doing a reset in response to an ioctl
* that changes the channel so update any state that
* might change as a result.
*/
ath_chan_change(sc, ic->ic_curchan);
if (sc->sc_beacons) { /* restart beacons */
#ifdef IEEE80211_SUPPORT_TDMA
if (sc->sc_tdma)
ath_tdma_config(sc, NULL);
else
#endif
ath_beacon_config(sc, NULL);
}
/*
* Release the reset lock and re-enable interrupts here.
* If an interrupt was being processed in ath_intr(),
* it would disable interrupts at this point. So we have
* to atomically enable interrupts and decrement the
* reset counter - this way ath_intr() doesn't end up
* disabling interrupts without a corresponding enable
* in the rest or channel change path.
*
* Grab the TX reference in case we need to transmit.
* That way a parallel transmit doesn't.
*/
ATH_PCU_LOCK(sc);
sc->sc_inreset_cnt--;
sc->sc_txstart_cnt++;
/* XXX only do this if sc_inreset_cnt == 0? */
ath_hal_intrset(ah, sc->sc_imask);
ATH_PCU_UNLOCK(sc);
/*
* TX and RX can be started here. If it were started with
* sc_inreset_cnt > 0, the TX and RX path would abort.
* Thus if this is a nested call through the reset or
* channel change code, TX completion will occur but
* RX completion and ath_start / ath_tx_start will not
* run.
*/
/* Restart TX/RX as needed */
ath_txrx_start(sc);
/* XXX TODO: we need to hold the tx refcount here! */
/* Restart TX completion and pending TX */
if (reset_type == ATH_RESET_NOLOSS) {
for (i = 0; i < HAL_NUM_TX_QUEUES; i++) {
if (ATH_TXQ_SETUP(sc, i)) {
ATH_TXQ_LOCK(&sc->sc_txq[i]);
ath_txq_restart_dma(sc, &sc->sc_txq[i]);
ATH_TXQ_UNLOCK(&sc->sc_txq[i]);
ATH_TX_LOCK(sc);
ath_txq_sched(sc, &sc->sc_txq[i]);
ATH_TX_UNLOCK(sc);
}
}
}
/*
* This may have been set during an ath_start() call which
* set this once it detected a concurrent TX was going on.
* So, clear it.
*/
IF_LOCK(&ifp->if_snd);
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
IF_UNLOCK(&ifp->if_snd);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
ATH_PCU_LOCK(sc);
sc->sc_txstart_cnt--;
ATH_PCU_UNLOCK(sc);
/* Handle any frames in the TX queue */
/*
* XXX should this be done by the caller, rather than
* ath_reset() ?
*/
ath_tx_kick(sc); /* restart xmit */
return 0;
}
static int
ath_reset_vap(struct ieee80211vap *vap, u_long cmd)
{
struct ieee80211com *ic = vap->iv_ic;
struct ifnet *ifp = ic->ic_ifp;
struct ath_softc *sc = ifp->if_softc;
struct ath_hal *ah = sc->sc_ah;
switch (cmd) {
case IEEE80211_IOC_TXPOWER:
/*
* If per-packet TPC is enabled, then we have nothing
* to do; otherwise we need to force the global limit.
* All this can happen directly; no need to reset.
*/
if (!ath_hal_gettpc(ah))
ath_hal_settxpowlimit(ah, ic->ic_txpowlimit);
return 0;
}
/* XXX? Full or NOLOSS? */
return ath_reset(ifp, ATH_RESET_FULL);
}
struct ath_buf *
_ath_getbuf_locked(struct ath_softc *sc, ath_buf_type_t btype)
{
struct ath_buf *bf;
ATH_TXBUF_LOCK_ASSERT(sc);
if (btype == ATH_BUFTYPE_MGMT)
bf = TAILQ_FIRST(&sc->sc_txbuf_mgmt);
else
bf = TAILQ_FIRST(&sc->sc_txbuf);
if (bf == NULL) {
sc->sc_stats.ast_tx_getnobuf++;
} else {
if (bf->bf_flags & ATH_BUF_BUSY) {
sc->sc_stats.ast_tx_getbusybuf++;
bf = NULL;
}
}
if (bf != NULL && (bf->bf_flags & ATH_BUF_BUSY) == 0) {
if (btype == ATH_BUFTYPE_MGMT)
TAILQ_REMOVE(&sc->sc_txbuf_mgmt, bf, bf_list);
else {
TAILQ_REMOVE(&sc->sc_txbuf, bf, bf_list);
sc->sc_txbuf_cnt--;
/*
* This shuldn't happen; however just to be
* safe print a warning and fudge the txbuf
* count.
*/
if (sc->sc_txbuf_cnt < 0) {
device_printf(sc->sc_dev,
"%s: sc_txbuf_cnt < 0?\n",
__func__);
sc->sc_txbuf_cnt = 0;
}
}
} else
bf = NULL;
if (bf == NULL) {
/* XXX should check which list, mgmt or otherwise */
DPRINTF(sc, ATH_DEBUG_XMIT, "%s: %s\n", __func__,
TAILQ_FIRST(&sc->sc_txbuf) == NULL ?
"out of xmit buffers" : "xmit buffer busy");
return NULL;
}
/* XXX TODO: should do this at buffer list initialisation */
/* XXX (then, ensure the buffer has the right flag set) */
bf->bf_flags = 0;
if (btype == ATH_BUFTYPE_MGMT)
bf->bf_flags |= ATH_BUF_MGMT;
else
bf->bf_flags &= (~ATH_BUF_MGMT);
/* Valid bf here; clear some basic fields */
bf->bf_next = NULL; /* XXX just to be sure */
bf->bf_last = NULL; /* XXX again, just to be sure */
bf->bf_comp = NULL; /* XXX again, just to be sure */
bzero(&bf->bf_state, sizeof(bf->bf_state));
/*
* Track the descriptor ID only if doing EDMA
*/
if (sc->sc_isedma) {
bf->bf_descid = sc->sc_txbuf_descid;
sc->sc_txbuf_descid++;
}
return bf;
}
/*
* When retrying a software frame, buffers marked ATH_BUF_BUSY
* can't be thrown back on the queue as they could still be
* in use by the hardware.
*
* This duplicates the buffer, or returns NULL.
*
* The descriptor is also copied but the link pointers and
* the DMA segments aren't copied; this frame should thus
* be again passed through the descriptor setup/chain routines
* so the link is correct.
*
* The caller must free the buffer using ath_freebuf().
*/
struct ath_buf *
ath_buf_clone(struct ath_softc *sc, struct ath_buf *bf)
{
struct ath_buf *tbf;
tbf = ath_getbuf(sc,
(bf->bf_flags & ATH_BUF_MGMT) ?
ATH_BUFTYPE_MGMT : ATH_BUFTYPE_NORMAL);
if (tbf == NULL)
return NULL; /* XXX failure? Why? */
/* Copy basics */
tbf->bf_next = NULL;
tbf->bf_nseg = bf->bf_nseg;
tbf->bf_flags = bf->bf_flags & ATH_BUF_FLAGS_CLONE;
tbf->bf_status = bf->bf_status;
tbf->bf_m = bf->bf_m;
tbf->bf_node = bf->bf_node;
KASSERT((bf->bf_node != NULL), ("%s: bf_node=NULL!", __func__));
/* will be setup by the chain/setup function */
tbf->bf_lastds = NULL;
/* for now, last == self */
tbf->bf_last = tbf;
tbf->bf_comp = bf->bf_comp;
/* NOTE: DMA segments will be setup by the setup/chain functions */
/* The caller has to re-init the descriptor + links */
/*
* Free the DMA mapping here, before we NULL the mbuf.
* We must only call bus_dmamap_unload() once per mbuf chain
* or behaviour is undefined.
*/
if (bf->bf_m != NULL) {
/*
* XXX is this POSTWRITE call required?
*/
bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
}
bf->bf_m = NULL;
bf->bf_node = NULL;
/* Copy state */
memcpy(&tbf->bf_state, &bf->bf_state, sizeof(bf->bf_state));
return tbf;
}
struct ath_buf *
ath_getbuf(struct ath_softc *sc, ath_buf_type_t btype)
{
struct ath_buf *bf;
ATH_TXBUF_LOCK(sc);
bf = _ath_getbuf_locked(sc, btype);
/*
* If a mgmt buffer was requested but we're out of those,
* try requesting a normal one.
*/
if (bf == NULL && btype == ATH_BUFTYPE_MGMT)
bf = _ath_getbuf_locked(sc, ATH_BUFTYPE_NORMAL);
ATH_TXBUF_UNLOCK(sc);
if (bf == NULL) {
struct ifnet *ifp = sc->sc_ifp;
DPRINTF(sc, ATH_DEBUG_XMIT, "%s: stop queue\n", __func__);
sc->sc_stats.ast_tx_qstop++;
IF_LOCK(&ifp->if_snd);
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IF_UNLOCK(&ifp->if_snd);
}
return bf;
}
static void
ath_qflush(struct ifnet *ifp)
{
/* XXX TODO */
}
/*
* Transmit a single frame.
*
* net80211 will free the node reference if the transmit
* fails, so don't free the node reference here.
*/
static int
ath_transmit(struct ifnet *ifp, struct mbuf *m)
{
struct ieee80211com *ic = ifp->if_l2com;
struct ath_softc *sc = ic->ic_ifp->if_softc;
struct ieee80211_node *ni;
struct mbuf *next;
struct ath_buf *bf;
ath_bufhead frags;
int retval = 0;
/*
* Tell the reset path that we're currently transmitting.
*/
ATH_PCU_LOCK(sc);
if (sc->sc_inreset_cnt > 0) {
DPRINTF(sc, ATH_DEBUG_XMIT,
"%s: sc_inreset_cnt > 0; bailing\n", __func__);
ATH_PCU_UNLOCK(sc);
IF_LOCK(&ifp->if_snd);
sc->sc_stats.ast_tx_qstop++;
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IF_UNLOCK(&ifp->if_snd);
ATH_KTR(sc, ATH_KTR_TX, 0, "ath_start_task: OACTIVE, finish");
return (ENOBUFS); /* XXX should be EINVAL or? */
}
sc->sc_txstart_cnt++;
ATH_PCU_UNLOCK(sc);
/* Wake the hardware up already */
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ATH_KTR(sc, ATH_KTR_TX, 0, "ath_transmit: start");
/*
* Grab the TX lock - it's ok to do this here; we haven't
* yet started transmitting.
*/
ATH_TX_LOCK(sc);
/*
* Node reference, if there's one.
*/
ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
/*
* Enforce how deep a node queue can get.
*
* XXX it would be nicer if we kept an mbuf queue per
* node and only whacked them into ath_bufs when we
* are ready to schedule some traffic from them.
* .. that may come later.
*
* XXX we should also track the per-node hardware queue
* depth so it is easy to limit the _SUM_ of the swq and
* hwq frames. Since we only schedule two HWQ frames
* at a time, this should be OK for now.
*/
if ((!(m->m_flags & M_EAPOL)) &&
(ATH_NODE(ni)->an_swq_depth > sc->sc_txq_node_maxdepth)) {
sc->sc_stats.ast_tx_nodeq_overflow++;
m_freem(m);
m = NULL;
retval = ENOBUFS;
goto finish;
}
/*
* Check how many TX buffers are available.
*
* If this is for non-EAPOL traffic, just leave some
* space free in order for buffer cloning and raw
* frame transmission to occur.
*
* If it's for EAPOL traffic, ignore this for now.
* Management traffic will be sent via the raw transmit
* method which bypasses this check.
*
* This is needed to ensure that EAPOL frames during
* (re) keying have a chance to go out.
*
* See kern/138379 for more information.
*/
if ((!(m->m_flags & M_EAPOL)) &&
(sc->sc_txbuf_cnt <= sc->sc_txq_data_minfree)) {
sc->sc_stats.ast_tx_nobuf++;
m_freem(m);
m = NULL;
retval = ENOBUFS;
goto finish;
}
/*
* Grab a TX buffer and associated resources.
*
* If it's an EAPOL frame, allocate a MGMT ath_buf.
* That way even with temporary buffer exhaustion due to
* the data path doesn't leave us without the ability
* to transmit management frames.
*
* Otherwise allocate a normal buffer.
*/
if (m->m_flags & M_EAPOL)
bf = ath_getbuf(sc, ATH_BUFTYPE_MGMT);
else
bf = ath_getbuf(sc, ATH_BUFTYPE_NORMAL);
if (bf == NULL) {
/*
* If we failed to allocate a buffer, fail.
*
* We shouldn't fail normally, due to the check
* above.
*/
sc->sc_stats.ast_tx_nobuf++;
IF_LOCK(&ifp->if_snd);
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IF_UNLOCK(&ifp->if_snd);
m_freem(m);
m = NULL;
retval = ENOBUFS;
goto finish;
}
/*
* At this point we have a buffer; so we need to free it
* if we hit any error conditions.
*/
/*
* Check for fragmentation. If this frame
* has been broken up verify we have enough
* buffers to send all the fragments so all
* go out or none...
*/
TAILQ_INIT(&frags);
if ((m->m_flags & M_FRAG) &&
!ath_txfrag_setup(sc, &frags, m, ni)) {
DPRINTF(sc, ATH_DEBUG_XMIT,
"%s: out of txfrag buffers\n", __func__);
sc->sc_stats.ast_tx_nofrag++;
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
ath_freetx(m);
goto bad;
}
/*
* At this point if we have any TX fragments, then we will
* have bumped the node reference once for each of those.
*/
/*
* XXX Is there anything actually _enforcing_ that the
* fragments are being transmitted in one hit, rather than
* being interleaved with other transmissions on that
* hardware queue?
*
* The ATH TX output lock is the only thing serialising this
* right now.
*/
/*
* Calculate the "next fragment" length field in ath_buf
* in order to let the transmit path know enough about
* what to next write to the hardware.
*/
if (m->m_flags & M_FRAG) {
struct ath_buf *fbf = bf;
struct ath_buf *n_fbf = NULL;
struct mbuf *fm = m->m_nextpkt;
/*
* We need to walk the list of fragments and set
* the next size to the following buffer.
* However, the first buffer isn't in the frag
* list, so we have to do some gymnastics here.
*/
TAILQ_FOREACH(n_fbf, &frags, bf_list) {
fbf->bf_nextfraglen = fm->m_pkthdr.len;
fbf = n_fbf;
fm = fm->m_nextpkt;
}
}
/*
* Bump the ifp output counter.
*
* XXX should use atomics?
*/
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
nextfrag:
/*
* Pass the frame to the h/w for transmission.
* Fragmented frames have each frag chained together
* with m_nextpkt. We know there are sufficient ath_buf's
* to send all the frags because of work done by
* ath_txfrag_setup. We leave m_nextpkt set while
* calling ath_tx_start so it can use it to extend the
* the tx duration to cover the subsequent frag and
* so it can reclaim all the mbufs in case of an error;
* ath_tx_start clears m_nextpkt once it commits to
* handing the frame to the hardware.
*
* Note: if this fails, then the mbufs are freed but
* not the node reference.
*/
next = m->m_nextpkt;
if (ath_tx_start(sc, ni, bf, m)) {
bad:
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
reclaim:
bf->bf_m = NULL;
bf->bf_node = NULL;
ATH_TXBUF_LOCK(sc);
ath_returnbuf_head(sc, bf);
/*
* Free the rest of the node references and
* buffers for the fragment list.
*/
ath_txfrag_cleanup(sc, &frags, ni);
ATH_TXBUF_UNLOCK(sc);
retval = ENOBUFS;
goto finish;
}
/*
* Check here if the node is in power save state.
*/
ath_tx_update_tim(sc, ni, 1);
if (next != NULL) {
/*
* Beware of state changing between frags.
* XXX check sta power-save state?
*/
if (ni->ni_vap->iv_state != IEEE80211_S_RUN) {
DPRINTF(sc, ATH_DEBUG_XMIT,
"%s: flush fragmented packet, state %s\n",
__func__,
ieee80211_state_name[ni->ni_vap->iv_state]);
/* XXX dmamap */
ath_freetx(next);
goto reclaim;
}
m = next;
bf = TAILQ_FIRST(&frags);
KASSERT(bf != NULL, ("no buf for txfrag"));
TAILQ_REMOVE(&frags, bf, bf_list);
goto nextfrag;
}
/*
* Bump watchdog timer.
*/
sc->sc_wd_timer = 5;
finish:
ATH_TX_UNLOCK(sc);
/*
* Finished transmitting!
*/
ATH_PCU_LOCK(sc);
sc->sc_txstart_cnt--;
ATH_PCU_UNLOCK(sc);
/* Sleep the hardware if required */
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
ATH_KTR(sc, ATH_KTR_TX, 0, "ath_transmit: finished");
return (retval);
}
static int
ath_media_change(struct ifnet *ifp)
{
int error = ieee80211_media_change(ifp);
/* NB: only the fixed rate can change and that doesn't need a reset */
return (error == ENETRESET ? 0 : error);
}
/*
* Block/unblock tx+rx processing while a key change is done.
* We assume the caller serializes key management operations
* so we only need to worry about synchronization with other
* uses that originate in the driver.
*/
static void
ath_key_update_begin(struct ieee80211vap *vap)
{
struct ifnet *ifp = vap->iv_ic->ic_ifp;
struct ath_softc *sc = ifp->if_softc;
DPRINTF(sc, ATH_DEBUG_KEYCACHE, "%s:\n", __func__);
taskqueue_block(sc->sc_tq);
}
static void
ath_key_update_end(struct ieee80211vap *vap)
{
struct ifnet *ifp = vap->iv_ic->ic_ifp;
struct ath_softc *sc = ifp->if_softc;
DPRINTF(sc, ATH_DEBUG_KEYCACHE, "%s:\n", __func__);
taskqueue_unblock(sc->sc_tq);
}
static void
ath_update_promisc(struct ifnet *ifp)
{
struct ath_softc *sc = ifp->if_softc;
u_int32_t rfilt;
/* configure rx filter */
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
rfilt = ath_calcrxfilter(sc);
ath_hal_setrxfilter(sc->sc_ah, rfilt);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
DPRINTF(sc, ATH_DEBUG_MODE, "%s: RX filter 0x%x\n", __func__, rfilt);
}
/*
* Driver-internal mcast update call.
*
* Assumes the hardware is already awake.
*/
static void
ath_update_mcast_hw(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
u_int32_t mfilt[2];
/* calculate and install multicast filter */
if ((ifp->if_flags & IFF_ALLMULTI) == 0) {
struct ifmultiaddr *ifma;
/*
* Merge multicast addresses to form the hardware filter.
*/
mfilt[0] = mfilt[1] = 0;
if_maddr_rlock(ifp); /* XXX need some fiddling to remove? */
TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
caddr_t dl;
u_int32_t val;
u_int8_t pos;
/* calculate XOR of eight 6bit values */
dl = LLADDR((struct sockaddr_dl *) ifma->ifma_addr);
val = LE_READ_4(dl + 0);
pos = (val >> 18) ^ (val >> 12) ^ (val >> 6) ^ val;
val = LE_READ_4(dl + 3);
pos ^= (val >> 18) ^ (val >> 12) ^ (val >> 6) ^ val;
pos &= 0x3f;
mfilt[pos / 32] |= (1 << (pos % 32));
}
if_maddr_runlock(ifp);
} else
mfilt[0] = mfilt[1] = ~0;
ath_hal_setmcastfilter(sc->sc_ah, mfilt[0], mfilt[1]);
DPRINTF(sc, ATH_DEBUG_MODE, "%s: MC filter %08x:%08x\n",
__func__, mfilt[0], mfilt[1]);
}
/*
* Called from the net80211 layer - force the hardware
* awake before operating.
*/
static void
ath_update_mcast(struct ifnet *ifp)
{
struct ath_softc *sc = ifp->if_softc;
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ath_update_mcast_hw(sc);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
}
void
ath_mode_init(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ath_hal *ah = sc->sc_ah;
u_int32_t rfilt;
/* configure rx filter */
rfilt = ath_calcrxfilter(sc);
ath_hal_setrxfilter(ah, rfilt);
/* configure operational mode */
ath_hal_setopmode(ah);
DPRINTF(sc, ATH_DEBUG_STATE | ATH_DEBUG_MODE,
"%s: ah=%p, ifp=%p, if_addr=%p\n",
__func__,
ah,
ifp,
(ifp == NULL) ? NULL : ifp->if_addr);
/* handle any link-level address change */
ath_hal_setmac(ah, IF_LLADDR(ifp));
/* calculate and install multicast filter */
ath_update_mcast_hw(sc);
}
/*
* Set the slot time based on the current setting.
*/
void
ath_setslottime(struct ath_softc *sc)
{
struct ieee80211com *ic = sc->sc_ifp->if_l2com;
struct ath_hal *ah = sc->sc_ah;
u_int usec;
if (IEEE80211_IS_CHAN_HALF(ic->ic_curchan))
usec = 13;
else if (IEEE80211_IS_CHAN_QUARTER(ic->ic_curchan))
usec = 21;
else if (IEEE80211_IS_CHAN_ANYG(ic->ic_curchan)) {
/* honor short/long slot time only in 11g */
/* XXX shouldn't honor on pure g or turbo g channel */
if (ic->ic_flags & IEEE80211_F_SHSLOT)
usec = HAL_SLOT_TIME_9;
else
usec = HAL_SLOT_TIME_20;
} else
usec = HAL_SLOT_TIME_9;
DPRINTF(sc, ATH_DEBUG_RESET,
"%s: chan %u MHz flags 0x%x %s slot, %u usec\n",
__func__, ic->ic_curchan->ic_freq, ic->ic_curchan->ic_flags,
ic->ic_flags & IEEE80211_F_SHSLOT ? "short" : "long", usec);
/* Wake up the hardware first before updating the slot time */
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ath_hal_setslottime(ah, usec);
ath_power_restore_power_state(sc);
sc->sc_updateslot = OK;
ATH_UNLOCK(sc);
}
/*
* Callback from the 802.11 layer to update the
* slot time based on the current setting.
*/
static void
ath_updateslot(struct ifnet *ifp)
{
struct ath_softc *sc = ifp->if_softc;
struct ieee80211com *ic = ifp->if_l2com;
/*
* When not coordinating the BSS, change the hardware
* immediately. For other operation we defer the change
* until beacon updates have propagated to the stations.
*
* XXX sc_updateslot isn't changed behind a lock?
*/
if (ic->ic_opmode == IEEE80211_M_HOSTAP ||
ic->ic_opmode == IEEE80211_M_MBSS)
sc->sc_updateslot = UPDATE;
else
ath_setslottime(sc);
}
/*
* Append the contents of src to dst; both queues
* are assumed to be locked.
*/
void
ath_txqmove(struct ath_txq *dst, struct ath_txq *src)
{
ATH_TXQ_LOCK_ASSERT(src);
ATH_TXQ_LOCK_ASSERT(dst);
TAILQ_CONCAT(&dst->axq_q, &src->axq_q, bf_list);
dst->axq_link = src->axq_link;
src->axq_link = NULL;
dst->axq_depth += src->axq_depth;
dst->axq_aggr_depth += src->axq_aggr_depth;
src->axq_depth = 0;
src->axq_aggr_depth = 0;
}
/*
* Reset the hardware, with no loss.
*
* This can't be used for a general case reset.
*/
static void
ath_reset_proc(void *arg, int pending)
{
struct ath_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
#if 0
if_printf(ifp, "%s: resetting\n", __func__);
#endif
ath_reset(ifp, ATH_RESET_NOLOSS);
}
/*
* Reset the hardware after detecting beacons have stopped.
*/
static void
ath_bstuck_proc(void *arg, int pending)
{
struct ath_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
uint32_t hangs = 0;
if (ath_hal_gethangstate(sc->sc_ah, 0xff, &hangs) && hangs != 0)
if_printf(ifp, "bb hang detected (0x%x)\n", hangs);
#ifdef ATH_DEBUG_ALQ
if (if_ath_alq_checkdebug(&sc->sc_alq, ATH_ALQ_STUCK_BEACON))
if_ath_alq_post(&sc->sc_alq, ATH_ALQ_STUCK_BEACON, 0, NULL);
#endif
if_printf(ifp, "stuck beacon; resetting (bmiss count %u)\n",
sc->sc_bmisscount);
sc->sc_stats.ast_bstuck++;
/*
* This assumes that there's no simultaneous channel mode change
* occuring.
*/
ath_reset(ifp, ATH_RESET_NOLOSS);
}
static void
ath_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
{
bus_addr_t *paddr = (bus_addr_t*) arg;
KASSERT(error == 0, ("error %u on bus_dma callback", error));
*paddr = segs->ds_addr;
}
/*
* Allocate the descriptors and appropriate DMA tag/setup.
*
* For some situations (eg EDMA TX completion), there isn't a requirement
* for the ath_buf entries to be allocated.
*/
int
ath_descdma_alloc_desc(struct ath_softc *sc,
struct ath_descdma *dd, ath_bufhead *head,
const char *name, int ds_size, int ndesc)
{
#define DS2PHYS(_dd, _ds) \
((_dd)->dd_desc_paddr + ((caddr_t)(_ds) - (caddr_t)(_dd)->dd_desc))
#define ATH_DESC_4KB_BOUND_CHECK(_daddr, _len) \
((((u_int32_t)(_daddr) & 0xFFF) > (0x1000 - (_len))) ? 1 : 0)
struct ifnet *ifp = sc->sc_ifp;
int error;
dd->dd_descsize = ds_size;
DPRINTF(sc, ATH_DEBUG_RESET,
"%s: %s DMA: %u desc, %d bytes per descriptor\n",
__func__, name, ndesc, dd->dd_descsize);
dd->dd_name = name;
dd->dd_desc_len = dd->dd_descsize * ndesc;
/*
* Merlin work-around:
* Descriptors that cross the 4KB boundary can't be used.
* Assume one skipped descriptor per 4KB page.
*/
if (! ath_hal_split4ktrans(sc->sc_ah)) {
int numpages = dd->dd_desc_len / 4096;
dd->dd_desc_len += ds_size * numpages;
}
/*
* Setup DMA descriptor area.
*
* BUS_DMA_ALLOCNOW is not used; we never use bounce
* buffers for the descriptors themselves.
*/
error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
PAGE_SIZE, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
dd->dd_desc_len, /* maxsize */
1, /* nsegments */
dd->dd_desc_len, /* maxsegsize */
0, /* flags */
NULL, /* lockfunc */
NULL, /* lockarg */
&dd->dd_dmat);
if (error != 0) {
if_printf(ifp, "cannot allocate %s DMA tag\n", dd->dd_name);
return error;
}
/* allocate descriptors */
error = bus_dmamem_alloc(dd->dd_dmat, (void**) &dd->dd_desc,
BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
&dd->dd_dmamap);
if (error != 0) {
if_printf(ifp, "unable to alloc memory for %u %s descriptors, "
"error %u\n", ndesc, dd->dd_name, error);
goto fail1;
}
error = bus_dmamap_load(dd->dd_dmat, dd->dd_dmamap,
dd->dd_desc, dd->dd_desc_len,
ath_load_cb, &dd->dd_desc_paddr,
BUS_DMA_NOWAIT);
if (error != 0) {
if_printf(ifp, "unable to map %s descriptors, error %u\n",
dd->dd_name, error);
goto fail2;
}
DPRINTF(sc, ATH_DEBUG_RESET, "%s: %s DMA map: %p (%lu) -> %p (%lu)\n",
__func__, dd->dd_name, (uint8_t *) dd->dd_desc,
(u_long) dd->dd_desc_len, (caddr_t) dd->dd_desc_paddr,
/*XXX*/ (u_long) dd->dd_desc_len);
return (0);
fail2:
bus_dmamem_free(dd->dd_dmat, dd->dd_desc, dd->dd_dmamap);
fail1:
bus_dma_tag_destroy(dd->dd_dmat);
memset(dd, 0, sizeof(*dd));
return error;
#undef DS2PHYS
#undef ATH_DESC_4KB_BOUND_CHECK
}
int
ath_descdma_setup(struct ath_softc *sc,
struct ath_descdma *dd, ath_bufhead *head,
const char *name, int ds_size, int nbuf, int ndesc)
{
#define DS2PHYS(_dd, _ds) \
((_dd)->dd_desc_paddr + ((caddr_t)(_ds) - (caddr_t)(_dd)->dd_desc))
#define ATH_DESC_4KB_BOUND_CHECK(_daddr, _len) \
((((u_int32_t)(_daddr) & 0xFFF) > (0x1000 - (_len))) ? 1 : 0)
struct ifnet *ifp = sc->sc_ifp;
uint8_t *ds;
struct ath_buf *bf;
int i, bsize, error;
/* Allocate descriptors */
error = ath_descdma_alloc_desc(sc, dd, head, name, ds_size,
nbuf * ndesc);
/* Assume any errors during allocation were dealt with */
if (error != 0) {
return (error);
}
ds = (uint8_t *) dd->dd_desc;
/* allocate rx buffers */
bsize = sizeof(struct ath_buf) * nbuf;
bf = malloc(bsize, M_ATHDEV, M_NOWAIT | M_ZERO);
if (bf == NULL) {
if_printf(ifp, "malloc of %s buffers failed, size %u\n",
dd->dd_name, bsize);
goto fail3;
}
dd->dd_bufptr = bf;
TAILQ_INIT(head);
for (i = 0; i < nbuf; i++, bf++, ds += (ndesc * dd->dd_descsize)) {
bf->bf_desc = (struct ath_desc *) ds;
bf->bf_daddr = DS2PHYS(dd, ds);
if (! ath_hal_split4ktrans(sc->sc_ah)) {
/*
* Merlin WAR: Skip descriptor addresses which
* cause 4KB boundary crossing along any point
* in the descriptor.
*/
if (ATH_DESC_4KB_BOUND_CHECK(bf->bf_daddr,
dd->dd_descsize)) {
/* Start at the next page */
ds += 0x1000 - (bf->bf_daddr & 0xFFF);
bf->bf_desc = (struct ath_desc *) ds;
bf->bf_daddr = DS2PHYS(dd, ds);
}
}
error = bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
&bf->bf_dmamap);
if (error != 0) {
if_printf(ifp, "unable to create dmamap for %s "
"buffer %u, error %u\n", dd->dd_name, i, error);
ath_descdma_cleanup(sc, dd, head);
return error;
}
bf->bf_lastds = bf->bf_desc; /* Just an initial value */
TAILQ_INSERT_TAIL(head, bf, bf_list);
}
/*
* XXX TODO: ensure that ds doesn't overflow the descriptor
* allocation otherwise weird stuff will occur and crash your
* machine.
*/
return 0;
/* XXX this should likely just call ath_descdma_cleanup() */
fail3:
bus_dmamap_unload(dd->dd_dmat, dd->dd_dmamap);
bus_dmamem_free(dd->dd_dmat, dd->dd_desc, dd->dd_dmamap);
bus_dma_tag_destroy(dd->dd_dmat);
memset(dd, 0, sizeof(*dd));
return error;
#undef DS2PHYS
#undef ATH_DESC_4KB_BOUND_CHECK
}
/*
* Allocate ath_buf entries but no descriptor contents.
*
* This is for RX EDMA where the descriptors are the header part of
* the RX buffer.
*/
int
ath_descdma_setup_rx_edma(struct ath_softc *sc,
struct ath_descdma *dd, ath_bufhead *head,
const char *name, int nbuf, int rx_status_len)
{
struct ifnet *ifp = sc->sc_ifp;
struct ath_buf *bf;
int i, bsize, error;
DPRINTF(sc, ATH_DEBUG_RESET, "%s: %s DMA: %u buffers\n",
__func__, name, nbuf);
dd->dd_name = name;
/*
* This is (mostly) purely for show. We're not allocating any actual
* descriptors here as EDMA RX has the descriptor be part
* of the RX buffer.
*
* However, dd_desc_len is used by ath_descdma_free() to determine
* whether we have already freed this DMA mapping.
*/
dd->dd_desc_len = rx_status_len * nbuf;
dd->dd_descsize = rx_status_len;
/* allocate rx buffers */
bsize = sizeof(struct ath_buf) * nbuf;
bf = malloc(bsize, M_ATHDEV, M_NOWAIT | M_ZERO);
if (bf == NULL) {
if_printf(ifp, "malloc of %s buffers failed, size %u\n",
dd->dd_name, bsize);
error = ENOMEM;
goto fail3;
}
dd->dd_bufptr = bf;
TAILQ_INIT(head);
for (i = 0; i < nbuf; i++, bf++) {
bf->bf_desc = NULL;
bf->bf_daddr = 0;
bf->bf_lastds = NULL; /* Just an initial value */
error = bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
&bf->bf_dmamap);
if (error != 0) {
if_printf(ifp, "unable to create dmamap for %s "
"buffer %u, error %u\n", dd->dd_name, i, error);
ath_descdma_cleanup(sc, dd, head);
return error;
}
TAILQ_INSERT_TAIL(head, bf, bf_list);
}
return 0;
fail3:
memset(dd, 0, sizeof(*dd));
return error;
}
void
ath_descdma_cleanup(struct ath_softc *sc,
struct ath_descdma *dd, ath_bufhead *head)
{
struct ath_buf *bf;
struct ieee80211_node *ni;
int do_warning = 0;
if (dd->dd_dmamap != 0) {
bus_dmamap_unload(dd->dd_dmat, dd->dd_dmamap);
bus_dmamem_free(dd->dd_dmat, dd->dd_desc, dd->dd_dmamap);
bus_dma_tag_destroy(dd->dd_dmat);
}
if (head != NULL) {
TAILQ_FOREACH(bf, head, bf_list) {
if (bf->bf_m) {
/*
* XXX warn if there's buffers here.
* XXX it should have been freed by the
* owner!
*/
if (do_warning == 0) {
do_warning = 1;
device_printf(sc->sc_dev,
"%s: %s: mbuf should've been"
" unmapped/freed!\n",
__func__,
dd->dd_name);
}
bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
m_freem(bf->bf_m);
bf->bf_m = NULL;
}
if (bf->bf_dmamap != NULL) {
bus_dmamap_destroy(sc->sc_dmat, bf->bf_dmamap);
bf->bf_dmamap = NULL;
}
ni = bf->bf_node;
bf->bf_node = NULL;
if (ni != NULL) {
/*
* Reclaim node reference.
*/
ieee80211_free_node(ni);
}
}
}
if (head != NULL)
TAILQ_INIT(head);
if (dd->dd_bufptr != NULL)
free(dd->dd_bufptr, M_ATHDEV);
memset(dd, 0, sizeof(*dd));
}
static int
ath_desc_alloc(struct ath_softc *sc)
{
int error;
error = ath_descdma_setup(sc, &sc->sc_txdma, &sc->sc_txbuf,
"tx", sc->sc_tx_desclen, ath_txbuf, ATH_MAX_SCATTER);
if (error != 0) {
return error;
}
sc->sc_txbuf_cnt = ath_txbuf;
error = ath_descdma_setup(sc, &sc->sc_txdma_mgmt, &sc->sc_txbuf_mgmt,
"tx_mgmt", sc->sc_tx_desclen, ath_txbuf_mgmt,
ATH_TXDESC);
if (error != 0) {
ath_descdma_cleanup(sc, &sc->sc_txdma, &sc->sc_txbuf);
return error;
}
/*
* XXX mark txbuf_mgmt frames with ATH_BUF_MGMT, so the
* flag doesn't have to be set in ath_getbuf_locked().
*/
error = ath_descdma_setup(sc, &sc->sc_bdma, &sc->sc_bbuf,
"beacon", sc->sc_tx_desclen, ATH_BCBUF, 1);
if (error != 0) {
ath_descdma_cleanup(sc, &sc->sc_txdma, &sc->sc_txbuf);
ath_descdma_cleanup(sc, &sc->sc_txdma_mgmt,
&sc->sc_txbuf_mgmt);
return error;
}
return 0;
}
static void
ath_desc_free(struct ath_softc *sc)
{
if (sc->sc_bdma.dd_desc_len != 0)
ath_descdma_cleanup(sc, &sc->sc_bdma, &sc->sc_bbuf);
if (sc->sc_txdma.dd_desc_len != 0)
ath_descdma_cleanup(sc, &sc->sc_txdma, &sc->sc_txbuf);
if (sc->sc_txdma_mgmt.dd_desc_len != 0)
ath_descdma_cleanup(sc, &sc->sc_txdma_mgmt,
&sc->sc_txbuf_mgmt);
}
static struct ieee80211_node *
ath_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
{
struct ieee80211com *ic = vap->iv_ic;
struct ath_softc *sc = ic->ic_ifp->if_softc;
const size_t space = sizeof(struct ath_node) + sc->sc_rc->arc_space;
struct ath_node *an;
an = malloc(space, M_80211_NODE, M_NOWAIT|M_ZERO);
if (an == NULL) {
/* XXX stat+msg */
return NULL;
}
ath_rate_node_init(sc, an);
/* Setup the mutex - there's no associd yet so set the name to NULL */
snprintf(an->an_name, sizeof(an->an_name), "%s: node %p",
device_get_nameunit(sc->sc_dev), an);
mtx_init(&an->an_mtx, an->an_name, NULL, MTX_DEF);
/* XXX setup ath_tid */
ath_tx_tid_init(sc, an);
DPRINTF(sc, ATH_DEBUG_NODE, "%s: %6D: an %p\n", __func__, mac, ":", an);
return &an->an_node;
}
static void
ath_node_cleanup(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
struct ath_softc *sc = ic->ic_ifp->if_softc;
DPRINTF(sc, ATH_DEBUG_NODE, "%s: %6D: an %p\n", __func__,
ni->ni_macaddr, ":", ATH_NODE(ni));
/* Cleanup ath_tid, free unused bufs, unlink bufs in TXQ */
ath_tx_node_flush(sc, ATH_NODE(ni));
ath_rate_node_cleanup(sc, ATH_NODE(ni));
sc->sc_node_cleanup(ni);
}
static void
ath_node_free(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
struct ath_softc *sc = ic->ic_ifp->if_softc;
DPRINTF(sc, ATH_DEBUG_NODE, "%s: %6D: an %p\n", __func__,
ni->ni_macaddr, ":", ATH_NODE(ni));
mtx_destroy(&ATH_NODE(ni)->an_mtx);
sc->sc_node_free(ni);
}
static void
ath_node_getsignal(const struct ieee80211_node *ni, int8_t *rssi, int8_t *noise)
{
struct ieee80211com *ic = ni->ni_ic;
struct ath_softc *sc = ic->ic_ifp->if_softc;
struct ath_hal *ah = sc->sc_ah;
*rssi = ic->ic_node_getrssi(ni);
if (ni->ni_chan != IEEE80211_CHAN_ANYC)
*noise = ath_hal_getchannoise(ah, ni->ni_chan);
else
*noise = -95; /* nominally correct */
}
/*
* Set the default antenna.
*/
void
ath_setdefantenna(struct ath_softc *sc, u_int antenna)
{
struct ath_hal *ah = sc->sc_ah;
/* XXX block beacon interrupts */
ath_hal_setdefantenna(ah, antenna);
if (sc->sc_defant != antenna)
sc->sc_stats.ast_ant_defswitch++;
sc->sc_defant = antenna;
sc->sc_rxotherant = 0;
}
static void
ath_txq_init(struct ath_softc *sc, struct ath_txq *txq, int qnum)
{
txq->axq_qnum = qnum;
txq->axq_ac = 0;
txq->axq_depth = 0;
txq->axq_aggr_depth = 0;
txq->axq_intrcnt = 0;
txq->axq_link = NULL;
txq->axq_softc = sc;
TAILQ_INIT(&txq->axq_q);
TAILQ_INIT(&txq->axq_tidq);
TAILQ_INIT(&txq->fifo.axq_q);
ATH_TXQ_LOCK_INIT(sc, txq);
}
/*
* Setup a h/w transmit queue.
*/
static struct ath_txq *
ath_txq_setup(struct ath_softc *sc, int qtype, int subtype)
{
#define N(a) (sizeof(a)/sizeof(a[0]))
struct ath_hal *ah = sc->sc_ah;
HAL_TXQ_INFO qi;
int qnum;
memset(&qi, 0, sizeof(qi));
qi.tqi_subtype = subtype;
qi.tqi_aifs = HAL_TXQ_USEDEFAULT;
qi.tqi_cwmin = HAL_TXQ_USEDEFAULT;
qi.tqi_cwmax = HAL_TXQ_USEDEFAULT;
/*
* Enable interrupts only for EOL and DESC conditions.
* We mark tx descriptors to receive a DESC interrupt
* when a tx queue gets deep; otherwise waiting for the
* EOL to reap descriptors. Note that this is done to
* reduce interrupt load and this only defers reaping
* descriptors, never transmitting frames. Aside from
* reducing interrupts this also permits more concurrency.
* The only potential downside is if the tx queue backs
* up in which case the top half of the kernel may backup
* due to a lack of tx descriptors.
*/
if (sc->sc_isedma)
qi.tqi_qflags = HAL_TXQ_TXEOLINT_ENABLE |
HAL_TXQ_TXOKINT_ENABLE;
else
qi.tqi_qflags = HAL_TXQ_TXEOLINT_ENABLE |
HAL_TXQ_TXDESCINT_ENABLE;
qnum = ath_hal_setuptxqueue(ah, qtype, &qi);
if (qnum == -1) {
/*
* NB: don't print a message, this happens
* normally on parts with too few tx queues
*/
return NULL;
}
if (qnum >= N(sc->sc_txq)) {
device_printf(sc->sc_dev,
"hal qnum %u out of range, max %zu!\n",
qnum, N(sc->sc_txq));
ath_hal_releasetxqueue(ah, qnum);
return NULL;
}
if (!ATH_TXQ_SETUP(sc, qnum)) {
ath_txq_init(sc, &sc->sc_txq[qnum], qnum);
sc->sc_txqsetup |= 1<<qnum;
}
return &sc->sc_txq[qnum];
#undef N
}
/*
* Setup a hardware data transmit queue for the specified
* access control. The hal may not support all requested
* queues in which case it will return a reference to a
* previously setup queue. We record the mapping from ac's
* to h/w queues for use by ath_tx_start and also track
* the set of h/w queues being used to optimize work in the
* transmit interrupt handler and related routines.
*/
static int
ath_tx_setup(struct ath_softc *sc, int ac, int haltype)
{
#define N(a) (sizeof(a)/sizeof(a[0]))
struct ath_txq *txq;
if (ac >= N(sc->sc_ac2q)) {
device_printf(sc->sc_dev, "AC %u out of range, max %zu!\n",
ac, N(sc->sc_ac2q));
return 0;
}
txq = ath_txq_setup(sc, HAL_TX_QUEUE_DATA, haltype);
if (txq != NULL) {
txq->axq_ac = ac;
sc->sc_ac2q[ac] = txq;
return 1;
} else
return 0;
#undef N
}
/*
* Update WME parameters for a transmit queue.
*/
static int
ath_txq_update(struct ath_softc *sc, int ac)
{
#define ATH_EXPONENT_TO_VALUE(v) ((1<<v)-1)
#define ATH_TXOP_TO_US(v) (v<<5)
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct ath_txq *txq = sc->sc_ac2q[ac];
struct wmeParams *wmep = &ic->ic_wme.wme_chanParams.cap_wmeParams[ac];
struct ath_hal *ah = sc->sc_ah;
HAL_TXQ_INFO qi;
ath_hal_gettxqueueprops(ah, txq->axq_qnum, &qi);
#ifdef IEEE80211_SUPPORT_TDMA
if (sc->sc_tdma) {
/*
* AIFS is zero so there's no pre-transmit wait. The
* burst time defines the slot duration and is configured
* through net80211. The QCU is setup to not do post-xmit
* back off, lockout all lower-priority QCU's, and fire
* off the DMA beacon alert timer which is setup based
* on the slot configuration.
*/
qi.tqi_qflags = HAL_TXQ_TXOKINT_ENABLE
| HAL_TXQ_TXERRINT_ENABLE
| HAL_TXQ_TXURNINT_ENABLE
| HAL_TXQ_TXEOLINT_ENABLE
| HAL_TXQ_DBA_GATED
| HAL_TXQ_BACKOFF_DISABLE
| HAL_TXQ_ARB_LOCKOUT_GLOBAL
;
qi.tqi_aifs = 0;
/* XXX +dbaprep? */
qi.tqi_readyTime = sc->sc_tdmaslotlen;
qi.tqi_burstTime = qi.tqi_readyTime;
} else {
#endif
/*
* XXX shouldn't this just use the default flags
* used in the previous queue setup?
*/
qi.tqi_qflags = HAL_TXQ_TXOKINT_ENABLE
| HAL_TXQ_TXERRINT_ENABLE
| HAL_TXQ_TXDESCINT_ENABLE
| HAL_TXQ_TXURNINT_ENABLE
| HAL_TXQ_TXEOLINT_ENABLE
;
qi.tqi_aifs = wmep->wmep_aifsn;
qi.tqi_cwmin = ATH_EXPONENT_TO_VALUE(wmep->wmep_logcwmin);
qi.tqi_cwmax = ATH_EXPONENT_TO_VALUE(wmep->wmep_logcwmax);
qi.tqi_readyTime = 0;
qi.tqi_burstTime = ATH_TXOP_TO_US(wmep->wmep_txopLimit);
#ifdef IEEE80211_SUPPORT_TDMA
}
#endif
DPRINTF(sc, ATH_DEBUG_RESET,
"%s: Q%u qflags 0x%x aifs %u cwmin %u cwmax %u burstTime %u\n",
__func__, txq->axq_qnum, qi.tqi_qflags,
qi.tqi_aifs, qi.tqi_cwmin, qi.tqi_cwmax, qi.tqi_burstTime);
if (!ath_hal_settxqueueprops(ah, txq->axq_qnum, &qi)) {
if_printf(ifp, "unable to update hardware queue "
"parameters for %s traffic!\n",
ieee80211_wme_acnames[ac]);
return 0;
} else {
ath_hal_resettxqueue(ah, txq->axq_qnum); /* push to h/w */
return 1;
}
#undef ATH_TXOP_TO_US
#undef ATH_EXPONENT_TO_VALUE
}
/*
* Callback from the 802.11 layer to update WME parameters.
*/
int
ath_wme_update(struct ieee80211com *ic)
{
struct ath_softc *sc = ic->ic_ifp->if_softc;
return !ath_txq_update(sc, WME_AC_BE) ||
!ath_txq_update(sc, WME_AC_BK) ||
!ath_txq_update(sc, WME_AC_VI) ||
!ath_txq_update(sc, WME_AC_VO) ? EIO : 0;
}
/*
* Reclaim resources for a setup queue.
*/
static void
ath_tx_cleanupq(struct ath_softc *sc, struct ath_txq *txq)
{
ath_hal_releasetxqueue(sc->sc_ah, txq->axq_qnum);
sc->sc_txqsetup &= ~(1<<txq->axq_qnum);
ATH_TXQ_LOCK_DESTROY(txq);
}
/*
* Reclaim all tx queue resources.
*/
static void
ath_tx_cleanup(struct ath_softc *sc)
{
int i;
ATH_TXBUF_LOCK_DESTROY(sc);
for (i = 0; i < HAL_NUM_TX_QUEUES; i++)
if (ATH_TXQ_SETUP(sc, i))
ath_tx_cleanupq(sc, &sc->sc_txq[i]);
}
/*
* Return h/w rate index for an IEEE rate (w/o basic rate bit)
* using the current rates in sc_rixmap.
*/
int
ath_tx_findrix(const struct ath_softc *sc, uint8_t rate)
{
int rix = sc->sc_rixmap[rate];
/* NB: return lowest rix for invalid rate */
return (rix == 0xff ? 0 : rix);
}
static void
ath_tx_update_stats(struct ath_softc *sc, struct ath_tx_status *ts,
struct ath_buf *bf)
{
struct ieee80211_node *ni = bf->bf_node;
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
int sr, lr, pri;
if (ts->ts_status == 0) {
u_int8_t txant = ts->ts_antenna;
sc->sc_stats.ast_ant_tx[txant]++;
sc->sc_ant_tx[txant]++;
if (ts->ts_finaltsi != 0)
sc->sc_stats.ast_tx_altrate++;
pri = M_WME_GETAC(bf->bf_m);
if (pri >= WME_AC_VO)
ic->ic_wme.wme_hipri_traffic++;
if ((bf->bf_state.bfs_txflags & HAL_TXDESC_NOACK) == 0)
ni->ni_inact = ni->ni_inact_reload;
} else {
if (ts->ts_status & HAL_TXERR_XRETRY)
sc->sc_stats.ast_tx_xretries++;
if (ts->ts_status & HAL_TXERR_FIFO)
sc->sc_stats.ast_tx_fifoerr++;
if (ts->ts_status & HAL_TXERR_FILT)
sc->sc_stats.ast_tx_filtered++;
if (ts->ts_status & HAL_TXERR_XTXOP)
sc->sc_stats.ast_tx_xtxop++;
if (ts->ts_status & HAL_TXERR_TIMER_EXPIRED)
sc->sc_stats.ast_tx_timerexpired++;
if (bf->bf_m->m_flags & M_FF)
sc->sc_stats.ast_ff_txerr++;
}
/* XXX when is this valid? */
if (ts->ts_flags & HAL_TX_DESC_CFG_ERR)
sc->sc_stats.ast_tx_desccfgerr++;
/*
* This can be valid for successful frame transmission!
* If there's a TX FIFO underrun during aggregate transmission,
* the MAC will pad the rest of the aggregate with delimiters.
* If a BA is returned, the frame is marked as "OK" and it's up
* to the TX completion code to notice which frames weren't
* successfully transmitted.
*/
if (ts->ts_flags & HAL_TX_DATA_UNDERRUN)
sc->sc_stats.ast_tx_data_underrun++;
if (ts->ts_flags & HAL_TX_DELIM_UNDERRUN)
sc->sc_stats.ast_tx_delim_underrun++;
sr = ts->ts_shortretry;
lr = ts->ts_longretry;
sc->sc_stats.ast_tx_shortretry += sr;
sc->sc_stats.ast_tx_longretry += lr;
}
/*
* The default completion. If fail is 1, this means
* "please don't retry the frame, and just return -1 status
* to the net80211 stack.
*/
void
ath_tx_default_comp(struct ath_softc *sc, struct ath_buf *bf, int fail)
{
struct ath_tx_status *ts = &bf->bf_status.ds_txstat;
int st;
if (fail == 1)
st = -1;
else
st = ((bf->bf_state.bfs_txflags & HAL_TXDESC_NOACK) == 0) ?
ts->ts_status : HAL_TXERR_XRETRY;
#if 0
if (bf->bf_state.bfs_dobaw)
device_printf(sc->sc_dev,
"%s: bf %p: seqno %d: dobaw should've been cleared!\n",
__func__,
bf,
SEQNO(bf->bf_state.bfs_seqno));
#endif
if (bf->bf_next != NULL)
device_printf(sc->sc_dev,
"%s: bf %p: seqno %d: bf_next not NULL!\n",
__func__,
bf,
SEQNO(bf->bf_state.bfs_seqno));
/*
* Check if the node software queue is empty; if so
* then clear the TIM.
*
* This needs to be done before the buffer is freed as
* otherwise the node reference will have been released
* and the node may not actually exist any longer.
*
* XXX I don't like this belonging here, but it's cleaner
* to do it here right now then all the other places
* where ath_tx_default_comp() is called.
*
* XXX TODO: during drain, ensure that the callback is
* being called so we get a chance to update the TIM.
*/
if (bf->bf_node) {
ATH_TX_LOCK(sc);
ath_tx_update_tim(sc, bf->bf_node, 0);
ATH_TX_UNLOCK(sc);
}
/*
* Do any tx complete callback. Note this must
* be done before releasing the node reference.
* This will free the mbuf, release the net80211
* node and recycle the ath_buf.
*/
ath_tx_freebuf(sc, bf, st);
}
/*
* Update rate control with the given completion status.
*/
void
ath_tx_update_ratectrl(struct ath_softc *sc, struct ieee80211_node *ni,
struct ath_rc_series *rc, struct ath_tx_status *ts, int frmlen,
int nframes, int nbad)
{
struct ath_node *an;
/* Only for unicast frames */
if (ni == NULL)
return;
an = ATH_NODE(ni);
ATH_NODE_UNLOCK_ASSERT(an);
if ((ts->ts_status & HAL_TXERR_FILT) == 0) {
ATH_NODE_LOCK(an);
ath_rate_tx_complete(sc, an, rc, ts, frmlen, nframes, nbad);
ATH_NODE_UNLOCK(an);
}
}
/*
* Process the completion of the given buffer.
*
* This calls the rate control update and then the buffer completion.
* This will either free the buffer or requeue it. In any case, the
* bf pointer should be treated as invalid after this function is called.
*/
void
ath_tx_process_buf_completion(struct ath_softc *sc, struct ath_txq *txq,
struct ath_tx_status *ts, struct ath_buf *bf)
{
struct ieee80211_node *ni = bf->bf_node;
ATH_TX_UNLOCK_ASSERT(sc);
ATH_TXQ_UNLOCK_ASSERT(txq);
/* If unicast frame, update general statistics */
if (ni != NULL) {
/* update statistics */
ath_tx_update_stats(sc, ts, bf);
}
/*
* Call the completion handler.
* The completion handler is responsible for
* calling the rate control code.
*
* Frames with no completion handler get the
* rate control code called here.
*/
if (bf->bf_comp == NULL) {
if ((ts->ts_status & HAL_TXERR_FILT) == 0 &&
(bf->bf_state.bfs_txflags & HAL_TXDESC_NOACK) == 0) {
/*
* XXX assume this isn't an aggregate
* frame.
*/
ath_tx_update_ratectrl(sc, ni,
bf->bf_state.bfs_rc, ts,
bf->bf_state.bfs_pktlen, 1,
(ts->ts_status == 0 ? 0 : 1));
}
ath_tx_default_comp(sc, bf, 0);
} else
bf->bf_comp(sc, bf, 0);
}
/*
* Process completed xmit descriptors from the specified queue.
* Kick the packet scheduler if needed. This can occur from this
* particular task.
*/
static int
ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq, int dosched)
{
struct ath_hal *ah = sc->sc_ah;
struct ath_buf *bf;
struct ath_desc *ds;
struct ath_tx_status *ts;
struct ieee80211_node *ni;
#ifdef IEEE80211_SUPPORT_SUPERG
struct ieee80211com *ic = sc->sc_ifp->if_l2com;
#endif /* IEEE80211_SUPPORT_SUPERG */
int nacked;
HAL_STATUS status;
DPRINTF(sc, ATH_DEBUG_TX_PROC, "%s: tx queue %u head %p link %p\n",
__func__, txq->axq_qnum,
(caddr_t)(uintptr_t) ath_hal_gettxbuf(sc->sc_ah, txq->axq_qnum),
txq->axq_link);
ATH_KTR(sc, ATH_KTR_TXCOMP, 4,
"ath_tx_processq: txq=%u head %p link %p depth %p",
txq->axq_qnum,
(caddr_t)(uintptr_t) ath_hal_gettxbuf(sc->sc_ah, txq->axq_qnum),
txq->axq_link,
txq->axq_depth);
nacked = 0;
for (;;) {
ATH_TXQ_LOCK(txq);
txq->axq_intrcnt = 0; /* reset periodic desc intr count */
bf = TAILQ_FIRST(&txq->axq_q);
if (bf == NULL) {
ATH_TXQ_UNLOCK(txq);
break;
}
ds = bf->bf_lastds; /* XXX must be setup correctly! */
ts = &bf->bf_status.ds_txstat;
status = ath_hal_txprocdesc(ah, ds, ts);
#ifdef ATH_DEBUG
if (sc->sc_debug & ATH_DEBUG_XMIT_DESC)
ath_printtxbuf(sc, bf, txq->axq_qnum, 0,
status == HAL_OK);
else if ((sc->sc_debug & ATH_DEBUG_RESET) && (dosched == 0))
ath_printtxbuf(sc, bf, txq->axq_qnum, 0,
status == HAL_OK);
#endif
#ifdef ATH_DEBUG_ALQ
if (if_ath_alq_checkdebug(&sc->sc_alq,
ATH_ALQ_EDMA_TXSTATUS)) {
if_ath_alq_post(&sc->sc_alq, ATH_ALQ_EDMA_TXSTATUS,
sc->sc_tx_statuslen,
(char *) ds);
}
#endif
if (status == HAL_EINPROGRESS) {
ATH_KTR(sc, ATH_KTR_TXCOMP, 3,
"ath_tx_processq: txq=%u, bf=%p ds=%p, HAL_EINPROGRESS",
txq->axq_qnum, bf, ds);
ATH_TXQ_UNLOCK(txq);
break;
}
ATH_TXQ_REMOVE(txq, bf, bf_list);
/*
* Sanity check.
*/
if (txq->axq_qnum != bf->bf_state.bfs_tx_queue) {
device_printf(sc->sc_dev,
"%s: TXQ=%d: bf=%p, bfs_tx_queue=%d\n",
__func__,
txq->axq_qnum,
bf,
bf->bf_state.bfs_tx_queue);
}
if (txq->axq_qnum != bf->bf_last->bf_state.bfs_tx_queue) {
device_printf(sc->sc_dev,
"%s: TXQ=%d: bf_last=%p, bfs_tx_queue=%d\n",
__func__,
txq->axq_qnum,
bf->bf_last,
bf->bf_last->bf_state.bfs_tx_queue);
}
#if 0
if (txq->axq_depth > 0) {
/*
* More frames follow. Mark the buffer busy
* so it's not re-used while the hardware may
* still re-read the link field in the descriptor.
*
* Use the last buffer in an aggregate as that
* is where the hardware may be - intermediate
* descriptors won't be "busy".
*/
bf->bf_last->bf_flags |= ATH_BUF_BUSY;
} else
txq->axq_link = NULL;
#else
bf->bf_last->bf_flags |= ATH_BUF_BUSY;
#endif
if (bf->bf_state.bfs_aggr)
txq->axq_aggr_depth--;
ni = bf->bf_node;
ATH_KTR(sc, ATH_KTR_TXCOMP, 5,
"ath_tx_processq: txq=%u, bf=%p, ds=%p, ni=%p, ts_status=0x%08x",
txq->axq_qnum, bf, ds, ni, ts->ts_status);
/*
* If unicast frame was ack'd update RSSI,
* including the last rx time used to
* workaround phantom bmiss interrupts.
*/
if (ni != NULL && ts->ts_status == 0 &&
((bf->bf_state.bfs_txflags & HAL_TXDESC_NOACK) == 0)) {
nacked++;
sc->sc_stats.ast_tx_rssi = ts->ts_rssi;
ATH_RSSI_LPF(sc->sc_halstats.ns_avgtxrssi,
ts->ts_rssi);
}
ATH_TXQ_UNLOCK(txq);
/*
* Update statistics and call completion
*/
ath_tx_process_buf_completion(sc, txq, ts, bf);
/* XXX at this point, bf and ni may be totally invalid */
}
#ifdef IEEE80211_SUPPORT_SUPERG
/*
* Flush fast-frame staging queue when traffic slows.
*/
if (txq->axq_depth <= 1)
ieee80211_ff_flush(ic, txq->axq_ac);
#endif
/* Kick the software TXQ scheduler */
if (dosched) {
ATH_TX_LOCK(sc);
ath_txq_sched(sc, txq);
ATH_TX_UNLOCK(sc);
}
ATH_KTR(sc, ATH_KTR_TXCOMP, 1,
"ath_tx_processq: txq=%u: done",
txq->axq_qnum);
return nacked;
}
#define TXQACTIVE(t, q) ( (t) & (1 << (q)))
/*
* Deferred processing of transmit interrupt; special-cased
* for a single hardware transmit queue (e.g. 5210 and 5211).
*/
static void
ath_tx_proc_q0(void *arg, int npending)
{
struct ath_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
uint32_t txqs;
ATH_PCU_LOCK(sc);
sc->sc_txproc_cnt++;
txqs = sc->sc_txq_active;
sc->sc_txq_active &= ~txqs;
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ATH_KTR(sc, ATH_KTR_TXCOMP, 1,
"ath_tx_proc_q0: txqs=0x%08x", txqs);
if (TXQACTIVE(txqs, 0) && ath_tx_processq(sc, &sc->sc_txq[0], 1))
/* XXX why is lastrx updated in tx code? */
sc->sc_lastrx = ath_hal_gettsf64(sc->sc_ah);
if (TXQACTIVE(txqs, sc->sc_cabq->axq_qnum))
ath_tx_processq(sc, sc->sc_cabq, 1);
IF_LOCK(&ifp->if_snd);
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
IF_UNLOCK(&ifp->if_snd);
sc->sc_wd_timer = 0;
if (sc->sc_softled)
ath_led_event(sc, sc->sc_txrix);
ATH_PCU_LOCK(sc);
sc->sc_txproc_cnt--;
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
ath_tx_kick(sc);
}
/*
* Deferred processing of transmit interrupt; special-cased
* for four hardware queues, 0-3 (e.g. 5212 w/ WME support).
*/
static void
ath_tx_proc_q0123(void *arg, int npending)
{
struct ath_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
int nacked;
uint32_t txqs;
ATH_PCU_LOCK(sc);
sc->sc_txproc_cnt++;
txqs = sc->sc_txq_active;
sc->sc_txq_active &= ~txqs;
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ATH_KTR(sc, ATH_KTR_TXCOMP, 1,
"ath_tx_proc_q0123: txqs=0x%08x", txqs);
/*
* Process each active queue.
*/
nacked = 0;
if (TXQACTIVE(txqs, 0))
nacked += ath_tx_processq(sc, &sc->sc_txq[0], 1);
if (TXQACTIVE(txqs, 1))
nacked += ath_tx_processq(sc, &sc->sc_txq[1], 1);
if (TXQACTIVE(txqs, 2))
nacked += ath_tx_processq(sc, &sc->sc_txq[2], 1);
if (TXQACTIVE(txqs, 3))
nacked += ath_tx_processq(sc, &sc->sc_txq[3], 1);
if (TXQACTIVE(txqs, sc->sc_cabq->axq_qnum))
ath_tx_processq(sc, sc->sc_cabq, 1);
if (nacked)
sc->sc_lastrx = ath_hal_gettsf64(sc->sc_ah);
IF_LOCK(&ifp->if_snd);
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
IF_UNLOCK(&ifp->if_snd);
sc->sc_wd_timer = 0;
if (sc->sc_softled)
ath_led_event(sc, sc->sc_txrix);
ATH_PCU_LOCK(sc);
sc->sc_txproc_cnt--;
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
ath_tx_kick(sc);
}
/*
* Deferred processing of transmit interrupt.
*/
static void
ath_tx_proc(void *arg, int npending)
{
struct ath_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
int i, nacked;
uint32_t txqs;
ATH_PCU_LOCK(sc);
sc->sc_txproc_cnt++;
txqs = sc->sc_txq_active;
sc->sc_txq_active &= ~txqs;
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ATH_KTR(sc, ATH_KTR_TXCOMP, 1, "ath_tx_proc: txqs=0x%08x", txqs);
/*
* Process each active queue.
*/
nacked = 0;
for (i = 0; i < HAL_NUM_TX_QUEUES; i++)
if (ATH_TXQ_SETUP(sc, i) && TXQACTIVE(txqs, i))
nacked += ath_tx_processq(sc, &sc->sc_txq[i], 1);
if (nacked)
sc->sc_lastrx = ath_hal_gettsf64(sc->sc_ah);
/* XXX check this inside of IF_LOCK? */
IF_LOCK(&ifp->if_snd);
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
IF_UNLOCK(&ifp->if_snd);
sc->sc_wd_timer = 0;
if (sc->sc_softled)
ath_led_event(sc, sc->sc_txrix);
ATH_PCU_LOCK(sc);
sc->sc_txproc_cnt--;
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
ath_tx_kick(sc);
}
#undef TXQACTIVE
/*
* Deferred processing of TXQ rescheduling.
*/
static void
ath_txq_sched_tasklet(void *arg, int npending)
{
struct ath_softc *sc = arg;
int i;
/* XXX is skipping ok? */
ATH_PCU_LOCK(sc);
#if 0
if (sc->sc_inreset_cnt > 0) {
device_printf(sc->sc_dev,
"%s: sc_inreset_cnt > 0; skipping\n", __func__);
ATH_PCU_UNLOCK(sc);
return;
}
#endif
sc->sc_txproc_cnt++;
ATH_PCU_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ATH_TX_LOCK(sc);
for (i = 0; i < HAL_NUM_TX_QUEUES; i++) {
if (ATH_TXQ_SETUP(sc, i)) {
ath_txq_sched(sc, &sc->sc_txq[i]);
}
}
ATH_TX_UNLOCK(sc);
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
ATH_PCU_LOCK(sc);
sc->sc_txproc_cnt--;
ATH_PCU_UNLOCK(sc);
}
void
ath_returnbuf_tail(struct ath_softc *sc, struct ath_buf *bf)
{
ATH_TXBUF_LOCK_ASSERT(sc);
if (bf->bf_flags & ATH_BUF_MGMT)
TAILQ_INSERT_TAIL(&sc->sc_txbuf_mgmt, bf, bf_list);
else {
TAILQ_INSERT_TAIL(&sc->sc_txbuf, bf, bf_list);
sc->sc_txbuf_cnt++;
if (sc->sc_txbuf_cnt > ath_txbuf) {
device_printf(sc->sc_dev,
"%s: sc_txbuf_cnt > %d?\n",
__func__,
ath_txbuf);
sc->sc_txbuf_cnt = ath_txbuf;
}
}
}
void
ath_returnbuf_head(struct ath_softc *sc, struct ath_buf *bf)
{
ATH_TXBUF_LOCK_ASSERT(sc);
if (bf->bf_flags & ATH_BUF_MGMT)
TAILQ_INSERT_HEAD(&sc->sc_txbuf_mgmt, bf, bf_list);
else {
TAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
sc->sc_txbuf_cnt++;
if (sc->sc_txbuf_cnt > ATH_TXBUF) {
device_printf(sc->sc_dev,
"%s: sc_txbuf_cnt > %d?\n",
__func__,
ATH_TXBUF);
sc->sc_txbuf_cnt = ATH_TXBUF;
}
}
}
/*
* Free the holding buffer if it exists
*/
void
ath_txq_freeholdingbuf(struct ath_softc *sc, struct ath_txq *txq)
{
ATH_TXBUF_UNLOCK_ASSERT(sc);
ATH_TXQ_LOCK_ASSERT(txq);
if (txq->axq_holdingbf == NULL)
return;
txq->axq_holdingbf->bf_flags &= ~ATH_BUF_BUSY;
ATH_TXBUF_LOCK(sc);
ath_returnbuf_tail(sc, txq->axq_holdingbf);
ATH_TXBUF_UNLOCK(sc);
txq->axq_holdingbf = NULL;
}
/*
* Add this buffer to the holding queue, freeing the previous
* one if it exists.
*/
static void
ath_txq_addholdingbuf(struct ath_softc *sc, struct ath_buf *bf)
{
struct ath_txq *txq;
txq = &sc->sc_txq[bf->bf_state.bfs_tx_queue];
ATH_TXBUF_UNLOCK_ASSERT(sc);
ATH_TXQ_LOCK_ASSERT(txq);
/* XXX assert ATH_BUF_BUSY is set */
/* XXX assert the tx queue is under the max number */
if (bf->bf_state.bfs_tx_queue > HAL_NUM_TX_QUEUES) {
device_printf(sc->sc_dev, "%s: bf=%p: invalid tx queue (%d)\n",
__func__,
bf,
bf->bf_state.bfs_tx_queue);
bf->bf_flags &= ~ATH_BUF_BUSY;
ath_returnbuf_tail(sc, bf);
return;
}
ath_txq_freeholdingbuf(sc, txq);
txq->axq_holdingbf = bf;
}
/*
* Return a buffer to the pool and update the 'busy' flag on the
* previous 'tail' entry.
*
* This _must_ only be called when the buffer is involved in a completed
* TX. The logic is that if it was part of an active TX, the previous
* buffer on the list is now not involved in a halted TX DMA queue, waiting
* for restart (eg for TDMA.)
*
* The caller must free the mbuf and recycle the node reference.
*
* XXX This method of handling busy / holding buffers is insanely stupid.
* It requires bf_state.bfs_tx_queue to be correctly assigned. It would
* be much nicer if buffers in the processq() methods would instead be
* always completed there (pushed onto a txq or ath_bufhead) so we knew
* exactly what hardware queue they came from in the first place.
*/
void
ath_freebuf(struct ath_softc *sc, struct ath_buf *bf)
{
struct ath_txq *txq;
txq = &sc->sc_txq[bf->bf_state.bfs_tx_queue];
KASSERT((bf->bf_node == NULL), ("%s: bf->bf_node != NULL\n", __func__));
KASSERT((bf->bf_m == NULL), ("%s: bf->bf_m != NULL\n", __func__));
/*
* If this buffer is busy, push it onto the holding queue.
*/
if (bf->bf_flags & ATH_BUF_BUSY) {
ATH_TXQ_LOCK(txq);
ath_txq_addholdingbuf(sc, bf);
ATH_TXQ_UNLOCK(txq);
return;
}
/*
* Not a busy buffer, so free normally
*/
ATH_TXBUF_LOCK(sc);
ath_returnbuf_tail(sc, bf);
ATH_TXBUF_UNLOCK(sc);
}
/*
* This is currently used by ath_tx_draintxq() and
* ath_tx_tid_free_pkts().
*
* It recycles a single ath_buf.
*/
void
ath_tx_freebuf(struct ath_softc *sc, struct ath_buf *bf, int status)
{
struct ieee80211_node *ni = bf->bf_node;
struct mbuf *m0 = bf->bf_m;
/*
* Make sure that we only sync/unload if there's an mbuf.
* If not (eg we cloned a buffer), the unload will have already
* occured.
*/
if (bf->bf_m != NULL) {
bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
}
bf->bf_node = NULL;
bf->bf_m = NULL;
/* Free the buffer, it's not needed any longer */
ath_freebuf(sc, bf);
/* Pass the buffer back to net80211 - completing it */
ieee80211_tx_complete(ni, m0, status);
}
static struct ath_buf *
ath_tx_draintxq_get_one(struct ath_softc *sc, struct ath_txq *txq)
{
struct ath_buf *bf;
ATH_TXQ_LOCK_ASSERT(txq);
/*
* Drain the FIFO queue first, then if it's
* empty, move to the normal frame queue.
*/
bf = TAILQ_FIRST(&txq->fifo.axq_q);
if (bf != NULL) {
/*
* Is it the last buffer in this set?
* Decrement the FIFO counter.
*/
if (bf->bf_flags & ATH_BUF_FIFOEND) {
if (txq->axq_fifo_depth == 0) {
device_printf(sc->sc_dev,
"%s: Q%d: fifo_depth=0, fifo.axq_depth=%d?\n",
__func__,
txq->axq_qnum,
txq->fifo.axq_depth);
} else
txq->axq_fifo_depth--;
}
ATH_TXQ_REMOVE(&txq->fifo, bf, bf_list);
return (bf);
}
/*
* Debugging!
*/
if (txq->axq_fifo_depth != 0 || txq->fifo.axq_depth != 0) {
device_printf(sc->sc_dev,
"%s: Q%d: fifo_depth=%d, fifo.axq_depth=%d\n",
__func__,
txq->axq_qnum,
txq->axq_fifo_depth,
txq->fifo.axq_depth);
}
/*
* Now drain the pending queue.
*/
bf = TAILQ_FIRST(&txq->axq_q);
if (bf == NULL) {
txq->axq_link = NULL;
return (NULL);
}
ATH_TXQ_REMOVE(txq, bf, bf_list);
return (bf);
}
void
ath_tx_draintxq(struct ath_softc *sc, struct ath_txq *txq)
{
#ifdef ATH_DEBUG
struct ath_hal *ah = sc->sc_ah;
#endif
struct ath_buf *bf;
u_int ix;
/*
* NB: this assumes output has been stopped and
* we do not need to block ath_tx_proc
*/
for (ix = 0;; ix++) {
ATH_TXQ_LOCK(txq);
bf = ath_tx_draintxq_get_one(sc, txq);
if (bf == NULL) {
ATH_TXQ_UNLOCK(txq);
break;
}
if (bf->bf_state.bfs_aggr)
txq->axq_aggr_depth--;
#ifdef ATH_DEBUG
if (sc->sc_debug & ATH_DEBUG_RESET) {
struct ieee80211com *ic = sc->sc_ifp->if_l2com;
int status = 0;
/*
* EDMA operation has a TX completion FIFO
* separate from the TX descriptor, so this
* method of checking the "completion" status
* is wrong.
*/
if (! sc->sc_isedma) {
status = (ath_hal_txprocdesc(ah,
bf->bf_lastds,
&bf->bf_status.ds_txstat) == HAL_OK);
}
ath_printtxbuf(sc, bf, txq->axq_qnum, ix, status);
ieee80211_dump_pkt(ic, mtod(bf->bf_m, const uint8_t *),
bf->bf_m->m_len, 0, -1);
}
#endif /* ATH_DEBUG */
/*
* Since we're now doing magic in the completion
* functions, we -must- call it for aggregation
* destinations or BAW tracking will get upset.
*/
/*
* Clear ATH_BUF_BUSY; the completion handler
* will free the buffer.
*/
ATH_TXQ_UNLOCK(txq);
bf->bf_flags &= ~ATH_BUF_BUSY;
if (bf->bf_comp)
bf->bf_comp(sc, bf, 1);
else
ath_tx_default_comp(sc, bf, 1);
}
/*
* Free the holding buffer if it exists
*/
ATH_TXQ_LOCK(txq);
ath_txq_freeholdingbuf(sc, txq);
ATH_TXQ_UNLOCK(txq);
/*
* Drain software queued frames which are on
* active TIDs.
*/
ath_tx_txq_drain(sc, txq);
}
static void
ath_tx_stopdma(struct ath_softc *sc, struct ath_txq *txq)
{
struct ath_hal *ah = sc->sc_ah;
ATH_TXQ_LOCK_ASSERT(txq);
DPRINTF(sc, ATH_DEBUG_RESET,
"%s: tx queue [%u] %p, active=%d, hwpending=%d, flags 0x%08x, "
"link %p, holdingbf=%p\n",
__func__,
txq->axq_qnum,
(caddr_t)(uintptr_t) ath_hal_gettxbuf(ah, txq->axq_qnum),
(int) (!! ath_hal_txqenabled(ah, txq->axq_qnum)),
(int) ath_hal_numtxpending(ah, txq->axq_qnum),
txq->axq_flags,
txq->axq_link,
txq->axq_holdingbf);
(void) ath_hal_stoptxdma(ah, txq->axq_qnum);
/* We've stopped TX DMA, so mark this as stopped. */
txq->axq_flags &= ~ATH_TXQ_PUTRUNNING;
#ifdef ATH_DEBUG
if ((sc->sc_debug & ATH_DEBUG_RESET)
&& (txq->axq_holdingbf != NULL)) {
ath_printtxbuf(sc, txq->axq_holdingbf, txq->axq_qnum, 0, 0);
}
#endif
}
int
ath_stoptxdma(struct ath_softc *sc)
{
struct ath_hal *ah = sc->sc_ah;
int i;
/* XXX return value */
if (sc->sc_invalid)
return 0;
if (!sc->sc_invalid) {
/* don't touch the hardware if marked invalid */
DPRINTF(sc, ATH_DEBUG_RESET, "%s: tx queue [%u] %p, link %p\n",
__func__, sc->sc_bhalq,
(caddr_t)(uintptr_t) ath_hal_gettxbuf(ah, sc->sc_bhalq),
NULL);
/* stop the beacon queue */
(void) ath_hal_stoptxdma(ah, sc->sc_bhalq);
/* Stop the data queues */
for (i = 0; i < HAL_NUM_TX_QUEUES; i++) {
if (ATH_TXQ_SETUP(sc, i)) {
ATH_TXQ_LOCK(&sc->sc_txq[i]);
ath_tx_stopdma(sc, &sc->sc_txq[i]);
ATH_TXQ_UNLOCK(&sc->sc_txq[i]);
}
}
}
return 1;
}
#ifdef ATH_DEBUG
void
ath_tx_dump(struct ath_softc *sc, struct ath_txq *txq)
{
struct ath_hal *ah = sc->sc_ah;
struct ath_buf *bf;
int i = 0;
if (! (sc->sc_debug & ATH_DEBUG_RESET))
return;
device_printf(sc->sc_dev, "%s: Q%d: begin\n",
__func__, txq->axq_qnum);
TAILQ_FOREACH(bf, &txq->axq_q, bf_list) {
ath_printtxbuf(sc, bf, txq->axq_qnum, i,
ath_hal_txprocdesc(ah, bf->bf_lastds,
&bf->bf_status.ds_txstat) == HAL_OK);
i++;
}
device_printf(sc->sc_dev, "%s: Q%d: end\n",
__func__, txq->axq_qnum);
}
#endif /* ATH_DEBUG */
/*
* Drain the transmit queues and reclaim resources.
*/
void
ath_legacy_tx_drain(struct ath_softc *sc, ATH_RESET_TYPE reset_type)
{
struct ath_hal *ah = sc->sc_ah;
struct ifnet *ifp = sc->sc_ifp;
int i;
struct ath_buf *bf_last;
(void) ath_stoptxdma(sc);
/*
* Dump the queue contents
*/
for (i = 0; i < HAL_NUM_TX_QUEUES; i++) {
/*
* XXX TODO: should we just handle the completed TX frames
* here, whether or not the reset is a full one or not?
*/
if (ATH_TXQ_SETUP(sc, i)) {
#ifdef ATH_DEBUG
if (sc->sc_debug & ATH_DEBUG_RESET)
ath_tx_dump(sc, &sc->sc_txq[i]);
#endif /* ATH_DEBUG */
if (reset_type == ATH_RESET_NOLOSS) {
ath_tx_processq(sc, &sc->sc_txq[i], 0);
ATH_TXQ_LOCK(&sc->sc_txq[i]);
/*
* Free the holding buffer; DMA is now
* stopped.
*/
ath_txq_freeholdingbuf(sc, &sc->sc_txq[i]);
/*
* Setup the link pointer to be the
* _last_ buffer/descriptor in the list.
* If there's nothing in the list, set it
* to NULL.
*/
bf_last = ATH_TXQ_LAST(&sc->sc_txq[i],
axq_q_s);
if (bf_last != NULL) {
ath_hal_gettxdesclinkptr(ah,
bf_last->bf_lastds,
&sc->sc_txq[i].axq_link);
} else {
sc->sc_txq[i].axq_link = NULL;
}
ATH_TXQ_UNLOCK(&sc->sc_txq[i]);
} else
ath_tx_draintxq(sc, &sc->sc_txq[i]);
}
}
#ifdef ATH_DEBUG
if (sc->sc_debug & ATH_DEBUG_RESET) {
struct ath_buf *bf = TAILQ_FIRST(&sc->sc_bbuf);
if (bf != NULL && bf->bf_m != NULL) {
ath_printtxbuf(sc, bf, sc->sc_bhalq, 0,
ath_hal_txprocdesc(ah, bf->bf_lastds,
&bf->bf_status.ds_txstat) == HAL_OK);
ieee80211_dump_pkt(ifp->if_l2com,
mtod(bf->bf_m, const uint8_t *), bf->bf_m->m_len,
0, -1);
}
}
#endif /* ATH_DEBUG */
IF_LOCK(&ifp->if_snd);
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
IF_UNLOCK(&ifp->if_snd);
sc->sc_wd_timer = 0;
}
/*
* Update internal state after a channel change.
*/
static void
ath_chan_change(struct ath_softc *sc, struct ieee80211_channel *chan)
{
enum ieee80211_phymode mode;
/*
* Change channels and update the h/w rate map
* if we're switching; e.g. 11a to 11b/g.
*/
mode = ieee80211_chan2mode(chan);
if (mode != sc->sc_curmode)
ath_setcurmode(sc, mode);
sc->sc_curchan = chan;
}
/*
* Set/change channels. If the channel is really being changed,
* it's done by resetting the chip. To accomplish this we must
* first cleanup any pending DMA, then restart stuff after a la
* ath_init.
*/
static int
ath_chan_set(struct ath_softc *sc, struct ieee80211_channel *chan)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct ath_hal *ah = sc->sc_ah;
int ret = 0;
/* Treat this as an interface reset */
ATH_PCU_UNLOCK_ASSERT(sc);
ATH_UNLOCK_ASSERT(sc);
/* (Try to) stop TX/RX from occuring */
taskqueue_block(sc->sc_tq);
ATH_PCU_LOCK(sc);
/* Disable interrupts */
ath_hal_intrset(ah, 0);
/* Stop new RX/TX/interrupt completion */
if (ath_reset_grablock(sc, 1) == 0) {
device_printf(sc->sc_dev, "%s: concurrent reset! Danger!\n",
__func__);
}
/* Stop pending RX/TX completion */
ath_txrx_stop_locked(sc);
ATH_PCU_UNLOCK(sc);
DPRINTF(sc, ATH_DEBUG_RESET, "%s: %u (%u MHz, flags 0x%x)\n",
__func__, ieee80211_chan2ieee(ic, chan),
chan->ic_freq, chan->ic_flags);
if (chan != sc->sc_curchan) {
HAL_STATUS status;
/*
* To switch channels clear any pending DMA operations;
* wait long enough for the RX fifo to drain, reset the
* hardware at the new frequency, and then re-enable
* the relevant bits of the h/w.
*/
#if 0
ath_hal_intrset(ah, 0); /* disable interrupts */
#endif
ath_stoprecv(sc, 1); /* turn off frame recv */
/*
* First, handle completed TX/RX frames.
*/
ath_rx_flush(sc);
ath_draintxq(sc, ATH_RESET_NOLOSS);
/*
* Next, flush the non-scheduled frames.
*/
ath_draintxq(sc, ATH_RESET_FULL); /* clear pending tx frames */
ath_update_chainmasks(sc, chan);
ath_hal_setchainmasks(sc->sc_ah, sc->sc_cur_txchainmask,
sc->sc_cur_rxchainmask);
if (!ath_hal_reset(ah, sc->sc_opmode, chan, AH_TRUE, &status)) {
if_printf(ifp, "%s: unable to reset "
"channel %u (%u MHz, flags 0x%x), hal status %u\n",
__func__, ieee80211_chan2ieee(ic, chan),
chan->ic_freq, chan->ic_flags, status);
ret = EIO;
goto finish;
}
sc->sc_diversity = ath_hal_getdiversity(ah);
ATH_RX_LOCK(sc);
sc->sc_rx_stopped = 1;
sc->sc_rx_resetted = 1;
ATH_RX_UNLOCK(sc);
/* Let DFS at it in case it's a DFS channel */
ath_dfs_radar_enable(sc, chan);
/* Let spectral at in case spectral is enabled */
ath_spectral_enable(sc, chan);
/*
* Let bluetooth coexistence at in case it's needed for this
* channel
*/
ath_btcoex_enable(sc, ic->ic_curchan);
/*
* If we're doing TDMA, enforce the TXOP limitation for chips
* that support it.
*/
if (sc->sc_hasenforcetxop && sc->sc_tdma)
ath_hal_setenforcetxop(sc->sc_ah, 1);
else
ath_hal_setenforcetxop(sc->sc_ah, 0);
/*
* Re-enable rx framework.
*/
if (ath_startrecv(sc) != 0) {
if_printf(ifp, "%s: unable to restart recv logic\n",
__func__);
ret = EIO;
goto finish;
}
/*
* Change channels and update the h/w rate map
* if we're switching; e.g. 11a to 11b/g.
*/
ath_chan_change(sc, chan);
/*
* Reset clears the beacon timers; reset them
* here if needed.
*/
if (sc->sc_beacons) { /* restart beacons */
#ifdef IEEE80211_SUPPORT_TDMA
if (sc->sc_tdma)
ath_tdma_config(sc, NULL);
else
#endif
ath_beacon_config(sc, NULL);
}
/*
* Re-enable interrupts.
*/
#if 0
ath_hal_intrset(ah, sc->sc_imask);
#endif
}
finish:
ATH_PCU_LOCK(sc);
sc->sc_inreset_cnt--;
/* XXX only do this if sc_inreset_cnt == 0? */
ath_hal_intrset(ah, sc->sc_imask);
ATH_PCU_UNLOCK(sc);
IF_LOCK(&ifp->if_snd);
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
IF_UNLOCK(&ifp->if_snd);
ath_txrx_start(sc);
/* XXX ath_start? */
return ret;
}
/*
* Periodically recalibrate the PHY to account
* for temperature/environment changes.
*/
static void
ath_calibrate(void *arg)
{
struct ath_softc *sc = arg;
struct ath_hal *ah = sc->sc_ah;
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
HAL_BOOL longCal, isCalDone = AH_TRUE;
HAL_BOOL aniCal, shortCal = AH_FALSE;
int nextcal;
ATH_LOCK_ASSERT(sc);
/*
* Force the hardware awake for ANI work.
*/
ath_power_set_power_state(sc, HAL_PM_AWAKE);
/* Skip trying to do this if we're in reset */
if (sc->sc_inreset_cnt)
goto restart;
if (ic->ic_flags & IEEE80211_F_SCAN) /* defer, off channel */
goto restart;
longCal = (ticks - sc->sc_lastlongcal >= ath_longcalinterval*hz);
aniCal = (ticks - sc->sc_lastani >= ath_anicalinterval*hz/1000);
if (sc->sc_doresetcal)
shortCal = (ticks - sc->sc_lastshortcal >= ath_shortcalinterval*hz/1000);
DPRINTF(sc, ATH_DEBUG_CALIBRATE, "%s: shortCal=%d; longCal=%d; aniCal=%d\n", __func__, shortCal, longCal, aniCal);
if (aniCal) {
sc->sc_stats.ast_ani_cal++;
sc->sc_lastani = ticks;
ath_hal_ani_poll(ah, sc->sc_curchan);
}
if (longCal) {
sc->sc_stats.ast_per_cal++;
sc->sc_lastlongcal = ticks;
if (ath_hal_getrfgain(ah) == HAL_RFGAIN_NEED_CHANGE) {
/*
* Rfgain is out of bounds, reset the chip
* to load new gain values.
*/
DPRINTF(sc, ATH_DEBUG_CALIBRATE,
"%s: rfgain change\n", __func__);
sc->sc_stats.ast_per_rfgain++;
sc->sc_resetcal = 0;
sc->sc_doresetcal = AH_TRUE;
taskqueue_enqueue(sc->sc_tq, &sc->sc_resettask);
callout_reset(&sc->sc_cal_ch, 1, ath_calibrate, sc);
ath_power_restore_power_state(sc);
return;
}
/*
* If this long cal is after an idle period, then
* reset the data collection state so we start fresh.
*/
if (sc->sc_resetcal) {
(void) ath_hal_calreset(ah, sc->sc_curchan);
sc->sc_lastcalreset = ticks;
sc->sc_lastshortcal = ticks;
sc->sc_resetcal = 0;
sc->sc_doresetcal = AH_TRUE;
}
}
/* Only call if we're doing a short/long cal, not for ANI calibration */
if (shortCal || longCal) {
isCalDone = AH_FALSE;
if (ath_hal_calibrateN(ah, sc->sc_curchan, longCal, &isCalDone)) {
if (longCal) {
/*
* Calibrate noise floor data again in case of change.
*/
ath_hal_process_noisefloor(ah);
}
} else {
DPRINTF(sc, ATH_DEBUG_ANY,
"%s: calibration of channel %u failed\n",
__func__, sc->sc_curchan->ic_freq);
sc->sc_stats.ast_per_calfail++;
}
if (shortCal)
sc->sc_lastshortcal = ticks;
}
if (!isCalDone) {
restart:
/*
* Use a shorter interval to potentially collect multiple
* data samples required to complete calibration. Once
* we're told the work is done we drop back to a longer
* interval between requests. We're more aggressive doing
* work when operating as an AP to improve operation right
* after startup.
*/
sc->sc_lastshortcal = ticks;
nextcal = ath_shortcalinterval*hz/1000;
if (sc->sc_opmode != HAL_M_HOSTAP)
nextcal *= 10;
sc->sc_doresetcal = AH_TRUE;
} else {
/* nextcal should be the shortest time for next event */
nextcal = ath_longcalinterval*hz;
if (sc->sc_lastcalreset == 0)
sc->sc_lastcalreset = sc->sc_lastlongcal;
else if (ticks - sc->sc_lastcalreset >= ath_resetcalinterval*hz)
sc->sc_resetcal = 1; /* setup reset next trip */
sc->sc_doresetcal = AH_FALSE;
}
/* ANI calibration may occur more often than short/long/resetcal */
if (ath_anicalinterval > 0)
nextcal = MIN(nextcal, ath_anicalinterval*hz/1000);
if (nextcal != 0) {
DPRINTF(sc, ATH_DEBUG_CALIBRATE, "%s: next +%u (%sisCalDone)\n",
__func__, nextcal, isCalDone ? "" : "!");
callout_reset(&sc->sc_cal_ch, nextcal, ath_calibrate, sc);
} else {
DPRINTF(sc, ATH_DEBUG_CALIBRATE, "%s: calibration disabled\n",
__func__);
/* NB: don't rearm timer */
}
/*
* Restore power state now that we're done.
*/
ath_power_restore_power_state(sc);
}
static void
ath_scan_start(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
struct ath_softc *sc = ifp->if_softc;
struct ath_hal *ah = sc->sc_ah;
u_int32_t rfilt;
/* XXX calibration timer? */
ATH_LOCK(sc);
sc->sc_scanning = 1;
sc->sc_syncbeacon = 0;
rfilt = ath_calcrxfilter(sc);
ATH_UNLOCK(sc);
ATH_PCU_LOCK(sc);
ath_hal_setrxfilter(ah, rfilt);
ath_hal_setassocid(ah, ifp->if_broadcastaddr, 0);
ATH_PCU_UNLOCK(sc);
DPRINTF(sc, ATH_DEBUG_STATE, "%s: RX filter 0x%x bssid %s aid 0\n",
__func__, rfilt, ether_sprintf(ifp->if_broadcastaddr));
}
static void
ath_scan_end(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
struct ath_softc *sc = ifp->if_softc;
struct ath_hal *ah = sc->sc_ah;
u_int32_t rfilt;
ATH_LOCK(sc);
sc->sc_scanning = 0;
rfilt = ath_calcrxfilter(sc);
ATH_UNLOCK(sc);
ATH_PCU_LOCK(sc);
ath_hal_setrxfilter(ah, rfilt);
ath_hal_setassocid(ah, sc->sc_curbssid, sc->sc_curaid);
ath_hal_process_noisefloor(ah);
ATH_PCU_UNLOCK(sc);
DPRINTF(sc, ATH_DEBUG_STATE, "%s: RX filter 0x%x bssid %s aid 0x%x\n",
__func__, rfilt, ether_sprintf(sc->sc_curbssid),
sc->sc_curaid);
}
#ifdef ATH_ENABLE_11N
/*
* For now, just do a channel change.
*
* Later, we'll go through the hard slog of suspending tx/rx, changing rate
* control state and resetting the hardware without dropping frames out
* of the queue.
*
* The unfortunate trouble here is making absolutely sure that the
* channel width change has propagated enough so the hardware
* absolutely isn't handed bogus frames for it's current operating
* mode. (Eg, 40MHz frames in 20MHz mode.) Since TX and RX can and
* does occur in parallel, we need to make certain we've blocked
* any further ongoing TX (and RX, that can cause raw TX)
* before we do this.
*/
static void
ath_update_chw(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
struct ath_softc *sc = ifp->if_softc;
DPRINTF(sc, ATH_DEBUG_STATE, "%s: called\n", __func__);
ath_set_channel(ic);
}
#endif /* ATH_ENABLE_11N */
static void
ath_set_channel(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
struct ath_softc *sc = ifp->if_softc;
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
(void) ath_chan_set(sc, ic->ic_curchan);
/*
* If we are returning to our bss channel then mark state
* so the next recv'd beacon's tsf will be used to sync the
* beacon timers. Note that since we only hear beacons in
* sta/ibss mode this has no effect in other operating modes.
*/
ATH_LOCK(sc);
if (!sc->sc_scanning && ic->ic_curchan == ic->ic_bsschan)
sc->sc_syncbeacon = 1;
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
}
/*
* Walk the vap list and check if there any vap's in RUN state.
*/
static int
ath_isanyrunningvaps(struct ieee80211vap *this)
{
struct ieee80211com *ic = this->iv_ic;
struct ieee80211vap *vap;
IEEE80211_LOCK_ASSERT(ic);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
if (vap != this && vap->iv_state >= IEEE80211_S_RUN)
return 1;
}
return 0;
}
static int
ath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
{
struct ieee80211com *ic = vap->iv_ic;
struct ath_softc *sc = ic->ic_ifp->if_softc;
struct ath_vap *avp = ATH_VAP(vap);
struct ath_hal *ah = sc->sc_ah;
struct ieee80211_node *ni = NULL;
int i, error, stamode;
u_int32_t rfilt;
int csa_run_transition = 0;
enum ieee80211_state ostate = vap->iv_state;
static const HAL_LED_STATE leds[] = {
HAL_LED_INIT, /* IEEE80211_S_INIT */
HAL_LED_SCAN, /* IEEE80211_S_SCAN */
HAL_LED_AUTH, /* IEEE80211_S_AUTH */
HAL_LED_ASSOC, /* IEEE80211_S_ASSOC */
HAL_LED_RUN, /* IEEE80211_S_CAC */
HAL_LED_RUN, /* IEEE80211_S_RUN */
HAL_LED_RUN, /* IEEE80211_S_CSA */
HAL_LED_RUN, /* IEEE80211_S_SLEEP */
};
DPRINTF(sc, ATH_DEBUG_STATE, "%s: %s -> %s\n", __func__,
ieee80211_state_name[ostate],
ieee80211_state_name[nstate]);
/*
* net80211 _should_ have the comlock asserted at this point.
* There are some comments around the calls to vap->iv_newstate
* which indicate that it (newstate) may end up dropping the
* lock. This and the subsequent lock assert check after newstate
* are an attempt to catch these and figure out how/why.
*/
IEEE80211_LOCK_ASSERT(ic);
/* Before we touch the hardware - wake it up */
ATH_LOCK(sc);
/*
* If the NIC is in anything other than SLEEP state,
* we need to ensure that self-generated frames are
* set for PWRMGT=0. Otherwise we may end up with
* strange situations.
*
* XXX TODO: is this actually the case? :-)
*/
if (nstate != IEEE80211_S_SLEEP)
ath_power_setselfgen(sc, HAL_PM_AWAKE);
/*
* Now, wake the thing up.
*/
ath_power_set_power_state(sc, HAL_PM_AWAKE);
/*
* And stop the calibration callout whilst we have
* ATH_LOCK held.
*/
callout_stop(&sc->sc_cal_ch);
ATH_UNLOCK(sc);
if (ostate == IEEE80211_S_CSA && nstate == IEEE80211_S_RUN)
csa_run_transition = 1;
ath_hal_setledstate(ah, leds[nstate]); /* set LED */
if (nstate == IEEE80211_S_SCAN) {
/*
* Scanning: turn off beacon miss and don't beacon.
* Mark beacon state so when we reach RUN state we'll
* [re]setup beacons. Unblock the task q thread so
* deferred interrupt processing is done.
*/
/* Ensure we stay awake during scan */
ATH_LOCK(sc);
ath_power_setselfgen(sc, HAL_PM_AWAKE);
ath_power_setpower(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
ath_hal_intrset(ah,
sc->sc_imask &~ (HAL_INT_SWBA | HAL_INT_BMISS));
sc->sc_imask &= ~(HAL_INT_SWBA | HAL_INT_BMISS);
sc->sc_beacons = 0;
taskqueue_unblock(sc->sc_tq);
}
ni = ieee80211_ref_node(vap->iv_bss);
rfilt = ath_calcrxfilter(sc);
stamode = (vap->iv_opmode == IEEE80211_M_STA ||
vap->iv_opmode == IEEE80211_M_AHDEMO ||
vap->iv_opmode == IEEE80211_M_IBSS);
/*
* XXX Dont need to do this (and others) if we've transitioned
* from SLEEP->RUN.
*/
if (stamode && nstate == IEEE80211_S_RUN) {
sc->sc_curaid = ni->ni_associd;
IEEE80211_ADDR_COPY(sc->sc_curbssid, ni->ni_bssid);
ath_hal_setassocid(ah, sc->sc_curbssid, sc->sc_curaid);
}
DPRINTF(sc, ATH_DEBUG_STATE, "%s: RX filter 0x%x bssid %s aid 0x%x\n",
__func__, rfilt, ether_sprintf(sc->sc_curbssid), sc->sc_curaid);
ath_hal_setrxfilter(ah, rfilt);
/* XXX is this to restore keycache on resume? */
if (vap->iv_opmode != IEEE80211_M_STA &&
(vap->iv_flags & IEEE80211_F_PRIVACY)) {
for (i = 0; i < IEEE80211_WEP_NKID; i++)
if (ath_hal_keyisvalid(ah, i))
ath_hal_keysetmac(ah, i, ni->ni_bssid);
}
/*
* Invoke the parent method to do net80211 work.
*/
error = avp->av_newstate(vap, nstate, arg);
if (error != 0)
goto bad;
/*
* See above: ensure av_newstate() doesn't drop the lock
* on us.
*/
IEEE80211_LOCK_ASSERT(ic);
if (nstate == IEEE80211_S_RUN) {
/* NB: collect bss node again, it may have changed */
ieee80211_free_node(ni);
ni = ieee80211_ref_node(vap->iv_bss);
DPRINTF(sc, ATH_DEBUG_STATE,
"%s(RUN): iv_flags 0x%08x bintvl %d bssid %s "
"capinfo 0x%04x chan %d\n", __func__,
vap->iv_flags, ni->ni_intval, ether_sprintf(ni->ni_bssid),
ni->ni_capinfo, ieee80211_chan2ieee(ic, ic->ic_curchan));
switch (vap->iv_opmode) {
#ifdef IEEE80211_SUPPORT_TDMA
case IEEE80211_M_AHDEMO:
if ((vap->iv_caps & IEEE80211_C_TDMA) == 0)
break;
/* fall thru... */
#endif
case IEEE80211_M_HOSTAP:
case IEEE80211_M_IBSS:
case IEEE80211_M_MBSS:
/*
* Allocate and setup the beacon frame.
*
* Stop any previous beacon DMA. This may be
* necessary, for example, when an ibss merge
* causes reconfiguration; there will be a state
* transition from RUN->RUN that means we may
* be called with beacon transmission active.
*/
ath_hal_stoptxdma(ah, sc->sc_bhalq);
error = ath_beacon_alloc(sc, ni);
if (error != 0)
goto bad;
/*
* If joining an adhoc network defer beacon timer
* configuration to the next beacon frame so we
* have a current TSF to use. Otherwise we're
* starting an ibss/bss so there's no need to delay;
* if this is the first vap moving to RUN state, then
* beacon state needs to be [re]configured.
*/
if (vap->iv_opmode == IEEE80211_M_IBSS &&
ni->ni_tstamp.tsf != 0) {
sc->sc_syncbeacon = 1;
} else if (!sc->sc_beacons) {
#ifdef IEEE80211_SUPPORT_TDMA
if (vap->iv_caps & IEEE80211_C_TDMA)
ath_tdma_config(sc, vap);
else
#endif
ath_beacon_config(sc, vap);
sc->sc_beacons = 1;
}
break;
case IEEE80211_M_STA:
/*
* Defer beacon timer configuration to the next
* beacon frame so we have a current TSF to use
* (any TSF collected when scanning is likely old).
* However if it's due to a CSA -> RUN transition,
* force a beacon update so we pick up a lack of
* beacons from an AP in CAC and thus force a
* scan.
*
* And, there's also corner cases here where
* after a scan, the AP may have disappeared.
* In that case, we may not receive an actual
* beacon to update the beacon timer and thus we
* won't get notified of the missing beacons.
*/
if (ostate != IEEE80211_S_RUN &&
ostate != IEEE80211_S_SLEEP) {
DPRINTF(sc, ATH_DEBUG_BEACON,
"%s: STA; syncbeacon=1\n", __func__);
sc->sc_syncbeacon = 1;
if (csa_run_transition)
ath_beacon_config(sc, vap);
/*
* PR: kern/175227
*
* Reconfigure beacons during reset; as otherwise
* we won't get the beacon timers reprogrammed
* after a reset and thus we won't pick up a
* beacon miss interrupt.
*
* Hopefully we'll see a beacon before the BMISS
* timer fires (too often), leading to a STA
* disassociation.
*/
sc->sc_beacons = 1;
}
break;
case IEEE80211_M_MONITOR:
/*
* Monitor mode vaps have only INIT->RUN and RUN->RUN
* transitions so we must re-enable interrupts here to
* handle the case of a single monitor mode vap.
*/
ath_hal_intrset(ah, sc->sc_imask);
break;
case IEEE80211_M_WDS:
break;
default:
break;
}
/*
* Let the hal process statistics collected during a
* scan so it can provide calibrated noise floor data.
*/
ath_hal_process_noisefloor(ah);
/*
* Reset rssi stats; maybe not the best place...
*/
sc->sc_halstats.ns_avgbrssi = ATH_RSSI_DUMMY_MARKER;
sc->sc_halstats.ns_avgrssi = ATH_RSSI_DUMMY_MARKER;
sc->sc_halstats.ns_avgtxrssi = ATH_RSSI_DUMMY_MARKER;
/*
* Force awake for RUN mode.
*/
ATH_LOCK(sc);
ath_power_setselfgen(sc, HAL_PM_AWAKE);
ath_power_setpower(sc, HAL_PM_AWAKE);
/*
* Finally, start any timers and the task q thread
* (in case we didn't go through SCAN state).
*/
if (ath_longcalinterval != 0) {
/* start periodic recalibration timer */
callout_reset(&sc->sc_cal_ch, 1, ath_calibrate, sc);
} else {
DPRINTF(sc, ATH_DEBUG_CALIBRATE,
"%s: calibration disabled\n", __func__);
}
ATH_UNLOCK(sc);
taskqueue_unblock(sc->sc_tq);
} else if (nstate == IEEE80211_S_INIT) {
/*
* If there are no vaps left in RUN state then
* shutdown host/driver operation:
* o disable interrupts
* o disable the task queue thread
* o mark beacon processing as stopped
*/
if (!ath_isanyrunningvaps(vap)) {
sc->sc_imask &= ~(HAL_INT_SWBA | HAL_INT_BMISS);
/* disable interrupts */
ath_hal_intrset(ah, sc->sc_imask &~ HAL_INT_GLOBAL);
taskqueue_block(sc->sc_tq);
sc->sc_beacons = 0;
}
#ifdef IEEE80211_SUPPORT_TDMA
ath_hal_setcca(ah, AH_TRUE);
#endif
} else if (nstate == IEEE80211_S_SLEEP) {
/* We're going to sleep, so transition appropriately */
/* For now, only do this if we're a single STA vap */
if (sc->sc_nvaps == 1 &&
vap->iv_opmode == IEEE80211_M_STA) {
DPRINTF(sc, ATH_DEBUG_BEACON, "%s: syncbeacon=%d\n", __func__, sc->sc_syncbeacon);
ATH_LOCK(sc);
/*
* Always at least set the self-generated
* frame config to set PWRMGT=1.
*/
ath_power_setselfgen(sc, HAL_PM_NETWORK_SLEEP);
/*
* If we're not syncing beacons, transition
* to NETWORK_SLEEP.
*
* We stay awake if syncbeacon > 0 in case
* we need to listen for some beacons otherwise
* our beacon timer config may be wrong.
*/
if (sc->sc_syncbeacon == 0) {
ath_power_setpower(sc, HAL_PM_NETWORK_SLEEP);
}
ATH_UNLOCK(sc);
}
}
bad:
ieee80211_free_node(ni);
/*
* Restore the power state - either to what it was, or
* to network_sleep if it's alright.
*/
ATH_LOCK(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
return error;
}
/*
* Allocate a key cache slot to the station so we can
* setup a mapping from key index to node. The key cache
* slot is needed for managing antenna state and for
* compression when stations do not use crypto. We do
* it uniliaterally here; if crypto is employed this slot
* will be reassigned.
*/
static void
ath_setup_stationkey(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ath_softc *sc = vap->iv_ic->ic_ifp->if_softc;
ieee80211_keyix keyix, rxkeyix;
/* XXX should take a locked ref to vap->iv_bss */
if (!ath_key_alloc(vap, &ni->ni_ucastkey, &keyix, &rxkeyix)) {
/*
* Key cache is full; we'll fall back to doing
* the more expensive lookup in software. Note
* this also means no h/w compression.
*/
/* XXX msg+statistic */
} else {
/* XXX locking? */
ni->ni_ucastkey.wk_keyix = keyix;
ni->ni_ucastkey.wk_rxkeyix = rxkeyix;
/* NB: must mark device key to get called back on delete */
ni->ni_ucastkey.wk_flags |= IEEE80211_KEY_DEVKEY;
IEEE80211_ADDR_COPY(ni->ni_ucastkey.wk_macaddr, ni->ni_macaddr);
/* NB: this will create a pass-thru key entry */
ath_keyset(sc, vap, &ni->ni_ucastkey, vap->iv_bss);
}
}
/*
* Setup driver-specific state for a newly associated node.
* Note that we're called also on a re-associate, the isnew
* param tells us if this is the first time or not.
*/
static void
ath_newassoc(struct ieee80211_node *ni, int isnew)
{
struct ath_node *an = ATH_NODE(ni);
struct ieee80211vap *vap = ni->ni_vap;
struct ath_softc *sc = vap->iv_ic->ic_ifp->if_softc;
const struct ieee80211_txparam *tp = ni->ni_txparms;
an->an_mcastrix = ath_tx_findrix(sc, tp->mcastrate);
an->an_mgmtrix = ath_tx_findrix(sc, tp->mgmtrate);
DPRINTF(sc, ATH_DEBUG_NODE, "%s: %6D: reassoc; isnew=%d, is_powersave=%d\n",
__func__,
ni->ni_macaddr,
":",
isnew,
an->an_is_powersave);
ATH_NODE_LOCK(an);
ath_rate_newassoc(sc, an, isnew);
ATH_NODE_UNLOCK(an);
if (isnew &&
(vap->iv_flags & IEEE80211_F_PRIVACY) == 0 && sc->sc_hasclrkey &&
ni->ni_ucastkey.wk_keyix == IEEE80211_KEYIX_NONE)
ath_setup_stationkey(ni);
/*
* If we're reassociating, make sure that any paused queues
* get unpaused.
*
* Now, we may hvae frames in the hardware queue for this node.
* So if we are reassociating and there are frames in the queue,
* we need to go through the cleanup path to ensure that they're
* marked as non-aggregate.
*/
if (! isnew) {
DPRINTF(sc, ATH_DEBUG_NODE,
"%s: %6D: reassoc; is_powersave=%d\n",
__func__,
ni->ni_macaddr,
":",
an->an_is_powersave);
/* XXX for now, we can't hold the lock across assoc */
ath_tx_node_reassoc(sc, an);
/* XXX for now, we can't hold the lock across wakeup */
if (an->an_is_powersave)
ath_tx_node_wakeup(sc, an);
}
}
static int
ath_setregdomain(struct ieee80211com *ic, struct ieee80211_regdomain *reg,
int nchans, struct ieee80211_channel chans[])
{
struct ath_softc *sc = ic->ic_ifp->if_softc;
struct ath_hal *ah = sc->sc_ah;
HAL_STATUS status;
DPRINTF(sc, ATH_DEBUG_REGDOMAIN,
"%s: rd %u cc %u location %c%s\n",
__func__, reg->regdomain, reg->country, reg->location,
reg->ecm ? " ecm" : "");
status = ath_hal_set_channels(ah, chans, nchans,
reg->country, reg->regdomain);
if (status != HAL_OK) {
DPRINTF(sc, ATH_DEBUG_REGDOMAIN, "%s: failed, status %u\n",
__func__, status);
return EINVAL; /* XXX */
}
return 0;
}
static void
ath_getradiocaps(struct ieee80211com *ic,
int maxchans, int *nchans, struct ieee80211_channel chans[])
{
struct ath_softc *sc = ic->ic_ifp->if_softc;
struct ath_hal *ah = sc->sc_ah;
DPRINTF(sc, ATH_DEBUG_REGDOMAIN, "%s: use rd %u cc %d\n",
__func__, SKU_DEBUG, CTRY_DEFAULT);
/* XXX check return */
(void) ath_hal_getchannels(ah, chans, maxchans, nchans,
HAL_MODE_ALL, CTRY_DEFAULT, SKU_DEBUG, AH_TRUE);
}
static int
ath_getchannels(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct ath_hal *ah = sc->sc_ah;
HAL_STATUS status;
/*
* Collect channel set based on EEPROM contents.
*/
status = ath_hal_init_channels(ah, ic->ic_channels, IEEE80211_CHAN_MAX,
&ic->ic_nchans, HAL_MODE_ALL, CTRY_DEFAULT, SKU_NONE, AH_TRUE);
if (status != HAL_OK) {
if_printf(ifp, "%s: unable to collect channel list from hal, "
"status %d\n", __func__, status);
return EINVAL;
}
(void) ath_hal_getregdomain(ah, &sc->sc_eerd);
ath_hal_getcountrycode(ah, &sc->sc_eecc); /* NB: cannot fail */
/* XXX map Atheros sku's to net80211 SKU's */
/* XXX net80211 types too small */
ic->ic_regdomain.regdomain = (uint16_t) sc->sc_eerd;
ic->ic_regdomain.country = (uint16_t) sc->sc_eecc;
ic->ic_regdomain.isocc[0] = ' '; /* XXX don't know */
ic->ic_regdomain.isocc[1] = ' ';
ic->ic_regdomain.ecm = 1;
ic->ic_regdomain.location = 'I';
DPRINTF(sc, ATH_DEBUG_REGDOMAIN,
"%s: eeprom rd %u cc %u (mapped rd %u cc %u) location %c%s\n",
__func__, sc->sc_eerd, sc->sc_eecc,
ic->ic_regdomain.regdomain, ic->ic_regdomain.country,
ic->ic_regdomain.location, ic->ic_regdomain.ecm ? " ecm" : "");
return 0;
}
static int
ath_rate_setup(struct ath_softc *sc, u_int mode)
{
struct ath_hal *ah = sc->sc_ah;
const HAL_RATE_TABLE *rt;
switch (mode) {
case IEEE80211_MODE_11A:
rt = ath_hal_getratetable(ah, HAL_MODE_11A);
break;
case IEEE80211_MODE_HALF:
rt = ath_hal_getratetable(ah, HAL_MODE_11A_HALF_RATE);
break;
case IEEE80211_MODE_QUARTER:
rt = ath_hal_getratetable(ah, HAL_MODE_11A_QUARTER_RATE);
break;
case IEEE80211_MODE_11B:
rt = ath_hal_getratetable(ah, HAL_MODE_11B);
break;
case IEEE80211_MODE_11G:
rt = ath_hal_getratetable(ah, HAL_MODE_11G);
break;
case IEEE80211_MODE_TURBO_A:
rt = ath_hal_getratetable(ah, HAL_MODE_108A);
break;
case IEEE80211_MODE_TURBO_G:
rt = ath_hal_getratetable(ah, HAL_MODE_108G);
break;
case IEEE80211_MODE_STURBO_A:
rt = ath_hal_getratetable(ah, HAL_MODE_TURBO);
break;
case IEEE80211_MODE_11NA:
rt = ath_hal_getratetable(ah, HAL_MODE_11NA_HT20);
break;
case IEEE80211_MODE_11NG:
rt = ath_hal_getratetable(ah, HAL_MODE_11NG_HT20);
break;
default:
DPRINTF(sc, ATH_DEBUG_ANY, "%s: invalid mode %u\n",
__func__, mode);
return 0;
}
sc->sc_rates[mode] = rt;
return (rt != NULL);
}
static void
ath_setcurmode(struct ath_softc *sc, enum ieee80211_phymode mode)
{
#define N(a) (sizeof(a)/sizeof(a[0]))
/* NB: on/off times from the Atheros NDIS driver, w/ permission */
static const struct {
u_int rate; /* tx/rx 802.11 rate */
u_int16_t timeOn; /* LED on time (ms) */
u_int16_t timeOff; /* LED off time (ms) */
} blinkrates[] = {
{ 108, 40, 10 },
{ 96, 44, 11 },
{ 72, 50, 13 },
{ 48, 57, 14 },
{ 36, 67, 16 },
{ 24, 80, 20 },
{ 22, 100, 25 },
{ 18, 133, 34 },
{ 12, 160, 40 },
{ 10, 200, 50 },
{ 6, 240, 58 },
{ 4, 267, 66 },
{ 2, 400, 100 },
{ 0, 500, 130 },
/* XXX half/quarter rates */
};
const HAL_RATE_TABLE *rt;
int i, j;
memset(sc->sc_rixmap, 0xff, sizeof(sc->sc_rixmap));
rt = sc->sc_rates[mode];
KASSERT(rt != NULL, ("no h/w rate set for phy mode %u", mode));
for (i = 0; i < rt->rateCount; i++) {
uint8_t ieeerate = rt->info[i].dot11Rate & IEEE80211_RATE_VAL;
if (rt->info[i].phy != IEEE80211_T_HT)
sc->sc_rixmap[ieeerate] = i;
else
sc->sc_rixmap[ieeerate | IEEE80211_RATE_MCS] = i;
}
memset(sc->sc_hwmap, 0, sizeof(sc->sc_hwmap));
for (i = 0; i < N(sc->sc_hwmap); i++) {
if (i >= rt->rateCount) {
sc->sc_hwmap[i].ledon = (500 * hz) / 1000;
sc->sc_hwmap[i].ledoff = (130 * hz) / 1000;
continue;
}
sc->sc_hwmap[i].ieeerate =
rt->info[i].dot11Rate & IEEE80211_RATE_VAL;
if (rt->info[i].phy == IEEE80211_T_HT)
sc->sc_hwmap[i].ieeerate |= IEEE80211_RATE_MCS;
sc->sc_hwmap[i].txflags = IEEE80211_RADIOTAP_F_DATAPAD;
if (rt->info[i].shortPreamble ||
rt->info[i].phy == IEEE80211_T_OFDM)
sc->sc_hwmap[i].txflags |= IEEE80211_RADIOTAP_F_SHORTPRE;
sc->sc_hwmap[i].rxflags = sc->sc_hwmap[i].txflags;
for (j = 0; j < N(blinkrates)-1; j++)
if (blinkrates[j].rate == sc->sc_hwmap[i].ieeerate)
break;
/* NB: this uses the last entry if the rate isn't found */
/* XXX beware of overlow */
sc->sc_hwmap[i].ledon = (blinkrates[j].timeOn * hz) / 1000;
sc->sc_hwmap[i].ledoff = (blinkrates[j].timeOff * hz) / 1000;
}
sc->sc_currates = rt;
sc->sc_curmode = mode;
/*
* All protection frames are transmited at 2Mb/s for
* 11g, otherwise at 1Mb/s.
*/
if (mode == IEEE80211_MODE_11G)
sc->sc_protrix = ath_tx_findrix(sc, 2*2);
else
sc->sc_protrix = ath_tx_findrix(sc, 2*1);
/* NB: caller is responsible for resetting rate control state */
#undef N
}
static void
ath_watchdog(void *arg)
{
struct ath_softc *sc = arg;
int do_reset = 0;
ATH_LOCK_ASSERT(sc);
if (sc->sc_wd_timer != 0 && --sc->sc_wd_timer == 0) {
struct ifnet *ifp = sc->sc_ifp;
uint32_t hangs;
ath_power_set_power_state(sc, HAL_PM_AWAKE);
if (ath_hal_gethangstate(sc->sc_ah, 0xffff, &hangs) &&
hangs != 0) {
if_printf(ifp, "%s hang detected (0x%x)\n",
hangs & 0xff ? "bb" : "mac", hangs);
} else
if_printf(ifp, "device timeout\n");
do_reset = 1;
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
sc->sc_stats.ast_watchdog++;
ath_power_restore_power_state(sc);
}
/*
* We can't hold the lock across the ath_reset() call.
*
* And since this routine can't hold a lock and sleep,
* do the reset deferred.
*/
if (do_reset) {
taskqueue_enqueue(sc->sc_tq, &sc->sc_resettask);
}
callout_schedule(&sc->sc_wd_ch, hz);
}
/*
* Fetch the rate control statistics for the given node.
*/
static int
ath_ioctl_ratestats(struct ath_softc *sc, struct ath_rateioctl *rs)
{
struct ath_node *an;
struct ieee80211com *ic = sc->sc_ifp->if_l2com;
struct ieee80211_node *ni;
int error = 0;
/* Perform a lookup on the given node */
ni = ieee80211_find_node(&ic->ic_sta, rs->is_u.macaddr);
if (ni == NULL) {
error = EINVAL;
goto bad;
}
/* Lock the ath_node */
an = ATH_NODE(ni);
ATH_NODE_LOCK(an);
/* Fetch the rate control stats for this node */
error = ath_rate_fetch_node_stats(sc, an, rs);
/* No matter what happens here, just drop through */
/* Unlock the ath_node */
ATH_NODE_UNLOCK(an);
/* Unref the node */
ieee80211_node_decref(ni);
bad:
return (error);
}
#ifdef ATH_DIAGAPI
/*
* Diagnostic interface to the HAL. This is used by various
* tools to do things like retrieve register contents for
* debugging. The mechanism is intentionally opaque so that
* it can change frequently w/o concern for compatiblity.
*/
static int
ath_ioctl_diag(struct ath_softc *sc, struct ath_diag *ad)
{
struct ath_hal *ah = sc->sc_ah;
u_int id = ad->ad_id & ATH_DIAG_ID;
void *indata = NULL;
void *outdata = NULL;
u_int32_t insize = ad->ad_in_size;
u_int32_t outsize = ad->ad_out_size;
int error = 0;
if (ad->ad_id & ATH_DIAG_IN) {
/*
* Copy in data.
*/
indata = malloc(insize, M_TEMP, M_NOWAIT);
if (indata == NULL) {
error = ENOMEM;
goto bad;
}
error = copyin(ad->ad_in_data, indata, insize);
if (error)
goto bad;
}
if (ad->ad_id & ATH_DIAG_DYN) {
/*
* Allocate a buffer for the results (otherwise the HAL
* returns a pointer to a buffer where we can read the
* results). Note that we depend on the HAL leaving this
* pointer for us to use below in reclaiming the buffer;
* may want to be more defensive.
*/
outdata = malloc(outsize, M_TEMP, M_NOWAIT);
if (outdata == NULL) {
error = ENOMEM;
goto bad;
}
}
ATH_LOCK(sc);
if (id != HAL_DIAG_REGS)
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ATH_UNLOCK(sc);
if (ath_hal_getdiagstate(ah, id, indata, insize, &outdata, &outsize)) {
if (outsize < ad->ad_out_size)
ad->ad_out_size = outsize;
if (outdata != NULL)
error = copyout(outdata, ad->ad_out_data,
ad->ad_out_size);
} else {
error = EINVAL;
}
ATH_LOCK(sc);
if (id != HAL_DIAG_REGS)
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
bad:
if ((ad->ad_id & ATH_DIAG_IN) && indata != NULL)
free(indata, M_TEMP);
if ((ad->ad_id & ATH_DIAG_DYN) && outdata != NULL)
free(outdata, M_TEMP);
return error;
}
#endif /* ATH_DIAGAPI */
static int
ath_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
#define IS_RUNNING(ifp) \
((ifp->if_flags & IFF_UP) && (ifp->if_drv_flags & IFF_DRV_RUNNING))
struct ath_softc *sc = ifp->if_softc;
struct ieee80211com *ic = ifp->if_l2com;
struct ifreq *ifr = (struct ifreq *)data;
const HAL_RATE_TABLE *rt;
int error = 0;
switch (cmd) {
case SIOCSIFFLAGS:
if (IS_RUNNING(ifp)) {
/*
* To avoid rescanning another access point,
* do not call ath_init() here. Instead,
* only reflect promisc mode settings.
*/
ATH_LOCK(sc);
ath_power_set_power_state(sc, HAL_PM_AWAKE);
ath_mode_init(sc);
ath_power_restore_power_state(sc);
ATH_UNLOCK(sc);
} else if (ifp->if_flags & IFF_UP) {
/*
* Beware of being called during attach/detach
* to reset promiscuous mode. In that case we
* will still be marked UP but not RUNNING.
* However trying to re-init the interface
* is the wrong thing to do as we've already
* torn down much of our state. There's
* probably a better way to deal with this.
*/
if (!sc->sc_invalid)
ath_init(sc); /* XXX lose error */
} else {
ATH_LOCK(sc);
ath_stop_locked(ifp);
if (!sc->sc_invalid)
ath_power_setpower(sc, HAL_PM_FULL_SLEEP);
ATH_UNLOCK(sc);
}
break;
case SIOCGIFMEDIA:
case SIOCSIFMEDIA:
error = ifmedia_ioctl(ifp, ifr, &ic->ic_media, cmd);
break;
case SIOCGATHSTATS:
/* NB: embed these numbers to get a consistent view */
sc->sc_stats.ast_tx_packets = ifp->if_get_counter(ifp,
IFCOUNTER_OPACKETS);
sc->sc_stats.ast_rx_packets = ifp->if_get_counter(ifp,
IFCOUNTER_IPACKETS);
sc->sc_stats.ast_tx_rssi = ATH_RSSI(sc->sc_halstats.ns_avgtxrssi);
sc->sc_stats.ast_rx_rssi = ATH_RSSI(sc->sc_halstats.ns_avgrssi);
#ifdef IEEE80211_SUPPORT_TDMA
sc->sc_stats.ast_tdma_tsfadjp = TDMA_AVG(sc->sc_avgtsfdeltap);
sc->sc_stats.ast_tdma_tsfadjm = TDMA_AVG(sc->sc_avgtsfdeltam);
#endif
rt = sc->sc_currates;
sc->sc_stats.ast_tx_rate =
rt->info[sc->sc_txrix].dot11Rate &~ IEEE80211_RATE_BASIC;
if (rt->info[sc->sc_txrix].phy & IEEE80211_T_HT)
sc->sc_stats.ast_tx_rate |= IEEE80211_RATE_MCS;
return copyout(&sc->sc_stats,
ifr->ifr_data, sizeof (sc->sc_stats));
case SIOCGATHAGSTATS:
return copyout(&sc->sc_aggr_stats,
ifr->ifr_data, sizeof (sc->sc_aggr_stats));
case SIOCZATHSTATS:
error = priv_check(curthread, PRIV_DRIVER);
if (error == 0) {
memset(&sc->sc_stats, 0, sizeof(sc->sc_stats));
memset(&sc->sc_aggr_stats, 0,
sizeof(sc->sc_aggr_stats));
memset(&sc->sc_intr_stats, 0,
sizeof(sc->sc_intr_stats));
}
break;
#ifdef ATH_DIAGAPI
case SIOCGATHDIAG:
error = ath_ioctl_diag(sc, (struct ath_diag *) ifr);
break;
case SIOCGATHPHYERR:
error = ath_ioctl_phyerr(sc,(struct ath_diag*) ifr);
break;
#endif
case SIOCGATHSPECTRAL:
error = ath_ioctl_spectral(sc,(struct ath_diag*) ifr);
break;
case SIOCGATHNODERATESTATS:
error = ath_ioctl_ratestats(sc, (struct ath_rateioctl *) ifr);
break;
case SIOCGIFADDR:
error = ether_ioctl(ifp, cmd, data);
break;
default:
error = EINVAL;
break;
}
return error;
#undef IS_RUNNING
}
/*
* Announce various information on device/driver attach.
*/
static void
ath_announce(struct ath_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ath_hal *ah = sc->sc_ah;
if_printf(ifp, "AR%s mac %d.%d RF%s phy %d.%d\n",
ath_hal_mac_name(ah), ah->ah_macVersion, ah->ah_macRev,
ath_hal_rf_name(ah), ah->ah_phyRev >> 4, ah->ah_phyRev & 0xf);
if_printf(ifp, "2GHz radio: 0x%.4x; 5GHz radio: 0x%.4x\n",
ah->ah_analog2GhzRev, ah->ah_analog5GhzRev);
if (bootverbose) {
int i;
for (i = 0; i <= WME_AC_VO; i++) {
struct ath_txq *txq = sc->sc_ac2q[i];
if_printf(ifp, "Use hw queue %u for %s traffic\n",
txq->axq_qnum, ieee80211_wme_acnames[i]);
}
if_printf(ifp, "Use hw queue %u for CAB traffic\n",
sc->sc_cabq->axq_qnum);
if_printf(ifp, "Use hw queue %u for beacons\n", sc->sc_bhalq);
}
if (ath_rxbuf != ATH_RXBUF)
if_printf(ifp, "using %u rx buffers\n", ath_rxbuf);
if (ath_txbuf != ATH_TXBUF)
if_printf(ifp, "using %u tx buffers\n", ath_txbuf);
if (sc->sc_mcastkey && bootverbose)
if_printf(ifp, "using multicast key search\n");
}
static void
ath_dfs_tasklet(void *p, int npending)
{
struct ath_softc *sc = (struct ath_softc *) p;
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
/*
* If previous processing has found a radar event,
* signal this to the net80211 layer to begin DFS
* processing.
*/
if (ath_dfs_process_radar_event(sc, sc->sc_curchan)) {
/* DFS event found, initiate channel change */
/*
* XXX doesn't currently tell us whether the event
* XXX was found in the primary or extension
* XXX channel!
*/
IEEE80211_LOCK(ic);
ieee80211_dfs_notify_radar(ic, sc->sc_curchan);
IEEE80211_UNLOCK(ic);
}
}
/*
* Enable/disable power save. This must be called with
* no TX driver locks currently held, so it should only
* be called from the RX path (which doesn't hold any
* TX driver locks.)
*/
static void
ath_node_powersave(struct ieee80211_node *ni, int enable)
{
#ifdef ATH_SW_PSQ
struct ath_node *an = ATH_NODE(ni);
struct ieee80211com *ic = ni->ni_ic;
struct ath_softc *sc = ic->ic_ifp->if_softc;
struct ath_vap *avp = ATH_VAP(ni->ni_vap);
/* XXX and no TXQ locks should be held here */
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE, "%s: %6D: enable=%d\n",
__func__,
ni->ni_macaddr,
":",
!! enable);
/* Suspend or resume software queue handling */
if (enable)
ath_tx_node_sleep(sc, an);
else
ath_tx_node_wakeup(sc, an);
/* Update net80211 state */
avp->av_node_ps(ni, enable);
#else
struct ath_vap *avp = ATH_VAP(ni->ni_vap);
/* Update net80211 state */
avp->av_node_ps(ni, enable);
#endif/* ATH_SW_PSQ */
}
/*
* Notification from net80211 that the powersave queue state has
* changed.
*
* Since the software queue also may have some frames:
*
* + if the node software queue has frames and the TID state
* is 0, we set the TIM;
* + if the node and the stack are both empty, we clear the TIM bit.
* + If the stack tries to set the bit, always set it.
* + If the stack tries to clear the bit, only clear it if the
* software queue in question is also cleared.
*
* TODO: this is called during node teardown; so let's ensure this
* is all correctly handled and that the TIM bit is cleared.
* It may be that the node flush is called _AFTER_ the net80211
* stack clears the TIM.
*
* Here is the racy part. Since it's possible >1 concurrent,
* overlapping TXes will appear complete with a TX completion in
* another thread, it's possible that the concurrent TIM calls will
* clash. We can't hold the node lock here because setting the
* TIM grabs the net80211 comlock and this may cause a LOR.
* The solution is either to totally serialise _everything_ at
* this point (ie, all TX, completion and any reset/flush go into
* one taskqueue) or a new "ath TIM lock" needs to be created that
* just wraps the driver state change and this call to avp->av_set_tim().
*
* The same race exists in the net80211 power save queue handling
* as well. Since multiple transmitting threads may queue frames
* into the driver, as well as ps-poll and the driver transmitting
* frames (and thus clearing the psq), it's quite possible that
* a packet entering the PSQ and a ps-poll being handled will
* race, causing the TIM to be cleared and not re-set.
*/
static int
ath_node_set_tim(struct ieee80211_node *ni, int enable)
{
#ifdef ATH_SW_PSQ
struct ieee80211com *ic = ni->ni_ic;
struct ath_softc *sc = ic->ic_ifp->if_softc;
struct ath_node *an = ATH_NODE(ni);
struct ath_vap *avp = ATH_VAP(ni->ni_vap);
int changed = 0;
ATH_TX_LOCK(sc);
an->an_stack_psq = enable;
/*
* This will get called for all operating modes,
* even if avp->av_set_tim is unset.
* It's currently set for hostap/ibss modes; but
* the same infrastructure is used for both STA
* and AP/IBSS node power save.
*/
if (avp->av_set_tim == NULL) {
ATH_TX_UNLOCK(sc);
return (0);
}
/*
* If setting the bit, always set it here.
* If clearing the bit, only clear it if the
* software queue is also empty.
*
* If the node has left power save, just clear the TIM
* bit regardless of the state of the power save queue.
*
* XXX TODO: although atomics are used, it's quite possible
* that a race will occur between this and setting/clearing
* in another thread. TX completion will occur always in
* one thread, however setting/clearing the TIM bit can come
* from a variety of different process contexts!
*/
if (enable && an->an_tim_set == 1) {
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: enable=%d, tim_set=1, ignoring\n",
__func__,
ni->ni_macaddr,
":",
enable);
ATH_TX_UNLOCK(sc);
} else if (enable) {
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: enable=%d, enabling TIM\n",
__func__,
ni->ni_macaddr,
":",
enable);
an->an_tim_set = 1;
ATH_TX_UNLOCK(sc);
changed = avp->av_set_tim(ni, enable);
} else if (an->an_swq_depth == 0) {
/* disable */
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: enable=%d, an_swq_depth == 0, disabling\n",
__func__,
ni->ni_macaddr,
":",
enable);
an->an_tim_set = 0;
ATH_TX_UNLOCK(sc);
changed = avp->av_set_tim(ni, enable);
} else if (! an->an_is_powersave) {
/*
* disable regardless; the node isn't in powersave now
*/
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: enable=%d, an_pwrsave=0, disabling\n",
__func__,
ni->ni_macaddr,
":",
enable);
an->an_tim_set = 0;
ATH_TX_UNLOCK(sc);
changed = avp->av_set_tim(ni, enable);
} else {
/*
* psq disable, node is currently in powersave, node
* software queue isn't empty, so don't clear the TIM bit
* for now.
*/
ATH_TX_UNLOCK(sc);
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: enable=%d, an_swq_depth > 0, ignoring\n",
__func__,
ni->ni_macaddr,
":",
enable);
changed = 0;
}
return (changed);
#else
struct ath_vap *avp = ATH_VAP(ni->ni_vap);
/*
* Some operating modes don't set av_set_tim(), so don't
* update it here.
*/
if (avp->av_set_tim == NULL)
return (0);
return (avp->av_set_tim(ni, enable));
#endif /* ATH_SW_PSQ */
}
/*
* Set or update the TIM from the software queue.
*
* Check the software queue depth before attempting to do lock
* anything; that avoids trying to obtain the lock. Then,
* re-check afterwards to ensure nothing has changed in the
* meantime.
*
* set: This is designed to be called from the TX path, after
* a frame has been queued; to see if the swq > 0.
*
* clear: This is designed to be called from the buffer completion point
* (right now it's ath_tx_default_comp()) where the state of
* a software queue has changed.
*
* It makes sense to place it at buffer free / completion rather
* than after each software queue operation, as there's no real
* point in churning the TIM bit as the last frames in the software
* queue are transmitted. If they fail and we retry them, we'd
* just be setting the TIM bit again anyway.
*/
void
ath_tx_update_tim(struct ath_softc *sc, struct ieee80211_node *ni,
int enable)
{
#ifdef ATH_SW_PSQ
struct ath_node *an;
struct ath_vap *avp;
/* Don't do this for broadcast/etc frames */
if (ni == NULL)
return;
an = ATH_NODE(ni);
avp = ATH_VAP(ni->ni_vap);
/*
* And for operating modes without the TIM handler set, let's
* just skip those.
*/
if (avp->av_set_tim == NULL)
return;
ATH_TX_LOCK_ASSERT(sc);
if (enable) {
if (an->an_is_powersave &&
an->an_tim_set == 0 &&
an->an_swq_depth != 0) {
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: swq_depth>0, tim_set=0, set!\n",
__func__,
ni->ni_macaddr,
":");
an->an_tim_set = 1;
(void) avp->av_set_tim(ni, 1);
}
} else {
/*
* Don't bother grabbing the lock unless the queue is empty.
*/
if (an->an_swq_depth != 0)
return;
if (an->an_is_powersave &&
an->an_stack_psq == 0 &&
an->an_tim_set == 1 &&
an->an_swq_depth == 0) {
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: swq_depth=0, tim_set=1, psq_set=0,"
" clear!\n",
__func__,
ni->ni_macaddr,
":");
an->an_tim_set = 0;
(void) avp->av_set_tim(ni, 0);
}
}
#else
return;
#endif /* ATH_SW_PSQ */
}
/*
* Received a ps-poll frame from net80211.
*
* Here we get a chance to serve out a software-queued frame ourselves
* before we punt it to net80211 to transmit us one itself - either
* because there's traffic in the net80211 psq, or a NULL frame to
* indicate there's nothing else.
*/
static void
ath_node_recv_pspoll(struct ieee80211_node *ni, struct mbuf *m)
{
#ifdef ATH_SW_PSQ
struct ath_node *an;
struct ath_vap *avp;
struct ieee80211com *ic = ni->ni_ic;
struct ath_softc *sc = ic->ic_ifp->if_softc;
int tid;
/* Just paranoia */
if (ni == NULL)
return;
/*
* Unassociated (temporary node) station.
*/
if (ni->ni_associd == 0)
return;
/*
* We do have an active node, so let's begin looking into it.
*/
an = ATH_NODE(ni);
avp = ATH_VAP(ni->ni_vap);
/*
* For now, we just call the original ps-poll method.
* Once we're ready to flip this on:
*
* + Set leak to 1, as no matter what we're going to have
* to send a frame;
* + Check the software queue and if there's something in it,
* schedule the highest TID thas has traffic from this node.
* Then make sure we schedule the software scheduler to
* run so it picks up said frame.
*
* That way whatever happens, we'll at least send _a_ frame
* to the given node.
*
* Again, yes, it's crappy QoS if the node has multiple
* TIDs worth of traffic - but let's get it working first
* before we optimise it.
*
* Also yes, there's definitely latency here - we're not
* direct dispatching to the hardware in this path (and
* we're likely being called from the packet receive path,
* so going back into TX may be a little hairy!) but again
* I'd like to get this working first before optimising
* turn-around time.
*/
ATH_TX_LOCK(sc);
/*
* Legacy - we're called and the node isn't asleep.
* Immediately punt.
*/
if (! an->an_is_powersave) {
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: not in powersave?\n",
__func__,
ni->ni_macaddr,
":");
ATH_TX_UNLOCK(sc);
avp->av_recv_pspoll(ni, m);
return;
}
/*
* We're in powersave.
*
* Leak a frame.
*/
an->an_leak_count = 1;
/*
* Now, if there's no frames in the node, just punt to
* recv_pspoll.
*
* Don't bother checking if the TIM bit is set, we really
* only care if there are any frames here!
*/
if (an->an_swq_depth == 0) {
ATH_TX_UNLOCK(sc);
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: SWQ empty; punting to net80211\n",
__func__,
ni->ni_macaddr,
":");
avp->av_recv_pspoll(ni, m);
return;
}
/*
* Ok, let's schedule the highest TID that has traffic
* and then schedule something.
*/
for (tid = IEEE80211_TID_SIZE - 1; tid >= 0; tid--) {
struct ath_tid *atid = &an->an_tid[tid];
/*
* No frames? Skip.
*/
if (atid->axq_depth == 0)
continue;
ath_tx_tid_sched(sc, atid);
/*
* XXX we could do a direct call to the TXQ
* scheduler code here to optimise latency
* at the expense of a REALLY deep callstack.
*/
ATH_TX_UNLOCK(sc);
taskqueue_enqueue(sc->sc_tq, &sc->sc_txqtask);
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: leaking frame to TID %d\n",
__func__,
ni->ni_macaddr,
":",
tid);
return;
}
ATH_TX_UNLOCK(sc);
/*
* XXX nothing in the TIDs at this point? Eek.
*/
DPRINTF(sc, ATH_DEBUG_NODE_PWRSAVE,
"%s: %6D: TIDs empty, but ath_node showed traffic?!\n",
__func__,
ni->ni_macaddr,
":");
avp->av_recv_pspoll(ni, m);
#else
avp->av_recv_pspoll(ni, m);
#endif /* ATH_SW_PSQ */
}
MODULE_VERSION(if_ath, 1);
MODULE_DEPEND(if_ath, wlan, 1, 1, 1); /* 802.11 media layer */
#if defined(IEEE80211_ALQ) || defined(AH_DEBUG_ALQ) || defined(ATH_DEBUG_ALQ)
MODULE_DEPEND(if_ath, alq, 1, 1, 1);
#endif
Index: head/sys/dev/ce/if_ce.c
===================================================================
--- head/sys/dev/ce/if_ce.c (revision 283290)
+++ head/sys/dev/ce/if_ce.c (revision 283291)
@@ -1,2648 +1,2644 @@
/*
* Cronyx-Tau32-PCI adapter driver for FreeBSD.
*
* Copyright (C) 2003-2005 Cronyx Engineering.
* Copyright (C) 2003-2005 Kurakin Roman, <rik@FreeBSD.org>
*
* This software is distributed with NO WARRANTIES, not even the implied
* warranties for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* Authors grant any other persons or organisations a permission to use,
* modify and redistribute this software in source and binary forms,
* as long as this message is kept with the software, all derivative
* works or modified versions.
*
* $Cronyx: if_ce.c,v 1.9.2.8 2005/11/21 14:17:44 rik Exp $
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#if __FreeBSD_version >= 500000
# define NPCI 1
#else
# include "pci.h"
#endif
#if NPCI > 0
#include <sys/ucred.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/conf.h>
#include <sys/malloc.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#if __FreeBSD_version >= 504000
#include <sys/sysctl.h>
#endif
#include <sys/tty.h>
#include <sys/bus.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <net/if.h>
#include <net/if_var.h>
#if __FreeBSD_version > 501000
# include <dev/pci/pcivar.h>
# include <dev/pci/pcireg.h>
#else
# include <pci/pcivar.h>
# include <pci/pcireg.h>
#endif
#include <machine/bus.h>
#include <sys/rman.h>
#include "opt_ng_cronyx.h"
#ifdef NETGRAPH_CRONYX
# include "opt_netgraph.h"
# ifndef NETGRAPH
# error #option NETGRAPH missed from configuration
# endif
# include <netgraph/ng_message.h>
# include <netgraph/netgraph.h>
# include <dev/ce/ng_ce.h>
#else
# include <net/if_types.h>
# include <net/if_sppp.h>
# define PP_CISCO IFF_LINK2
# include <net/bpf.h>
#endif
#include <dev/cx/machdep.h>
#include <dev/ce/ceddk.h>
#include <machine/cserial.h>
#include <machine/resource.h>
#include <machine/pmap.h>
/* If we don't have Cronyx's sppp version, we don't have fr support via sppp */
#ifndef PP_FR
#define PP_FR 0
#endif
#ifndef IFP2SP
#define IFP2SP(ifp) ((struct sppp*)ifp)
#endif
#ifndef SP2IFP
#define SP2IFP(sp) ((struct ifnet*)sp)
#endif
#ifndef PCIR_BAR
#define PCIR_BAR(x) (PCIR_MAPS + (x) * 4)
#endif
/* define as our previous return value */
#ifndef BUS_PROBE_DEFAULT
#define BUS_PROBE_DEFAULT 0
#endif
#define CE_DEBUG(d,s) ({if (d->chan->debug) {\
printf ("%s: ", d->name); printf s;}})
#define CE_DEBUG2(d,s) ({if (d->chan->debug>1) {\
printf ("%s: ", d->name); printf s;}})
-#ifndef CALLOUT_MPSAFE
-#define CALLOUT_MPSAFE 0
-#endif
-
#ifndef IF_DRAIN
#define IF_DRAIN(ifq) do { \
struct mbuf *m; \
for (;;) { \
IF_DEQUEUE(ifq, m); \
if (m == NULL) \
break; \
m_freem(m); \
} \
} while (0)
#endif
#ifndef _IF_QLEN
#define _IF_QLEN(ifq) ((ifq)->ifq_len)
#endif
#ifndef callout_drain
#define callout_drain callout_stop
#endif
#define CE_LOCK_NAME "ceX"
#define CE_LOCK(_bd) mtx_lock (&(_bd)->ce_mtx)
#define CE_UNLOCK(_bd) mtx_unlock (&(_bd)->ce_mtx)
#define CE_LOCK_ASSERT(_bd) mtx_assert (&(_bd)->ce_mtx, MA_OWNED)
#define CDEV_MAJOR 185
static int ce_probe __P((device_t));
static int ce_attach __P((device_t));
static int ce_detach __P((device_t));
static device_method_t ce_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, ce_probe),
DEVMETHOD(device_attach, ce_attach),
DEVMETHOD(device_detach, ce_detach),
DEVMETHOD_END
};
typedef struct _ce_dma_mem_t {
unsigned long phys;
void *virt;
size_t size;
#if __FreeBSD_version >= 500000
bus_dma_tag_t dmat;
bus_dmamap_t mapp;
#endif
} ce_dma_mem_t;
typedef struct _drv_t {
char name [8];
int running;
ce_board_t *board;
ce_chan_t *chan;
struct ifqueue rqueue;
#ifdef NETGRAPH
char nodename [NG_NODESIZE];
hook_p hook;
hook_p debug_hook;
node_p node;
struct ifqueue queue;
struct ifqueue hi_queue;
#else
struct ifnet *ifp;
#endif
short timeout;
struct callout timeout_handle;
#if __FreeBSD_version >= 500000
struct cdev *devt;
#else /* __FreeBSD_version < 500000 */
dev_t devt;
#endif
ce_dma_mem_t dmamem;
} drv_t;
typedef struct _bdrv_t {
ce_board_t *board;
struct resource *ce_res;
struct resource *ce_irq;
void *ce_intrhand;
ce_dma_mem_t dmamem;
drv_t channel [NCHAN];
#if __FreeBSD_version >= 504000
struct mtx ce_mtx;
#endif
} bdrv_t;
static driver_t ce_driver = {
"ce",
ce_methods,
sizeof(bdrv_t),
};
static devclass_t ce_devclass;
static void ce_receive (ce_chan_t *c, unsigned char *data, int len);
static void ce_transmit (ce_chan_t *c, void *attachment, int len);
static void ce_error (ce_chan_t *c, int data);
static void ce_up (drv_t *d);
static void ce_start (drv_t *d);
static void ce_down (drv_t *d);
static void ce_watchdog (drv_t *d);
static void ce_watchdog_timer (void *arg);
#ifdef NETGRAPH
extern struct ng_type typestruct;
#else
static void ce_ifstart (struct ifnet *ifp);
static void ce_tlf (struct sppp *sp);
static void ce_tls (struct sppp *sp);
static int ce_sioctl (struct ifnet *ifp, u_long cmd, caddr_t data);
static void ce_initialize (void *softc);
#endif
static ce_board_t *adapter [NBRD];
static drv_t *channel [NBRD*NCHAN];
static struct callout led_timo [NBRD];
static struct callout timeout_handle;
static int ce_destroy = 0;
#if __FreeBSD_version < 500000
static int ce_open (dev_t dev, int oflags, int devtype, struct proc *p);
static int ce_close (dev_t dev, int fflag, int devtype, struct proc *p);
static int ce_ioctl (dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p);
#else
static int ce_open (struct cdev *dev, int oflags, int devtype, struct thread *td);
static int ce_close (struct cdev *dev, int fflag, int devtype, struct thread *td);
static int ce_ioctl (struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td);
#endif
#if __FreeBSD_version < 500000
static struct cdevsw ce_cdevsw = {
ce_open, ce_close, noread, nowrite,
ce_ioctl, nopoll, nommap, nostrategy,
"ce", CDEV_MAJOR, nodump, nopsize,
D_NAGGED, -1
};
#elif __FreeBSD_version == 500000
static struct cdevsw ce_cdevsw = {
ce_open, ce_close, noread, nowrite,
ce_ioctl, nopoll, nommap, nostrategy,
"ce", CDEV_MAJOR, nodump, nopsize,
D_NAGGED,
};
#elif __FreeBSD_version <= 501000
static struct cdevsw ce_cdevsw = {
.d_open = ce_open,
.d_close = ce_close,
.d_read = noread,
.d_write = nowrite,
.d_ioctl = ce_ioctl,
.d_poll = nopoll,
.d_mmap = nommap,
.d_strategy = nostrategy,
.d_name = "ce",
.d_maj = CDEV_MAJOR,
.d_dump = nodump,
.d_flags = D_NAGGED,
};
#elif __FreeBSD_version < 502103
static struct cdevsw ce_cdevsw = {
.d_open = ce_open,
.d_close = ce_close,
.d_ioctl = ce_ioctl,
.d_name = "ce",
.d_maj = CDEV_MAJOR,
.d_flags = D_NAGGED,
};
#elif __FreeBSD_version < 600000
static struct cdevsw ce_cdevsw = {
.d_version = D_VERSION,
.d_open = ce_open,
.d_close = ce_close,
.d_ioctl = ce_ioctl,
.d_name = "ce",
.d_maj = CDEV_MAJOR,
.d_flags = D_NEEDGIANT,
};
#else /* __FreeBSD_version >= 600000 */
static struct cdevsw ce_cdevsw = {
.d_version = D_VERSION,
.d_open = ce_open,
.d_close = ce_close,
.d_ioctl = ce_ioctl,
.d_name = "ce",
};
#endif
/*
* Make an mbuf from data.
*/
static struct mbuf *makembuf (void *buf, unsigned len)
{
struct mbuf *m;
MGETHDR (m, M_NOWAIT, MT_DATA);
if (! m)
return 0;
if (!(MCLGET(m, M_NOWAIT))) {
m_freem (m);
return 0;
}
m->m_pkthdr.len = m->m_len = len;
bcopy (buf, mtod (m, caddr_t), len);
return m;
}
static int ce_probe (device_t dev)
{
if ((pci_get_vendor (dev) == TAU32_PCI_VENDOR_ID) &&
(pci_get_device (dev) == TAU32_PCI_DEVICE_ID)) {
device_set_desc (dev, "Cronyx-Tau32-PCI serial adapter");
return BUS_PROBE_DEFAULT;
}
return ENXIO;
}
static void ce_timeout (void *arg)
{
drv_t *d;
int s, i, k;
for (i = 0; i < NBRD; ++i) {
if (adapter[i] == NULL)
continue;
for (k = 0; k < NCHAN; ++k) {
s = splimp ();
if (ce_destroy) {
splx (s);
return;
}
d = channel[i * NCHAN + k];
if (!d) {
splx (s);
continue;
}
CE_LOCK ((bdrv_t *)d->board->sys);
switch (d->chan->type) {
case T_E1:
ce_e1_timer (d->chan);
break;
default:
break;
}
CE_UNLOCK ((bdrv_t *)d->board->sys);
splx (s);
}
}
s = splimp ();
if (!ce_destroy)
callout_reset (&timeout_handle, hz, ce_timeout, 0);
splx (s);
}
static void ce_led_off (void *arg)
{
ce_board_t *b = arg;
bdrv_t *bd = (bdrv_t *) b->sys;
int s;
s = splimp ();
if (ce_destroy) {
splx (s);
return;
}
CE_LOCK (bd);
TAU32_LedSet (b->ddk.pControllerObject, 0);
CE_UNLOCK (bd);
splx (s);
}
static void ce_intr (void *arg)
{
bdrv_t *bd = arg;
ce_board_t *b = bd->board;
int s;
int i;
#if __FreeBSD_version >= 500000 && defined NETGRAPH
int error;
#endif
s = splimp ();
if (ce_destroy) {
splx (s);
return;
}
CE_LOCK (bd);
/* Turn LED on. */
TAU32_LedSet (b->ddk.pControllerObject, 1);
TAU32_HandleInterrupt (b->ddk.pControllerObject);
/* Turn LED off 50 msec later. */
callout_reset (&led_timo[b->num], hz/20, ce_led_off, b);
CE_UNLOCK (bd);
splx (s);
/* Pass packets in a lock-free state */
for (i = 0; i < NCHAN && b->chan[i].type; i++) {
drv_t *d = b->chan[i].sys;
struct mbuf *m;
if (!d || !d->running)
continue;
while (_IF_QLEN(&d->rqueue)) {
IF_DEQUEUE (&d->rqueue,m);
if (!m)
continue;
#ifdef NETGRAPH
if (d->hook) {
#if __FreeBSD_version >= 500000
NG_SEND_DATA_ONLY (error, d->hook, m);
#else
ng_queue_data (d->hook, m, 0);
#endif
} else {
IF_DRAIN (&d->rqueue);
}
#else
sppp_input (d->ifp, m);
#endif
}
}
}
#if __FreeBSD_version >= 500000
static void
ce_bus_dmamap_addr (void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
unsigned long *addr;
if (error)
return;
KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg));
addr = arg;
*addr = segs->ds_addr;
}
#ifndef BUS_DMA_ZERO
#define BUS_DMA_ZERO 0
#endif
static int
ce_bus_dma_mem_alloc (int bnum, int cnum, ce_dma_mem_t *dmem)
{
int error;
error = bus_dma_tag_create (NULL, 16, 0, BUS_SPACE_MAXADDR_32BIT,
BUS_SPACE_MAXADDR, NULL, NULL, dmem->size, 1,
dmem->size, 0,
#if __FreeBSD_version >= 502000
NULL, NULL,
#endif
&dmem->dmat);
if (error) {
if (cnum >= 0) printf ("ce%d-%d: ", bnum, cnum);
else printf ("ce%d: ", bnum);
printf ("couldn't allocate tag for dma memory\n");
return 0;
}
error = bus_dmamem_alloc (dmem->dmat, (void **)&dmem->virt,
BUS_DMA_NOWAIT | BUS_DMA_ZERO, &dmem->mapp);
if (error) {
if (cnum >= 0) printf ("ce%d-%d: ", bnum, cnum);
else printf ("ce%d: ", bnum);
printf ("couldn't allocate mem for dma memory\n");
bus_dma_tag_destroy (dmem->dmat);
return 0;
}
error = bus_dmamap_load (dmem->dmat, dmem->mapp, dmem->virt,
dmem->size, ce_bus_dmamap_addr, &dmem->phys, 0);
if (error) {
if (cnum >= 0) printf ("ce%d-%d: ", bnum, cnum);
else printf ("ce%d: ", bnum);
printf ("couldn't load mem map for dma memory\n");
bus_dmamem_free (dmem->dmat, dmem->virt, dmem->mapp);
bus_dma_tag_destroy (dmem->dmat);
return 0;
}
#if __FreeBSD_version >= 502000
bzero (dmem->virt, dmem->size);
#endif
return 1;
}
static void
ce_bus_dma_mem_free (ce_dma_mem_t *dmem)
{
bus_dmamap_unload (dmem->dmat, dmem->mapp);
bus_dmamem_free (dmem->dmat, dmem->virt, dmem->mapp);
bus_dma_tag_destroy (dmem->dmat);
}
#else
static int
ce_bus_dma_mem_alloc (int bnum, int cnum, ce_dma_mem_t *dmem)
{
dmem->virt = contigmalloc (dmem->size, M_DEVBUF, M_WAITOK,
0x100000, 0xffffffff, 16, 0);
if (dmem->virt == NULL) {
if (cnum >= 0) printf ("ce%d-%d: ", bnum, cnum);
else printf ("ce%d: ", bnum);
printf ("couldn't allocate dma memory\n");
return 0;
}
dmem->phys = vtophys (dmem->virt);
bzero (dmem->virt, dmem->size);
return 1;
}
static void
ce_bus_dma_mem_free (ce_dma_mem_t *dmem)
{
contigfree (dmem->virt, dmem->size, M_DEVBUF);
}
#endif
/*
* Called if the probe succeeded.
*/
static int ce_attach (device_t dev)
{
bdrv_t *bd = device_get_softc (dev);
int unit = device_get_unit (dev);
#if __FreeBSD_version >= 504000
char *ce_ln = CE_LOCK_NAME;
#endif
vm_offset_t vbase;
int rid, error;
ce_board_t *b;
ce_chan_t *c;
drv_t *d;
int s;
b = malloc (sizeof(ce_board_t), M_DEVBUF, M_WAITOK);
if (!b) {
printf ("ce%d: couldn't allocate memory\n", unit);
return (ENXIO);
}
bzero (b, sizeof(ce_board_t));
b->ddk.sys = &b;
#if __FreeBSD_version >= 440000
pci_enable_busmaster (dev);
#endif
bd->dmamem.size = TAU32_ControllerObjectSize;
if (! ce_bus_dma_mem_alloc (unit, -1, &bd->dmamem)) {
free (b, M_DEVBUF);
return (ENXIO);
}
b->ddk.pControllerObject = bd->dmamem.virt;
bd->board = b;
b->sys = bd;
rid = PCIR_BAR(0);
bd->ce_res = bus_alloc_resource (dev, SYS_RES_MEMORY, &rid,
0, ~0, 1, RF_ACTIVE);
if (! bd->ce_res) {
printf ("ce%d: cannot map memory\n", unit);
ce_bus_dma_mem_free (&bd->dmamem);
free (b, M_DEVBUF);
return (ENXIO);
}
vbase = (vm_offset_t) rman_get_virtual (bd->ce_res);
b->ddk.PciBar1VirtualAddress = (void *)vbase;
b->ddk.ControllerObjectPhysicalAddress = bd->dmamem.phys;
b->ddk.pErrorNotifyCallback = ce_error_callback;
b->ddk.pStatusNotifyCallback = ce_status_callback;
b->num = unit;
TAU32_BeforeReset(&b->ddk);
pci_write_config (dev, TAU32_PCI_RESET_ADDRESS, TAU32_PCI_RESET_ON, 4);
pci_write_config (dev, TAU32_PCI_RESET_ADDRESS, TAU32_PCI_RESET_OFF, 4);
if(!TAU32_Initialize(&b->ddk, 0))
{
printf ("ce%d: init adapter error 0x%08x, bus dead bits 0x%08lx\n",
unit, b->ddk.InitErrors, b->ddk.DeadBits);
bus_release_resource (dev, SYS_RES_MEMORY, PCIR_BAR(0), bd->ce_res);
ce_bus_dma_mem_free (&bd->dmamem);
free (b, M_DEVBUF);
return (ENXIO);
}
s = splimp ();
ce_init_board (b);
rid = 0;
bd->ce_irq = bus_alloc_resource (dev, SYS_RES_IRQ, &rid, 0, ~0, 1,
RF_SHAREABLE | RF_ACTIVE);
if (! bd->ce_irq) {
printf ("ce%d: cannot map interrupt\n", unit);
bus_release_resource (dev, SYS_RES_MEMORY, PCIR_BAR(0), bd->ce_res);
ce_bus_dma_mem_free (&bd->dmamem);
free (b, M_DEVBUF);
splx (s);
return (ENXIO);
}
#if __FreeBSD_version >= 500000
- callout_init (&led_timo[unit], CALLOUT_MPSAFE);
+ callout_init (&led_timo[unit], 1);
#else
callout_init (&led_timo[unit]);
#endif
error = bus_setup_intr (dev, bd->ce_irq,
#if __FreeBSD_version >= 500013
INTR_TYPE_NET|INTR_MPSAFE,
#else
INTR_TYPE_NET,
#endif
NULL, ce_intr, bd, &bd->ce_intrhand);
if (error) {
printf ("ce%d: cannot set up irq\n", unit);
bus_release_resource (dev, SYS_RES_IRQ, 0, bd->ce_irq);
bus_release_resource (dev, SYS_RES_MEMORY,
PCIR_BAR(0), bd->ce_res);
ce_bus_dma_mem_free (&bd->dmamem);
free (b, M_DEVBUF);
splx (s);
return (ENXIO);
}
switch (b->ddk.Model) {
case 1: strcpy (b->name, TAU32_BASE_NAME); break;
case 2: strcpy (b->name, TAU32_LITE_NAME); break;
case 3: strcpy (b->name, TAU32_ADPCM_NAME); break;
default: strcpy (b->name, TAU32_UNKNOWN_NAME); break;
}
printf ("ce%d: %s\n", unit, b->name);
for (c = b->chan; c < b->chan + NCHAN; ++c) {
c->num = (c - b->chan);
c->board = b;
d = &bd->channel[c->num];
d->dmamem.size = sizeof(ce_buf_t);
if (! ce_bus_dma_mem_alloc (unit, c->num, &d->dmamem))
continue;
channel [b->num * NCHAN + c->num] = d;
sprintf (d->name, "ce%d.%d", b->num, c->num);
d->board = b;
d->chan = c;
c->sys = d;
}
for (c = b->chan; c < b->chan + NCHAN; ++c) {
if (c->sys == NULL)
continue;
d = c->sys;
- callout_init (&d->timeout_handle, CALLOUT_MPSAFE);
+ callout_init (&d->timeout_handle, 1);
#ifdef NETGRAPH
if (ng_make_node_common (&typestruct, &d->node) != 0) {
printf ("%s: cannot make common node\n", d->name);
d->node = NULL;
continue;
}
#if __FreeBSD_version >= 500000
NG_NODE_SET_PRIVATE (d->node, d);
#else
d->node->private = d;
#endif
sprintf (d->nodename, "%s%d", NG_CE_NODE_TYPE,
c->board->num * NCHAN + c->num);
if (ng_name_node (d->node, d->nodename)) {
printf ("%s: cannot name node\n", d->nodename);
#if __FreeBSD_version >= 500000
NG_NODE_UNREF (d->node);
#else
ng_rmnode (d->node);
ng_unref (d->node);
#endif
continue;
}
d->queue.ifq_maxlen = ifqmaxlen;
d->hi_queue.ifq_maxlen = ifqmaxlen;
d->rqueue.ifq_maxlen = ifqmaxlen;
#if __FreeBSD_version >= 500000
mtx_init (&d->queue.ifq_mtx, "ce_queue", NULL, MTX_DEF);
mtx_init (&d->hi_queue.ifq_mtx, "ce_queue_hi", NULL, MTX_DEF);
mtx_init (&d->rqueue.ifq_mtx, "ce_rqueue", NULL, MTX_DEF);
#endif
#else /*NETGRAPH*/
#if __FreeBSD_version >= 600031
d->ifp = if_alloc(IFT_PPP);
#else
d->ifp = malloc (sizeof(struct sppp), M_DEVBUF, M_WAITOK);
bzero (d->ifp, sizeof(struct sppp));
#endif
if (!d->ifp) {
printf ("%s: cannot if_alloc() interface\n", d->name);
continue;
}
d->ifp->if_softc = d;
#if __FreeBSD_version > 501000
if_initname (d->ifp, "ce", b->num * NCHAN + c->num);
#else
d->ifp->if_unit = b->num * NCHAN + c->num;
d->ifp->if_name = "ce";
#endif
d->ifp->if_mtu = PP_MTU;
d->ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
d->ifp->if_ioctl = ce_sioctl;
d->ifp->if_start = ce_ifstart;
d->ifp->if_init = ce_initialize;
d->rqueue.ifq_maxlen = ifqmaxlen;
#if __FreeBSD_version >= 500000
mtx_init (&d->rqueue.ifq_mtx, "ce_rqueue", NULL, MTX_DEF);
#endif
sppp_attach (d->ifp);
if_attach (d->ifp);
IFP2SP(d->ifp)->pp_tlf = ce_tlf;
IFP2SP(d->ifp)->pp_tls = ce_tls;
/* If BPF is in the kernel, call the attach for it.
* The header size of PPP or Cisco/HDLC is 4 bytes. */
bpfattach (d->ifp, DLT_PPP, 4);
#endif /*NETGRAPH*/
ce_start_chan (c, 1, 1, d->dmamem.virt, d->dmamem.phys);
/* Register callback functions. */
ce_register_transmit (c, &ce_transmit);
ce_register_receive (c, &ce_receive);
ce_register_error (c, &ce_error);
d->devt = make_dev (&ce_cdevsw, b->num*NCHAN+c->num, UID_ROOT,
GID_WHEEL, 0600, "ce%d", b->num*NCHAN+c->num);
}
#if __FreeBSD_version >= 504000
ce_ln[2] = '0' + unit;
mtx_init (&bd->ce_mtx, ce_ln, MTX_NETWORK_LOCK, MTX_DEF|MTX_RECURSE);
#endif
CE_LOCK (bd);
TAU32_EnableInterrupts(b->ddk.pControllerObject);
adapter[unit] = b;
CE_UNLOCK (bd);
splx (s);
return 0;
}
static int ce_detach (device_t dev)
{
bdrv_t *bd = device_get_softc (dev);
ce_board_t *b = bd->board;
ce_chan_t *c;
int s;
#if __FreeBSD_version >= 504000
KASSERT (mtx_initialized (&bd->ce_mtx), ("ce mutex not initialized"));
#endif
s = splimp ();
CE_LOCK (bd);
/* Check if the device is busy (open). */
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
/* XXX Non existen chan! */
if (! d || ! d->chan)
continue;
if (d->running) {
CE_UNLOCK (bd);
splx (s);
return EBUSY;
}
}
/* Ok, we can unload driver */
/* At first we should disable interrupts */
ce_destroy = 1;
TAU32_DisableInterrupts(b->ddk.pControllerObject);
callout_stop (&led_timo[b->num]);
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (! d || ! d->chan)
continue;
callout_stop (&d->timeout_handle);
#ifndef NETGRAPH
/* Detach from the packet filter list of interfaces. */
bpfdetach (d->ifp);
/* Detach from the sync PPP list. */
sppp_detach (d->ifp);
/* Detach from the system list of interfaces. */
if_detach (d->ifp);
#if __FreeBSD_version > 600031
if_free(d->ifp);
#else
free (d->ifp, M_DEVBUF);
#endif
IF_DRAIN (&d->rqueue);
#if __FreeBSD_version >= 500000
mtx_destroy (&d->rqueue.ifq_mtx);
#endif
#else
#if __FreeBSD_version >= 500000
if (d->node) {
ng_rmnode_self (d->node);
NG_NODE_UNREF (d->node);
d->node = NULL;
}
IF_DRAIN (&d->rqueue);
mtx_destroy (&d->queue.ifq_mtx);
mtx_destroy (&d->hi_queue.ifq_mtx);
mtx_destroy (&d->rqueue.ifq_mtx);
#else
ng_rmnode (d->node);
d->node = 0;
#endif
#endif
destroy_dev (d->devt);
}
CE_UNLOCK (bd);
splx (s);
callout_drain (&led_timo[b->num]);
/* Disable the interrupt request. */
bus_teardown_intr (dev, bd->ce_irq, bd->ce_intrhand);
bus_release_resource (dev, SYS_RES_IRQ, 0, bd->ce_irq);
TAU32_DestructiveHalt (b->ddk.pControllerObject, 0);
bus_release_resource (dev, SYS_RES_MEMORY, PCIR_BAR(0), bd->ce_res);
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (! d || ! d->chan)
continue;
callout_drain (&d->timeout_handle);
channel [b->num * NCHAN + c->num] = 0;
/* Deallocate buffers. */
ce_bus_dma_mem_free (&d->dmamem);
}
adapter [b->num] = 0;
ce_bus_dma_mem_free (&bd->dmamem);
free (b, M_DEVBUF);
#if __FreeBSD_version >= 504000
mtx_destroy (&bd->ce_mtx);
#endif
return 0;
}
#ifndef NETGRAPH
static void ce_ifstart (struct ifnet *ifp)
{
drv_t *d = ifp->if_softc;
bdrv_t *bd = d->board->sys;
CE_LOCK (bd);
ce_start (d);
CE_UNLOCK (bd);
}
static void ce_tlf (struct sppp *sp)
{
drv_t *d = SP2IFP(sp)->if_softc;
CE_DEBUG2 (d, ("ce_tlf\n"));
sp->pp_down (sp);
}
static void ce_tls (struct sppp *sp)
{
drv_t *d = SP2IFP(sp)->if_softc;
CE_DEBUG2 (d, ("ce_tls\n"));
sp->pp_up (sp);
}
/*
* Process an ioctl request.
*/
static int ce_sioctl (struct ifnet *ifp, u_long cmd, caddr_t data)
{
drv_t *d = ifp->if_softc;
bdrv_t *bd = d->board->sys;
int error, s, was_up, should_be_up;
#if __FreeBSD_version >= 600034
was_up = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
#else
was_up = (ifp->if_flags & IFF_RUNNING) != 0;
#endif
error = sppp_ioctl (ifp, cmd, data);
if (error)
return error;
if (! (ifp->if_flags & IFF_DEBUG))
d->chan->debug = 0;
else
d->chan->debug = d->chan->debug_shadow;
switch (cmd) {
default: CE_DEBUG2 (d, ("ioctl 0x%lx\n", cmd)); return 0;
case SIOCADDMULTI: CE_DEBUG2 (d, ("ioctl SIOCADDMULTI\n")); return 0;
case SIOCDELMULTI: CE_DEBUG2 (d, ("ioctl SIOCDELMULTI\n")); return 0;
case SIOCSIFFLAGS: CE_DEBUG2 (d, ("ioctl SIOCSIFFLAGS\n")); break;
case SIOCSIFADDR: CE_DEBUG2 (d, ("ioctl SIOCSIFADDR\n")); break;
}
/* We get here only in case of SIFFLAGS or SIFADDR. */
s = splimp ();
CE_LOCK (bd);
#if __FreeBSD_version >= 600034
should_be_up = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
#else
should_be_up = (ifp->if_flags & IFF_RUNNING) != 0;
#endif
if (! was_up && should_be_up) {
/* Interface goes up -- start it. */
ce_up (d);
ce_start (d);
} else if (was_up && ! should_be_up) {
/* Interface is going down -- stop it. */
/* if ((IFP2SP(ifp)->pp_flags & PP_FR) || (ifp->if_flags & PP_CISCO))*/
ce_down (d);
}
CE_DEBUG (d, ("ioctl 0x%lx p4\n", cmd));
CE_UNLOCK (bd);
splx (s);
return 0;
}
/*
* Initialization of interface.
* It seems to be never called by upper level?
*/
static void ce_initialize (void *softc)
{
drv_t *d = softc;
CE_DEBUG (d, ("ce_initialize\n"));
}
#endif /*NETGRAPH*/
/*
* Stop the interface. Called on splimp().
*/
static void ce_down (drv_t *d)
{
CE_DEBUG (d, ("ce_down\n"));
/* Interface is going down -- stop it. */
ce_set_dtr (d->chan, 0);
ce_set_rts (d->chan, 0);
d->running = 0;
callout_stop (&d->timeout_handle);
}
/*
* Start the interface. Called on splimp().
*/
static void ce_up (drv_t *d)
{
CE_DEBUG (d, ("ce_up\n"));
ce_set_dtr (d->chan, 1);
ce_set_rts (d->chan, 1);
d->running = 1;
}
/*
* Start output on the interface. Get another datagram to send
* off of the interface queue, and copy it to the interface
* before starting the output.
*/
static void ce_send (drv_t *d)
{
struct mbuf *m;
u_short len;
CE_DEBUG2 (d, ("ce_send\n"));
/* No output if the interface is down. */
if (! d->running)
return;
while (ce_transmit_space (d->chan)) {
/* Get the packet to send. */
#ifdef NETGRAPH
IF_DEQUEUE (&d->hi_queue, m);
if (! m)
IF_DEQUEUE (&d->queue, m);
#else
m = sppp_dequeue (d->ifp);
#endif
if (! m)
return;
#ifndef NETGRAPH
#if __FreeBSD_version >= 500000
BPF_MTAP (d->ifp, m);
#else
if (d->ifp->if_bpf)
bpf_mtap (d->ifp, m);
#endif
#endif
#if __FreeBSD_version >= 490000
len = m_length (m, NULL);
#else
len = m->m_pkthdr.len;
#endif
if (len >= BUFSZ)
printf ("%s: too long packet: %d bytes: ",
d->name, len);
else if (! m->m_next)
ce_send_packet (d->chan, (u_char*) mtod (m, caddr_t), len, 0);
else {
ce_buf_item_t *item = (ce_buf_item_t*)d->chan->tx_queue;
m_copydata (m, 0, len, item->buf);
ce_send_packet (d->chan, item->buf, len, 0);
}
m_freem (m);
/* Set up transmit timeout, if the transmit ring is not empty.*/
d->timeout = 10;
}
#ifndef NETGRAPH
#if __FreeBSD_version >= 600034
d->ifp->if_flags |= IFF_DRV_OACTIVE;
#else
d->ifp->if_flags |= IFF_OACTIVE;
#endif
#endif
}
/*
* Start output on the interface.
* Always called on splimp().
*/
static void ce_start (drv_t *d)
{
if (d->running) {
if (! d->chan->dtr)
ce_set_dtr (d->chan, 1);
if (! d->chan->rts)
ce_set_rts (d->chan, 1);
ce_send (d);
callout_reset (&d->timeout_handle, hz, ce_watchdog_timer, d);
}
}
/*
* Handle transmit timeouts.
* Recover after lost transmit interrupts.
* Always called on splimp().
*/
static void ce_watchdog (drv_t *d)
{
CE_DEBUG (d, ("device timeout\n"));
if (d->running) {
ce_set_dtr (d->chan, 0);
ce_set_rts (d->chan, 0);
/* ce_stop_chan (d->chan);*/
/* ce_start_chan (d->chan, 1, 1, 0, 0);*/
ce_set_dtr (d->chan, 1);
ce_set_rts (d->chan, 1);
ce_start (d);
}
}
static void ce_watchdog_timer (void *arg)
{
drv_t *d = arg;
bdrv_t *bd = d->board->sys;
CE_LOCK(bd);
if (d->timeout == 1)
ce_watchdog (d);
if (d->timeout)
d->timeout--;
callout_reset (&d->timeout_handle, hz, ce_watchdog_timer, d);
CE_UNLOCK(bd);
}
static void ce_transmit (ce_chan_t *c, void *attachment, int len)
{
drv_t *d = c->sys;
d->timeout = 0;
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_OPACKETS, 1);
#if __FreeBSD_version >= 600034
d->ifp->if_flags &= ~IFF_DRV_OACTIVE;
#else
d->ifp->if_flags &= ~IFF_OACTIVE;
#endif
#endif
ce_start (d);
}
static void ce_receive (ce_chan_t *c, unsigned char *data, int len)
{
drv_t *d = c->sys;
struct mbuf *m;
if (! d->running)
return;
m = makembuf (data, len);
if (! m) {
CE_DEBUG (d, ("no memory for packet\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IQDROPS, 1);
#endif
return;
}
if (c->debug > 1)
m_print (m, 0);
#ifdef NETGRAPH
m->m_pkthdr.rcvif = 0;
IF_ENQUEUE(&d->rqueue, m);
#else
if_inc_counter(d->ifp, IFCOUNTER_IPACKETS, 1);
m->m_pkthdr.rcvif = d->ifp;
/* Check if there's a BPF listener on this interface.
* If so, hand off the raw packet to bpf. */
BPF_MTAP(d->ifp, m);
IF_ENQUEUE(&d->rqueue, m);
#endif
}
static void ce_error (ce_chan_t *c, int data)
{
drv_t *d = c->sys;
switch (data) {
case CE_FRAME:
CE_DEBUG (d, ("frame error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CE_CRC:
CE_DEBUG (d, ("crc error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CE_OVERRUN:
CE_DEBUG (d, ("overrun error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_COLLISIONS, 1);
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CE_OVERFLOW:
CE_DEBUG (d, ("overflow error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CE_UNDERRUN:
CE_DEBUG (d, ("underrun error\n"));
d->timeout = 0;
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_OERRORS, 1);
#if __FreeBSD_version >= 600034
d->ifp->if_flags &= ~IFF_DRV_OACTIVE;
#else
d->ifp->if_flags &= ~IFF_OACTIVE;
#endif
#endif
ce_start (d);
break;
default:
CE_DEBUG (d, ("error #%d\n", data));
break;
}
}
/*
* You also need read, write, open, close routines.
* This should get you started
*/
#if __FreeBSD_version < 500000
static int ce_open (dev_t dev, int oflags, int devtype, struct proc *p)
#else
static int ce_open (struct cdev *dev, int oflags, int devtype, struct thread *td)
#endif
{
int unit = dev2unit (dev);
drv_t *d;
if (unit >= NBRD*NCHAN || ! (d = channel[unit]))
return ENXIO;
CE_DEBUG2 (d, ("ce_open\n"));
return 0;
}
/*
* Only called on the LAST close.
*/
#if __FreeBSD_version < 500000
static int ce_close (dev_t dev, int fflag, int devtype, struct proc *p)
#else
static int ce_close (struct cdev *dev, int fflag, int devtype, struct thread *td)
#endif
{
drv_t *d = channel [dev2unit (dev)];
CE_DEBUG2 (d, ("ce_close\n"));
return 0;
}
static int ce_modem_status (ce_chan_t *c)
{
drv_t *d = c->sys;
bdrv_t *bd = d->board->sys;
int status, s;
status = d->running ? TIOCM_LE : 0;
s = splimp ();
CE_LOCK (bd);
if (ce_get_cd (c)) status |= TIOCM_CD;
if (ce_get_cts (c)) status |= TIOCM_CTS;
if (ce_get_dsr (c)) status |= TIOCM_DSR;
if (c->dtr) status |= TIOCM_DTR;
if (c->rts) status |= TIOCM_RTS;
CE_UNLOCK (bd);
splx (s);
return status;
}
#if __FreeBSD_version < 500000
static int ce_ioctl (dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
#else
static int ce_ioctl (struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td)
#endif
{
drv_t *d = channel [dev2unit (dev)];
bdrv_t *bd = d->board->sys;
ce_chan_t *c = d->chan;
struct serial_statistics *st;
struct e1_statistics *opte1;
int error, s;
char mask[16];
switch (cmd) {
case SERIAL_GETREGISTERED:
CE_DEBUG2 (d, ("ioctl: getregistered\n"));
bzero (mask, sizeof(mask));
for (s=0; s<NBRD*NCHAN; ++s)
if (channel [s])
mask [s/8] |= 1 << (s & 7);
bcopy (mask, data, sizeof (mask));
return 0;
#ifndef NETGRAPH
case SERIAL_GETPROTO:
CE_DEBUG2 (d, ("ioctl: getproto\n"));
strcpy ((char*)data, (IFP2SP(d->ifp)->pp_flags & PP_FR) ? "fr" :
(d->ifp->if_flags & PP_CISCO) ? "cisco" : "ppp");
return 0;
case SERIAL_SETPROTO:
CE_DEBUG2 (d, ("ioctl: setproto\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
#if __FreeBSD_version >= 600034
if (d->ifp->if_flags & IFF_DRV_RUNNING)
#else
if (d->ifp->if_flags & IFF_RUNNING)
#endif
return EBUSY;
if (! strcmp ("cisco", (char*)data)) {
IFP2SP(d->ifp)->pp_flags &= ~(PP_FR);
IFP2SP(d->ifp)->pp_flags |= PP_KEEPALIVE;
d->ifp->if_flags |= PP_CISCO;
#if PP_FR != 0
} else if (! strcmp ("fr", (char*)data)) {
d->ifp->if_flags &= ~(PP_CISCO);
IFP2SP(d->ifp)->pp_flags |= PP_FR | PP_KEEPALIVE;
#endif
} else if (! strcmp ("ppp", (char*)data)) {
IFP2SP(d->ifp)->pp_flags &= ~PP_FR;
IFP2SP(d->ifp)->pp_flags &= ~PP_KEEPALIVE;
d->ifp->if_flags &= ~(PP_CISCO);
} else
return EINVAL;
return 0;
case SERIAL_GETKEEPALIVE:
CE_DEBUG2 (d, ("ioctl: getkeepalive\n"));
if ((IFP2SP(d->ifp)->pp_flags & PP_FR) ||
(d->ifp->if_flags & PP_CISCO))
return EINVAL;
*(int*)data = (IFP2SP(d->ifp)->pp_flags & PP_KEEPALIVE) ? 1 : 0;
return 0;
case SERIAL_SETKEEPALIVE:
CE_DEBUG2 (d, ("ioctl: setkeepalive\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
if ((IFP2SP(d->ifp)->pp_flags & PP_FR) ||
(d->ifp->if_flags & PP_CISCO))
return EINVAL;
s = splimp ();
CE_LOCK (bd);
if (*(int*)data)
IFP2SP(d->ifp)->pp_flags |= PP_KEEPALIVE;
else
IFP2SP(d->ifp)->pp_flags &= ~PP_KEEPALIVE;
CE_UNLOCK (bd);
splx (s);
return 0;
#endif /*NETGRAPH*/
case SERIAL_GETMODE:
CE_DEBUG2 (d, ("ioctl: getmode\n"));
*(int*)data = SERIAL_HDLC;
return 0;
case SERIAL_SETMODE:
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
if (*(int*)data != SERIAL_HDLC)
return EINVAL;
return 0;
case SERIAL_GETCFG:
CE_DEBUG2 (d, ("ioctl: getcfg\n"));
*(char*)data = 'c';
return 0;
case SERIAL_SETCFG:
CE_DEBUG2 (d, ("ioctl: setcfg\n"));
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
if (*((char*)data) != 'c')
return EINVAL;
return 0;
case SERIAL_GETSTAT:
CE_DEBUG2 (d, ("ioctl: getstat\n"));
st = (struct serial_statistics*) data;
st->rintr = c->rintr;
st->tintr = c->tintr;
st->mintr = 0;
st->ibytes = c->ibytes;
st->ipkts = c->ipkts;
st->obytes = c->obytes;
st->opkts = c->opkts;
st->ierrs = c->overrun + c->frame + c->crc;
st->oerrs = c->underrun;
return 0;
case SERIAL_GETESTAT:
CE_DEBUG2 (d, ("ioctl: getestat\n"));
if (c->type != T_E1)
return EINVAL;
opte1 = (struct e1_statistics*) data;
opte1->status = 0;
if (c->status & ESTS_NOALARM)
opte1->status |= E1_NOALARM;
if (c->status & ESTS_LOS)
opte1->status |= E1_LOS;
if (c->status & ESTS_LOF)
opte1->status |= E1_LOF;
if (c->status & ESTS_AIS)
opte1->status |= E1_AIS;
if (c->status & ESTS_LOMF)
opte1->status |= E1_LOMF;
if (c->status & ESTS_AIS16)
opte1->status |= E1_AIS16;
if (c->status & ESTS_FARLOF)
opte1->status |= E1_FARLOF;
if (c->status & ESTS_FARLOMF)
opte1->status |= E1_FARLOMF;
if (c->status & ESTS_TSTREQ)
opte1->status |= E1_TSTREQ;
if (c->status & ESTS_TSTERR)
opte1->status |= E1_TSTERR;
opte1->cursec = c->cursec;
opte1->totsec = c->totsec + c->cursec;
opte1->currnt.bpv = c->currnt.bpv;
opte1->currnt.fse = c->currnt.fse;
opte1->currnt.crce = c->currnt.crce;
opte1->currnt.rcrce = c->currnt.rcrce;
opte1->currnt.uas = c->currnt.uas;
opte1->currnt.les = c->currnt.les;
opte1->currnt.es = c->currnt.es;
opte1->currnt.bes = c->currnt.bes;
opte1->currnt.ses = c->currnt.ses;
opte1->currnt.oofs = c->currnt.oofs;
opte1->currnt.css = c->currnt.css;
opte1->currnt.dm = c->currnt.dm;
opte1->total.bpv = c->total.bpv + c->currnt.bpv;
opte1->total.fse = c->total.fse + c->currnt.fse;
opte1->total.crce = c->total.crce + c->currnt.crce;
opte1->total.rcrce = c->total.rcrce + c->currnt.rcrce;
opte1->total.uas = c->total.uas + c->currnt.uas;
opte1->total.les = c->total.les + c->currnt.les;
opte1->total.es = c->total.es + c->currnt.es;
opte1->total.bes = c->total.bes + c->currnt.bes;
opte1->total.ses = c->total.ses + c->currnt.ses;
opte1->total.oofs = c->total.oofs + c->currnt.oofs;
opte1->total.css = c->total.css + c->currnt.css;
opte1->total.dm = c->total.dm + c->currnt.dm;
for (s=0; s<48; ++s) {
opte1->interval[s].bpv = c->interval[s].bpv;
opte1->interval[s].fse = c->interval[s].fse;
opte1->interval[s].crce = c->interval[s].crce;
opte1->interval[s].rcrce = c->interval[s].rcrce;
opte1->interval[s].uas = c->interval[s].uas;
opte1->interval[s].les = c->interval[s].les;
opte1->interval[s].es = c->interval[s].es;
opte1->interval[s].bes = c->interval[s].bes;
opte1->interval[s].ses = c->interval[s].ses;
opte1->interval[s].oofs = c->interval[s].oofs;
opte1->interval[s].css = c->interval[s].css;
opte1->interval[s].dm = c->interval[s].dm;
}
return 0;
case SERIAL_CLRSTAT:
CE_DEBUG2 (d, ("ioctl: clrstat\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
c->rintr = 0;
c->tintr = 0;
c->ibytes = 0;
c->obytes = 0;
c->ipkts = 0;
c->opkts = 0;
c->overrun = 0;
c->frame = 0;
c->crc = 0;
c->underrun = 0;
bzero (&c->currnt, sizeof (c->currnt));
bzero (&c->total, sizeof (c->total));
bzero (c->interval, sizeof (c->interval));
return 0;
case SERIAL_GETLOOP:
CE_DEBUG2 (d, ("ioctl: getloop\n"));
if (c->type != T_E1)
return EINVAL;
*(int*)data = c->lloop;
return 0;
case SERIAL_SETLOOP:
CE_DEBUG2 (d, ("ioctl: setloop\n"));
if (c->type != T_E1)
return EINVAL;
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
s = splimp ();
CE_LOCK (bd);
ce_set_lloop (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETRLOOP:
CE_DEBUG2 (d, ("ioctl: getrloop\n"));
if (c->type != T_E1)
return EINVAL;
*(int*)data = c->rloop;
return 0;
case SERIAL_SETRLOOP:
CE_DEBUG2 (d, ("ioctl: setloop\n"));
if (c->type != T_E1)
return EINVAL;
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
s = splimp ();
CE_LOCK (bd);
ce_set_rloop (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETDEBUG:
CE_DEBUG2 (d, ("ioctl: getdebug\n"));
*(int*)data = d->chan->debug;
return 0;
case SERIAL_SETDEBUG:
CE_DEBUG2 (d, ("ioctl: setdebug\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
#ifndef NETGRAPH
/*
* The debug_shadow is always greater than zero for logic
* simplicity. For switching debug off the IFF_DEBUG is
* responsible.
*/
d->chan->debug_shadow = (*(int*)data) ? (*(int*)data) : 1;
if (d->ifp->if_flags & IFF_DEBUG)
d->chan->debug = d->chan->debug_shadow;
#else
d->chan->debug = *(int*)data;
#endif
return 0;
case SERIAL_GETBAUD:
CE_DEBUG2 (d, ("ioctl: getbaud\n"));
*(long*)data = c->baud;
return 0;
case SERIAL_SETBAUD:
CE_DEBUG2 (d, ("ioctl: setbaud\n"));
if (c->type != T_E1 || !c->unfram)
return EINVAL;
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
s = splimp ();
CE_LOCK (bd);
ce_set_baud (c, *(long*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETTIMESLOTS:
CE_DEBUG2 (d, ("ioctl: gettimeslots\n"));
if ((c->type != T_E1 || c->unfram) && c->type != T_DATA)
return EINVAL;
*(u_long*)data = c->ts;
return 0;
case SERIAL_SETTIMESLOTS:
CE_DEBUG2 (d, ("ioctl: settimeslots\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
if ((c->type != T_E1 || c->unfram) && c->type != T_DATA)
return EINVAL;
s = splimp ();
CE_LOCK (bd);
ce_set_ts (c, *(u_long*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETHIGAIN:
CE_DEBUG2 (d, ("ioctl: gethigain\n"));
if (c->type != T_E1)
return EINVAL;
*(int*)data = c->higain;
return 0;
case SERIAL_SETHIGAIN:
CE_DEBUG2 (d, ("ioctl: sethigain\n"));
if (c->type != T_E1)
return EINVAL;
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
s = splimp ();
CE_LOCK (bd);
ce_set_higain (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETPHONY:
CE_DEBUG2 (d, ("ioctl: getphony\n"));
*(int*)data = c->phony;
return 0;
case SERIAL_SETPHONY:
CE_DEBUG2 (d, ("ioctl: setphony\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
s = splimp ();
CE_LOCK (bd);
ce_set_phony (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETUNFRAM:
CE_DEBUG2 (d, ("ioctl: getunfram\n"));
if (c->type != T_E1 || c->num != 0)
return EINVAL;
*(int*)data = c->unfram;
return 0;
case SERIAL_SETUNFRAM:
CE_DEBUG2 (d, ("ioctl: setunfram\n"));
if (c->type != T_E1 || c->num != 0)
return EINVAL;
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
s = splimp ();
CE_LOCK (bd);
ce_set_unfram (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETSCRAMBLER:
CE_DEBUG2 (d, ("ioctl: getscrambler\n"));
if (!c->unfram)
return EINVAL;
*(int*)data = c->scrambler;
return 0;
case SERIAL_SETSCRAMBLER:
CE_DEBUG2 (d, ("ioctl: setscrambler\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
if (!c->unfram)
return EINVAL;
s = splimp ();
CE_LOCK (bd);
ce_set_scrambler (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETMONITOR:
CE_DEBUG2 (d, ("ioctl: getmonitor\n"));
if (c->type != T_E1)
return EINVAL;
*(int*)data = c->monitor;
return 0;
case SERIAL_SETMONITOR:
CE_DEBUG2 (d, ("ioctl: setmonitor\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CE_LOCK (bd);
ce_set_monitor (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETUSE16:
CE_DEBUG2 (d, ("ioctl: getuse16\n"));
if (c->type != T_E1 || c->unfram)
return EINVAL;
*(int*)data = c->use16;
return 0;
case SERIAL_SETUSE16:
CE_DEBUG2 (d, ("ioctl: setuse16\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CE_LOCK (bd);
ce_set_use16 (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETCRC4:
CE_DEBUG2 (d, ("ioctl: getcrc4\n"));
if (c->type != T_E1 || c->unfram)
return EINVAL;
*(int*)data = c->crc4;
return 0;
case SERIAL_SETCRC4:
CE_DEBUG2 (d, ("ioctl: setcrc4\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
if (c->type != T_E1 || c->unfram)
return EINVAL;
s = splimp ();
CE_LOCK (bd);
ce_set_crc4 (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETCLK:
CE_DEBUG2 (d, ("ioctl: getclk\n"));
if (c->type != T_E1)
return EINVAL;
switch (c->gsyn) {
default: *(int*)data = E1CLK_INTERNAL; break;
case GSYN_RCV: *(int*)data = E1CLK_RECEIVE; break;
case GSYN_RCV0: *(int*)data = E1CLK_RECEIVE_CHAN0; break;
case GSYN_RCV1: *(int*)data = E1CLK_RECEIVE_CHAN1; break;
}
return 0;
case SERIAL_SETCLK:
CE_DEBUG2 (d, ("ioctl: setclk\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CE_LOCK (bd);
switch (*(int*)data) {
default: ce_set_gsyn (c, GSYN_INT); break;
case E1CLK_RECEIVE: ce_set_gsyn (c, GSYN_RCV); break;
case E1CLK_RECEIVE_CHAN0: ce_set_gsyn (c, GSYN_RCV0); break;
case E1CLK_RECEIVE_CHAN1: ce_set_gsyn (c, GSYN_RCV1); break;
}
CE_UNLOCK (bd);
splx (s);
return 0;
#if 0
case SERIAL_RESET:
CE_DEBUG2 (d, ("ioctl: reset\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
s = splimp ();
CE_LOCK (bd);
/* ce_reset (c->board, 0, 0);*/
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_HARDRESET:
CE_DEBUG2 (d, ("ioctl: hardreset\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
s = splimp ();
CE_LOCK (bd);
/* hard_reset (c->board); */
CE_UNLOCK (bd);
splx (s);
return 0;
#endif
case SERIAL_GETCABLE:
CE_DEBUG2 (d, ("ioctl: getcable\n"));
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CE_LOCK (bd);
*(int*)data = CABLE_TP;
CE_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETDIR:
CE_DEBUG2 (d, ("ioctl: getdir\n"));
if (c->type != T_E1 && c->type != T_DATA)
return EINVAL;
*(int*)data = c->dir;
return 0;
case SERIAL_SETDIR:
CE_DEBUG2 (d, ("ioctl: setdir\n"));
/* Only for superuser! */
#if __FreeBSD_version < 500000
error = suser (p);
#elif __FreeBSD_version < 700000
error = suser (td);
#else
error = priv_check (td, PRIV_DRIVER);
#endif
if (error)
return error;
s = splimp ();
CE_LOCK (bd);
ce_set_dir (c, *(int*)data);
CE_UNLOCK (bd);
splx (s);
return 0;
case TIOCSDTR: /* Set DTR */
s = splimp ();
CE_LOCK (bd);
ce_set_dtr (c, 1);
CE_UNLOCK (bd);
splx (s);
return 0;
case TIOCCDTR: /* Clear DTR */
s = splimp ();
CE_LOCK (bd);
ce_set_dtr (c, 0);
CE_UNLOCK (bd);
splx (s);
return 0;
case TIOCMSET: /* Set DTR/RTS */
s = splimp ();
CE_LOCK (bd);
ce_set_dtr (c, (*(int*)data & TIOCM_DTR) ? 1 : 0);
ce_set_rts (c, (*(int*)data & TIOCM_RTS) ? 1 : 0);
CE_UNLOCK (bd);
splx (s);
return 0;
case TIOCMBIS: /* Add DTR/RTS */
s = splimp ();
CE_LOCK (bd);
if (*(int*)data & TIOCM_DTR) ce_set_dtr (c, 1);
if (*(int*)data & TIOCM_RTS) ce_set_rts (c, 1);
CE_UNLOCK (bd);
splx (s);
return 0;
case TIOCMBIC: /* Clear DTR/RTS */
s = splimp ();
CE_LOCK (bd);
if (*(int*)data & TIOCM_DTR) ce_set_dtr (c, 0);
if (*(int*)data & TIOCM_RTS) ce_set_rts (c, 0);
CE_UNLOCK (bd);
splx (s);
return 0;
case TIOCMGET: /* Get modem status */
*(int*)data = ce_modem_status (c);
return 0;
}
return ENOTTY;
}
#ifdef NETGRAPH
#if __FreeBSD_version >= 500000
static int ng_ce_constructor (node_p node)
{
drv_t *d = NG_NODE_PRIVATE (node);
#else
static int ng_ce_constructor (node_p *node)
{
drv_t *d = (*node)->private;
#endif
CE_DEBUG (d, ("Constructor\n"));
return EINVAL;
}
static int ng_ce_newhook (node_p node, hook_p hook, const char *name)
{
int s;
#if __FreeBSD_version >= 500000
drv_t *d = NG_NODE_PRIVATE (node);
#else
drv_t *d = node->private;
#endif
bdrv_t *bd = d->board->sys;
CE_DEBUG (d, ("Newhook\n"));
/* Attach debug hook */
if (strcmp (name, NG_CE_HOOK_DEBUG) == 0) {
#if __FreeBSD_version >= 500000
NG_HOOK_SET_PRIVATE (hook, NULL);
#else
hook->private = 0;
#endif
d->debug_hook = hook;
return 0;
}
/* Check for raw hook */
if (strcmp (name, NG_CE_HOOK_RAW) != 0)
return EINVAL;
#if __FreeBSD_version >= 500000
NG_HOOK_SET_PRIVATE (hook, d);
#else
hook->private = d;
#endif
d->hook = hook;
s = splimp ();
CE_LOCK (bd);
ce_up (d);
CE_UNLOCK (bd);
splx (s);
return 0;
}
static char *format_timeslots (u_long s)
{
static char buf [100];
char *p = buf;
int i;
for (i=1; i<32; ++i)
if ((s >> i) & 1) {
int prev = (i > 1) & (s >> (i-1));
int next = (i < 31) & (s >> (i+1));
if (prev) {
if (next)
continue;
*p++ = '-';
} else if (p > buf)
*p++ = ',';
if (i >= 10)
*p++ = '0' + i / 10;
*p++ = '0' + i % 10;
}
*p = 0;
return buf;
}
static int print_modems (char *s, ce_chan_t *c, int need_header)
{
int status = ce_modem_status (c);
int length = 0;
if (need_header)
length += sprintf (s + length, " LE DTR DSR RTS CTS CD\n");
length += sprintf (s + length, "%4s %4s %4s %4s %4s %4s\n",
status & TIOCM_LE ? "On" : "-",
status & TIOCM_DTR ? "On" : "-",
status & TIOCM_DSR ? "On" : "-",
status & TIOCM_RTS ? "On" : "-",
status & TIOCM_CTS ? "On" : "-",
status & TIOCM_CD ? "On" : "-");
return length;
}
static int print_stats (char *s, ce_chan_t *c, int need_header)
{
int length = 0;
if (need_header)
length += sprintf (s + length, " Rintr Tintr Mintr Ibytes Ipkts Ierrs Obytes Opkts Oerrs\n");
length += sprintf (s + length, "%7ld %7ld %7ld %8lu %7ld %7ld %8lu %7ld %7ld\n",
c->rintr, c->tintr, 0l, (unsigned long) c->ibytes,
c->ipkts, c->overrun + c->frame + c->crc,
(unsigned long) c->obytes, c->opkts, c->underrun);
return length;
}
static char *format_e1_status (u_char status)
{
static char buf [80];
if (status & E1_NOALARM)
return "Ok";
buf[0] = 0;
if (status & E1_LOS) strcat (buf, ",LOS");
if (status & E1_AIS) strcat (buf, ",AIS");
if (status & E1_LOF) strcat (buf, ",LOF");
if (status & E1_LOMF) strcat (buf, ",LOMF");
if (status & E1_FARLOF) strcat (buf, ",FARLOF");
if (status & E1_AIS16) strcat (buf, ",AIS16");
if (status & E1_FARLOMF) strcat (buf, ",FARLOMF");
if (status & E1_TSTREQ) strcat (buf, ",TSTREQ");
if (status & E1_TSTERR) strcat (buf, ",TSTERR");
if (buf[0] == ',')
return buf+1;
return "Unknown";
}
static int print_frac (char *s, int leftalign, u_long numerator, u_long divider)
{
int n, length = 0;
if (numerator < 1 || divider < 1) {
length += sprintf (s+length, leftalign ? "/- " : " -");
return length;
}
n = (int) (0.5 + 1000.0 * numerator / divider);
if (n < 1000) {
length += sprintf (s+length, leftalign ? "/.%-3d" : " .%03d", n);
return length;
}
*(s + length) = leftalign ? '/' : ' ';
length ++;
if (n >= 1000000) n = (n+500) / 1000 * 1000;
else if (n >= 100000) n = (n+50) / 100 * 100;
else if (n >= 10000) n = (n+5) / 10 * 10;
switch (n) {
case 1000: length += printf (s+length, ".999"); return length;
case 10000: n = 9990; break;
case 100000: n = 99900; break;
case 1000000: n = 999000; break;
}
if (n < 10000) length += sprintf (s+length, "%d.%d", n/1000, n/10%100);
else if (n < 100000) length += sprintf (s+length, "%d.%d", n/1000, n/100%10);
else if (n < 1000000) length += sprintf (s+length, "%d.", n/1000);
else length += sprintf (s+length, "%d", n/1000);
return length;
}
static int print_e1_stats (char *s, ce_chan_t *c)
{
struct e1_counters total;
u_long totsec;
int length = 0;
totsec = c->totsec + c->cursec;
total.bpv = c->total.bpv + c->currnt.bpv;
total.fse = c->total.fse + c->currnt.fse;
total.crce = c->total.crce + c->currnt.crce;
total.rcrce = c->total.rcrce + c->currnt.rcrce;
total.uas = c->total.uas + c->currnt.uas;
total.les = c->total.les + c->currnt.les;
total.es = c->total.es + c->currnt.es;
total.bes = c->total.bes + c->currnt.bes;
total.ses = c->total.ses + c->currnt.ses;
total.oofs = c->total.oofs + c->currnt.oofs;
total.css = c->total.css + c->currnt.css;
total.dm = c->total.dm + c->currnt.dm;
length += sprintf (s + length, " Unav/Degr Bpv/Fsyn CRC/RCRC Err/Lerr Sev/Bur Oof/Slp Status\n");
/* Unavailable seconds, degraded minutes */
length += print_frac (s + length, 0, c->currnt.uas, c->cursec);
length += print_frac (s + length, 1, 60 * c->currnt.dm, c->cursec);
/* Bipolar violations, frame sync errors */
length += print_frac (s + length, 0, c->currnt.bpv, c->cursec);
length += print_frac (s + length, 1, c->currnt.fse, c->cursec);
/* CRC errors, remote CRC errors (E-bit) */
length += print_frac (s + length, 0, c->currnt.crce, c->cursec);
length += print_frac (s + length, 1, c->currnt.rcrce, c->cursec);
/* Errored seconds, line errored seconds */
length += print_frac (s + length, 0, c->currnt.es, c->cursec);
length += print_frac (s + length, 1, c->currnt.les, c->cursec);
/* Severely errored seconds, burst errored seconds */
length += print_frac (s + length, 0, c->currnt.ses, c->cursec);
length += print_frac (s + length, 1, c->currnt.bes, c->cursec);
/* Out of frame seconds, controlled slip seconds */
length += print_frac (s + length, 0, c->currnt.oofs, c->cursec);
length += print_frac (s + length, 1, c->currnt.css, c->cursec);
length += sprintf (s + length, " %s\n", format_e1_status (c->status));
/* Print total statistics. */
length += print_frac (s + length, 0, total.uas, totsec);
length += print_frac (s + length, 1, 60 * total.dm, totsec);
length += print_frac (s + length, 0, total.bpv, totsec);
length += print_frac (s + length, 1, total.fse, totsec);
length += print_frac (s + length, 0, total.crce, totsec);
length += print_frac (s + length, 1, total.rcrce, totsec);
length += print_frac (s + length, 0, total.es, totsec);
length += print_frac (s + length, 1, total.les, totsec);
length += print_frac (s + length, 0, total.ses, totsec);
length += print_frac (s + length, 1, total.bes, totsec);
length += print_frac (s + length, 0, total.oofs, totsec);
length += print_frac (s + length, 1, total.css, totsec);
length += sprintf (s + length, " -- Total\n");
return length;
}
static int print_chan (char *s, ce_chan_t *c)
{
drv_t *d = c->sys;
int length = 0;
length += sprintf (s + length, "ce%d", c->board->num * NCHAN + c->num);
if (d->chan->debug)
length += sprintf (s + length, " debug=%d", d->chan->debug);
if (c->board->mux) {
length += sprintf (s + length, " cfg=C");
} else {
length += sprintf (s + length, " cfg=A");
}
if (c->baud)
length += sprintf (s + length, " %ld", c->baud);
else
length += sprintf (s + length, " extclock");
if (c->type == T_E1)
switch (c->gsyn) {
case GSYN_INT : length += sprintf (s + length, " syn=int"); break;
case GSYN_RCV : length += sprintf (s + length, " syn=rcv"); break;
case GSYN_RCV0 : length += sprintf (s + length, " syn=rcv0"); break;
case GSYN_RCV1 : length += sprintf (s + length, " syn=rcv1"); break;
}
if (c->type == T_E1)
length += sprintf (s + length, " higain=%s", c->higain ? "on" : "off");
length += sprintf (s + length, " loop=%s", c->lloop ? "on" : "off");
if (c->type == T_E1)
length += sprintf (s + length, " ts=%s", format_timeslots (c->ts));
length += sprintf (s + length, "\n");
return length;
}
#if __FreeBSD_version >= 500000
static int ng_ce_rcvmsg (node_p node, item_p item, hook_p lasthook)
{
drv_t *d = NG_NODE_PRIVATE (node);
struct ng_mesg *msg;
#else
static int ng_ce_rcvmsg (node_p node, struct ng_mesg *msg,
const char *retaddr, struct ng_mesg **rptr)
{
drv_t *d = node->private;
#endif
struct ng_mesg *resp = NULL;
int error = 0;
CE_DEBUG (d, ("Rcvmsg\n"));
#if __FreeBSD_version >= 500000
NGI_GET_MSG (item, msg);
#endif
switch (msg->header.typecookie) {
default:
error = EINVAL;
break;
case NGM_CE_COOKIE:
printf ("Not implemented yet\n");
error = EINVAL;
break;
case NGM_GENERIC_COOKIE:
switch (msg->header.cmd) {
default:
error = EINVAL;
break;
case NGM_TEXT_STATUS: {
char *s;
int l = 0;
int dl = sizeof (struct ng_mesg) + 730;
#if __FreeBSD_version >= 500000
NG_MKRESPONSE (resp, msg, dl, M_NOWAIT);
if (! resp) {
error = ENOMEM;
break;
}
#else
resp = malloc (M_NETGRAPH, M_NOWAIT);
if (! resp) {
error = ENOMEM;
break;
}
bzero (resp, dl);
#endif
s = (resp)->data;
if (d) {
l += print_chan (s + l, d->chan);
l += print_stats (s + l, d->chan, 1);
l += print_modems (s + l, d->chan, 1);
l += print_e1_stats (s + l, d->chan);
} else
l += sprintf (s + l, "Error: node not connect to channel");
#if __FreeBSD_version < 500000
(resp)->header.version = NG_VERSION;
(resp)->header.arglen = strlen (s) + 1;
(resp)->header.token = msg->header.token;
(resp)->header.typecookie = NGM_CE_COOKIE;
(resp)->header.cmd = msg->header.cmd;
#endif
strncpy ((resp)->header.cmdstr, "status", NG_CMDSTRSIZ);
}
break;
}
break;
}
#if __FreeBSD_version >= 500000
NG_RESPOND_MSG (error, node, item, resp);
NG_FREE_MSG (msg);
#else
*rptr = resp;
free (msg, M_NETGRAPH);
#endif
return error;
}
#if __FreeBSD_version >= 500000
static int ng_ce_rcvdata (hook_p hook, item_p item)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE(hook));
struct mbuf *m;
#if __FreeBSD_version < 502120
meta_p meta;
#else
struct ng_tag_prio *ptag;
#endif
#else
static int ng_ce_rcvdata (hook_p hook, struct mbuf *m, meta_p meta)
{
drv_t *d = hook->node->private;
#endif
bdrv_t *bd = d->board->sys;
struct ifqueue *q;
int s;
CE_DEBUG2 (d, ("Rcvdata\n"));
#if __FreeBSD_version >= 500000
NGI_GET_M (item, m);
#if __FreeBSD_version < 502120
NGI_GET_META (item, meta);
#endif
NG_FREE_ITEM (item);
if (! NG_HOOK_PRIVATE (hook) || ! d) {
NG_FREE_M (m);
#if __FreeBSD_version < 502120
NG_FREE_META (meta);
#endif
#else
if (! hook->private || ! d) {
NG_FREE_DATA (m,meta);
#endif
return ENETDOWN;
}
#if __FreeBSD_version >= 502120
/* Check for high priority data */
if ((ptag = (struct ng_tag_prio *)m_tag_locate(m, NGM_GENERIC_COOKIE,
NG_TAG_PRIO, NULL)) != NULL && (ptag->priority > NG_PRIO_CUTOFF) )
q = &d->hi_queue;
else
q = &d->queue;
#else
q = (meta && meta->priority > 0) ? &d->hi_queue : &d->queue;
#endif
s = splimp ();
CE_LOCK (bd);
#if __FreeBSD_version >= 500000
IF_LOCK (q);
if (_IF_QFULL (q)) {
IF_UNLOCK (q);
CE_UNLOCK (bd);
splx (s);
NG_FREE_M (m);
#if __FreeBSD_version < 502120
NG_FREE_META (meta);
#endif
return ENOBUFS;
}
_IF_ENQUEUE (q, m);
IF_UNLOCK (q);
#else
if (IF_QFULL (q)) {
IF_DROP (q);
CE_UNLOCK (bd);
splx (s);
NG_FREE_DATA (m, meta);
return ENOBUFS;
}
IF_ENQUEUE (q, m);
#endif
ce_start (d);
CE_UNLOCK (bd);
splx (s);
return 0;
}
static int ng_ce_rmnode (node_p node)
{
#if __FreeBSD_version >= 500000
drv_t *d = NG_NODE_PRIVATE (node);
CE_DEBUG (d, ("Rmnode\n"));
if (d && d->running) {
bdrv_t *bd = d->board->sys;
int s = splimp ();
CE_LOCK (bd);
ce_down (d);
CE_UNLOCK (bd);
splx (s);
}
#ifdef KLD_MODULE
#if __FreeBSD_version >= 502120
if (node->nd_flags & NGF_REALLY_DIE) {
#else
if (node->nd_flags & NG_REALLY_DIE) {
#endif
NG_NODE_SET_PRIVATE (node, NULL);
NG_NODE_UNREF (node);
}
#if __FreeBSD_version >= 502120
NG_NODE_REVIVE(node); /* Persistant node */
#else
node->nd_flags &= ~NG_INVALID;
#endif
#endif
#else /* __FreeBSD_version < 500000 */
drv_t *d = node->private;
if (d && d->running) {
bdrv_t *bd = d->board->sys;
int s = splimp ();
CE_LOCK (bd);
ce_down (d);
CE_UNLOCK (bd);
splx (s);
}
node->flags |= NG_INVALID;
ng_cutlinks (node);
#ifdef KLD_MODULE
ng_unname (node);
ng_unref (node);
#endif
#endif
return 0;
}
static int ng_ce_connect (hook_p hook)
{
#if __FreeBSD_version >= 500000
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE (hook));
#else
drv_t *d = hook->node->private;
#endif
if (d) {
CE_DEBUG (d, ("Connect\n"));
callout_reset (&d->timeout_handle, hz, ce_watchdog_timer, d);
}
return 0;
}
static int ng_ce_disconnect (hook_p hook)
{
#if __FreeBSD_version >= 500000
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE (hook));
#else
drv_t *d = hook->node->private;
#endif
if (d) {
CE_DEBUG (d, ("Disconnect\n"));
#if __FreeBSD_version >= 500000
if (NG_HOOK_PRIVATE (hook))
#else
if (hook->private)
#endif
{
bdrv_t *bd = d->board->sys;
int s = splimp ();
CE_LOCK (bd);
ce_down (d);
CE_UNLOCK (bd);
splx (s);
}
/* If we were wait it than it reasserted now, just stop it. */
if (!callout_drain (&d->timeout_handle))
callout_stop (&d->timeout_handle);
}
return 0;
}
#endif
static int ce_modevent (module_t mod, int type, void *unused)
{
#if __FreeBSD_version < 500000
dev_t dev;
struct cdevsw *cdsw;
#endif
static int load_count = 0;
#if __FreeBSD_version < 500000
dev = makedev (CDEV_MAJOR, 0);
#endif
switch (type) {
case MOD_LOAD:
#if __FreeBSD_version < 500000
if (dev != NODEV &&
(cdsw = devsw (dev)) &&
cdsw->d_maj == CDEV_MAJOR) {
printf ("Tau32-PCI driver is already in system\n");
return (ENXIO);
}
#endif
#if __FreeBSD_version >= 500000 && defined NETGRAPH
if (ng_newtype (&typestruct))
printf ("Failed to register ng_ce\n");
#endif
++load_count;
#if __FreeBSD_version <= 500000
cdevsw_add (&ce_cdevsw);
#endif
#if __FreeBSD_version >= 500000
- callout_init (&timeout_handle, CALLOUT_MPSAFE);
+ callout_init (&timeout_handle, 1);
#else
callout_init (&timeout_handle);
#endif
callout_reset (&timeout_handle, hz*5, ce_timeout, 0);
break;
case MOD_UNLOAD:
if (load_count == 1) {
printf ("Removing device entry for Tau32-PCI\n");
#if __FreeBSD_version <= 500000
cdevsw_remove (&ce_cdevsw);
#endif
#if __FreeBSD_version >= 500000 && defined NETGRAPH
ng_rmtype (&typestruct);
#endif
}
/* If we were wait it than it reasserted now, just stop it.
* Actually we shouldn't get this condition. But code could be
* changed in the future, so just be a litle paranoid.
*/
if (!callout_drain (&timeout_handle))
callout_stop (&timeout_handle);
--load_count;
break;
case MOD_SHUTDOWN:
break;
}
return 0;
}
#ifdef NETGRAPH
#if __FreeBSD_version >= 502100
static struct ng_type typestruct = {
.version = NG_ABI_VERSION,
.name = NG_CE_NODE_TYPE,
.constructor = ng_ce_constructor,
.rcvmsg = ng_ce_rcvmsg,
.shutdown = ng_ce_rmnode,
.newhook = ng_ce_newhook,
.connect = ng_ce_connect,
.rcvdata = ng_ce_rcvdata,
.disconnect = ng_ce_disconnect,
};
#else /* __FreeBSD_version < 502100 */
static struct ng_type typestruct = {
#if __FreeBSD_version >= 500000
NG_ABI_VERSION,
#else
NG_VERSION,
#endif
NG_CE_NODE_TYPE,
ce_modevent,
ng_ce_constructor,
ng_ce_rcvmsg,
ng_ce_rmnode,
ng_ce_newhook,
NULL,
ng_ce_connect,
ng_ce_rcvdata,
#if __FreeBSD_version < 500000
NULL,
#endif
ng_ce_disconnect,
NULL
};
#endif /* __FreeBSD_version < 502100 */
#endif /*NETGRAPH*/
#if __FreeBSD_version >= 500000
#ifdef NETGRAPH
MODULE_DEPEND (ng_ce, netgraph, NG_ABI_VERSION, NG_ABI_VERSION, NG_ABI_VERSION);
#else
MODULE_DEPEND (ce, sppp, 1, 1, 1);
#endif
#ifdef KLD_MODULE
DRIVER_MODULE (cemod, pci, ce_driver, ce_devclass, ce_modevent, NULL);
#else
DRIVER_MODULE (ce, pci, ce_driver, ce_devclass, ce_modevent, NULL);
#endif
#else /* if __FreeBSD_version < 500000*/
#ifdef NETGRAPH
DRIVER_MODULE (ce, pci, ce_driver, ce_devclass, ng_mod_event, &typestruct);
#else
DRIVER_MODULE (ce, pci, ce_driver, ce_devclass, ce_modevent, NULL);
#endif
#endif /* __FreeBSD_version < 500000 */
#endif /* NPCI */
Index: head/sys/dev/cp/if_cp.c
===================================================================
--- head/sys/dev/cp/if_cp.c (revision 283290)
+++ head/sys/dev/cp/if_cp.c (revision 283291)
@@ -1,2270 +1,2270 @@
/*-
* Cronyx-Tau-PCI adapter driver for FreeBSD.
* Supports PPP/HDLC, Cisco/HDLC and FrameRelay protocol in synchronous mode,
* and asynchronous channels with full modem control.
* Keepalive protocol implemented in both Cisco and PPP modes.
*
* Copyright (C) 1999-2004 Cronyx Engineering.
* Author: Kurakin Roman, <rik@cronyx.ru>
*
* Copyright (C) 1999-2002 Cronyx Engineering.
* Author: Serge Vakulenko, <vak@cronyx.ru>
*
* This software is distributed with NO WARRANTIES, not even the implied
* warranties for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* Authors grant any other persons or organisations a permission to use,
* modify and redistribute this software in source and binary forms,
* as long as this message is kept with the software, all derivative
* works or modified versions.
*
* Cronyx Id: if_cp.c,v 1.1.2.41 2004/06/23 17:09:13 rik Exp $
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/ucred.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/conf.h>
#include <sys/malloc.h>
#include <sys/priv.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <sys/tty.h>
#include <sys/bus.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <net/if.h>
#include <net/if_var.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
#include <machine/bus.h>
#include <sys/rman.h>
#include "opt_ng_cronyx.h"
#ifdef NETGRAPH_CRONYX
# include "opt_netgraph.h"
# ifndef NETGRAPH
# error #option NETGRAPH missed from configuration
# endif
# include <netgraph/ng_message.h>
# include <netgraph/netgraph.h>
# include <dev/cp/ng_cp.h>
#else
# include <net/if_sppp.h>
# include <net/if_types.h>
#include <dev/pci/pcivar.h>
# define PP_CISCO IFF_LINK2
# include <net/bpf.h>
#endif
#include <dev/cx/machdep.h>
#include <dev/cp/cpddk.h>
#include <machine/cserial.h>
#include <machine/resource.h>
#include <machine/pmap.h>
/* If we don't have Cronyx's sppp version, we don't have fr support via sppp */
#ifndef PP_FR
#define PP_FR 0
#endif
#define CP_DEBUG(d,s) ({if (d->chan->debug) {\
printf ("%s: ", d->name); printf s;}})
#define CP_DEBUG2(d,s) ({if (d->chan->debug>1) {\
printf ("%s: ", d->name); printf s;}})
#define CP_LOCK_NAME "cpX"
#define CP_LOCK(_bd) mtx_lock (&(_bd)->cp_mtx)
#define CP_UNLOCK(_bd) mtx_unlock (&(_bd)->cp_mtx)
#define CP_LOCK_ASSERT(_bd) mtx_assert (&(_bd)->cp_mtx, MA_OWNED)
static int cp_probe __P((device_t));
static int cp_attach __P((device_t));
static int cp_detach __P((device_t));
static device_method_t cp_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, cp_probe),
DEVMETHOD(device_attach, cp_attach),
DEVMETHOD(device_detach, cp_detach),
DEVMETHOD_END
};
typedef struct _cp_dma_mem_t {
unsigned long phys;
void *virt;
size_t size;
bus_dma_tag_t dmat;
bus_dmamap_t mapp;
} cp_dma_mem_t;
typedef struct _drv_t {
char name [8];
int running;
cp_chan_t *chan;
cp_board_t *board;
cp_dma_mem_t dmamem;
#ifdef NETGRAPH
char nodename [NG_NODESIZE];
hook_p hook;
hook_p debug_hook;
node_p node;
struct ifqueue queue;
struct ifqueue hi_queue;
#else
struct ifqueue queue;
struct ifnet *ifp;
#endif
short timeout;
struct callout timeout_handle;
struct cdev *devt;
} drv_t;
typedef struct _bdrv_t {
cp_board_t *board;
struct resource *cp_res;
struct resource *cp_irq;
void *cp_intrhand;
cp_dma_mem_t dmamem;
drv_t channel [NCHAN];
struct mtx cp_mtx;
} bdrv_t;
static driver_t cp_driver = {
"cp",
cp_methods,
sizeof(bdrv_t),
};
static devclass_t cp_devclass;
static void cp_receive (cp_chan_t *c, unsigned char *data, int len);
static void cp_transmit (cp_chan_t *c, void *attachment, int len);
static void cp_error (cp_chan_t *c, int data);
static void cp_up (drv_t *d);
static void cp_start (drv_t *d);
static void cp_down (drv_t *d);
static void cp_watchdog (drv_t *d);
static void cp_watchdog_timer (void *arg);
#ifdef NETGRAPH
extern struct ng_type typestruct;
#else
static void cp_ifstart (struct ifnet *ifp);
static void cp_tlf (struct sppp *sp);
static void cp_tls (struct sppp *sp);
static int cp_sioctl (struct ifnet *ifp, u_long cmd, caddr_t data);
static void cp_initialize (void *softc);
#endif
static cp_board_t *adapter [NBRD];
static drv_t *channel [NBRD*NCHAN];
static struct callout led_timo [NBRD];
static struct callout timeout_handle;
static int cp_destroy = 0;
static int cp_open (struct cdev *dev, int oflags, int devtype, struct thread *td);
static int cp_close (struct cdev *dev, int fflag, int devtype, struct thread *td);
static int cp_ioctl (struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td);
static struct cdevsw cp_cdevsw = {
.d_version = D_VERSION,
.d_open = cp_open,
.d_close = cp_close,
.d_ioctl = cp_ioctl,
.d_name = "cp",
};
/*
* Make an mbuf from data.
*/
static struct mbuf *makembuf (void *buf, unsigned len)
{
struct mbuf *m;
MGETHDR (m, M_NOWAIT, MT_DATA);
if (! m)
return 0;
if (!(MCLGET (m, M_NOWAIT))) {
m_freem (m);
return 0;
}
m->m_pkthdr.len = m->m_len = len;
bcopy (buf, mtod (m, caddr_t), len);
return m;
}
static int cp_probe (device_t dev)
{
if ((pci_get_vendor (dev) == cp_vendor_id) &&
(pci_get_device (dev) == cp_device_id)) {
device_set_desc (dev, "Cronyx-Tau-PCI serial adapter");
return BUS_PROBE_DEFAULT;
}
return ENXIO;
}
static void cp_timeout (void *arg)
{
drv_t *d;
int s, i, k;
for (i = 0; i < NBRD; ++i) {
if (adapter[i] == NULL)
continue;
for (k = 0; k < NCHAN; ++k) {
s = splimp ();
if (cp_destroy) {
splx (s);
return;
}
d = channel[i * NCHAN + k];
if (!d) {
splx (s);
continue;
}
CP_LOCK ((bdrv_t *)d->board->sys);
switch (d->chan->type) {
case T_G703:
cp_g703_timer (d->chan);
break;
case T_E1:
cp_e1_timer (d->chan);
break;
case T_E3:
case T_T3:
case T_STS1:
cp_e3_timer (d->chan);
break;
default:
break;
}
CP_UNLOCK ((bdrv_t *)d->board->sys);
splx (s);
}
}
s = splimp ();
if (!cp_destroy)
callout_reset (&timeout_handle, hz, cp_timeout, 0);
splx (s);
}
static void cp_led_off (void *arg)
{
cp_board_t *b = arg;
bdrv_t *bd = (bdrv_t *) b->sys;
int s;
s = splimp ();
if (cp_destroy) {
splx (s);
return;
}
CP_LOCK (bd);
cp_led (b, 0);
CP_UNLOCK (bd);
splx (s);
}
static void cp_intr (void *arg)
{
bdrv_t *bd = arg;
cp_board_t *b = bd->board;
#ifndef NETGRAPH
int i;
#endif
int s = splimp ();
if (cp_destroy) {
splx (s);
return;
}
CP_LOCK (bd);
/* Check if we are ready */
if (b->sys == NULL) {
/* Not we are not, just cleanup. */
cp_interrupt_poll (b, 1);
CP_UNLOCK (bd);
return;
}
/* Turn LED on. */
cp_led (b, 1);
cp_interrupt (b);
/* Turn LED off 50 msec later. */
callout_reset (&led_timo[b->num], hz/20, cp_led_off, b);
CP_UNLOCK (bd);
splx (s);
#ifndef NETGRAPH
/* Pass packets in a lock-free state */
for (i = 0; i < NCHAN && b->chan[i].type; i++) {
drv_t *d = b->chan[i].sys;
struct mbuf *m;
if (!d || !d->running)
continue;
while (_IF_QLEN(&d->queue)) {
IF_DEQUEUE (&d->queue,m);
if (!m)
continue;
sppp_input (d->ifp, m);
}
}
#endif
}
static void
cp_bus_dmamap_addr (void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
unsigned long *addr;
if (error)
return;
KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg));
addr = arg;
*addr = segs->ds_addr;
}
static int
cp_bus_dma_mem_alloc (int bnum, int cnum, cp_dma_mem_t *dmem)
{
int error;
error = bus_dma_tag_create (NULL, 16, 0, BUS_SPACE_MAXADDR_32BIT,
BUS_SPACE_MAXADDR, NULL, NULL, dmem->size, 1,
dmem->size, 0, NULL, NULL, &dmem->dmat);
if (error) {
if (cnum >= 0) printf ("cp%d-%d: ", bnum, cnum);
else printf ("cp%d: ", bnum);
printf ("couldn't allocate tag for dma memory\n");
return 0;
}
error = bus_dmamem_alloc (dmem->dmat, (void **)&dmem->virt,
BUS_DMA_NOWAIT | BUS_DMA_ZERO, &dmem->mapp);
if (error) {
if (cnum >= 0) printf ("cp%d-%d: ", bnum, cnum);
else printf ("cp%d: ", bnum);
printf ("couldn't allocate mem for dma memory\n");
bus_dma_tag_destroy (dmem->dmat);
return 0;
}
error = bus_dmamap_load (dmem->dmat, dmem->mapp, dmem->virt,
dmem->size, cp_bus_dmamap_addr, &dmem->phys, 0);
if (error) {
if (cnum >= 0) printf ("cp%d-%d: ", bnum, cnum);
else printf ("cp%d: ", bnum);
printf ("couldn't load mem map for dma memory\n");
bus_dmamem_free (dmem->dmat, dmem->virt, dmem->mapp);
bus_dma_tag_destroy (dmem->dmat);
return 0;
}
return 1;
}
static void
cp_bus_dma_mem_free (cp_dma_mem_t *dmem)
{
bus_dmamap_unload (dmem->dmat, dmem->mapp);
bus_dmamem_free (dmem->dmat, dmem->virt, dmem->mapp);
bus_dma_tag_destroy (dmem->dmat);
}
/*
* Called if the probe succeeded.
*/
static int cp_attach (device_t dev)
{
bdrv_t *bd = device_get_softc (dev);
int unit = device_get_unit (dev);
char *cp_ln = CP_LOCK_NAME;
unsigned short res;
vm_offset_t vbase;
int rid, error;
cp_board_t *b;
cp_chan_t *c;
drv_t *d;
int s = splimp ();
b = malloc (sizeof(cp_board_t), M_DEVBUF, M_WAITOK);
if (!b) {
printf ("cp%d: couldn't allocate memory\n", unit);
splx (s);
return (ENXIO);
}
bzero (b, sizeof(cp_board_t));
bd->board = b;
rid = PCIR_BAR(0);
bd->cp_res = bus_alloc_resource (dev, SYS_RES_MEMORY, &rid,
0, ~0, 1, RF_ACTIVE);
if (! bd->cp_res) {
printf ("cp%d: cannot map memory\n", unit);
free (b, M_DEVBUF);
splx (s);
return (ENXIO);
}
vbase = (vm_offset_t) rman_get_virtual (bd->cp_res);
cp_ln[2] = '0' + unit;
mtx_init (&bd->cp_mtx, cp_ln, MTX_NETWORK_LOCK, MTX_DEF|MTX_RECURSE);
res = cp_init (b, unit, (u_char*) vbase);
if (res) {
printf ("cp%d: can't init, error code:%x\n", unit, res);
bus_release_resource (dev, SYS_RES_MEMORY, PCIR_BAR(0), bd->cp_res);
free (b, M_DEVBUF);
splx (s);
return (ENXIO);
}
bd->dmamem.size = sizeof(cp_qbuf_t);
if (! cp_bus_dma_mem_alloc (unit, -1, &bd->dmamem)) {
free (b, M_DEVBUF);
splx (s);
return (ENXIO);
}
CP_LOCK (bd);
cp_reset (b, bd->dmamem.virt, bd->dmamem.phys);
CP_UNLOCK (bd);
rid = 0;
bd->cp_irq = bus_alloc_resource (dev, SYS_RES_IRQ, &rid, 0, ~0, 1,
RF_SHAREABLE | RF_ACTIVE);
if (! bd->cp_irq) {
cp_destroy = 1;
printf ("cp%d: cannot map interrupt\n", unit);
bus_release_resource (dev, SYS_RES_MEMORY,
PCIR_BAR(0), bd->cp_res);
mtx_destroy (&bd->cp_mtx);
free (b, M_DEVBUF);
splx (s);
return (ENXIO);
}
- callout_init (&led_timo[unit], CALLOUT_MPSAFE);
+ callout_init (&led_timo[unit], 1);
error = bus_setup_intr (dev, bd->cp_irq,
INTR_TYPE_NET|INTR_MPSAFE,
NULL, cp_intr, bd, &bd->cp_intrhand);
if (error) {
cp_destroy = 1;
printf ("cp%d: cannot set up irq\n", unit);
bus_release_resource (dev, SYS_RES_IRQ, 0, bd->cp_irq);
bus_release_resource (dev, SYS_RES_MEMORY,
PCIR_BAR(0), bd->cp_res);
mtx_destroy (&bd->cp_mtx);
free (b, M_DEVBUF);
splx (s);
return (ENXIO);
}
printf ("cp%d: %s, clock %ld MHz\n", unit, b->name, b->osc / 1000000);
for (c = b->chan; c < b->chan + NCHAN; ++c) {
if (! c->type)
continue;
d = &bd->channel[c->num];
d->dmamem.size = sizeof(cp_buf_t);
if (! cp_bus_dma_mem_alloc (unit, c->num, &d->dmamem))
continue;
channel [b->num*NCHAN + c->num] = d;
sprintf (d->name, "cp%d.%d", b->num, c->num);
d->board = b;
d->chan = c;
c->sys = d;
- callout_init (&d->timeout_handle, CALLOUT_MPSAFE);
+ callout_init (&d->timeout_handle, 1);
#ifdef NETGRAPH
if (ng_make_node_common (&typestruct, &d->node) != 0) {
printf ("%s: cannot make common node\n", d->name);
d->node = NULL;
continue;
}
NG_NODE_SET_PRIVATE (d->node, d);
sprintf (d->nodename, "%s%d", NG_CP_NODE_TYPE,
c->board->num*NCHAN + c->num);
if (ng_name_node (d->node, d->nodename)) {
printf ("%s: cannot name node\n", d->nodename);
NG_NODE_UNREF (d->node);
continue;
}
d->queue.ifq_maxlen = ifqmaxlen;
d->hi_queue.ifq_maxlen = ifqmaxlen;
mtx_init (&d->queue.ifq_mtx, "cp_queue", NULL, MTX_DEF);
mtx_init (&d->hi_queue.ifq_mtx, "cp_queue_hi", NULL, MTX_DEF);
#else /*NETGRAPH*/
d->ifp = if_alloc(IFT_PPP);
if (d->ifp == NULL) {
printf ("%s: cannot if_alloc() interface\n", d->name);
continue;
}
d->ifp->if_softc = d;
if_initname (d->ifp, "cp", b->num * NCHAN + c->num);
d->ifp->if_mtu = PP_MTU;
d->ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
d->ifp->if_ioctl = cp_sioctl;
d->ifp->if_start = cp_ifstart;
d->ifp->if_init = cp_initialize;
d->queue.ifq_maxlen = NRBUF;
mtx_init (&d->queue.ifq_mtx, "cp_queue", NULL, MTX_DEF);
sppp_attach (d->ifp);
if_attach (d->ifp);
IFP2SP(d->ifp)->pp_tlf = cp_tlf;
IFP2SP(d->ifp)->pp_tls = cp_tls;
/* If BPF is in the kernel, call the attach for it.
* The header size of PPP or Cisco/HDLC is 4 bytes. */
bpfattach (d->ifp, DLT_PPP, 4);
#endif /*NETGRAPH*/
cp_start_e1 (c);
cp_start_chan (c, 1, 1, d->dmamem.virt, d->dmamem.phys);
/* Register callback functions. */
cp_register_transmit (c, &cp_transmit);
cp_register_receive (c, &cp_receive);
cp_register_error (c, &cp_error);
d->devt = make_dev (&cp_cdevsw, b->num*NCHAN+c->num, UID_ROOT,
GID_WHEEL, 0600, "cp%d", b->num*NCHAN+c->num);
}
CP_LOCK (bd);
b->sys = bd;
adapter[unit] = b;
CP_UNLOCK (bd);
splx (s);
return 0;
}
static int cp_detach (device_t dev)
{
bdrv_t *bd = device_get_softc (dev);
cp_board_t *b = bd->board;
cp_chan_t *c;
int s;
KASSERT (mtx_initialized (&bd->cp_mtx), ("cp mutex not initialized"));
s = splimp ();
CP_LOCK (bd);
/* Check if the device is busy (open). */
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (! d || ! d->chan->type)
continue;
if (d->running) {
CP_UNLOCK (bd);
splx (s);
return EBUSY;
}
}
/* Ok, we can unload driver */
/* At first we should stop all channels */
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (! d || ! d->chan->type)
continue;
cp_stop_chan (c);
cp_stop_e1 (c);
cp_set_dtr (d->chan, 0);
cp_set_rts (d->chan, 0);
}
/* Reset the adapter. */
cp_destroy = 1;
cp_interrupt_poll (b, 1);
cp_led_off (b);
cp_reset (b, 0 ,0);
callout_stop (&led_timo[b->num]);
/* Disable the interrupt request. */
bus_teardown_intr (dev, bd->cp_irq, bd->cp_intrhand);
for (c=b->chan; c<b->chan+NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (! d || ! d->chan->type)
continue;
callout_stop (&d->timeout_handle);
#ifndef NETGRAPH
/* Detach from the packet filter list of interfaces. */
bpfdetach (d->ifp);
/* Detach from the sync PPP list. */
sppp_detach (d->ifp);
/* Detach from the system list of interfaces. */
if_detach (d->ifp);
if_free (d->ifp);
IF_DRAIN (&d->queue);
mtx_destroy (&d->queue.ifq_mtx);
#else
if (d->node) {
ng_rmnode_self (d->node);
NG_NODE_UNREF (d->node);
d->node = NULL;
}
mtx_destroy (&d->queue.ifq_mtx);
mtx_destroy (&d->hi_queue.ifq_mtx);
#endif
destroy_dev (d->devt);
}
b->sys = NULL;
CP_UNLOCK (bd);
bus_release_resource (dev, SYS_RES_IRQ, 0, bd->cp_irq);
bus_release_resource (dev, SYS_RES_MEMORY, PCIR_BAR(0), bd->cp_res);
CP_LOCK (bd);
cp_led_off (b);
CP_UNLOCK (bd);
callout_drain (&led_timo[b->num]);
splx (s);
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (! d || ! d->chan->type)
continue;
callout_drain (&d->timeout_handle);
channel [b->num*NCHAN + c->num] = 0;
/* Deallocate buffers. */
cp_bus_dma_mem_free (&d->dmamem);
}
adapter [b->num] = 0;
cp_bus_dma_mem_free (&bd->dmamem);
free (b, M_DEVBUF);
mtx_destroy (&bd->cp_mtx);
return 0;
}
#ifndef NETGRAPH
static void cp_ifstart (struct ifnet *ifp)
{
drv_t *d = ifp->if_softc;
bdrv_t *bd = d->board->sys;
CP_LOCK (bd);
cp_start (d);
CP_UNLOCK (bd);
}
static void cp_tlf (struct sppp *sp)
{
drv_t *d = SP2IFP(sp)->if_softc;
CP_DEBUG2 (d, ("cp_tlf\n"));
/* XXXRIK: Don't forget to protect them by LOCK, or kill them. */
/* cp_set_dtr (d->chan, 0);*/
/* cp_set_rts (d->chan, 0);*/
if (!(sp->pp_flags & PP_FR) && !(d->ifp->if_flags & PP_CISCO))
sp->pp_down (sp);
}
static void cp_tls (struct sppp *sp)
{
drv_t *d = SP2IFP(sp)->if_softc;
CP_DEBUG2 (d, ("cp_tls\n"));
if (!(sp->pp_flags & PP_FR) && !(d->ifp->if_flags & PP_CISCO))
sp->pp_up (sp);
}
/*
* Process an ioctl request.
*/
static int cp_sioctl (struct ifnet *ifp, u_long cmd, caddr_t data)
{
drv_t *d = ifp->if_softc;
bdrv_t *bd = d->board->sys;
int error, s, was_up, should_be_up;
was_up = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
error = sppp_ioctl (ifp, cmd, data);
if (error)
return error;
if (! (ifp->if_flags & IFF_DEBUG))
d->chan->debug = 0;
else
d->chan->debug = d->chan->debug_shadow;
switch (cmd) {
default: CP_DEBUG2 (d, ("ioctl 0x%lx\n", cmd)); return 0;
case SIOCADDMULTI: CP_DEBUG2 (d, ("ioctl SIOCADDMULTI\n")); return 0;
case SIOCDELMULTI: CP_DEBUG2 (d, ("ioctl SIOCDELMULTI\n")); return 0;
case SIOCSIFFLAGS: CP_DEBUG2 (d, ("ioctl SIOCSIFFLAGS\n")); break;
case SIOCSIFADDR: CP_DEBUG2 (d, ("ioctl SIOCSIFADDR\n")); break;
}
/* We get here only in case of SIFFLAGS or SIFADDR. */
s = splimp ();
CP_LOCK (bd);
should_be_up = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
if (! was_up && should_be_up) {
/* Interface goes up -- start it. */
cp_up (d);
cp_start (d);
} else if (was_up && ! should_be_up) {
/* Interface is going down -- stop it. */
/* if ((IFP2SP(ifp)->pp_flags & PP_FR) || (ifp->if_flags & PP_CISCO))*/
cp_down (d);
}
CP_DEBUG (d, ("ioctl 0x%lx p4\n", cmd));
CP_UNLOCK (bd);
splx (s);
return 0;
}
/*
* Initialization of interface.
* It seems to be never called by upper level?
*/
static void cp_initialize (void *softc)
{
drv_t *d = softc;
CP_DEBUG (d, ("cp_initialize\n"));
}
#endif /*NETGRAPH*/
/*
* Stop the interface. Called on splimp().
*/
static void cp_down (drv_t *d)
{
CP_DEBUG (d, ("cp_down\n"));
/* Interface is going down -- stop it. */
cp_set_dtr (d->chan, 0);
cp_set_rts (d->chan, 0);
d->running = 0;
callout_stop (&d->timeout_handle);
}
/*
* Start the interface. Called on splimp().
*/
static void cp_up (drv_t *d)
{
CP_DEBUG (d, ("cp_up\n"));
cp_set_dtr (d->chan, 1);
cp_set_rts (d->chan, 1);
d->running = 1;
}
/*
* Start output on the interface. Get another datagram to send
* off of the interface queue, and copy it to the interface
* before starting the output.
*/
static void cp_send (drv_t *d)
{
struct mbuf *m;
u_short len;
CP_DEBUG2 (d, ("cp_send, tn=%d te=%d\n", d->chan->tn, d->chan->te));
/* No output if the interface is down. */
if (! d->running)
return;
/* No output if the modem is off. */
if (! (d->chan->lloop || d->chan->type != T_SERIAL ||
cp_get_dsr (d->chan)))
return;
while (cp_transmit_space (d->chan)) {
/* Get the packet to send. */
#ifdef NETGRAPH
IF_DEQUEUE (&d->hi_queue, m);
if (! m)
IF_DEQUEUE (&d->queue, m);
#else
m = sppp_dequeue (d->ifp);
#endif
if (! m)
return;
#ifndef NETGRAPH
BPF_MTAP (d->ifp, m);
#endif
len = m_length (m, NULL);
if (len >= BUFSZ)
printf ("%s: too long packet: %d bytes: ",
d->name, len);
else if (! m->m_next)
cp_send_packet (d->chan, (u_char*) mtod (m, caddr_t), len, 0);
else {
u_char *buf = d->chan->tbuf[d->chan->te];
m_copydata (m, 0, len, buf);
cp_send_packet (d->chan, buf, len, 0);
}
m_freem (m);
/* Set up transmit timeout, if the transmit ring is not empty.*/
d->timeout = 10;
}
#ifndef NETGRAPH
d->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
#endif
}
/*
* Start output on the interface.
* Always called on splimp().
*/
static void cp_start (drv_t *d)
{
if (d->running) {
if (! d->chan->dtr)
cp_set_dtr (d->chan, 1);
if (! d->chan->rts)
cp_set_rts (d->chan, 1);
cp_send (d);
callout_reset (&d->timeout_handle, hz, cp_watchdog_timer, d);
}
}
/*
* Handle transmit timeouts.
* Recover after lost transmit interrupts.
* Always called on splimp().
*/
static void cp_watchdog (drv_t *d)
{
CP_DEBUG (d, ("device timeout\n"));
if (d->running) {
cp_stop_chan (d->chan);
cp_stop_e1 (d->chan);
cp_start_e1 (d->chan);
cp_start_chan (d->chan, 1, 1, 0, 0);
cp_set_dtr (d->chan, 1);
cp_set_rts (d->chan, 1);
cp_start (d);
}
}
static void cp_watchdog_timer (void *arg)
{
drv_t *d = arg;
bdrv_t *bd = d->board->sys;
CP_LOCK (bd);
if (d->timeout == 1)
cp_watchdog (d);
if (d->timeout)
d->timeout--;
callout_reset (&d->timeout_handle, hz, cp_watchdog_timer, d);
CP_UNLOCK (bd);
}
static void cp_transmit (cp_chan_t *c, void *attachment, int len)
{
drv_t *d = c->sys;
d->timeout = 0;
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_OPACKETS, 1);
d->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
#endif
cp_start (d);
}
static void cp_receive (cp_chan_t *c, unsigned char *data, int len)
{
drv_t *d = c->sys;
struct mbuf *m;
#ifdef NETGRAPH
int error;
#endif
if (! d->running)
return;
m = makembuf (data, len);
if (! m) {
CP_DEBUG (d, ("no memory for packet\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IQDROPS, 1);
#endif
return;
}
if (c->debug > 1)
m_print (m, 0);
#ifdef NETGRAPH
m->m_pkthdr.rcvif = 0;
NG_SEND_DATA_ONLY (error, d->hook, m);
#else
if_inc_counter(d->ifp, IFCOUNTER_IPACKETS, 1);
m->m_pkthdr.rcvif = d->ifp;
/* Check if there's a BPF listener on this interface.
* If so, hand off the raw packet to bpf. */
BPF_MTAP(d->ifp, m);
IF_ENQUEUE (&d->queue, m);
#endif
}
static void cp_error (cp_chan_t *c, int data)
{
drv_t *d = c->sys;
switch (data) {
case CP_FRAME:
CP_DEBUG (d, ("frame error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CP_CRC:
CP_DEBUG (d, ("crc error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CP_OVERRUN:
CP_DEBUG (d, ("overrun error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_COLLISIONS, 1);
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CP_OVERFLOW:
CP_DEBUG (d, ("overflow error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CP_UNDERRUN:
CP_DEBUG (d, ("underrun error\n"));
d->timeout = 0;
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_OERRORS, 1);
d->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
#endif
cp_start (d);
break;
default:
CP_DEBUG (d, ("error #%d\n", data));
break;
}
}
/*
* You also need read, write, open, close routines.
* This should get you started
*/
static int cp_open (struct cdev *dev, int oflags, int devtype, struct thread *td)
{
int unit = dev2unit (dev);
drv_t *d;
if (unit >= NBRD*NCHAN || ! (d = channel[unit]))
return ENXIO;
CP_DEBUG2 (d, ("cp_open\n"));
return 0;
}
/*
* Only called on the LAST close.
*/
static int cp_close (struct cdev *dev, int fflag, int devtype, struct thread *td)
{
drv_t *d = channel [dev2unit (dev)];
CP_DEBUG2 (d, ("cp_close\n"));
return 0;
}
static int cp_modem_status (cp_chan_t *c)
{
drv_t *d = c->sys;
bdrv_t *bd = d->board->sys;
int status, s;
status = d->running ? TIOCM_LE : 0;
s = splimp ();
CP_LOCK (bd);
if (cp_get_cd (c)) status |= TIOCM_CD;
if (cp_get_cts (c)) status |= TIOCM_CTS;
if (cp_get_dsr (c)) status |= TIOCM_DSR;
if (c->dtr) status |= TIOCM_DTR;
if (c->rts) status |= TIOCM_RTS;
CP_UNLOCK (bd);
splx (s);
return status;
}
static int cp_ioctl (struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td)
{
drv_t *d = channel [dev2unit (dev)];
bdrv_t *bd = d->board->sys;
cp_chan_t *c = d->chan;
struct serial_statistics *st;
struct e1_statistics *opte1;
struct e3_statistics *opte3;
int error, s;
char mask[16];
switch (cmd) {
case SERIAL_GETREGISTERED:
CP_DEBUG2 (d, ("ioctl: getregistered\n"));
bzero (mask, sizeof(mask));
for (s=0; s<NBRD*NCHAN; ++s)
if (channel [s])
mask [s/8] |= 1 << (s & 7);
bcopy (mask, data, sizeof (mask));
return 0;
#ifndef NETGRAPH
case SERIAL_GETPROTO:
CP_DEBUG2 (d, ("ioctl: getproto\n"));
strcpy ((char*)data, (IFP2SP(d->ifp)->pp_flags & PP_FR) ? "fr" :
(d->ifp->if_flags & PP_CISCO) ? "cisco" : "ppp");
return 0;
case SERIAL_SETPROTO:
CP_DEBUG2 (d, ("ioctl: setproto\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (d->ifp->if_drv_flags & IFF_DRV_RUNNING)
return EBUSY;
if (! strcmp ("cisco", (char*)data)) {
IFP2SP(d->ifp)->pp_flags &= ~(PP_FR);
IFP2SP(d->ifp)->pp_flags |= PP_KEEPALIVE;
d->ifp->if_flags |= PP_CISCO;
#if PP_FR != 0
} else if (! strcmp ("fr", (char*)data)) {
d->ifp->if_flags &= ~(PP_CISCO);
IFP2SP(d->ifp)->pp_flags |= PP_FR | PP_KEEPALIVE;
#endif
} else if (! strcmp ("ppp", (char*)data)) {
IFP2SP(d->ifp)->pp_flags &= ~PP_FR;
IFP2SP(d->ifp)->pp_flags &= ~PP_KEEPALIVE;
d->ifp->if_flags &= ~(PP_CISCO);
} else
return EINVAL;
return 0;
case SERIAL_GETKEEPALIVE:
CP_DEBUG2 (d, ("ioctl: getkeepalive\n"));
if ((IFP2SP(d->ifp)->pp_flags & PP_FR) ||
(d->ifp->if_flags & PP_CISCO))
return EINVAL;
*(int*)data = (IFP2SP(d->ifp)->pp_flags & PP_KEEPALIVE) ? 1 : 0;
return 0;
case SERIAL_SETKEEPALIVE:
CP_DEBUG2 (d, ("ioctl: setkeepalive\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if ((IFP2SP(d->ifp)->pp_flags & PP_FR) ||
(d->ifp->if_flags & PP_CISCO))
return EINVAL;
s = splimp ();
CP_LOCK (bd);
if (*(int*)data)
IFP2SP(d->ifp)->pp_flags |= PP_KEEPALIVE;
else
IFP2SP(d->ifp)->pp_flags &= ~PP_KEEPALIVE;
CP_UNLOCK (bd);
splx (s);
return 0;
#endif /*NETGRAPH*/
case SERIAL_GETMODE:
CP_DEBUG2 (d, ("ioctl: getmode\n"));
*(int*)data = SERIAL_HDLC;
return 0;
case SERIAL_SETMODE:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (*(int*)data != SERIAL_HDLC)
return EINVAL;
return 0;
case SERIAL_GETCFG:
CP_DEBUG2 (d, ("ioctl: getcfg\n"));
if (c->type != T_E1 || c->unfram)
return EINVAL;
*(char*)data = c->board->mux ? 'c' : 'a';
return 0;
case SERIAL_SETCFG:
CP_DEBUG2 (d, ("ioctl: setcfg\n"));
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_mux (c->board, *((char*)data) == 'c');
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETSTAT:
CP_DEBUG2 (d, ("ioctl: getstat\n"));
st = (struct serial_statistics*) data;
st->rintr = c->rintr;
st->tintr = c->tintr;
st->mintr = 0;
st->ibytes = c->ibytes;
st->ipkts = c->ipkts;
st->obytes = c->obytes;
st->opkts = c->opkts;
st->ierrs = c->overrun + c->frame + c->crc;
st->oerrs = c->underrun;
return 0;
case SERIAL_GETESTAT:
CP_DEBUG2 (d, ("ioctl: getestat\n"));
if (c->type != T_E1 && c->type != T_G703)
return EINVAL;
opte1 = (struct e1_statistics*) data;
opte1->status = c->status;
opte1->cursec = c->cursec;
opte1->totsec = c->totsec + c->cursec;
opte1->currnt.bpv = c->currnt.bpv;
opte1->currnt.fse = c->currnt.fse;
opte1->currnt.crce = c->currnt.crce;
opte1->currnt.rcrce = c->currnt.rcrce;
opte1->currnt.uas = c->currnt.uas;
opte1->currnt.les = c->currnt.les;
opte1->currnt.es = c->currnt.es;
opte1->currnt.bes = c->currnt.bes;
opte1->currnt.ses = c->currnt.ses;
opte1->currnt.oofs = c->currnt.oofs;
opte1->currnt.css = c->currnt.css;
opte1->currnt.dm = c->currnt.dm;
opte1->total.bpv = c->total.bpv + c->currnt.bpv;
opte1->total.fse = c->total.fse + c->currnt.fse;
opte1->total.crce = c->total.crce + c->currnt.crce;
opte1->total.rcrce = c->total.rcrce + c->currnt.rcrce;
opte1->total.uas = c->total.uas + c->currnt.uas;
opte1->total.les = c->total.les + c->currnt.les;
opte1->total.es = c->total.es + c->currnt.es;
opte1->total.bes = c->total.bes + c->currnt.bes;
opte1->total.ses = c->total.ses + c->currnt.ses;
opte1->total.oofs = c->total.oofs + c->currnt.oofs;
opte1->total.css = c->total.css + c->currnt.css;
opte1->total.dm = c->total.dm + c->currnt.dm;
for (s=0; s<48; ++s) {
opte1->interval[s].bpv = c->interval[s].bpv;
opte1->interval[s].fse = c->interval[s].fse;
opte1->interval[s].crce = c->interval[s].crce;
opte1->interval[s].rcrce = c->interval[s].rcrce;
opte1->interval[s].uas = c->interval[s].uas;
opte1->interval[s].les = c->interval[s].les;
opte1->interval[s].es = c->interval[s].es;
opte1->interval[s].bes = c->interval[s].bes;
opte1->interval[s].ses = c->interval[s].ses;
opte1->interval[s].oofs = c->interval[s].oofs;
opte1->interval[s].css = c->interval[s].css;
opte1->interval[s].dm = c->interval[s].dm;
}
return 0;
case SERIAL_GETE3STAT:
CP_DEBUG2 (d, ("ioctl: gete3stat\n"));
if (c->type != T_E3 && c->type != T_T3 && c->type != T_STS1)
return EINVAL;
opte3 = (struct e3_statistics*) data;
opte3->status = c->e3status;
opte3->cursec = (c->e3csec_5 * 2 + 1) / 10;
opte3->totsec = c->e3tsec + opte3->cursec;
opte3->ccv = c->e3ccv;
opte3->tcv = c->e3tcv + opte3->ccv;
for (s = 0; s < 48; ++s) {
opte3->icv[s] = c->e3icv[s];
}
return 0;
case SERIAL_CLRSTAT:
CP_DEBUG2 (d, ("ioctl: clrstat\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
c->rintr = 0;
c->tintr = 0;
c->ibytes = 0;
c->obytes = 0;
c->ipkts = 0;
c->opkts = 0;
c->overrun = 0;
c->frame = 0;
c->crc = 0;
c->underrun = 0;
bzero (&c->currnt, sizeof (c->currnt));
bzero (&c->total, sizeof (c->total));
bzero (c->interval, sizeof (c->interval));
c->e3ccv = 0;
c->e3tcv = 0;
bzero (c->e3icv, sizeof (c->e3icv));
return 0;
case SERIAL_GETBAUD:
CP_DEBUG2 (d, ("ioctl: getbaud\n"));
*(long*)data = c->baud;
return 0;
case SERIAL_SETBAUD:
CP_DEBUG2 (d, ("ioctl: setbaud\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CP_LOCK (bd);
cp_set_baud (c, *(long*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETLOOP:
CP_DEBUG2 (d, ("ioctl: getloop\n"));
*(int*)data = c->lloop;
return 0;
case SERIAL_SETLOOP:
CP_DEBUG2 (d, ("ioctl: setloop\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CP_LOCK (bd);
cp_set_lloop (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETDPLL:
CP_DEBUG2 (d, ("ioctl: getdpll\n"));
if (c->type != T_SERIAL)
return EINVAL;
*(int*)data = c->dpll;
return 0;
case SERIAL_SETDPLL:
CP_DEBUG2 (d, ("ioctl: setdpll\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_SERIAL)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_dpll (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETNRZI:
CP_DEBUG2 (d, ("ioctl: getnrzi\n"));
if (c->type != T_SERIAL)
return EINVAL;
*(int*)data = c->nrzi;
return 0;
case SERIAL_SETNRZI:
CP_DEBUG2 (d, ("ioctl: setnrzi\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_SERIAL)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_nrzi (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETDEBUG:
CP_DEBUG2 (d, ("ioctl: getdebug\n"));
*(int*)data = d->chan->debug;
return 0;
case SERIAL_SETDEBUG:
CP_DEBUG2 (d, ("ioctl: setdebug\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
#ifndef NETGRAPH
/*
* The debug_shadow is always greater than zero for logic
* simplicity. For switching debug off the IFF_DEBUG is
* responsible.
*/
d->chan->debug_shadow = (*(int*)data) ? (*(int*)data) : 1;
if (d->ifp->if_flags & IFF_DEBUG)
d->chan->debug = d->chan->debug_shadow;
#else
d->chan->debug = *(int*)data;
#endif
return 0;
case SERIAL_GETHIGAIN:
CP_DEBUG2 (d, ("ioctl: gethigain\n"));
if (c->type != T_E1)
return EINVAL;
*(int*)data = c->higain;
return 0;
case SERIAL_SETHIGAIN:
CP_DEBUG2 (d, ("ioctl: sethigain\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_higain (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETPHONY:
CP_DEBUG2 (d, ("ioctl: getphony\n"));
if (c->type != T_E1)
return EINVAL;
*(int*)data = c->phony;
return 0;
case SERIAL_SETPHONY:
CP_DEBUG2 (d, ("ioctl: setphony\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_phony (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETUNFRAM:
CP_DEBUG2 (d, ("ioctl: getunfram\n"));
if (c->type != T_E1)
return EINVAL;
*(int*)data = c->unfram;
return 0;
case SERIAL_SETUNFRAM:
CP_DEBUG2 (d, ("ioctl: setunfram\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_unfram (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETSCRAMBLER:
CP_DEBUG2 (d, ("ioctl: getscrambler\n"));
if (c->type != T_G703 && !c->unfram)
return EINVAL;
*(int*)data = c->scrambler;
return 0;
case SERIAL_SETSCRAMBLER:
CP_DEBUG2 (d, ("ioctl: setscrambler\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_G703 && !c->unfram)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_scrambler (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETMONITOR:
CP_DEBUG2 (d, ("ioctl: getmonitor\n"));
if (c->type != T_E1 &&
c->type != T_E3 &&
c->type != T_T3 &&
c->type != T_STS1)
return EINVAL;
*(int*)data = c->monitor;
return 0;
case SERIAL_SETMONITOR:
CP_DEBUG2 (d, ("ioctl: setmonitor\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_monitor (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETUSE16:
CP_DEBUG2 (d, ("ioctl: getuse16\n"));
if (c->type != T_E1 || c->unfram)
return EINVAL;
*(int*)data = c->use16;
return 0;
case SERIAL_SETUSE16:
CP_DEBUG2 (d, ("ioctl: setuse16\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_use16 (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETCRC4:
CP_DEBUG2 (d, ("ioctl: getcrc4\n"));
if (c->type != T_E1 || c->unfram)
return EINVAL;
*(int*)data = c->crc4;
return 0;
case SERIAL_SETCRC4:
CP_DEBUG2 (d, ("ioctl: setcrc4\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_E1)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_crc4 (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETCLK:
CP_DEBUG2 (d, ("ioctl: getclk\n"));
if (c->type != T_E1 &&
c->type != T_G703 &&
c->type != T_E3 &&
c->type != T_T3 &&
c->type != T_STS1)
return EINVAL;
switch (c->gsyn) {
default: *(int*)data = E1CLK_INTERNAL; break;
case GSYN_RCV: *(int*)data = E1CLK_RECEIVE; break;
case GSYN_RCV0: *(int*)data = E1CLK_RECEIVE_CHAN0; break;
case GSYN_RCV1: *(int*)data = E1CLK_RECEIVE_CHAN1; break;
case GSYN_RCV2: *(int*)data = E1CLK_RECEIVE_CHAN2; break;
case GSYN_RCV3: *(int*)data = E1CLK_RECEIVE_CHAN3; break;
}
return 0;
case SERIAL_SETCLK:
CP_DEBUG2 (d, ("ioctl: setclk\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_E1 &&
c->type != T_G703 &&
c->type != T_E3 &&
c->type != T_T3 &&
c->type != T_STS1)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
switch (*(int*)data) {
default: cp_set_gsyn (c, GSYN_INT); break;
case E1CLK_RECEIVE: cp_set_gsyn (c, GSYN_RCV); break;
case E1CLK_RECEIVE_CHAN0: cp_set_gsyn (c, GSYN_RCV0); break;
case E1CLK_RECEIVE_CHAN1: cp_set_gsyn (c, GSYN_RCV1); break;
case E1CLK_RECEIVE_CHAN2: cp_set_gsyn (c, GSYN_RCV2); break;
case E1CLK_RECEIVE_CHAN3: cp_set_gsyn (c, GSYN_RCV3); break;
}
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETTIMESLOTS:
CP_DEBUG2 (d, ("ioctl: gettimeslots\n"));
if ((c->type != T_E1 || c->unfram) && c->type != T_DATA)
return EINVAL;
*(u_long*)data = c->ts;
return 0;
case SERIAL_SETTIMESLOTS:
CP_DEBUG2 (d, ("ioctl: settimeslots\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if ((c->type != T_E1 || c->unfram) && c->type != T_DATA)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_ts (c, *(u_long*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETINVCLK:
CP_DEBUG2 (d, ("ioctl: getinvclk\n"));
#if 1
return EINVAL;
#else
if (c->type != T_SERIAL)
return EINVAL;
*(int*)data = c->invtxc;
return 0;
#endif
case SERIAL_SETINVCLK:
CP_DEBUG2 (d, ("ioctl: setinvclk\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_SERIAL)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_invtxc (c, *(int*)data);
cp_set_invrxc (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETINVTCLK:
CP_DEBUG2 (d, ("ioctl: getinvtclk\n"));
if (c->type != T_SERIAL)
return EINVAL;
*(int*)data = c->invtxc;
return 0;
case SERIAL_SETINVTCLK:
CP_DEBUG2 (d, ("ioctl: setinvtclk\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_SERIAL)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_invtxc (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETINVRCLK:
CP_DEBUG2 (d, ("ioctl: getinvrclk\n"));
if (c->type != T_SERIAL)
return EINVAL;
*(int*)data = c->invrxc;
return 0;
case SERIAL_SETINVRCLK:
CP_DEBUG2 (d, ("ioctl: setinvrclk\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->type != T_SERIAL)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
cp_set_invrxc (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETLEVEL:
CP_DEBUG2 (d, ("ioctl: getlevel\n"));
if (c->type != T_G703)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
*(int*)data = cp_get_lq (c);
CP_UNLOCK (bd);
splx (s);
return 0;
#if 0
case SERIAL_RESET:
CP_DEBUG2 (d, ("ioctl: reset\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CP_LOCK (bd);
cp_reset (c->board, 0, 0);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_HARDRESET:
CP_DEBUG2 (d, ("ioctl: hardreset\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CP_LOCK (bd);
/* hard_reset (c->board); */
CP_UNLOCK (bd);
splx (s);
return 0;
#endif
case SERIAL_GETCABLE:
CP_DEBUG2 (d, ("ioctl: getcable\n"));
if (c->type != T_SERIAL)
return EINVAL;
s = splimp ();
CP_LOCK (bd);
*(int*)data = cp_get_cable (c);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETDIR:
CP_DEBUG2 (d, ("ioctl: getdir\n"));
if (c->type != T_E1 && c->type != T_DATA)
return EINVAL;
*(int*)data = c->dir;
return 0;
case SERIAL_SETDIR:
CP_DEBUG2 (d, ("ioctl: setdir\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CP_LOCK (bd);
cp_set_dir (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETRLOOP:
CP_DEBUG2 (d, ("ioctl: getrloop\n"));
if (c->type != T_G703 &&
c->type != T_E3 &&
c->type != T_T3 &&
c->type != T_STS1)
return EINVAL;
*(int*)data = cp_get_rloop (c);
return 0;
case SERIAL_SETRLOOP:
CP_DEBUG2 (d, ("ioctl: setloop\n"));
if (c->type != T_E3 && c->type != T_T3 && c->type != T_STS1)
return EINVAL;
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CP_LOCK (bd);
cp_set_rloop (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETCABLEN:
CP_DEBUG2 (d, ("ioctl: getcablen\n"));
if (c->type != T_T3 && c->type != T_STS1)
return EINVAL;
*(int*)data = c->cablen;
return 0;
case SERIAL_SETCABLEN:
CP_DEBUG2 (d, ("ioctl: setloop\n"));
if (c->type != T_T3 && c->type != T_STS1)
return EINVAL;
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CP_LOCK (bd);
cp_set_cablen (c, *(int*)data);
CP_UNLOCK (bd);
splx (s);
return 0;
case TIOCSDTR: /* Set DTR */
s = splimp ();
CP_LOCK (bd);
cp_set_dtr (c, 1);
CP_UNLOCK (bd);
splx (s);
return 0;
case TIOCCDTR: /* Clear DTR */
s = splimp ();
CP_LOCK (bd);
cp_set_dtr (c, 0);
CP_UNLOCK (bd);
splx (s);
return 0;
case TIOCMSET: /* Set DTR/RTS */
s = splimp ();
CP_LOCK (bd);
cp_set_dtr (c, (*(int*)data & TIOCM_DTR) ? 1 : 0);
cp_set_rts (c, (*(int*)data & TIOCM_RTS) ? 1 : 0);
CP_UNLOCK (bd);
splx (s);
return 0;
case TIOCMBIS: /* Add DTR/RTS */
s = splimp ();
CP_LOCK (bd);
if (*(int*)data & TIOCM_DTR) cp_set_dtr (c, 1);
if (*(int*)data & TIOCM_RTS) cp_set_rts (c, 1);
CP_UNLOCK (bd);
splx (s);
return 0;
case TIOCMBIC: /* Clear DTR/RTS */
s = splimp ();
CP_LOCK (bd);
if (*(int*)data & TIOCM_DTR) cp_set_dtr (c, 0);
if (*(int*)data & TIOCM_RTS) cp_set_rts (c, 0);
CP_UNLOCK (bd);
splx (s);
return 0;
case TIOCMGET: /* Get modem status */
*(int*)data = cp_modem_status (c);
return 0;
}
return ENOTTY;
}
#ifdef NETGRAPH
static int ng_cp_constructor (node_p node)
{
drv_t *d = NG_NODE_PRIVATE (node);
CP_DEBUG (d, ("Constructor\n"));
return EINVAL;
}
static int ng_cp_newhook (node_p node, hook_p hook, const char *name)
{
int s;
drv_t *d = NG_NODE_PRIVATE (node);
bdrv_t *bd = d->board->sys;
CP_DEBUG (d, ("Newhook\n"));
/* Attach debug hook */
if (strcmp (name, NG_CP_HOOK_DEBUG) == 0) {
NG_HOOK_SET_PRIVATE (hook, NULL);
d->debug_hook = hook;
return 0;
}
/* Check for raw hook */
if (strcmp (name, NG_CP_HOOK_RAW) != 0)
return EINVAL;
NG_HOOK_SET_PRIVATE (hook, d);
d->hook = hook;
s = splimp ();
CP_LOCK (bd);
cp_up (d);
CP_UNLOCK (bd);
splx (s);
return 0;
}
static char *format_timeslots (u_long s)
{
static char buf [100];
char *p = buf;
int i;
for (i=1; i<32; ++i)
if ((s >> i) & 1) {
int prev = (i > 1) & (s >> (i-1));
int next = (i < 31) & (s >> (i+1));
if (prev) {
if (next)
continue;
*p++ = '-';
} else if (p > buf)
*p++ = ',';
if (i >= 10)
*p++ = '0' + i / 10;
*p++ = '0' + i % 10;
}
*p = 0;
return buf;
}
static int print_modems (char *s, cp_chan_t *c, int need_header)
{
int status = cp_modem_status (c);
int length = 0;
if (need_header)
length += sprintf (s + length, " LE DTR DSR RTS CTS CD\n");
length += sprintf (s + length, "%4s %4s %4s %4s %4s %4s\n",
status & TIOCM_LE ? "On" : "-",
status & TIOCM_DTR ? "On" : "-",
status & TIOCM_DSR ? "On" : "-",
status & TIOCM_RTS ? "On" : "-",
status & TIOCM_CTS ? "On" : "-",
status & TIOCM_CD ? "On" : "-");
return length;
}
static int print_stats (char *s, cp_chan_t *c, int need_header)
{
int length = 0;
if (need_header)
length += sprintf (s + length, " Rintr Tintr Mintr Ibytes Ipkts Ierrs Obytes Opkts Oerrs\n");
length += sprintf (s + length, "%7ld %7ld %7ld %8lu %7ld %7ld %8lu %7ld %7ld\n",
c->rintr, c->tintr, 0l, (unsigned long) c->ibytes,
c->ipkts, c->overrun + c->frame + c->crc,
(unsigned long) c->obytes, c->opkts, c->underrun);
return length;
}
static char *format_e1_status (u_char status)
{
static char buf [80];
if (status & E1_NOALARM)
return "Ok";
buf[0] = 0;
if (status & E1_LOS) strcat (buf, ",LOS");
if (status & E1_AIS) strcat (buf, ",AIS");
if (status & E1_LOF) strcat (buf, ",LOF");
if (status & E1_LOMF) strcat (buf, ",LOMF");
if (status & E1_FARLOF) strcat (buf, ",FARLOF");
if (status & E1_AIS16) strcat (buf, ",AIS16");
if (status & E1_FARLOMF) strcat (buf, ",FARLOMF");
if (status & E1_TSTREQ) strcat (buf, ",TSTREQ");
if (status & E1_TSTERR) strcat (buf, ",TSTERR");
if (buf[0] == ',')
return buf+1;
return "Unknown";
}
static int print_frac (char *s, int leftalign, u_long numerator, u_long divider)
{
int n, length = 0;
if (numerator < 1 || divider < 1) {
length += sprintf (s+length, leftalign ? "/- " : " -");
return length;
}
n = (int) (0.5 + 1000.0 * numerator / divider);
if (n < 1000) {
length += sprintf (s+length, leftalign ? "/.%-3d" : " .%03d", n);
return length;
}
*(s + length) = leftalign ? '/' : ' ';
length ++;
if (n >= 1000000) n = (n+500) / 1000 * 1000;
else if (n >= 100000) n = (n+50) / 100 * 100;
else if (n >= 10000) n = (n+5) / 10 * 10;
switch (n) {
case 1000: length += printf (s+length, ".999"); return length;
case 10000: n = 9990; break;
case 100000: n = 99900; break;
case 1000000: n = 999000; break;
}
if (n < 10000) length += sprintf (s+length, "%d.%d", n/1000, n/10%100);
else if (n < 100000) length += sprintf (s+length, "%d.%d", n/1000, n/100%10);
else if (n < 1000000) length += sprintf (s+length, "%d.", n/1000);
else length += sprintf (s+length, "%d", n/1000);
return length;
}
static int print_e1_stats (char *s, cp_chan_t *c)
{
struct e1_counters total;
u_long totsec;
int length = 0;
totsec = c->totsec + c->cursec;
total.bpv = c->total.bpv + c->currnt.bpv;
total.fse = c->total.fse + c->currnt.fse;
total.crce = c->total.crce + c->currnt.crce;
total.rcrce = c->total.rcrce + c->currnt.rcrce;
total.uas = c->total.uas + c->currnt.uas;
total.les = c->total.les + c->currnt.les;
total.es = c->total.es + c->currnt.es;
total.bes = c->total.bes + c->currnt.bes;
total.ses = c->total.ses + c->currnt.ses;
total.oofs = c->total.oofs + c->currnt.oofs;
total.css = c->total.css + c->currnt.css;
total.dm = c->total.dm + c->currnt.dm;
length += sprintf (s + length, " Unav/Degr Bpv/Fsyn CRC/RCRC Err/Lerr Sev/Bur Oof/Slp Status\n");
/* Unavailable seconds, degraded minutes */
length += print_frac (s + length, 0, c->currnt.uas, c->cursec);
length += print_frac (s + length, 1, 60 * c->currnt.dm, c->cursec);
/* Bipolar violations, frame sync errors */
length += print_frac (s + length, 0, c->currnt.bpv, c->cursec);
length += print_frac (s + length, 1, c->currnt.fse, c->cursec);
/* CRC errors, remote CRC errors (E-bit) */
length += print_frac (s + length, 0, c->currnt.crce, c->cursec);
length += print_frac (s + length, 1, c->currnt.rcrce, c->cursec);
/* Errored seconds, line errored seconds */
length += print_frac (s + length, 0, c->currnt.es, c->cursec);
length += print_frac (s + length, 1, c->currnt.les, c->cursec);
/* Severely errored seconds, burst errored seconds */
length += print_frac (s + length, 0, c->currnt.ses, c->cursec);
length += print_frac (s + length, 1, c->currnt.bes, c->cursec);
/* Out of frame seconds, controlled slip seconds */
length += print_frac (s + length, 0, c->currnt.oofs, c->cursec);
length += print_frac (s + length, 1, c->currnt.css, c->cursec);
length += sprintf (s + length, " %s\n", format_e1_status (c->status));
/* Print total statistics. */
length += print_frac (s + length, 0, total.uas, totsec);
length += print_frac (s + length, 1, 60 * total.dm, totsec);
length += print_frac (s + length, 0, total.bpv, totsec);
length += print_frac (s + length, 1, total.fse, totsec);
length += print_frac (s + length, 0, total.crce, totsec);
length += print_frac (s + length, 1, total.rcrce, totsec);
length += print_frac (s + length, 0, total.es, totsec);
length += print_frac (s + length, 1, total.les, totsec);
length += print_frac (s + length, 0, total.ses, totsec);
length += print_frac (s + length, 1, total.bes, totsec);
length += print_frac (s + length, 0, total.oofs, totsec);
length += print_frac (s + length, 1, total.css, totsec);
length += sprintf (s + length, " -- Total\n");
return length;
}
static int print_chan (char *s, cp_chan_t *c)
{
drv_t *d = c->sys;
bdrv_t *bd = d->board->sys;
int length = 0;
length += sprintf (s + length, "cp%d", c->board->num * NCHAN + c->num);
if (d->chan->debug)
length += sprintf (s + length, " debug=%d", d->chan->debug);
if (c->board->mux) {
length += sprintf (s + length, " cfg=C");
} else {
length += sprintf (s + length, " cfg=A");
}
if (c->baud)
length += sprintf (s + length, " %ld", c->baud);
else
length += sprintf (s + length, " extclock");
if (c->type == T_E1 || c->type == T_G703)
switch (c->gsyn) {
case GSYN_INT : length += sprintf (s + length, " syn=int"); break;
case GSYN_RCV : length += sprintf (s + length, " syn=rcv"); break;
case GSYN_RCV0 : length += sprintf (s + length, " syn=rcv0"); break;
case GSYN_RCV1 : length += sprintf (s + length, " syn=rcv1"); break;
case GSYN_RCV2 : length += sprintf (s + length, " syn=rcv2"); break;
case GSYN_RCV3 : length += sprintf (s + length, " syn=rcv3"); break;
}
if (c->type == T_SERIAL) {
length += sprintf (s + length, " dpll=%s", c->dpll ? "on" : "off");
length += sprintf (s + length, " nrzi=%s", c->nrzi ? "on" : "off");
length += sprintf (s + length, " invclk=%s", c->invtxc ? "on" : "off");
}
if (c->type == T_E1)
length += sprintf (s + length, " higain=%s", c->higain ? "on" : "off");
length += sprintf (s + length, " loop=%s", c->lloop ? "on" : "off");
if (c->type == T_E1)
length += sprintf (s + length, " ts=%s", format_timeslots (c->ts));
if (c->type == T_G703) {
int lq, x;
x = splimp ();
CP_LOCK (bd);
lq = cp_get_lq (c);
CP_UNLOCK (bd);
splx (x);
length += sprintf (s + length, " (level=-%.1fdB)", lq / 10.0);
}
length += sprintf (s + length, "\n");
return length;
}
static int ng_cp_rcvmsg (node_p node, item_p item, hook_p lasthook)
{
drv_t *d = NG_NODE_PRIVATE (node);
struct ng_mesg *msg;
struct ng_mesg *resp = NULL;
int error = 0;
CP_DEBUG (d, ("Rcvmsg\n"));
NGI_GET_MSG (item, msg);
switch (msg->header.typecookie) {
default:
error = EINVAL;
break;
case NGM_CP_COOKIE:
printf ("Not implemented yet\n");
error = EINVAL;
break;
case NGM_GENERIC_COOKIE:
switch (msg->header.cmd) {
default:
error = EINVAL;
break;
case NGM_TEXT_STATUS: {
char *s;
int l = 0;
int dl = sizeof (struct ng_mesg) + 730;
NG_MKRESPONSE (resp, msg, dl, M_NOWAIT);
if (! resp) {
error = ENOMEM;
break;
}
s = (resp)->data;
if (d) {
l += print_chan (s + l, d->chan);
l += print_stats (s + l, d->chan, 1);
l += print_modems (s + l, d->chan, 1);
l += print_e1_stats (s + l, d->chan);
} else
l += sprintf (s + l, "Error: node not connect to channel");
strncpy ((resp)->header.cmdstr, "status", NG_CMDSTRSIZ);
}
break;
}
break;
}
NG_RESPOND_MSG (error, node, item, resp);
NG_FREE_MSG (msg);
return error;
}
static int ng_cp_rcvdata (hook_p hook, item_p item)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE(hook));
struct mbuf *m;
struct ng_tag_prio *ptag;
bdrv_t *bd = d->board->sys;
struct ifqueue *q;
int s;
CP_DEBUG2 (d, ("Rcvdata\n"));
NGI_GET_M (item, m);
NG_FREE_ITEM (item);
if (! NG_HOOK_PRIVATE (hook) || ! d) {
NG_FREE_M (m);
return ENETDOWN;
}
/* Check for high priority data */
if ((ptag = (struct ng_tag_prio *)m_tag_locate(m, NGM_GENERIC_COOKIE,
NG_TAG_PRIO, NULL)) != NULL && (ptag->priority > NG_PRIO_CUTOFF) )
q = &d->hi_queue;
else
q = &d->queue;
s = splimp ();
CP_LOCK (bd);
IF_LOCK (q);
if (_IF_QFULL (q)) {
IF_UNLOCK (q);
CP_UNLOCK (bd);
splx (s);
NG_FREE_M (m);
return ENOBUFS;
}
_IF_ENQUEUE (q, m);
IF_UNLOCK (q);
cp_start (d);
CP_UNLOCK (bd);
splx (s);
return 0;
}
static int ng_cp_rmnode (node_p node)
{
drv_t *d = NG_NODE_PRIVATE (node);
CP_DEBUG (d, ("Rmnode\n"));
if (d && d->running) {
bdrv_t *bd = d->board->sys;
int s = splimp ();
CP_LOCK (bd);
cp_down (d);
CP_UNLOCK (bd);
splx (s);
}
#ifdef KLD_MODULE
if (node->nd_flags & NGF_REALLY_DIE) {
NG_NODE_SET_PRIVATE (node, NULL);
NG_NODE_UNREF (node);
}
NG_NODE_REVIVE(node); /* Persistant node */
#endif
return 0;
}
static int ng_cp_connect (hook_p hook)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE (hook));
if (d) {
CP_DEBUG (d, ("Connect\n"));
callout_reset (&d->timeout_handle, hz, cp_watchdog_timer, d);
}
return 0;
}
static int ng_cp_disconnect (hook_p hook)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE (hook));
if (d) {
CP_DEBUG (d, ("Disconnect\n"));
if (NG_HOOK_PRIVATE (hook))
{
bdrv_t *bd = d->board->sys;
int s = splimp ();
CP_LOCK (bd);
cp_down (d);
CP_UNLOCK (bd);
splx (s);
}
/* If we were wait it than it reasserted now, just stop it. */
if (!callout_drain (&d->timeout_handle))
callout_stop (&d->timeout_handle);
}
return 0;
}
#endif
static int cp_modevent (module_t mod, int type, void *unused)
{
static int load_count = 0;
switch (type) {
case MOD_LOAD:
#ifdef NETGRAPH
if (ng_newtype (&typestruct))
printf ("Failed to register ng_cp\n");
#endif
++load_count;
- callout_init (&timeout_handle, CALLOUT_MPSAFE);
+ callout_init (&timeout_handle, 1);
callout_reset (&timeout_handle, hz*5, cp_timeout, 0);
break;
case MOD_UNLOAD:
if (load_count == 1) {
printf ("Removing device entry for Tau-PCI\n");
#ifdef NETGRAPH
ng_rmtype (&typestruct);
#endif
}
/* If we were wait it than it reasserted now, just stop it.
* Actually we shouldn't get this condition. But code could be
* changed in the future, so just be a litle paranoid.
*/
if (!callout_drain (&timeout_handle))
callout_stop (&timeout_handle);
--load_count;
break;
case MOD_SHUTDOWN:
break;
}
return 0;
}
#ifdef NETGRAPH
static struct ng_type typestruct = {
.version = NG_ABI_VERSION,
.name = NG_CP_NODE_TYPE,
.constructor = ng_cp_constructor,
.rcvmsg = ng_cp_rcvmsg,
.shutdown = ng_cp_rmnode,
.newhook = ng_cp_newhook,
.connect = ng_cp_connect,
.rcvdata = ng_cp_rcvdata,
.disconnect = ng_cp_disconnect,
};
#endif /*NETGRAPH*/
#ifdef NETGRAPH
MODULE_DEPEND (ng_cp, netgraph, NG_ABI_VERSION, NG_ABI_VERSION, NG_ABI_VERSION);
#else
MODULE_DEPEND (cp, sppp, 1, 1, 1);
#endif
DRIVER_MODULE (cp, pci, cp_driver, cp_devclass, cp_modevent, NULL);
MODULE_VERSION (cp, 1);
Index: head/sys/dev/ctau/if_ct.c
===================================================================
--- head/sys/dev/ctau/if_ct.c (revision 283290)
+++ head/sys/dev/ctau/if_ct.c (revision 283291)
@@ -1,2206 +1,2206 @@
/*-
* Cronyx-Tau adapter driver for FreeBSD.
* Supports PPP/HDLC and Cisco/HDLC protocol in synchronous mode,
* and asynchronous channels with full modem control.
* Keepalive protocol implemented in both Cisco and PPP modes.
*
* Copyright (C) 1994-2002 Cronyx Engineering.
* Author: Serge Vakulenko, <vak@cronyx.ru>
*
* Copyright (C) 1999-2004 Cronyx Engineering.
* Author: Roman Kurakin, <rik@cronyx.ru>
*
* This software is distributed with NO WARRANTIES, not even the implied
* warranties for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* Authors grant any other persons or organisations a permission to use,
* modify and redistribute this software in source and binary forms,
* as long as this message is kept with the software, all derivative
* works or modified versions.
*
* Cronyx Id: if_ct.c,v 1.1.2.31 2004/06/23 17:09:13 rik Exp $
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/mbuf.h>
#include <sys/sockio.h>
#include <sys/malloc.h>
#include <sys/priv.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <sys/conf.h>
#include <sys/errno.h>
#include <sys/tty.h>
#include <sys/bus.h>
#include <machine/bus.h>
#include <sys/rman.h>
#include <isa/isavar.h>
#include <sys/interrupt.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <net/if.h>
#include <net/if_var.h>
#include <machine/cpufunc.h>
#include <machine/cserial.h>
#include <machine/resource.h>
#include <dev/cx/machdep.h>
#include <dev/ctau/ctddk.h>
#include <dev/cx/cronyxfw.h>
#include "opt_ng_cronyx.h"
#ifdef NETGRAPH_CRONYX
# include "opt_netgraph.h"
# include <netgraph/ng_message.h>
# include <netgraph/netgraph.h>
# include <dev/ctau/ng_ct.h>
#else
# include <net/if_types.h>
# include <net/if_sppp.h>
# define PP_CISCO IFF_LINK2
# include <net/bpf.h>
#endif
#define NCTAU 1
/* If we don't have Cronyx's sppp version, we don't have fr support via sppp */
#ifndef PP_FR
#define PP_FR 0
#endif
#define CT_DEBUG(d,s) ({if (d->chan->debug) {\
printf ("%s: ", d->name); printf s;}})
#define CT_DEBUG2(d,s) ({if (d->chan->debug>1) {\
printf ("%s: ", d->name); printf s;}})
#define CT_LOCK_NAME "ctX"
#define CT_LOCK(_bd) mtx_lock (&(_bd)->ct_mtx)
#define CT_UNLOCK(_bd) mtx_unlock (&(_bd)->ct_mtx)
#define CT_LOCK_ASSERT(_bd) mtx_assert (&(_bd)->ct_mtx, MA_OWNED)
static void ct_identify __P((driver_t *, device_t));
static int ct_probe __P((device_t));
static int ct_attach __P((device_t));
static int ct_detach __P((device_t));
static device_method_t ct_isa_methods [] = {
DEVMETHOD(device_identify, ct_identify),
DEVMETHOD(device_probe, ct_probe),
DEVMETHOD(device_attach, ct_attach),
DEVMETHOD(device_detach, ct_detach),
DEVMETHOD_END
};
typedef struct _ct_dma_mem_t {
unsigned long phys;
void *virt;
size_t size;
bus_dma_tag_t dmat;
bus_dmamap_t mapp;
} ct_dma_mem_t;
typedef struct _drv_t {
char name [8];
ct_chan_t *chan;
ct_board_t *board;
struct _bdrv_t *bd;
ct_dma_mem_t dmamem;
int running;
#ifdef NETGRAPH
char nodename [NG_NODESIZ];
hook_p hook;
hook_p debug_hook;
node_p node;
struct ifqueue queue;
struct ifqueue hi_queue;
#else
struct ifqueue queue;
struct ifnet *ifp;
#endif
short timeout;
struct callout timeout_handle;
struct cdev *devt;
} drv_t;
typedef struct _bdrv_t {
ct_board_t *board;
struct resource *base_res;
struct resource *drq_res;
struct resource *irq_res;
int base_rid;
int drq_rid;
int irq_rid;
void *intrhand;
drv_t channel [NCHAN];
struct mtx ct_mtx;
} bdrv_t;
static driver_t ct_isa_driver = {
"ct",
ct_isa_methods,
sizeof (bdrv_t),
};
static devclass_t ct_devclass;
static void ct_receive (ct_chan_t *c, char *data, int len);
static void ct_transmit (ct_chan_t *c, void *attachment, int len);
static void ct_error (ct_chan_t *c, int data);
static void ct_up (drv_t *d);
static void ct_start (drv_t *d);
static void ct_down (drv_t *d);
static void ct_watchdog (drv_t *d);
static void ct_watchdog_timer (void *arg);
#ifdef NETGRAPH
extern struct ng_type typestruct;
#else
static void ct_ifstart (struct ifnet *ifp);
static void ct_tlf (struct sppp *sp);
static void ct_tls (struct sppp *sp);
static int ct_sioctl (struct ifnet *ifp, u_long cmd, caddr_t data);
static void ct_initialize (void *softc);
#endif
static ct_board_t *adapter [NCTAU];
static drv_t *channel [NCTAU*NCHAN];
static struct callout led_timo [NCTAU];
static struct callout timeout_handle;
static int ct_open (struct cdev *dev, int oflags, int devtype, struct thread *td);
static int ct_close (struct cdev *dev, int fflag, int devtype, struct thread *td);
static int ct_ioctl (struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td);
static struct cdevsw ct_cdevsw = {
.d_version = D_VERSION,
.d_open = ct_open,
.d_close = ct_close,
.d_ioctl = ct_ioctl,
.d_name = "ct",
};
/*
* Make an mbuf from data.
*/
static struct mbuf *makembuf (void *buf, u_int len)
{
struct mbuf *m;
MGETHDR (m, M_NOWAIT, MT_DATA);
if (! m)
return 0;
if (!(MCLGET(m, M_NOWAIT))) {
m_freem (m);
return 0;
}
m->m_pkthdr.len = m->m_len = len;
bcopy (buf, mtod (m, caddr_t), len);
return m;
}
static void ct_timeout (void *arg)
{
drv_t *d;
int s, i, k;
for (i = 0; i < NCTAU; ++i) {
if (adapter[i] == NULL)
continue;
for (k = 0; k < NCHAN; k++) {
d = channel[i * NCHAN + k];
if (! d)
continue;
if (d->chan->mode != M_G703)
continue;
s = splimp ();
CT_LOCK ((bdrv_t *)d->bd);
ct_g703_timer (d->chan);
CT_UNLOCK ((bdrv_t *)d->bd);
splx (s);
}
}
callout_reset (&timeout_handle, hz, ct_timeout, 0);
}
static void ct_led_off (void *arg)
{
ct_board_t *b = arg;
bdrv_t *bd = ((drv_t *)b->chan->sys)->bd;
int s = splimp ();
CT_LOCK (bd);
ct_led (b, 0);
CT_UNLOCK (bd);
splx (s);
}
/*
* Activate interrupt handler from DDK.
*/
static void ct_intr (void *arg)
{
bdrv_t *bd = arg;
ct_board_t *b = bd->board;
#ifndef NETGRAPH
int i;
#endif
int s = splimp ();
CT_LOCK (bd);
/* Turn LED on. */
ct_led (b, 1);
ct_int_handler (b);
/* Turn LED off 50 msec later. */
callout_reset (&led_timo[b->num], hz/20, ct_led_off, b);
CT_UNLOCK (bd);
splx (s);
#ifndef NETGRAPH
/* Pass packets in a lock-free state */
for (i = 0; i < NCHAN && b->chan[i].type; i++) {
drv_t *d = b->chan[i].sys;
struct mbuf *m;
if (!d || !d->running)
continue;
while (_IF_QLEN(&d->queue)) {
IF_DEQUEUE (&d->queue,m);
if (!m)
continue;
sppp_input (d->ifp, m);
}
}
#endif
}
static int probe_irq (ct_board_t *b, int irq)
{
int mask, busy, cnt;
/* Clear pending irq, if any. */
ct_probe_irq (b, -irq);
DELAY (100);
for (cnt=0; cnt<5; ++cnt) {
/* Get the mask of pending irqs, assuming they are busy.
* Activate the adapter on given irq. */
busy = ct_probe_irq (b, irq);
DELAY (1000);
/* Get the mask of active irqs.
* Deactivate our irq. */
mask = ct_probe_irq (b, -irq);
DELAY (100);
if ((mask & ~busy) == 1 << irq) {
ct_probe_irq (b, 0);
/* printf ("ct%d: irq %d ok, mask=0x%04x, busy=0x%04x\n",
b->num, irq, mask, busy); */
return 1;
}
}
/* printf ("ct%d: irq %d not functional, mask=0x%04x, busy=0x%04x\n",
b->num, irq, mask, busy); */
ct_probe_irq (b, 0);
return 0;
}
static short porttab [] = {
0x200, 0x220, 0x240, 0x260, 0x280, 0x2a0, 0x2c0, 0x2e0,
0x300, 0x320, 0x340, 0x360, 0x380, 0x3a0, 0x3c0, 0x3e0, 0
};
static char dmatab [] = { 7, 6, 5, 0 };
static char irqtab [] = { 5, 10, 11, 7, 3, 15, 12, 0 };
static int ct_is_free_res (device_t dev, int rid, int type, u_long start,
u_long end, u_long count)
{
struct resource *res;
if (!(res = bus_alloc_resource (dev, type, &rid, start, end, count, 0)))
return 0;
bus_release_resource (dev, type, rid, res);
return 1;
}
static void ct_identify (driver_t *driver, device_t dev)
{
u_long iobase, rescount;
int devcount;
device_t *devices;
device_t child;
devclass_t my_devclass;
int i, k;
if ((my_devclass = devclass_find ("ct")) == NULL)
return;
devclass_get_devices (my_devclass, &devices, &devcount);
if (devcount == 0) {
/* We should find all devices by our self. We could alter other
* devices, but we don't have a choise
*/
for (i = 0; (iobase = porttab [i]) != 0; i++) {
if (!ct_is_free_res (dev, 0, SYS_RES_IOPORT,
iobase, iobase + NPORT, NPORT))
continue;
if (ct_probe_board (iobase, -1, -1) == 0)
continue;
devcount++;
child = BUS_ADD_CHILD (dev, ISA_ORDER_SPECULATIVE, "ct",
-1);
if (child == NULL)
return;
device_set_desc_copy (child, "Cronyx Tau-ISA");
device_set_driver (child, driver);
bus_set_resource (child, SYS_RES_IOPORT, 0,
iobase, NPORT);
if (devcount >= NCTAU)
break;
}
} else {
static short porttab [] = {
0x200, 0x220, 0x240, 0x260, 0x280, 0x2a0, 0x2c0, 0x2e0,
0x300, 0x320, 0x340, 0x360, 0x380, 0x3a0, 0x3c0, 0x3e0, 0
};
/* Lets check user choise.
*/
for (k = 0; k < devcount; k++) {
if (bus_get_resource (devices[k], SYS_RES_IOPORT, 0,
&iobase, &rescount) != 0)
continue;
for (i = 0; porttab [i] != 0; i++) {
if (porttab [i] != iobase)
continue;
if (!ct_is_free_res (devices[k], 0, SYS_RES_IOPORT,
iobase, iobase + NPORT, NPORT))
continue;
if (ct_probe_board (iobase, -1, -1) == 0)
continue;
porttab [i] = -1;
device_set_desc_copy (devices[k], "Cronyx Tau-ISA");
break;
}
if (porttab [i] == 0) {
device_delete_child (
device_get_parent (devices[k]),
devices [k]);
devices[k] = 0;
continue;
}
}
for (k = 0; k < devcount; k++) {
if (devices[k] == 0)
continue;
if (bus_get_resource (devices[k], SYS_RES_IOPORT, 0,
&iobase, &rescount) == 0)
continue;
for (i = 0; (iobase = porttab [i]) != 0; i++) {
if (porttab [i] == -1)
continue;
if (!ct_is_free_res (devices[k], 0, SYS_RES_IOPORT,
iobase, iobase + NPORT, NPORT))
continue;
if (ct_probe_board (iobase, -1, -1) == 0)
continue;
bus_set_resource (devices[k], SYS_RES_IOPORT, 0,
iobase, NPORT);
porttab [i] = -1;
device_set_desc_copy (devices[k], "Cronyx Tau-ISA");
break;
}
if (porttab [i] == 0) {
device_delete_child (
device_get_parent (devices[k]),
devices [k]);
}
}
free (devices, M_TEMP);
}
return;
}
static int ct_probe (device_t dev)
{
int unit = device_get_unit (dev);
u_long iobase, rescount;
if (!device_get_desc (dev) ||
strcmp (device_get_desc (dev), "Cronyx Tau-ISA"))
return ENXIO;
/* KASSERT ((bd != NULL), ("ct%d: NULL device softc\n", unit));*/
if (bus_get_resource (dev, SYS_RES_IOPORT, 0, &iobase, &rescount) != 0) {
printf ("ct%d: Couldn't get IOPORT\n", unit);
return ENXIO;
}
if (!ct_is_free_res (dev, 0, SYS_RES_IOPORT,
iobase, iobase + NPORT, NPORT)) {
printf ("ct%d: Resource IOPORT isn't free\n", unit);
return ENXIO;
}
if (!ct_probe_board (iobase, -1, -1)) {
printf ("ct%d: probing for Tau-ISA at %lx faild\n", unit, iobase);
return ENXIO;
}
return 0;
}
static void
ct_bus_dmamap_addr (void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
unsigned long *addr;
if (error)
return;
KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg));
addr = arg;
*addr = segs->ds_addr;
}
static int
ct_bus_dma_mem_alloc (int bnum, int cnum, ct_dma_mem_t *dmem)
{
int error;
error = bus_dma_tag_create (NULL, 16, 0, BUS_SPACE_MAXADDR_24BIT,
BUS_SPACE_MAXADDR, NULL, NULL, dmem->size, 1,
dmem->size, 0, NULL, NULL, &dmem->dmat);
if (error) {
if (cnum >= 0) printf ("ct%d-%d: ", bnum, cnum);
else printf ("ct%d: ", bnum);
printf ("couldn't allocate tag for dma memory\n");
return 0;
}
error = bus_dmamem_alloc (dmem->dmat, (void **)&dmem->virt,
BUS_DMA_NOWAIT | BUS_DMA_ZERO, &dmem->mapp);
if (error) {
if (cnum >= 0) printf ("ct%d-%d: ", bnum, cnum);
else printf ("ct%d: ", bnum);
printf ("couldn't allocate mem for dma memory\n");
bus_dma_tag_destroy (dmem->dmat);
return 0;
}
error = bus_dmamap_load (dmem->dmat, dmem->mapp, dmem->virt,
dmem->size, ct_bus_dmamap_addr, &dmem->phys, 0);
if (error) {
if (cnum >= 0) printf ("ct%d-%d: ", bnum, cnum);
else printf ("ct%d: ", bnum);
printf ("couldn't load mem map for dma memory\n");
bus_dmamem_free (dmem->dmat, dmem->virt, dmem->mapp);
bus_dma_tag_destroy (dmem->dmat);
return 0;
}
return 1;
}
static void
ct_bus_dma_mem_free (ct_dma_mem_t *dmem)
{
bus_dmamap_unload (dmem->dmat, dmem->mapp);
bus_dmamem_free (dmem->dmat, dmem->virt, dmem->mapp);
bus_dma_tag_destroy (dmem->dmat);
}
/*
* The adapter is present, initialize the driver structures.
*/
static int ct_attach (device_t dev)
{
bdrv_t *bd = device_get_softc (dev);
u_long iobase, drq, irq, rescount;
int unit = device_get_unit (dev);
char *ct_ln = CT_LOCK_NAME;
ct_board_t *b;
ct_chan_t *c;
drv_t *d;
int i;
int s;
KASSERT ((bd != NULL), ("ct%d: NULL device softc\n", unit));
bus_get_resource (dev, SYS_RES_IOPORT, 0, &iobase, &rescount);
bd->base_rid = 0;
bd->base_res = bus_alloc_resource (dev, SYS_RES_IOPORT, &bd->base_rid,
iobase, iobase + NPORT, NPORT, RF_ACTIVE);
if (! bd->base_res) {
printf ("ct%d: cannot alloc base address\n", unit);
return ENXIO;
}
if (bus_get_resource (dev, SYS_RES_DRQ, 0, &drq, &rescount) != 0) {
for (i = 0; (drq = dmatab [i]) != 0; i++) {
if (!ct_is_free_res (dev, 0, SYS_RES_DRQ,
drq, drq + 1, 1))
continue;
bus_set_resource (dev, SYS_RES_DRQ, 0, drq, 1);
break;
}
if (dmatab[i] == 0) {
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
printf ("ct%d: Couldn't get DRQ\n", unit);
return ENXIO;
}
}
bd->drq_rid = 0;
bd->drq_res = bus_alloc_resource (dev, SYS_RES_DRQ, &bd->drq_rid,
drq, drq + 1, 1, RF_ACTIVE);
if (! bd->drq_res) {
printf ("ct%d: cannot allocate drq\n", unit);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
return ENXIO;
}
if (bus_get_resource (dev, SYS_RES_IRQ, 0, &irq, &rescount) != 0) {
for (i = 0; (irq = irqtab [i]) != 0; i++) {
if (!ct_is_free_res (dev, 0, SYS_RES_IRQ,
irq, irq + 1, 1))
continue;
bus_set_resource (dev, SYS_RES_IRQ, 0, irq, 1);
break;
}
if (irqtab[i] == 0) {
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
printf ("ct%d: Couldn't get IRQ\n", unit);
return ENXIO;
}
}
bd->irq_rid = 0;
bd->irq_res = bus_alloc_resource (dev, SYS_RES_IRQ, &bd->irq_rid,
irq, irq + 1, 1, RF_ACTIVE);
if (! bd->irq_res) {
printf ("ct%d: Couldn't allocate irq\n", unit);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
return ENXIO;
}
b = malloc (sizeof (ct_board_t), M_DEVBUF, M_WAITOK);
if (!b) {
printf ("ct:%d: Couldn't allocate memory\n", unit);
return (ENXIO);
}
adapter[unit] = b;
bzero (b, sizeof(ct_board_t));
if (! ct_open_board (b, unit, iobase, irq, drq)) {
printf ("ct%d: error loading firmware\n", unit);
free (b, M_DEVBUF);
bus_release_resource (dev, SYS_RES_IRQ, bd->irq_rid,
bd->irq_res);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
return ENXIO;
}
bd->board = b;
ct_ln[2] = '0' + unit;
mtx_init (&bd->ct_mtx, ct_ln, MTX_NETWORK_LOCK, MTX_DEF|MTX_RECURSE);
if (! probe_irq (b, irq)) {
printf ("ct%d: irq %ld not functional\n", unit, irq);
bd->board = 0;
adapter [unit] = 0;
free (b, M_DEVBUF);
bus_release_resource (dev, SYS_RES_IRQ, bd->irq_rid,
bd->irq_res);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
mtx_destroy (&bd->ct_mtx);
return ENXIO;
}
- callout_init (&led_timo[unit], CALLOUT_MPSAFE);
+ callout_init (&led_timo[unit], 1);
s = splimp ();
if (bus_setup_intr (dev, bd->irq_res,
INTR_TYPE_NET|INTR_MPSAFE,
NULL, ct_intr, bd, &bd->intrhand)) {
printf ("ct%d: Can't setup irq %ld\n", unit, irq);
bd->board = 0;
adapter [unit] = 0;
free (b, M_DEVBUF);
bus_release_resource (dev, SYS_RES_IRQ, bd->irq_rid,
bd->irq_res);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
mtx_destroy (&bd->ct_mtx);
splx (s);
return ENXIO;
}
CT_LOCK (bd);
ct_init_board (b, b->num, b->port, irq, drq, b->type, b->osc);
ct_setup_board (b, 0, 0, 0);
CT_UNLOCK (bd);
printf ("ct%d: <Cronyx-%s>, clock %s MHz\n", b->num, b->name,
b->osc == 20000000 ? "20" : "16.384");
for (c = b->chan; c < b->chan + NCHAN; ++c) {
d = &bd->channel[c->num];
d->dmamem.size = sizeof(ct_buf_t);
if (! ct_bus_dma_mem_alloc (unit, c->num, &d->dmamem))
continue;
d->board = b;
d->chan = c;
d->bd = bd;
c->sys = d;
channel [b->num*NCHAN + c->num] = d;
sprintf (d->name, "ct%d.%d", b->num, c->num);
- callout_init (&d->timeout_handle, CALLOUT_MPSAFE);
+ callout_init (&d->timeout_handle, 1);
#ifdef NETGRAPH
if (ng_make_node_common (&typestruct, &d->node) != 0) {
printf ("%s: cannot make common node\n", d->name);
channel [b->num*NCHAN + c->num] = 0;
c->sys = 0;
ct_bus_dma_mem_free (&d->dmamem);
continue;
}
NG_NODE_SET_PRIVATE (d->node, d);
sprintf (d->nodename, "%s%d", NG_CT_NODE_TYPE,
c->board->num*NCHAN + c->num);
if (ng_name_node (d->node, d->nodename)) {
printf ("%s: cannot name node\n", d->nodename);
NG_NODE_UNREF (d->node);
channel [b->num*NCHAN + c->num] = 0;
c->sys = 0;
ct_bus_dma_mem_free (&d->dmamem);
continue;
}
d->queue.ifq_maxlen = ifqmaxlen;
d->hi_queue.ifq_maxlen = ifqmaxlen;
mtx_init (&d->queue.ifq_mtx, "ct_queue", NULL, MTX_DEF);
mtx_init (&d->hi_queue.ifq_mtx, "ct_queue_hi", NULL, MTX_DEF);
#else /*NETGRAPH*/
d->ifp = if_alloc(IFT_PPP);
if (d->ifp == NULL) {
printf ("%s: cannot if_alloc common interface\n",
d->name);
channel [b->num*NCHAN + c->num] = 0;
c->sys = 0;
ct_bus_dma_mem_free (&d->dmamem);
continue;
}
d->ifp->if_softc = d;
if_initname (d->ifp, "ct", b->num * NCHAN + c->num);
d->ifp->if_mtu = PP_MTU;
d->ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
d->ifp->if_ioctl = ct_sioctl;
d->ifp->if_start = ct_ifstart;
d->ifp->if_init = ct_initialize;
d->queue.ifq_maxlen = NBUF;
mtx_init (&d->queue.ifq_mtx, "ct_queue", NULL, MTX_DEF);
sppp_attach (d->ifp);
if_attach (d->ifp);
IFP2SP(d->ifp)->pp_tlf = ct_tlf;
IFP2SP(d->ifp)->pp_tls = ct_tls;
/* If BPF is in the kernel, call the attach for it.
* Header size is 4 bytes. */
bpfattach (d->ifp, DLT_PPP, 4);
#endif /*NETGRAPH*/
CT_LOCK (bd);
ct_start_chan (c, d->dmamem.virt, d->dmamem.phys);
ct_register_receive (c, &ct_receive);
ct_register_transmit (c, &ct_transmit);
ct_register_error (c, &ct_error);
CT_UNLOCK (bd);
d->devt = make_dev (&ct_cdevsw, b->num*NCHAN+c->num, UID_ROOT,
GID_WHEEL, 0600, "ct%d", b->num*NCHAN+c->num);
}
splx (s);
return 0;
}
static int ct_detach (device_t dev)
{
bdrv_t *bd = device_get_softc (dev);
ct_board_t *b = bd->board;
ct_chan_t *c;
int s;
KASSERT (mtx_initialized (&bd->ct_mtx), ("ct mutex not initialized"));
s = splimp ();
CT_LOCK (bd);
/* Check if the device is busy (open). */
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (!d || !d->chan->type)
continue;
if (d->running) {
CT_UNLOCK (bd);
splx (s);
return EBUSY;
}
}
/* Deactivate the timeout routine. */
callout_stop (&led_timo[b->num]);
CT_UNLOCK (bd);
bus_teardown_intr (dev, bd->irq_res, bd->intrhand);
bus_release_resource (dev, SYS_RES_IRQ, bd->irq_rid, bd->irq_res);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid, bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid, bd->base_res);
CT_LOCK (bd);
ct_close_board (b);
CT_UNLOCK (bd);
/* Detach the interfaces, free buffer memory. */
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (!d || !d->chan->type)
continue;
callout_stop (&d->timeout_handle);
#ifdef NETGRAPH
if (d->node) {
ng_rmnode_self (d->node);
NG_NODE_UNREF (d->node);
d->node = NULL;
}
mtx_destroy (&d->queue.ifq_mtx);
mtx_destroy (&d->hi_queue.ifq_mtx);
#else
/* Detach from the packet filter list of interfaces. */
bpfdetach (d->ifp);
/* Detach from the sync PPP list. */
sppp_detach (d->ifp);
if_detach (d->ifp);
if_free (d->ifp);
IF_DRAIN (&d->queue);
mtx_destroy (&d->queue.ifq_mtx);
#endif
destroy_dev (d->devt);
}
CT_LOCK (bd);
ct_led_off (b);
CT_UNLOCK (bd);
callout_drain (&led_timo[b->num]);
splx (s);
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (!d || !d->chan->type)
continue;
callout_drain(&d->timeout_handle);
/* Deallocate buffers. */
ct_bus_dma_mem_free (&d->dmamem);
}
bd->board = 0;
adapter [b->num] = 0;
free (b, M_DEVBUF);
mtx_destroy (&bd->ct_mtx);
return 0;
}
#ifndef NETGRAPH
static void ct_ifstart (struct ifnet *ifp)
{
drv_t *d = ifp->if_softc;
bdrv_t *bd = d->bd;
CT_LOCK (bd);
ct_start (d);
CT_UNLOCK (bd);
}
static void ct_tlf (struct sppp *sp)
{
drv_t *d = SP2IFP(sp)->if_softc;
CT_DEBUG (d, ("ct_tlf\n"));
/* ct_set_dtr (d->chan, 0);*/
/* ct_set_rts (d->chan, 0);*/
if (!(sp->pp_flags & PP_FR) && !(d->ifp->if_flags & PP_CISCO))
sp->pp_down (sp);
}
static void ct_tls (struct sppp *sp)
{
drv_t *d = SP2IFP(sp)->if_softc;
CT_DEBUG (d, ("ct_tls\n"));
if (!(sp->pp_flags & PP_FR) && !(d->ifp->if_flags & PP_CISCO))
sp->pp_up (sp);
}
/*
* Initialization of interface.
* Ii seems to be never called by upper level.
*/
static void ct_initialize (void *softc)
{
drv_t *d = softc;
CT_DEBUG (d, ("ct_initialize\n"));
}
/*
* Process an ioctl request.
*/
static int ct_sioctl (struct ifnet *ifp, u_long cmd, caddr_t data)
{
drv_t *d = ifp->if_softc;
bdrv_t *bd = d->bd;
int error, s, was_up, should_be_up;
was_up = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
error = sppp_ioctl (ifp, cmd, data);
if (error)
return error;
if (! (ifp->if_flags & IFF_DEBUG))
d->chan->debug = 0;
else
d->chan->debug = d->chan->debug_shadow;
switch (cmd) {
default: CT_DEBUG2 (d, ("ioctl 0x%lx\n", cmd)); return 0;
case SIOCADDMULTI: CT_DEBUG2 (d, ("SIOCADDMULTI\n")); return 0;
case SIOCDELMULTI: CT_DEBUG2 (d, ("SIOCDELMULTI\n")); return 0;
case SIOCSIFFLAGS: CT_DEBUG2 (d, ("SIOCSIFFLAGS\n")); break;
case SIOCSIFADDR: CT_DEBUG2 (d, ("SIOCSIFADDR\n")); break;
}
/* We get here only in case of SIFFLAGS or SIFADDR. */
s = splimp ();
CT_LOCK (bd);
should_be_up = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
if (! was_up && should_be_up) {
/* Interface goes up -- start it. */
ct_up (d);
ct_start (d);
} else if (was_up && ! should_be_up) {
/* Interface is going down -- stop it. */
/* if ((IFP2SP(d->ifp)->pp_flags & PP_FR) || (ifp->if_flags & PP_CISCO))*/
ct_down (d);
}
CT_UNLOCK (bd);
splx (s);
return 0;
}
#endif /*NETGRAPH*/
/*
* Stop the interface. Called on splimp().
*/
static void ct_down (drv_t *d)
{
int s = splimp ();
CT_DEBUG (d, ("ct_down\n"));
ct_set_dtr (d->chan, 0);
ct_set_rts (d->chan, 0);
d->running = 0;
callout_stop (&d->timeout_handle);
splx (s);
}
/*
* Start the interface. Called on splimp().
*/
static void ct_up (drv_t *d)
{
int s = splimp ();
CT_DEBUG (d, ("ct_up\n"));
ct_set_dtr (d->chan, 1);
ct_set_rts (d->chan, 1);
d->running = 1;
splx (s);
}
/*
* Start output on the (slave) interface. Get another datagram to send
* off of the interface queue, and copy it to the interface
* before starting the output.
*/
static void ct_send (drv_t *d)
{
struct mbuf *m;
u_short len;
CT_DEBUG2 (d, ("ct_send, tn=%d\n", d->chan->tn));
/* No output if the interface is down. */
if (! d->running)
return;
/* No output if the modem is off. */
if (! ct_get_dsr (d->chan) && !ct_get_loop (d->chan))
return;
while (ct_buf_free (d->chan)) {
/* Get the packet to send. */
#ifdef NETGRAPH
IF_DEQUEUE (&d->hi_queue, m);
if (! m)
IF_DEQUEUE (&d->queue, m);
#else
m = sppp_dequeue (d->ifp);
#endif
if (! m)
return;
#ifndef NETGRAPH
BPF_MTAP (d->ifp, m);
#endif
len = m_length (m, NULL);
if (! m->m_next)
ct_send_packet (d->chan, (u_char*)mtod (m, caddr_t),
len, 0);
else {
m_copydata (m, 0, len, d->chan->tbuf[d->chan->te]);
ct_send_packet (d->chan, d->chan->tbuf[d->chan->te],
len, 0);
}
m_freem (m);
/* Set up transmit timeout, if the transmit ring is not empty.
* Transmit timeout is 10 seconds. */
d->timeout = 10;
}
#ifndef NETGRAPH
d->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
#endif
}
/*
* Start output on the interface.
* Always called on splimp().
*/
static void ct_start (drv_t *d)
{
int s = splimp ();
if (d->running) {
if (! d->chan->dtr)
ct_set_dtr (d->chan, 1);
if (! d->chan->rts)
ct_set_rts (d->chan, 1);
ct_send (d);
callout_reset (&d->timeout_handle, hz, ct_watchdog_timer, d);
}
splx (s);
}
/*
* Handle transmit timeouts.
* Recover after lost transmit interrupts.
* Always called on splimp().
*/
static void ct_watchdog (drv_t *d)
{
CT_DEBUG (d, ("device timeout\n"));
if (d->running) {
ct_setup_chan (d->chan);
ct_start_chan (d->chan, 0, 0);
ct_set_dtr (d->chan, 1);
ct_set_rts (d->chan, 1);
ct_start (d);
}
}
static void ct_watchdog_timer (void *arg)
{
drv_t *d = arg;
bdrv_t *bd = d->bd;
CT_LOCK (bd);
if (d->timeout == 1)
ct_watchdog (d);
if (d->timeout)
d->timeout--;
callout_reset (&d->timeout_handle, hz, ct_watchdog_timer, d);
CT_UNLOCK (bd);
}
/*
* Transmit callback function.
*/
static void ct_transmit (ct_chan_t *c, void *attachment, int len)
{
drv_t *d = c->sys;
if (!d)
return;
d->timeout = 0;
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_OPACKETS, 1);
d->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
#endif
ct_start (d);
}
/*
* Process the received packet.
*/
static void ct_receive (ct_chan_t *c, char *data, int len)
{
drv_t *d = c->sys;
struct mbuf *m;
#ifdef NETGRAPH
int error;
#endif
if (!d || !d->running)
return;
m = makembuf (data, len);
if (! m) {
CT_DEBUG (d, ("no memory for packet\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IQDROPS, 1);
#endif
return;
}
if (c->debug > 1)
m_print (m, 0);
#ifdef NETGRAPH
m->m_pkthdr.rcvif = 0;
NG_SEND_DATA_ONLY (error, d->hook, m);
#else
if_inc_counter(d->ifp, IFCOUNTER_IPACKETS, 1);
m->m_pkthdr.rcvif = d->ifp;
/* Check if there's a BPF listener on this interface.
* If so, hand off the raw packet to bpf. */
BPF_MTAP(d->ifp, m);
IF_ENQUEUE (&d->queue, m);
#endif
}
/*
* Error callback function.
*/
static void ct_error (ct_chan_t *c, int data)
{
drv_t *d = c->sys;
if (!d)
return;
switch (data) {
case CT_FRAME:
CT_DEBUG (d, ("frame error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CT_CRC:
CT_DEBUG (d, ("crc error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CT_OVERRUN:
CT_DEBUG (d, ("overrun error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_COLLISIONS, 1);
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CT_OVERFLOW:
CT_DEBUG (d, ("overflow error\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CT_UNDERRUN:
CT_DEBUG (d, ("underrun error\n"));
d->timeout = 0;
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_OERRORS, 1);
d->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
#endif
ct_start (d);
break;
default:
CT_DEBUG (d, ("error #%d\n", data));
}
}
static int ct_open (struct cdev *dev, int oflags, int devtype, struct thread *td)
{
drv_t *d;
if (dev2unit(dev) >= NCTAU*NCHAN || ! (d = channel[dev2unit(dev)]))
return ENXIO;
CT_DEBUG2 (d, ("ct_open\n"));
return 0;
}
static int ct_close (struct cdev *dev, int fflag, int devtype, struct thread *td)
{
drv_t *d = channel [dev2unit(dev)];
if (!d)
return 0;
CT_DEBUG2 (d, ("ct_close\n"));
return 0;
}
static int ct_modem_status (ct_chan_t *c)
{
drv_t *d = c->sys;
bdrv_t *bd;
int status, s;
if (!d)
return 0;
bd = d->bd;
status = d->running ? TIOCM_LE : 0;
s = splimp ();
CT_LOCK (bd);
if (ct_get_cd (c)) status |= TIOCM_CD;
if (ct_get_cts (c)) status |= TIOCM_CTS;
if (ct_get_dsr (c)) status |= TIOCM_DSR;
if (c->dtr) status |= TIOCM_DTR;
if (c->rts) status |= TIOCM_RTS;
CT_UNLOCK (bd);
splx (s);
return status;
}
/*
* Process an ioctl request on /dev/cronyx/ctauN.
*/
static int ct_ioctl (struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td)
{
drv_t *d = channel [dev2unit (dev)];
bdrv_t *bd;
ct_chan_t *c;
struct serial_statistics *st;
struct e1_statistics *opte1;
int error, s;
char mask[16];
if (!d || !d->chan)
return 0;
bd = d->bd;
c = d->chan;
switch (cmd) {
case SERIAL_GETREGISTERED:
bzero (mask, sizeof(mask));
for (s=0; s<NCTAU*NCHAN; ++s)
if (channel [s])
mask [s/8] |= 1 << (s & 7);
bcopy (mask, data, sizeof (mask));
return 0;
#ifndef NETGRAPH
case SERIAL_GETPROTO:
strcpy ((char*)data, (IFP2SP(d->ifp)->pp_flags & PP_FR) ? "fr" :
(d->ifp->if_flags & PP_CISCO) ? "cisco" : "ppp");
return 0;
case SERIAL_SETPROTO:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (d->ifp->if_drv_flags & IFF_DRV_RUNNING)
return EBUSY;
if (! strcmp ("cisco", (char*)data)) {
IFP2SP(d->ifp)->pp_flags &= ~(PP_FR);
IFP2SP(d->ifp)->pp_flags |= PP_KEEPALIVE;
d->ifp->if_flags |= PP_CISCO;
} else if (! strcmp ("fr", (char*)data)) {
d->ifp->if_flags &= ~(PP_CISCO);
IFP2SP(d->ifp)->pp_flags |= PP_FR | PP_KEEPALIVE;
} else if (! strcmp ("ppp", (char*)data)) {
IFP2SP(d->ifp)->pp_flags &= ~(PP_FR | PP_KEEPALIVE);
d->ifp->if_flags &= ~(PP_CISCO);
} else
return EINVAL;
return 0;
case SERIAL_GETKEEPALIVE:
if ((IFP2SP(d->ifp)->pp_flags & PP_FR) ||
(d->ifp->if_flags & PP_CISCO))
return EINVAL;
*(int*)data = (IFP2SP(d->ifp)->pp_flags & PP_KEEPALIVE) ? 1 : 0;
return 0;
case SERIAL_SETKEEPALIVE:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if ((IFP2SP(d->ifp)->pp_flags & PP_FR) ||
(d->ifp->if_flags & PP_CISCO))
return EINVAL;
if (*(int*)data)
IFP2SP(d->ifp)->pp_flags |= PP_KEEPALIVE;
else
IFP2SP(d->ifp)->pp_flags &= ~PP_KEEPALIVE;
return 0;
#endif /*NETGRAPH*/
case SERIAL_GETMODE:
*(int*)data = SERIAL_HDLC;
return 0;
case SERIAL_GETCFG:
if (c->mode == M_HDLC)
return EINVAL;
switch (ct_get_config (c->board)) {
default: *(char*)data = 'a'; break;
case CFG_B: *(char*)data = 'b'; break;
case CFG_C: *(char*)data = 'c'; break;
}
return 0;
case SERIAL_SETCFG:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_HDLC)
return EINVAL;
s = splimp ();
CT_LOCK (bd);
switch (*(char*)data) {
case 'a': ct_set_config (c->board, CFG_A); break;
case 'b': ct_set_config (c->board, CFG_B); break;
case 'c': ct_set_config (c->board, CFG_C); break;
}
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETSTAT:
st = (struct serial_statistics*) data;
st->rintr = c->rintr;
st->tintr = c->tintr;
st->mintr = c->mintr;
st->ibytes = c->ibytes;
st->ipkts = c->ipkts;
st->ierrs = c->ierrs;
st->obytes = c->obytes;
st->opkts = c->opkts;
st->oerrs = c->oerrs;
return 0;
case SERIAL_GETESTAT:
opte1 = (struct e1_statistics*)data;
opte1->status = c->status;
opte1->cursec = c->cursec;
opte1->totsec = c->totsec + c->cursec;
opte1->currnt.bpv = c->currnt.bpv;
opte1->currnt.fse = c->currnt.fse;
opte1->currnt.crce = c->currnt.crce;
opte1->currnt.rcrce = c->currnt.rcrce;
opte1->currnt.uas = c->currnt.uas;
opte1->currnt.les = c->currnt.les;
opte1->currnt.es = c->currnt.es;
opte1->currnt.bes = c->currnt.bes;
opte1->currnt.ses = c->currnt.ses;
opte1->currnt.oofs = c->currnt.oofs;
opte1->currnt.css = c->currnt.css;
opte1->currnt.dm = c->currnt.dm;
opte1->total.bpv = c->total.bpv + c->currnt.bpv;
opte1->total.fse = c->total.fse + c->currnt.fse;
opte1->total.crce = c->total.crce + c->currnt.crce;
opte1->total.rcrce = c->total.rcrce + c->currnt.rcrce;
opte1->total.uas = c->total.uas + c->currnt.uas;
opte1->total.les = c->total.les + c->currnt.les;
opte1->total.es = c->total.es + c->currnt.es;
opte1->total.bes = c->total.bes + c->currnt.bes;
opte1->total.ses = c->total.ses + c->currnt.ses;
opte1->total.oofs = c->total.oofs + c->currnt.oofs;
opte1->total.css = c->total.css + c->currnt.css;
opte1->total.dm = c->total.dm + c->currnt.dm;
for (s=0; s<48; ++s) {
opte1->interval[s].bpv = c->interval[s].bpv;
opte1->interval[s].fse = c->interval[s].fse;
opte1->interval[s].crce = c->interval[s].crce;
opte1->interval[s].rcrce = c->interval[s].rcrce;
opte1->interval[s].uas = c->interval[s].uas;
opte1->interval[s].les = c->interval[s].les;
opte1->interval[s].es = c->interval[s].es;
opte1->interval[s].bes = c->interval[s].bes;
opte1->interval[s].ses = c->interval[s].ses;
opte1->interval[s].oofs = c->interval[s].oofs;
opte1->interval[s].css = c->interval[s].css;
opte1->interval[s].dm = c->interval[s].dm;
}
return 0;
case SERIAL_CLRSTAT:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
c->rintr = 0;
c->tintr = 0;
c->mintr = 0;
c->ibytes = 0;
c->ipkts = 0;
c->ierrs = 0;
c->obytes = 0;
c->opkts = 0;
c->oerrs = 0;
bzero (&c->currnt, sizeof (c->currnt));
bzero (&c->total, sizeof (c->total));
bzero (c->interval, sizeof (c->interval));
return 0;
case SERIAL_GETBAUD:
*(long*)data = ct_get_baud(c);
return 0;
case SERIAL_SETBAUD:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CT_LOCK (bd);
ct_set_baud (c, *(long*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETLOOP:
*(int*)data = ct_get_loop (c);
return 0;
case SERIAL_SETLOOP:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CT_LOCK (bd);
ct_set_loop (c, *(int*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETDPLL:
if (c->mode == M_E1 || c->mode == M_G703)
return EINVAL;
*(int*)data = ct_get_dpll (c);
return 0;
case SERIAL_SETDPLL:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_E1 || c->mode == M_G703)
return EINVAL;
s = splimp ();
CT_LOCK (bd);
ct_set_dpll (c, *(int*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETNRZI:
if (c->mode == M_E1 || c->mode == M_G703)
return EINVAL;
*(int*)data = ct_get_nrzi (c);
return 0;
case SERIAL_SETNRZI:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_E1 || c->mode == M_G703)
return EINVAL;
s = splimp ();
CT_LOCK (bd);
ct_set_nrzi (c, *(int*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETDEBUG:
*(int*)data = c->debug;
return 0;
case SERIAL_SETDEBUG:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
#ifndef NETGRAPH
/*
* The debug_shadow is always greater than zero for logic
* simplicity. For switching debug off the IFF_DEBUG is
* responsible.
*/
c->debug_shadow = (*(int*)data) ? (*(int*)data) : 1;
if (d->ifp->if_flags & IFF_DEBUG)
c->debug = c->debug_shadow;
#else
c->debug = *(int*)data;
#endif
return 0;
case SERIAL_GETHIGAIN:
if (c->mode != M_E1)
return EINVAL;
*(int*)data = ct_get_higain (c);
return 0;
case SERIAL_SETHIGAIN:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CT_LOCK (bd);
ct_set_higain (c, *(int*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETPHONY:
CT_DEBUG2 (d, ("ioctl: getphony\n"));
if (c->mode != M_E1)
return EINVAL;
*(int*)data = c->gopt.phony;
return 0;
case SERIAL_SETPHONY:
CT_DEBUG2 (d, ("ioctl: setphony\n"));
if (c->mode != M_E1)
return EINVAL;
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CT_LOCK (bd);
ct_set_phony (c, *(int*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETCLK:
if (c->mode != M_E1 && c->mode != M_G703)
return EINVAL;
switch (ct_get_clk(c)) {
default: *(int*)data = E1CLK_INTERNAL; break;
case GCLK_RCV: *(int*)data = E1CLK_RECEIVE; break;
case GCLK_RCLKO: *(int*)data = c->num ?
E1CLK_RECEIVE_CHAN0 : E1CLK_RECEIVE_CHAN1; break;
}
return 0;
case SERIAL_SETCLK:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CT_LOCK (bd);
switch (*(int*)data) {
default: ct_set_clk (c, GCLK_INT); break;
case E1CLK_RECEIVE: ct_set_clk (c, GCLK_RCV); break;
case E1CLK_RECEIVE_CHAN0:
case E1CLK_RECEIVE_CHAN1:
ct_set_clk (c, GCLK_RCLKO); break;
}
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETTIMESLOTS:
if (c->mode != M_E1)
return EINVAL;
*(long*)data = ct_get_ts (c);
return 0;
case SERIAL_SETTIMESLOTS:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CT_LOCK (bd);
ct_set_ts (c, *(long*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETSUBCHAN:
if (c->mode != M_E1)
return EINVAL;
*(long*)data = ct_get_subchan (c->board);
return 0;
case SERIAL_SETSUBCHAN:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splimp ();
CT_LOCK (bd);
ct_set_subchan (c->board, *(long*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETINVCLK:
case SERIAL_GETINVTCLK:
if (c->mode == M_E1 || c->mode == M_G703)
return EINVAL;
*(int*)data = ct_get_invtxc (c);
return 0;
case SERIAL_GETINVRCLK:
if (c->mode == M_E1 || c->mode == M_G703)
return EINVAL;
*(int*)data = ct_get_invrxc (c);
return 0;
case SERIAL_SETINVCLK:
case SERIAL_SETINVTCLK:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_E1 || c->mode == M_G703)
return EINVAL;
s = splimp ();
CT_LOCK (bd);
ct_set_invtxc (c, *(int*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_SETINVRCLK:
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_E1 || c->mode == M_G703)
return EINVAL;
s = splimp ();
CT_LOCK (bd);
ct_set_invrxc (c, *(int*)data);
CT_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETLEVEL:
if (c->mode != M_G703)
return EINVAL;
s = splimp ();
CT_LOCK (bd);
*(int*)data = ct_get_lq (c);
CT_UNLOCK (bd);
splx (s);
return 0;
case TIOCSDTR: /* Set DTR */
s = splimp ();
CT_LOCK (bd);
ct_set_dtr (c, 1);
CT_UNLOCK (bd);
splx (s);
return 0;
case TIOCCDTR: /* Clear DTR */
s = splimp ();
CT_LOCK (bd);
ct_set_dtr (c, 0);
CT_UNLOCK (bd);
splx (s);
return 0;
case TIOCMSET: /* Set DTR/RTS */
s = splimp ();
CT_LOCK (bd);
ct_set_dtr (c, (*(int*)data & TIOCM_DTR) ? 1 : 0);
ct_set_rts (c, (*(int*)data & TIOCM_RTS) ? 1 : 0);
CT_UNLOCK (bd);
splx (s);
return 0;
case TIOCMBIS: /* Add DTR/RTS */
s = splimp ();
CT_LOCK (bd);
if (*(int*)data & TIOCM_DTR) ct_set_dtr (c, 1);
if (*(int*)data & TIOCM_RTS) ct_set_rts (c, 1);
CT_UNLOCK (bd);
splx (s);
return 0;
case TIOCMBIC: /* Clear DTR/RTS */
s = splimp ();
CT_LOCK (bd);
if (*(int*)data & TIOCM_DTR) ct_set_dtr (c, 0);
if (*(int*)data & TIOCM_RTS) ct_set_rts (c, 0);
CT_UNLOCK (bd);
splx (s);
return 0;
case TIOCMGET: /* Get modem status */
*(int*)data = ct_modem_status (c);
return 0;
}
return ENOTTY;
}
#ifdef NETGRAPH
static int ng_ct_constructor (node_p node)
{
drv_t *d = NG_NODE_PRIVATE (node);
CT_DEBUG (d, ("Constructor\n"));
return EINVAL;
}
static int ng_ct_newhook (node_p node, hook_p hook, const char *name)
{
int s;
drv_t *d = NG_NODE_PRIVATE (node);
if (!d)
return EINVAL;
bdrv_t *bd = d->bd;
/* Attach debug hook */
if (strcmp (name, NG_CT_HOOK_DEBUG) == 0) {
NG_HOOK_SET_PRIVATE (hook, NULL);
d->debug_hook = hook;
return 0;
}
/* Check for raw hook */
if (strcmp (name, NG_CT_HOOK_RAW) != 0)
return EINVAL;
NG_HOOK_SET_PRIVATE (hook, d);
d->hook = hook;
s = splimp ();
CT_LOCK (bd);
ct_up (d);
CT_UNLOCK (bd);
splx (s);
return 0;
}
static char *format_timeslots (u_long s)
{
static char buf [100];
char *p = buf;
int i;
for (i=1; i<32; ++i)
if ((s >> i) & 1) {
int prev = (i > 1) & (s >> (i-1));
int next = (i < 31) & (s >> (i+1));
if (prev) {
if (next)
continue;
*p++ = '-';
} else if (p > buf)
*p++ = ',';
if (i >= 10)
*p++ = '0' + i / 10;
*p++ = '0' + i % 10;
}
*p = 0;
return buf;
}
static int print_modems (char *s, ct_chan_t *c, int need_header)
{
int status = ct_modem_status (c);
int length = 0;
if (need_header)
length += sprintf (s + length, " LE DTR DSR RTS CTS CD\n");
length += sprintf (s + length, "%4s %4s %4s %4s %4s %4s\n",
status & TIOCM_LE ? "On" : "-",
status & TIOCM_DTR ? "On" : "-",
status & TIOCM_DSR ? "On" : "-",
status & TIOCM_RTS ? "On" : "-",
status & TIOCM_CTS ? "On" : "-",
status & TIOCM_CD ? "On" : "-");
return length;
}
static int print_stats (char *s, ct_chan_t *c, int need_header)
{
struct serial_statistics st;
int length = 0;
st.rintr = c->rintr;
st.tintr = c->tintr;
st.mintr = c->mintr;
st.ibytes = c->ibytes;
st.ipkts = c->ipkts;
st.ierrs = c->ierrs;
st.obytes = c->obytes;
st.opkts = c->opkts;
st.oerrs = c->oerrs;
if (need_header)
length += sprintf (s + length, " Rintr Tintr Mintr Ibytes Ipkts Ierrs Obytes Opkts Oerrs\n");
length += sprintf (s + length, "%7ld %7ld %7ld %8ld %7ld %7ld %8ld %7ld %7ld\n",
st.rintr, st.tintr, st.mintr, st.ibytes, st.ipkts,
st.ierrs, st.obytes, st.opkts, st.oerrs);
return length;
}
static char *format_e1_status (u_char status)
{
static char buf [80];
if (status & E1_NOALARM)
return "Ok";
buf[0] = 0;
if (status & E1_LOS) strcat (buf, ",LOS");
if (status & E1_AIS) strcat (buf, ",AIS");
if (status & E1_LOF) strcat (buf, ",LOF");
if (status & E1_LOMF) strcat (buf, ",LOMF");
if (status & E1_FARLOF) strcat (buf, ",FARLOF");
if (status & E1_AIS16) strcat (buf, ",AIS16");
if (status & E1_FARLOMF) strcat (buf, ",FARLOMF");
if (status & E1_TSTREQ) strcat (buf, ",TSTREQ");
if (status & E1_TSTERR) strcat (buf, ",TSTERR");
if (buf[0] == ',')
return buf+1;
return "Unknown";
}
static int print_frac (char *s, int leftalign, u_long numerator, u_long divider)
{
int n, length = 0;
if (numerator < 1 || divider < 1) {
length += sprintf (s+length, leftalign ? "/- " : " -");
return length;
}
n = (int) (0.5 + 1000.0 * numerator / divider);
if (n < 1000) {
length += sprintf (s+length, leftalign ? "/.%-3d" : " .%03d", n);
return length;
}
*(s + length) = leftalign ? '/' : ' ';
length ++;
if (n >= 1000000) n = (n+500) / 1000 * 1000;
else if (n >= 100000) n = (n+50) / 100 * 100;
else if (n >= 10000) n = (n+5) / 10 * 10;
switch (n) {
case 1000: length += printf (s+length, ".999"); return length;
case 10000: n = 9990; break;
case 100000: n = 99900; break;
case 1000000: n = 999000; break;
}
if (n < 10000) length += sprintf (s+length, "%d.%d", n/1000, n/10%100);
else if (n < 100000) length += sprintf (s+length, "%d.%d", n/1000, n/100%10);
else if (n < 1000000) length += sprintf (s+length, "%d.", n/1000);
else length += sprintf (s+length, "%d", n/1000);
return length;
}
static int print_e1_stats (char *s, ct_chan_t *c)
{
struct e1_counters total;
u_long totsec;
int length = 0;
totsec = c->totsec + c->cursec;
total.bpv = c->total.bpv + c->currnt.bpv;
total.fse = c->total.fse + c->currnt.fse;
total.crce = c->total.crce + c->currnt.crce;
total.rcrce = c->total.rcrce + c->currnt.rcrce;
total.uas = c->total.uas + c->currnt.uas;
total.les = c->total.les + c->currnt.les;
total.es = c->total.es + c->currnt.es;
total.bes = c->total.bes + c->currnt.bes;
total.ses = c->total.ses + c->currnt.ses;
total.oofs = c->total.oofs + c->currnt.oofs;
total.css = c->total.css + c->currnt.css;
total.dm = c->total.dm + c->currnt.dm;
length += sprintf (s + length, " Unav/Degr Bpv/Fsyn CRC/RCRC Err/Lerr Sev/Bur Oof/Slp Status\n");
/* Unavailable seconds, degraded minutes */
length += print_frac (s + length, 0, c->currnt.uas, c->cursec);
length += print_frac (s + length, 1, 60 * c->currnt.dm, c->cursec);
/* Bipolar violations, frame sync errors */
length += print_frac (s + length, 0, c->currnt.bpv, c->cursec);
length += print_frac (s + length, 1, c->currnt.fse, c->cursec);
/* CRC errors, remote CRC errors (E-bit) */
length += print_frac (s + length, 0, c->currnt.crce, c->cursec);
length += print_frac (s + length, 1, c->currnt.rcrce, c->cursec);
/* Errored seconds, line errored seconds */
length += print_frac (s + length, 0, c->currnt.es, c->cursec);
length += print_frac (s + length, 1, c->currnt.les, c->cursec);
/* Severely errored seconds, burst errored seconds */
length += print_frac (s + length, 0, c->currnt.ses, c->cursec);
length += print_frac (s + length, 1, c->currnt.bes, c->cursec);
/* Out of frame seconds, controlled slip seconds */
length += print_frac (s + length, 0, c->currnt.oofs, c->cursec);
length += print_frac (s + length, 1, c->currnt.css, c->cursec);
length += sprintf (s + length, " %s\n", format_e1_status (c->status));
/* Print total statistics. */
length += print_frac (s + length, 0, total.uas, totsec);
length += print_frac (s + length, 1, 60 * total.dm, totsec);
length += print_frac (s + length, 0, total.bpv, totsec);
length += print_frac (s + length, 1, total.fse, totsec);
length += print_frac (s + length, 0, total.crce, totsec);
length += print_frac (s + length, 1, total.rcrce, totsec);
length += print_frac (s + length, 0, total.es, totsec);
length += print_frac (s + length, 1, total.les, totsec);
length += print_frac (s + length, 0, total.ses, totsec);
length += print_frac (s + length, 1, total.bes, totsec);
length += print_frac (s + length, 0, total.oofs, totsec);
length += print_frac (s + length, 1, total.css, totsec);
length += sprintf (s + length, " -- Total\n");
return length;
}
static int print_chan (char *s, ct_chan_t *c)
{
drv_t *d = c->sys;
bdrv_t *bd = d->bd;
int length = 0;
length += sprintf (s + length, "ct%d", c->board->num * NCHAN + c->num);
if (d->chan->debug)
length += sprintf (s + length, " debug=%d", d->chan->debug);
switch (ct_get_config (c->board)) {
case CFG_A: length += sprintf (s + length, " cfg=A"); break;
case CFG_B: length += sprintf (s + length, " cfg=B"); break;
case CFG_C: length += sprintf (s + length, " cfg=C"); break;
default: length += sprintf (s + length, " cfg=unknown"); break;
}
if (ct_get_baud (c))
length += sprintf (s + length, " %ld", ct_get_baud (c));
else
length += sprintf (s + length, " extclock");
if (c->mode == M_E1 || c->mode == M_G703)
switch (ct_get_clk(c)) {
case GCLK_INT : length += sprintf (s + length, " syn=int"); break;
case GCLK_RCV : length += sprintf (s + length, " syn=rcv"); break;
case GCLK_RCLKO : length += sprintf (s + length, " syn=xrcv"); break;
}
if (c->mode == M_HDLC) {
length += sprintf (s + length, " dpll=%s", ct_get_dpll (c) ? "on" : "off");
length += sprintf (s + length, " nrzi=%s", ct_get_nrzi (c) ? "on" : "off");
length += sprintf (s + length, " invtclk=%s", ct_get_invtxc (c) ? "on" : "off");
length += sprintf (s + length, " invrclk=%s", ct_get_invrxc (c) ? "on" : "off");
}
if (c->mode == M_E1)
length += sprintf (s + length, " higain=%s", ct_get_higain (c)? "on" : "off");
length += sprintf (s + length, " loop=%s", ct_get_loop (c) ? "on" : "off");
if (c->mode == M_E1)
length += sprintf (s + length, " ts=%s", format_timeslots (ct_get_ts(c)));
if (c->mode == M_E1 && ct_get_config (c->board) != CFG_A)
length += sprintf (s + length, " pass=%s", format_timeslots (ct_get_subchan(c->board)));
if (c->mode == M_G703) {
int lq, x;
x = splimp ();
CT_LOCK (bd);
lq = ct_get_lq (c);
CT_UNLOCK (bd);
splx (x);
length += sprintf (s + length, " (level=-%.1fdB)", lq / 10.0);
}
length += sprintf (s + length, "\n");
return length;
}
static int ng_ct_rcvmsg (node_p node, item_p item, hook_p lasthook)
{
drv_t *d = NG_NODE_PRIVATE (node);
struct ng_mesg *msg;
struct ng_mesg *resp = NULL;
int error = 0;
if (!d)
return EINVAL;
CT_DEBUG (d, ("Rcvmsg\n"));
NGI_GET_MSG (item, msg);
switch (msg->header.typecookie) {
default:
error = EINVAL;
break;
case NGM_CT_COOKIE:
printf ("Don't forget to implement\n");
error = EINVAL;
break;
case NGM_GENERIC_COOKIE:
switch (msg->header.cmd) {
default:
error = EINVAL;
break;
case NGM_TEXT_STATUS: {
char *s;
int l = 0;
int dl = sizeof (struct ng_mesg) + 730;
NG_MKRESPONSE (resp, msg, dl, M_NOWAIT);
if (! resp) {
error = ENOMEM;
break;
}
s = (resp)->data;
l += print_chan (s + l, d->chan);
l += print_stats (s + l, d->chan, 1);
l += print_modems (s + l, d->chan, 1);
l += print_e1_stats (s + l, d->chan);
strncpy ((resp)->header.cmdstr, "status", NG_CMDSTRSIZ);
}
break;
}
break;
}
NG_RESPOND_MSG (error, node, item, resp);
NG_FREE_MSG (msg);
return error;
}
static int ng_ct_rcvdata (hook_p hook, item_p item)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE(hook));
struct mbuf *m;
struct ng_tag_prio *ptag;
bdrv_t *bd;
struct ifqueue *q;
int s;
if (!d)
return ENETDOWN;
bd = d->bd;
NGI_GET_M (item, m);
NG_FREE_ITEM (item);
if (! NG_HOOK_PRIVATE (hook) || ! d) {
NG_FREE_M (m);
return ENETDOWN;
}
/* Check for high priority data */
if ((ptag = (struct ng_tag_prio *)m_tag_locate(m, NGM_GENERIC_COOKIE,
NG_TAG_PRIO, NULL)) != NULL && (ptag->priority > NG_PRIO_CUTOFF) )
q = &d->hi_queue;
else
q = &d->queue;
s = splimp ();
CT_LOCK (bd);
IF_LOCK (q);
if (_IF_QFULL (q)) {
IF_UNLOCK (q);
CT_UNLOCK (bd);
splx (s);
NG_FREE_M (m);
return ENOBUFS;
}
_IF_ENQUEUE (q, m);
IF_UNLOCK (q);
ct_start (d);
CT_UNLOCK (bd);
splx (s);
return 0;
}
static int ng_ct_rmnode (node_p node)
{
drv_t *d = NG_NODE_PRIVATE (node);
bdrv_t *bd;
CT_DEBUG (d, ("Rmnode\n"));
if (d && d->running) {
bd = d->bd;
int s = splimp ();
CT_LOCK (bd);
ct_down (d);
CT_UNLOCK (bd);
splx (s);
}
#ifdef KLD_MODULE
if (node->nd_flags & NGF_REALLY_DIE) {
NG_NODE_SET_PRIVATE (node, NULL);
NG_NODE_UNREF (node);
}
NG_NODE_REVIVE(node); /* Persistant node */
#endif
return 0;
}
static int ng_ct_connect (hook_p hook)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE (hook));
if (!d)
return 0;
callout_reset (&d->timeout_handle, hz, ct_watchdog_timer, d);
return 0;
}
static int ng_ct_disconnect (hook_p hook)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE (hook));
bdrv_t *bd;
if (!d)
return 0;
bd = d->bd;
CT_LOCK (bd);
if (NG_HOOK_PRIVATE (hook))
ct_down (d);
CT_UNLOCK (bd);
/* If we were wait it than it reasserted now, just stop it. */
if (!callout_drain (&d->timeout_handle))
callout_stop (&d->timeout_handle);
return 0;
}
#endif
static int ct_modevent (module_t mod, int type, void *unused)
{
static int load_count = 0;
switch (type) {
case MOD_LOAD:
#ifdef NETGRAPH
if (ng_newtype (&typestruct))
printf ("Failed to register ng_ct\n");
#endif
++load_count;
- callout_init (&timeout_handle, CALLOUT_MPSAFE);
+ callout_init (&timeout_handle, 1);
callout_reset (&timeout_handle, hz*5, ct_timeout, 0);
break;
case MOD_UNLOAD:
if (load_count == 1) {
printf ("Removing device entry for Tau-ISA\n");
#ifdef NETGRAPH
ng_rmtype (&typestruct);
#endif
}
/* If we were wait it than it reasserted now, just stop it. */
if (!callout_drain (&timeout_handle))
callout_stop (&timeout_handle);
--load_count;
break;
case MOD_SHUTDOWN:
break;
}
return 0;
}
#ifdef NETGRAPH
static struct ng_type typestruct = {
.version = NG_ABI_VERSION,
.name = NG_CT_NODE_TYPE,
.constructor = ng_ct_constructor,
.rcvmsg = ng_ct_rcvmsg,
.shutdown = ng_ct_rmnode,
.newhook = ng_ct_newhook,
.connect = ng_ct_connect,
.rcvdata = ng_ct_rcvdata,
.disconnect = ng_ct_disconnect,
};
#endif /*NETGRAPH*/
#ifdef NETGRAPH
MODULE_DEPEND (ng_ct, netgraph, NG_ABI_VERSION, NG_ABI_VERSION, NG_ABI_VERSION);
#else
MODULE_DEPEND (ct, sppp, 1, 1, 1);
#endif
DRIVER_MODULE (ct, isa, ct_isa_driver, ct_devclass, ct_modevent, NULL);
MODULE_VERSION (ct, 1);
Index: head/sys/dev/cx/if_cx.c
===================================================================
--- head/sys/dev/cx/if_cx.c (revision 283290)
+++ head/sys/dev/cx/if_cx.c (revision 283291)
@@ -1,2545 +1,2545 @@
/*-
* Cronyx-Sigma adapter driver for FreeBSD.
* Supports PPP/HDLC and Cisco/HDLC protocol in synchronous mode,
* and asynchronous channels with full modem control.
* Keepalive protocol implemented in both Cisco and PPP modes.
*
* Copyright (C) 1994-2002 Cronyx Engineering.
* Author: Serge Vakulenko, <vak@cronyx.ru>
*
* Copyright (C) 1999-2004 Cronyx Engineering.
* Rewritten on DDK, ported to NETGRAPH, rewritten for FreeBSD 3.x-5.x by
* Kurakin Roman, <rik@cronyx.ru>
*
* This software is distributed with NO WARRANTIES, not even the implied
* warranties for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* Authors grant any other persons or organisations a permission to use,
* modify and redistribute this software in source and binary forms,
* as long as this message is kept with the software, all derivative
* works or modified versions.
*
* Cronyx Id: if_cx.c,v 1.1.2.34 2004/06/23 17:09:13 rik Exp $
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/mbuf.h>
#include <sys/sockio.h>
#include <sys/malloc.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <sys/conf.h>
#include <sys/errno.h>
#include <sys/serial.h>
#include <sys/tty.h>
#include <sys/bus.h>
#include <machine/bus.h>
#include <sys/rman.h>
#include <isa/isavar.h>
#include <sys/fcntl.h>
#include <sys/interrupt.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <net/if.h>
#include <net/if_var.h>
#include <machine/cpufunc.h>
#include <machine/cserial.h>
#include <machine/resource.h>
#include <dev/cx/machdep.h>
#include <dev/cx/cxddk.h>
#include <dev/cx/cronyxfw.h>
#include "opt_ng_cronyx.h"
#ifdef NETGRAPH_CRONYX
# include "opt_netgraph.h"
# include <netgraph/ng_message.h>
# include <netgraph/netgraph.h>
# include <dev/cx/ng_cx.h>
#else
# include <net/if_types.h>
# include <net/if_sppp.h>
# define PP_CISCO IFF_LINK2
# include <net/bpf.h>
#endif
#define NCX 1
/* If we don't have Cronyx's sppp version, we don't have fr support via sppp */
#ifndef PP_FR
#define PP_FR 0
#endif
#define CX_DEBUG(d,s) ({if (d->chan->debug) {\
printf ("%s: ", d->name); printf s;}})
#define CX_DEBUG2(d,s) ({if (d->chan->debug>1) {\
printf ("%s: ", d->name); printf s;}})
#define CX_LOCK_NAME "cxX"
#define CX_LOCK(_bd) mtx_lock (&(_bd)->cx_mtx)
#define CX_UNLOCK(_bd) mtx_unlock (&(_bd)->cx_mtx)
#define CX_LOCK_ASSERT(_bd) mtx_assert (&(_bd)->cx_mtx, MA_OWNED)
typedef struct _async_q {
int beg;
int end;
#define BF_SZ 14400
int buf[BF_SZ+1];
} async_q;
#define AQ_GSZ(q) ((BF_SZ + (q)->end - (q)->beg)%BF_SZ)
#define AQ_PUSH(q,c) {*((q)->buf + (q)->end) = c;\
(q)->end = ((q)->end + 1)%BF_SZ;}
#define AQ_POP(q,c) {c = *((q)->buf + (q)->beg);\
(q)->beg = ((q)->beg + 1)%BF_SZ;}
static void cx_identify __P((driver_t *, device_t));
static int cx_probe __P((device_t));
static int cx_attach __P((device_t));
static int cx_detach __P((device_t));
static t_open_t cx_topen;
static t_modem_t cx_tmodem;
static t_close_t cx_tclose;
static device_method_t cx_isa_methods [] = {
DEVMETHOD(device_identify, cx_identify),
DEVMETHOD(device_probe, cx_probe),
DEVMETHOD(device_attach, cx_attach),
DEVMETHOD(device_detach, cx_detach),
DEVMETHOD_END
};
typedef struct _cx_dma_mem_t {
unsigned long phys;
void *virt;
size_t size;
bus_dma_tag_t dmat;
bus_dmamap_t mapp;
} cx_dma_mem_t;
typedef struct _drv_t {
char name [8];
cx_chan_t *chan;
cx_board_t *board;
cx_dma_mem_t dmamem;
struct tty *tty;
struct callout dcd_timeout_handle;
unsigned callout;
unsigned lock;
int open_dev;
int cd;
int running;
#ifdef NETGRAPH
char nodename [NG_NODESIZ];
hook_p hook;
hook_p debug_hook;
node_p node;
struct ifqueue lo_queue;
struct ifqueue hi_queue;
#else
struct ifqueue queue;
struct ifnet *ifp;
#endif
short timeout;
struct callout timeout_handle;
struct cdev *devt;
async_q aqueue;
#define CX_READ 1
#define CX_WRITE 2
int intr_action;
short atimeout;
} drv_t;
typedef struct _bdrv_t {
cx_board_t *board;
struct resource *base_res;
struct resource *drq_res;
struct resource *irq_res;
int base_rid;
int drq_rid;
int irq_rid;
void *intrhand;
drv_t channel [NCHAN];
struct mtx cx_mtx;
} bdrv_t;
static driver_t cx_isa_driver = {
"cx",
cx_isa_methods,
sizeof (bdrv_t),
};
static devclass_t cx_devclass;
extern long csigma_fw_len;
extern const char *csigma_fw_version;
extern const char *csigma_fw_date;
extern const char *csigma_fw_copyright;
extern const cr_dat_tst_t csigma_fw_tvec[];
extern const u_char csigma_fw_data[];
static void cx_oproc (struct tty *tp);
static int cx_param (struct tty *tp, struct termios *t);
static void cx_stop (struct tty *tp, int flag);
static void cx_receive (cx_chan_t *c, char *data, int len);
static void cx_transmit (cx_chan_t *c, void *attachment, int len);
static void cx_error (cx_chan_t *c, int data);
static void cx_modem (cx_chan_t *c);
static void cx_up (drv_t *d);
static void cx_start (drv_t *d);
static void cx_softintr (void *);
static void *cx_fast_ih;
static void cx_down (drv_t *d);
static void cx_watchdog (drv_t *d);
static void cx_watchdog_timer (void *arg);
static void cx_carrier (void *arg);
#ifdef NETGRAPH
extern struct ng_type typestruct;
#else
static void cx_ifstart (struct ifnet *ifp);
static void cx_tlf (struct sppp *sp);
static void cx_tls (struct sppp *sp);
static int cx_sioctl (struct ifnet *ifp, u_long cmd, caddr_t data);
static void cx_initialize (void *softc);
#endif
static cx_board_t *adapter [NCX];
static drv_t *channel [NCX*NCHAN];
static struct callout led_timo [NCX];
static struct callout timeout_handle;
static int cx_open (struct cdev *dev, int flag, int mode, struct thread *td);
static int cx_close (struct cdev *dev, int flag, int mode, struct thread *td);
static int cx_ioctl (struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td);
static struct cdevsw cx_cdevsw = {
.d_version = D_VERSION,
.d_open = cx_open,
.d_close = cx_close,
.d_ioctl = cx_ioctl,
.d_name = "cx",
.d_flags = D_TTY,
};
static int MY_SOFT_INTR;
/*
* Make an mbuf from data.
*/
static struct mbuf *makembuf (void *buf, u_int len)
{
struct mbuf *m, *o, *p;
MGETHDR (m, M_NOWAIT, MT_DATA);
if (! m)
return 0;
if (len >= MINCLSIZE)
MCLGET (m, M_NOWAIT);
m->m_pkthdr.len = len;
m->m_len = 0;
p = m;
while (len) {
u_int n = M_TRAILINGSPACE (p);
if (n > len)
n = len;
if (! n) {
/* Allocate new mbuf. */
o = p;
MGET (p, M_NOWAIT, MT_DATA);
if (! p) {
m_freem (m);
return 0;
}
if (len >= MINCLSIZE)
MCLGET (p, M_NOWAIT);
p->m_len = 0;
o->m_next = p;
n = M_TRAILINGSPACE (p);
if (n > len)
n = len;
}
bcopy (buf, mtod (p, caddr_t) + p->m_len, n);
p->m_len += n;
buf = n + (char*) buf;
len -= n;
}
return m;
}
/*
* Recover after lost transmit interrupts.
*/
static void cx_timeout (void *arg)
{
drv_t *d;
int s, i, k;
for (i = 0; i < NCX; i++) {
if (adapter[i] == NULL)
continue;
for (k = 0; k < NCHAN; ++k) {
d = channel[i * NCHAN + k];
if (! d)
continue;
s = splhigh ();
CX_LOCK ((bdrv_t *)d->board->sys);
if (d->atimeout == 1 && d->tty && d->tty->t_state & TS_BUSY) {
d->tty->t_state &= ~TS_BUSY;
if (d->tty->t_dev) {
d->intr_action |= CX_WRITE;
MY_SOFT_INTR = 1;
swi_sched (cx_fast_ih, 0);
}
CX_DEBUG (d, ("cx_timeout\n"));
}
if (d->atimeout)
d->atimeout--;
CX_UNLOCK ((bdrv_t *)d->board->sys);
splx (s);
}
}
callout_reset (&timeout_handle, hz*5, cx_timeout, 0);
}
static void cx_led_off (void *arg)
{
cx_board_t *b = arg;
bdrv_t *bd = b->sys;
int s;
s = splhigh ();
CX_LOCK (bd);
cx_led (b, 0);
CX_UNLOCK (bd);
splx (s);
}
/*
* Activate interrupt handler from DDK.
*/
static void cx_intr (void *arg)
{
bdrv_t *bd = arg;
cx_board_t *b = bd->board;
#ifndef NETGRAPH
int i;
#endif
int s = splhigh ();
CX_LOCK (bd);
/* Turn LED on. */
cx_led (b, 1);
cx_int_handler (b);
/* Turn LED off 50 msec later. */
callout_reset (&led_timo[b->num], hz/20, cx_led_off, b);
CX_UNLOCK (bd);
splx (s);
#ifndef NETGRAPH
/* Pass packets in a lock-free state */
for (i = 0; i < NCHAN && b->chan[i].type; i++) {
drv_t *d = b->chan[i].sys;
struct mbuf *m;
if (!d || !d->running)
continue;
while (_IF_QLEN(&d->queue)) {
IF_DEQUEUE (&d->queue,m);
if (!m)
continue;
sppp_input (d->ifp, m);
}
}
#endif
}
static int probe_irq (cx_board_t *b, int irq)
{
int mask, busy, cnt;
/* Clear pending irq, if any. */
cx_probe_irq (b, -irq);
DELAY (100);
for (cnt=0; cnt<5; ++cnt) {
/* Get the mask of pending irqs, assuming they are busy.
* Activate the adapter on given irq. */
busy = cx_probe_irq (b, irq);
DELAY (100);
/* Get the mask of active irqs.
* Deactivate our irq. */
mask = cx_probe_irq (b, -irq);
DELAY (100);
if ((mask & ~busy) == 1 << irq) {
cx_probe_irq (b, 0);
/* printf ("cx%d: irq %d ok, mask=0x%04x, busy=0x%04x\n",
b->num, irq, mask, busy); */
return 1;
}
}
/* printf ("cx%d: irq %d not functional, mask=0x%04x, busy=0x%04x\n",
b->num, irq, mask, busy); */
cx_probe_irq (b, 0);
return 0;
}
static short porttab [] = {
0x200, 0x220, 0x240, 0x260, 0x280, 0x2a0, 0x2c0, 0x2e0,
0x300, 0x320, 0x340, 0x360, 0x380, 0x3a0, 0x3c0, 0x3e0, 0
};
static char dmatab [] = { 7, 6, 5, 0 };
static char irqtab [] = { 5, 10, 11, 7, 3, 15, 12, 0 };
static int cx_is_free_res (device_t dev, int rid, int type, u_long start,
u_long end, u_long count)
{
struct resource *res;
if (!(res = bus_alloc_resource (dev, type, &rid, start, end, count, 0)))
return 0;
bus_release_resource (dev, type, rid, res);
return 1;
}
static void cx_identify (driver_t *driver, device_t dev)
{
u_long iobase, rescount;
int devcount;
device_t *devices;
device_t child;
devclass_t my_devclass;
int i, k;
if ((my_devclass = devclass_find ("cx")) == NULL)
return;
devclass_get_devices (my_devclass, &devices, &devcount);
if (devcount == 0) {
/* We should find all devices by our self. We could alter other
* devices, but we don't have a choise
*/
for (i = 0; (iobase = porttab [i]) != 0; i++) {
if (!cx_is_free_res (dev, 0, SYS_RES_IOPORT,
iobase, iobase + NPORT, NPORT))
continue;
if (cx_probe_board (iobase, -1, -1) == 0)
continue;
devcount++;
child = BUS_ADD_CHILD (dev, ISA_ORDER_SPECULATIVE, "cx",
-1);
if (child == NULL)
return;
device_set_desc_copy (child, "Cronyx Sigma");
device_set_driver (child, driver);
bus_set_resource (child, SYS_RES_IOPORT, 0,
iobase, NPORT);
if (devcount >= NCX)
break;
}
} else {
static short porttab [] = {
0x200, 0x220, 0x240, 0x260, 0x280, 0x2a0, 0x2c0, 0x2e0,
0x300, 0x320, 0x340, 0x360, 0x380, 0x3a0, 0x3c0, 0x3e0, 0
};
/* Lets check user choise.
*/
for (k = 0; k < devcount; k++) {
if (bus_get_resource (devices[k], SYS_RES_IOPORT, 0,
&iobase, &rescount) != 0)
continue;
for (i = 0; porttab [i] != 0; i++) {
if (porttab [i] != iobase)
continue;
if (!cx_is_free_res (devices[k], 0, SYS_RES_IOPORT,
iobase, iobase + NPORT, NPORT))
continue;
if (cx_probe_board (iobase, -1, -1) == 0)
continue;
porttab [i] = -1;
device_set_desc_copy (devices[k], "Cronyx Sigma");
break;
}
if (porttab [i] == 0) {
device_delete_child (
device_get_parent (devices[k]),
devices [k]);
devices[k] = 0;
continue;
}
}
for (k = 0; k < devcount; k++) {
if (devices[k] == 0)
continue;
if (bus_get_resource (devices[k], SYS_RES_IOPORT, 0,
&iobase, &rescount) == 0)
continue;
for (i = 0; (iobase = porttab [i]) != 0; i++) {
if (porttab [i] == -1) {
continue;
}
if (!cx_is_free_res (devices[k], 0, SYS_RES_IOPORT,
iobase, iobase + NPORT, NPORT))
continue;
if (cx_probe_board (iobase, -1, -1) == 0)
continue;
bus_set_resource (devices[k], SYS_RES_IOPORT, 0,
iobase, NPORT);
porttab [i] = -1;
device_set_desc_copy (devices[k], "Cronyx Sigma");
break;
}
if (porttab [i] == 0) {
device_delete_child (
device_get_parent (devices[k]),
devices [k]);
}
}
free (devices, M_TEMP);
}
return;
}
static int cx_probe (device_t dev)
{
int unit = device_get_unit (dev);
int i;
u_long iobase, rescount;
if (!device_get_desc (dev) ||
strcmp (device_get_desc (dev), "Cronyx Sigma"))
return ENXIO;
if (bus_get_resource (dev, SYS_RES_IOPORT, 0, &iobase, &rescount) != 0) {
printf ("cx%d: Couldn't get IOPORT\n", unit);
return ENXIO;
}
if (!cx_is_free_res (dev, 0, SYS_RES_IOPORT,
iobase, iobase + NPORT, NPORT)) {
printf ("cx%d: Resource IOPORT isn't free %lx\n", unit, iobase);
return ENXIO;
}
for (i = 0; porttab [i] != 0; i++) {
if (porttab [i] == iobase) {
porttab [i] = -1;
break;
}
}
if (porttab [i] == 0) {
return ENXIO;
}
if (!cx_probe_board (iobase, -1, -1)) {
printf ("cx%d: probing for Sigma at %lx faild\n", unit, iobase);
return ENXIO;
}
return 0;
}
static void
cx_bus_dmamap_addr (void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
unsigned long *addr;
if (error)
return;
KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg));
addr = arg;
*addr = segs->ds_addr;
}
static int
cx_bus_dma_mem_alloc (int bnum, int cnum, cx_dma_mem_t *dmem)
{
int error;
error = bus_dma_tag_create (NULL, 16, 0, BUS_SPACE_MAXADDR_24BIT,
BUS_SPACE_MAXADDR, NULL, NULL, dmem->size, 1,
dmem->size, 0, NULL, NULL, &dmem->dmat);
if (error) {
if (cnum >= 0) printf ("cx%d-%d: ", bnum, cnum);
else printf ("cx%d: ", bnum);
printf ("couldn't allocate tag for dma memory\n");
return 0;
}
error = bus_dmamem_alloc (dmem->dmat, (void **)&dmem->virt,
BUS_DMA_NOWAIT | BUS_DMA_ZERO, &dmem->mapp);
if (error) {
if (cnum >= 0) printf ("cx%d-%d: ", bnum, cnum);
else printf ("cx%d: ", bnum);
printf ("couldn't allocate mem for dma memory\n");
bus_dma_tag_destroy (dmem->dmat);
return 0;
}
error = bus_dmamap_load (dmem->dmat, dmem->mapp, dmem->virt,
dmem->size, cx_bus_dmamap_addr, &dmem->phys, 0);
if (error) {
if (cnum >= 0) printf ("cx%d-%d: ", bnum, cnum);
else printf ("cx%d: ", bnum);
printf ("couldn't load mem map for dma memory\n");
bus_dmamem_free (dmem->dmat, dmem->virt, dmem->mapp);
bus_dma_tag_destroy (dmem->dmat);
return 0;
}
return 1;
}
static void
cx_bus_dma_mem_free (cx_dma_mem_t *dmem)
{
bus_dmamap_unload (dmem->dmat, dmem->mapp);
bus_dmamem_free (dmem->dmat, dmem->virt, dmem->mapp);
bus_dma_tag_destroy (dmem->dmat);
}
/*
* The adapter is present, initialize the driver structures.
*/
static int cx_attach (device_t dev)
{
bdrv_t *bd = device_get_softc (dev);
u_long iobase, drq, irq, rescount;
int unit = device_get_unit (dev);
char *cx_ln = CX_LOCK_NAME;
cx_board_t *b;
cx_chan_t *c;
drv_t *d;
int i;
int s;
KASSERT ((bd != NULL), ("cx%d: NULL device softc\n", unit));
bus_get_resource (dev, SYS_RES_IOPORT, 0, &iobase, &rescount);
bd->base_rid = 0;
bd->base_res = bus_alloc_resource (dev, SYS_RES_IOPORT, &bd->base_rid,
iobase, iobase + NPORT, NPORT, RF_ACTIVE);
if (! bd->base_res) {
printf ("cx%d: cannot allocate base address\n", unit);
return ENXIO;
}
if (bus_get_resource (dev, SYS_RES_DRQ, 0, &drq, &rescount) != 0) {
for (i = 0; (drq = dmatab [i]) != 0; i++) {
if (!cx_is_free_res (dev, 0, SYS_RES_DRQ,
drq, drq + 1, 1))
continue;
bus_set_resource (dev, SYS_RES_DRQ, 0, drq, 1);
break;
}
if (dmatab[i] == 0) {
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
printf ("cx%d: Couldn't get DRQ\n", unit);
return ENXIO;
}
}
bd->drq_rid = 0;
bd->drq_res = bus_alloc_resource (dev, SYS_RES_DRQ, &bd->drq_rid,
drq, drq + 1, 1, RF_ACTIVE);
if (! bd->drq_res) {
printf ("cx%d: cannot allocate drq\n", unit);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
return ENXIO;
}
if (bus_get_resource (dev, SYS_RES_IRQ, 0, &irq, &rescount) != 0) {
for (i = 0; (irq = irqtab [i]) != 0; i++) {
if (!cx_is_free_res (dev, 0, SYS_RES_IRQ,
irq, irq + 1, 1))
continue;
bus_set_resource (dev, SYS_RES_IRQ, 0, irq, 1);
break;
}
if (irqtab[i] == 0) {
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
printf ("cx%d: Couldn't get IRQ\n", unit);
return ENXIO;
}
}
bd->irq_rid = 0;
bd->irq_res = bus_alloc_resource (dev, SYS_RES_IRQ, &bd->irq_rid,
irq, irq + 1, 1, RF_ACTIVE);
if (! bd->irq_res) {
printf ("cx%d: Couldn't allocate irq\n", unit);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
return ENXIO;
}
b = malloc (sizeof (cx_board_t), M_DEVBUF, M_WAITOK);
if (!b) {
printf ("cx:%d: Couldn't allocate memory\n", unit);
return (ENXIO);
}
adapter[unit] = b;
bzero (b, sizeof(cx_board_t));
if (! cx_open_board (b, unit, iobase, irq, drq)) {
printf ("cx%d: error loading firmware\n", unit);
free (b, M_DEVBUF);
bus_release_resource (dev, SYS_RES_IRQ, bd->irq_rid,
bd->irq_res);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
return ENXIO;
}
bd->board = b;
cx_ln[2] = '0' + unit;
mtx_init (&bd->cx_mtx, cx_ln, MTX_NETWORK_LOCK, MTX_DEF|MTX_RECURSE);
if (! probe_irq (b, irq)) {
printf ("cx%d: irq %ld not functional\n", unit, irq);
bd->board = 0;
adapter [unit] = 0;
mtx_destroy (&bd->cx_mtx);
free (b, M_DEVBUF);
bus_release_resource (dev, SYS_RES_IRQ, bd->irq_rid,
bd->irq_res);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
return ENXIO;
}
b->sys = bd;
- callout_init (&led_timo[b->num], CALLOUT_MPSAFE);
+ callout_init (&led_timo[b->num], 1);
s = splhigh ();
if (bus_setup_intr (dev, bd->irq_res,
INTR_TYPE_NET|INTR_MPSAFE,
NULL, cx_intr, bd, &bd->intrhand)) {
printf ("cx%d: Can't setup irq %ld\n", unit, irq);
bd->board = 0;
b->sys = 0;
adapter [unit] = 0;
mtx_destroy (&bd->cx_mtx);
free (b, M_DEVBUF);
bus_release_resource (dev, SYS_RES_IRQ, bd->irq_rid,
bd->irq_res);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid,
bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid,
bd->base_res);
splx (s);
return ENXIO;
}
CX_LOCK (bd);
cx_init (b, b->num, b->port, irq, drq);
cx_setup_board (b, 0, 0, 0);
CX_UNLOCK (bd);
printf ("cx%d: <Cronyx-Sigma-%s>\n", b->num, b->name);
for (c=b->chan; c<b->chan+NCHAN; ++c) {
if (c->type == T_NONE)
continue;
d = &bd->channel[c->num];
d->dmamem.size = sizeof(cx_buf_t);
if (! cx_bus_dma_mem_alloc (unit, c->num, &d->dmamem))
continue;
d->board = b;
d->chan = c;
d->open_dev = 0;
c->sys = d;
channel [b->num*NCHAN + c->num] = d;
sprintf (d->name, "cx%d.%d", b->num, c->num);
switch (c->type) {
case T_SYNC_RS232:
case T_SYNC_V35:
case T_SYNC_RS449:
case T_UNIV:
case T_UNIV_RS232:
case T_UNIV_RS449:
case T_UNIV_V35:
- callout_init (&d->timeout_handle, CALLOUT_MPSAFE);
+ callout_init (&d->timeout_handle, 1);
#ifdef NETGRAPH
if (ng_make_node_common (&typestruct, &d->node) != 0) {
printf ("%s: cannot make common node\n", d->name);
channel [b->num*NCHAN + c->num] = 0;
c->sys = 0;
cx_bus_dma_mem_free (&d->dmamem);
continue;
}
NG_NODE_SET_PRIVATE (d->node, d);
sprintf (d->nodename, "%s%d", NG_CX_NODE_TYPE,
c->board->num*NCHAN + c->num);
if (ng_name_node (d->node, d->nodename)) {
printf ("%s: cannot name node\n", d->nodename);
NG_NODE_UNREF (d->node);
channel [b->num*NCHAN + c->num] = 0;
c->sys = 0;
cx_bus_dma_mem_free (&d->dmamem);
continue;
}
d->lo_queue.ifq_maxlen = ifqmaxlen;
d->hi_queue.ifq_maxlen = ifqmaxlen;
mtx_init (&d->lo_queue.ifq_mtx, "cx_queue_lo", NULL, MTX_DEF);
mtx_init (&d->hi_queue.ifq_mtx, "cx_queue_hi", NULL, MTX_DEF);
#else /*NETGRAPH*/
d->ifp = if_alloc(IFT_PPP);
if (d->ifp == NULL) {
printf ("%s: cannot if_alloc() common interface\n",
d->name);
channel [b->num*NCHAN + c->num] = 0;
c->sys = 0;
cx_bus_dma_mem_free (&d->dmamem);
continue;
}
d->ifp->if_softc = d;
if_initname (d->ifp, "cx", b->num * NCHAN + c->num);
d->ifp->if_mtu = PP_MTU;
d->ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
d->ifp->if_ioctl = cx_sioctl;
d->ifp->if_start = cx_ifstart;
d->ifp->if_init = cx_initialize;
d->queue.ifq_maxlen = 2;
mtx_init (&d->queue.ifq_mtx, "cx_queue", NULL, MTX_DEF);
sppp_attach (d->ifp);
if_attach (d->ifp);
IFP2SP(d->ifp)->pp_tlf = cx_tlf;
IFP2SP(d->ifp)->pp_tls = cx_tls;
/* If BPF is in the kernel, call the attach for it.
* Size of PPP header is 4 bytes. */
bpfattach (d->ifp, DLT_PPP, 4);
#endif /*NETGRAPH*/
}
d->tty = ttyalloc ();
d->tty->t_open = cx_topen;
d->tty->t_close = cx_tclose;
d->tty->t_param = cx_param;
d->tty->t_stop = cx_stop;
d->tty->t_modem = cx_tmodem;
d->tty->t_oproc = cx_oproc;
d->tty->t_sc = d;
CX_LOCK (bd);
cx_start_chan (c, d->dmamem.virt, d->dmamem.phys);
cx_register_receive (c, &cx_receive);
cx_register_transmit (c, &cx_transmit);
cx_register_error (c, &cx_error);
cx_register_modem (c, &cx_modem);
CX_UNLOCK (bd);
ttycreate(d->tty, TS_CALLOUT, "x%r%r", b->num, c->num);
d->devt = make_dev (&cx_cdevsw, b->num*NCHAN + c->num + 64, UID_ROOT, GID_WHEEL, 0600, "cx%d", b->num*NCHAN + c->num);
d->devt->si_drv1 = d;
- callout_init (&d->dcd_timeout_handle, CALLOUT_MPSAFE);
+ callout_init (&d->dcd_timeout_handle, 1);
}
splx (s);
return 0;
}
static int cx_detach (device_t dev)
{
bdrv_t *bd = device_get_softc (dev);
cx_board_t *b = bd->board;
cx_chan_t *c;
int s;
KASSERT (mtx_initialized (&bd->cx_mtx), ("cx mutex not initialized"));
s = splhigh ();
CX_LOCK (bd);
/* Check if the device is busy (open). */
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (!d || d->chan->type == T_NONE)
continue;
if (d->lock) {
CX_UNLOCK (bd);
splx (s);
return EBUSY;
}
if (c->mode == M_ASYNC && d->tty && (d->tty->t_state & TS_ISOPEN) &&
(d->open_dev|0x2)) {
CX_UNLOCK (bd);
splx (s);
return EBUSY;
}
if (d->running) {
CX_UNLOCK (bd);
splx (s);
return EBUSY;
}
}
/* Deactivate the timeout routine. And soft interrupt*/
callout_stop (&led_timo[b->num]);
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = c->sys;
if (!d || d->chan->type == T_NONE)
continue;
callout_stop (&d->dcd_timeout_handle);
}
CX_UNLOCK (bd);
bus_teardown_intr (dev, bd->irq_res, bd->intrhand);
bus_release_resource (dev, SYS_RES_IRQ, bd->irq_rid, bd->irq_res);
bus_release_resource (dev, SYS_RES_DRQ, bd->drq_rid, bd->drq_res);
bus_release_resource (dev, SYS_RES_IOPORT, bd->base_rid, bd->base_res);
CX_LOCK (bd);
cx_close_board (b);
/* Detach the interfaces, free buffer memory. */
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (!d || d->chan->type == T_NONE)
continue;
if (d->tty) {
ttyfree (d->tty);
d->tty = NULL;
}
callout_stop (&d->timeout_handle);
#ifdef NETGRAPH
if (d->node) {
ng_rmnode_self (d->node);
NG_NODE_UNREF (d->node);
d->node = NULL;
}
mtx_destroy (&d->lo_queue.ifq_mtx);
mtx_destroy (&d->hi_queue.ifq_mtx);
#else
/* Detach from the packet filter list of interfaces. */
bpfdetach (d->ifp);
/* Detach from the sync PPP list. */
sppp_detach (d->ifp);
if_detach (d->ifp);
if_free(d->ifp);
/* XXXRIK: check interconnection with irq handler */
IF_DRAIN (&d->queue);
mtx_destroy (&d->queue.ifq_mtx);
#endif
destroy_dev (d->devt);
}
cx_led_off (b);
CX_UNLOCK (bd);
callout_drain (&led_timo[b->num]);
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = c->sys;
if (!d || d->chan->type == T_NONE)
continue;
callout_drain (&d->dcd_timeout_handle);
callout_drain (&d->timeout_handle);
}
splx (s);
s = splhigh ();
for (c = b->chan; c < b->chan + NCHAN; ++c) {
drv_t *d = (drv_t*) c->sys;
if (!d || d->chan->type == T_NONE)
continue;
/* Deallocate buffers. */
cx_bus_dma_mem_free (&d->dmamem);
}
bd->board = 0;
adapter [b->num] = 0;
free (b, M_DEVBUF);
splx (s);
mtx_destroy (&bd->cx_mtx);
return 0;
}
#ifndef NETGRAPH
static void cx_ifstart (struct ifnet *ifp)
{
drv_t *d = ifp->if_softc;
bdrv_t *bd = d->board->sys;
CX_LOCK (bd);
cx_start (d);
CX_UNLOCK (bd);
}
static void cx_tlf (struct sppp *sp)
{
drv_t *d = SP2IFP(sp)->if_softc;
CX_DEBUG (d, ("cx_tlf\n"));
/* cx_set_dtr (d->chan, 0);*/
/* cx_set_rts (d->chan, 0);*/
if (!(IFP2SP(d->ifp)->pp_flags & PP_FR) && !(d->ifp->if_flags & PP_CISCO))
sp->pp_down (sp);
}
static void cx_tls (struct sppp *sp)
{
drv_t *d = SP2IFP(sp)->if_softc;
CX_DEBUG (d, ("cx_tls\n"));
if (!(IFP2SP(d->ifp)->pp_flags & PP_FR) && !(d->ifp->if_flags & PP_CISCO))
sp->pp_up (sp);
}
/*
* Initialization of interface.
* It seems to be never called by upper level.
*/
static void cx_initialize (void *softc)
{
drv_t *d = softc;
CX_DEBUG (d, ("cx_initialize\n"));
}
/*
* Process an ioctl request.
*/
static int cx_sioctl (struct ifnet *ifp, u_long cmd, caddr_t data)
{
drv_t *d = ifp->if_softc;
bdrv_t *bd = d->board->sys;
int error, s, was_up, should_be_up;
/* No socket ioctls while the channel is in async mode. */
if (d->chan->type == T_NONE || d->chan->mode == M_ASYNC)
return EBUSY;
/* Socket ioctls on slave subchannels are not allowed. */
was_up = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
error = sppp_ioctl (ifp, cmd, data);
if (error)
return error;
s = splhigh ();
CX_LOCK (bd);
if (! (ifp->if_flags & IFF_DEBUG))
d->chan->debug = 0;
else
d->chan->debug = d->chan->debug_shadow;
CX_UNLOCK (bd);
splx (s);
switch (cmd) {
default: CX_DEBUG2 (d, ("ioctl 0x%lx\n", cmd)); return 0;
case SIOCADDMULTI: CX_DEBUG2 (d, ("SIOCADDMULTI\n")); return 0;
case SIOCDELMULTI: CX_DEBUG2 (d, ("SIOCDELMULTI\n")); return 0;
case SIOCSIFFLAGS: CX_DEBUG2 (d, ("SIOCSIFFLAGS\n")); break;
case SIOCSIFADDR: CX_DEBUG2 (d, ("SIOCSIFADDR\n")); break;
}
/* We get here only in case of SIFFLAGS or SIFADDR. */
s = splhigh ();
CX_LOCK (bd);
should_be_up = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
if (!was_up && should_be_up) {
/* Interface goes up -- start it. */
cx_up (d);
cx_start (d);
} else if (was_up && !should_be_up) {
/* Interface is going down -- stop it. */
/* if ((IFP2SP(d->ifp)->pp_flags & PP_FR) || (ifp->if_flags & PP_CISCO))*/
cx_down (d);
}
CX_UNLOCK (bd);
splx (s);
return 0;
}
#endif /*NETGRAPH*/
/*
* Stop the interface. Called on splimp().
*/
static void cx_down (drv_t *d)
{
int s = splhigh ();
CX_DEBUG (d, ("cx_down\n"));
cx_set_dtr (d->chan, 0);
cx_set_rts (d->chan, 0);
d->running = 0;
callout_stop (&d->timeout_handle);
splx (s);
}
/*
* Start the interface. Called on splimp().
*/
static void cx_up (drv_t *d)
{
int s = splhigh ();
CX_DEBUG (d, ("cx_up\n"));
cx_set_dtr (d->chan, 1);
cx_set_rts (d->chan, 1);
d->running = 1;
splx (s);
}
/*
* Start output on the (slave) interface. Get another datagram to send
* off of the interface queue, and copy it to the interface
* before starting the output.
*/
static void cx_send (drv_t *d)
{
struct mbuf *m;
u_short len;
CX_DEBUG2 (d, ("cx_send\n"));
/* No output if the interface is down. */
if (! d->running)
return;
/* No output if the modem is off. */
if (! cx_get_dsr (d->chan) && ! cx_get_loop(d->chan))
return;
if (cx_buf_free (d->chan)) {
/* Get the packet to send. */
#ifdef NETGRAPH
IF_DEQUEUE (&d->hi_queue, m);
if (! m)
IF_DEQUEUE (&d->lo_queue, m);
#else
m = sppp_dequeue (d->ifp);
#endif
if (! m)
return;
#ifndef NETGRAPH
BPF_MTAP (d->ifp, m);
#endif
len = m_length (m, NULL);
if (! m->m_next)
cx_send_packet (d->chan, (u_char*)mtod (m, caddr_t),
len, 0);
else {
u_char buf [DMABUFSZ];
m_copydata (m, 0, len, buf);
cx_send_packet (d->chan, buf, len, 0);
}
m_freem (m);
/* Set up transmit timeout, 10 seconds. */
d->timeout = 10;
}
#ifndef NETGRAPH
d->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
#endif
}
/*
* Start output on the interface.
* Always called on splimp().
*/
static void cx_start (drv_t *d)
{
int s = splhigh ();
if (d->running) {
if (! d->chan->dtr)
cx_set_dtr (d->chan, 1);
if (! d->chan->rts)
cx_set_rts (d->chan, 1);
cx_send (d);
callout_reset (&d->timeout_handle, hz, cx_watchdog_timer, d);
}
splx (s);
}
/*
* Handle transmit timeouts.
* Recover after lost transmit interrupts.
* Always called on splimp().
*/
static void cx_watchdog (drv_t *d)
{
CX_DEBUG (d, ("device timeout\n"));
if (d->running) {
cx_setup_chan (d->chan);
cx_start_chan (d->chan, 0, 0);
cx_set_dtr (d->chan, 1);
cx_set_rts (d->chan, 1);
cx_start (d);
}
}
static void cx_watchdog_timer (void *arg)
{
drv_t *d = arg;
bdrv_t *bd = d->board->sys;
CX_LOCK (bd);
if (d->timeout == 1)
cx_watchdog (d);
if (d->timeout)
d->timeout--;
callout_reset (&d->timeout_handle, hz, cx_watchdog_timer, d);
CX_UNLOCK (bd);
}
/*
* Transmit callback function.
*/
static void cx_transmit (cx_chan_t *c, void *attachment, int len)
{
drv_t *d = c->sys;
if (!d)
return;
if (c->mode == M_ASYNC && d->tty) {
d->tty->t_state &= ~(TS_BUSY | TS_FLUSH);
d->atimeout = 0;
if (d->tty->t_dev) {
d->intr_action |= CX_WRITE;
MY_SOFT_INTR = 1;
swi_sched (cx_fast_ih, 0);
}
return;
}
d->timeout = 0;
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_OPACKETS, 1);
d->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
#endif
cx_start (d);
}
/*
* Process the received packet.
*/
static void cx_receive (cx_chan_t *c, char *data, int len)
{
drv_t *d = c->sys;
struct mbuf *m;
char *cc = data;
#ifdef NETGRAPH
int error;
#endif
if (!d)
return;
if (c->mode == M_ASYNC && d->tty) {
if (d->tty->t_state & TS_ISOPEN) {
async_q *q = &d->aqueue;
int size = BF_SZ - 1 - AQ_GSZ (q);
if (len <= 0 && !size)
return;
if (len > size) {
c->ierrs++;
cx_error (c, CX_OVERRUN);
len = size - 1;
}
while (len--) {
AQ_PUSH (q, *(unsigned char *)cc);
cc++;
}
d->intr_action |= CX_READ;
MY_SOFT_INTR = 1;
swi_sched (cx_fast_ih, 0);
}
return;
}
if (! d->running)
return;
m = makembuf (data, len);
if (! m) {
CX_DEBUG (d, ("no memory for packet\n"));
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_IQDROPS, 1);
#endif
return;
}
if (c->debug > 1)
m_print (m, 0);
#ifdef NETGRAPH
m->m_pkthdr.rcvif = 0;
NG_SEND_DATA_ONLY (error, d->hook, m);
#else
if_inc_counter(d->ifp, IFCOUNTER_IPACKETS, 1);
m->m_pkthdr.rcvif = d->ifp;
/* Check if there's a BPF listener on this interface.
* If so, hand off the raw packet to bpf. */
BPF_MTAP(d->ifp, m);
IF_ENQUEUE (&d->queue, m);
#endif
}
#define CONDITION(t,tp) (!(t->c_iflag & (ICRNL | IGNCR | IMAXBEL | INLCR | ISTRIP | IXON))\
&& (!(tp->t_iflag & BRKINT) || (tp->t_iflag & IGNBRK))\
&& (!(tp->t_iflag & PARMRK)\
|| (tp->t_iflag & (IGNPAR | IGNBRK)) == (IGNPAR | IGNBRK))\
&& !(t->c_lflag & (ECHO | ICANON | IEXTEN | ISIG | PENDIN))\
&& linesw[tp->t_line]->l_rint == ttyinput)
/*
* Error callback function.
*/
static void cx_error (cx_chan_t *c, int data)
{
drv_t *d = c->sys;
async_q *q;
if (!d)
return;
q = &(d->aqueue);
switch (data) {
case CX_FRAME:
CX_DEBUG (d, ("frame error\n"));
if (c->mode == M_ASYNC && d->tty && (d->tty->t_state & TS_ISOPEN)
&& (AQ_GSZ (q) < BF_SZ - 1)
&& (!CONDITION((&d->tty->t_termios), (d->tty))
|| !(d->tty->t_iflag & (IGNPAR | PARMRK)))) {
AQ_PUSH (q, TTY_FE);
d->intr_action |= CX_READ;
MY_SOFT_INTR = 1;
swi_sched (cx_fast_ih, 0);
}
#ifndef NETGRAPH
else
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CX_CRC:
CX_DEBUG (d, ("crc error\n"));
if (c->mode == M_ASYNC && d->tty && (d->tty->t_state & TS_ISOPEN)
&& (AQ_GSZ (q) < BF_SZ - 1)
&& (!CONDITION((&d->tty->t_termios), (d->tty))
|| !(d->tty->t_iflag & INPCK)
|| !(d->tty->t_iflag & (IGNPAR | PARMRK)))) {
AQ_PUSH (q, TTY_PE);
d->intr_action |= CX_READ;
MY_SOFT_INTR = 1;
swi_sched (cx_fast_ih, 0);
}
#ifndef NETGRAPH
else
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CX_OVERRUN:
CX_DEBUG (d, ("overrun error\n"));
#ifdef TTY_OE
if (c->mode == M_ASYNC && d->tty && (d->tty->t_state & TS_ISOPEN)
&& (AQ_GSZ (q) < BF_SZ - 1)
&& (!CONDITION((&d->tty->t_termios), (d->tty)))) {
AQ_PUSH (q, TTY_OE);
d->intr_action |= CX_READ;
MY_SOFT_INTR = 1;
swi_sched (cx_fast_ih, 0);
}
#endif
#ifndef NETGRAPH
else {
if_inc_counter(d->ifp, IFCOUNTER_COLLISIONS, 1);
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
}
#endif
break;
case CX_OVERFLOW:
CX_DEBUG (d, ("overflow error\n"));
#ifndef NETGRAPH
if (c->mode != M_ASYNC)
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
case CX_UNDERRUN:
CX_DEBUG (d, ("underrun error\n"));
if (c->mode != M_ASYNC) {
d->timeout = 0;
#ifndef NETGRAPH
if_inc_counter(d->ifp, IFCOUNTER_OERRORS, 1);
d->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
#endif
cx_start (d);
}
break;
case CX_BREAK:
CX_DEBUG (d, ("break error\n"));
if (c->mode == M_ASYNC && d->tty && (d->tty->t_state & TS_ISOPEN)
&& (AQ_GSZ (q) < BF_SZ - 1)
&& (!CONDITION((&d->tty->t_termios), (d->tty))
|| !(d->tty->t_iflag & (IGNBRK | BRKINT | PARMRK)))) {
AQ_PUSH (q, TTY_BI);
d->intr_action |= CX_READ;
MY_SOFT_INTR = 1;
swi_sched (cx_fast_ih, 0);
}
#ifndef NETGRAPH
else
if_inc_counter(d->ifp, IFCOUNTER_IERRORS, 1);
#endif
break;
default:
CX_DEBUG (d, ("error #%d\n", data));
}
}
static int cx_topen (struct tty *tp, struct cdev *dev)
{
bdrv_t *bd;
drv_t *d;
d = tp->t_sc;
CX_DEBUG2 (d, ("cx_open (serial)\n"));
bd = d->board->sys;
if (d->chan->mode != M_ASYNC)
return (EBUSY);
d->open_dev |= 0x2;
CX_LOCK (bd);
cx_start_chan (d->chan, 0, 0);
cx_set_dtr (d->chan, 1);
cx_set_rts (d->chan, 1);
d->cd = cx_get_cd (d->chan);
CX_UNLOCK (bd);
CX_DEBUG2 (d, ("cx_open done\n"));
return 0;
}
static void cx_tclose (struct tty *tp)
{
drv_t *d;
bdrv_t *bd;
d = tp->t_sc;
CX_DEBUG2 (d, ("cx_close\n"));
bd = d->board->sys;
CX_LOCK (bd);
/* Disable receiver.
* Transmitter continues sending the queued data. */
cx_enable_receive (d->chan, 0);
CX_UNLOCK (bd);
d->open_dev &= ~0x2;
}
static int cx_tmodem (struct tty *tp, int sigon, int sigoff)
{
drv_t *d;
bdrv_t *bd;
d = tp->t_sc;
bd = d->board->sys;
CX_LOCK (bd);
if (!sigon && !sigoff) {
if (cx_get_dsr (d->chan)) sigon |= SER_DSR;
if (cx_get_cd (d->chan)) sigon |= SER_DCD;
if (cx_get_cts (d->chan)) sigon |= SER_CTS;
if (d->chan->dtr) sigon |= SER_DTR;
if (d->chan->rts) sigon |= SER_RTS;
CX_UNLOCK (bd);
return sigon;
}
if (sigon & SER_DTR)
cx_set_dtr (d->chan, 1);
if (sigoff & SER_DTR)
cx_set_dtr (d->chan, 0);
if (sigon & SER_RTS)
cx_set_rts (d->chan, 1);
if (sigoff & SER_RTS)
cx_set_rts (d->chan, 0);
CX_UNLOCK (bd);
return (0);
}
static int cx_open (struct cdev *dev, int flag, int mode, struct thread *td)
{
int unit;
drv_t *d;
d = dev->si_drv1;
unit = d->chan->num;
CX_DEBUG2 (d, ("cx_open unit=%d, flag=0x%x, mode=0x%x\n",
unit, flag, mode));
d->open_dev |= 0x1;
CX_DEBUG2 (d, ("cx_open done\n"));
return 0;
}
static int cx_close (struct cdev *dev, int flag, int mode, struct thread *td)
{
drv_t *d;
d = dev->si_drv1;
CX_DEBUG2 (d, ("cx_close\n"));
d->open_dev &= ~0x1;
return 0;
}
static int cx_modem_status (drv_t *d)
{
bdrv_t *bd = d->board->sys;
int status = 0, s = splhigh ();
CX_LOCK (bd);
/* Already opened by someone or network interface is up? */
if ((d->chan->mode == M_ASYNC && d->tty && (d->tty->t_state & TS_ISOPEN) &&
(d->open_dev|0x2)) || (d->chan->mode != M_ASYNC && d->running))
status = TIOCM_LE; /* always enabled while open */
if (cx_get_dsr (d->chan)) status |= TIOCM_DSR;
if (cx_get_cd (d->chan)) status |= TIOCM_CD;
if (cx_get_cts (d->chan)) status |= TIOCM_CTS;
if (d->chan->dtr) status |= TIOCM_DTR;
if (d->chan->rts) status |= TIOCM_RTS;
CX_UNLOCK (bd);
splx (s);
return status;
}
static int cx_ioctl (struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td)
{
drv_t *d;
bdrv_t *bd;
cx_chan_t *c;
struct serial_statistics *st;
int error, s;
char mask[16];
d = dev->si_drv1;
c = d->chan;
bd = d->board->sys;
switch (cmd) {
case SERIAL_GETREGISTERED:
CX_DEBUG2 (d, ("ioctl: getregistered\n"));
bzero (mask, sizeof(mask));
for (s=0; s<NCX*NCHAN; ++s)
if (channel [s])
mask [s/8] |= 1 << (s & 7);
bcopy (mask, data, sizeof (mask));
return 0;
case SERIAL_GETPORT:
CX_DEBUG2 (d, ("ioctl: getport\n"));
s = splhigh ();
CX_LOCK (bd);
*(int *)data = cx_get_port (c);
CX_UNLOCK (bd);
splx (s);
if (*(int *)data<0)
return (EINVAL);
else
return 0;
case SERIAL_SETPORT:
CX_DEBUG2 (d, ("ioctl: setproto\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splhigh ();
CX_LOCK (bd);
cx_set_port (c, *(int *)data);
CX_UNLOCK (bd);
splx (s);
return 0;
#ifndef NETGRAPH
case SERIAL_GETPROTO:
CX_DEBUG2 (d, ("ioctl: getproto\n"));
s = splhigh ();
CX_LOCK (bd);
strcpy ((char*)data, (c->mode == M_ASYNC) ? "async" :
(IFP2SP(d->ifp)->pp_flags & PP_FR) ? "fr" :
(d->ifp->if_flags & PP_CISCO) ? "cisco" : "ppp");
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_SETPROTO:
CX_DEBUG2 (d, ("ioctl: setproto\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_ASYNC)
return EBUSY;
if (d->ifp->if_drv_flags & IFF_DRV_RUNNING)
return EBUSY;
if (! strcmp ("cisco", (char*)data)) {
IFP2SP(d->ifp)->pp_flags &= ~(PP_FR);
IFP2SP(d->ifp)->pp_flags |= PP_KEEPALIVE;
d->ifp->if_flags |= PP_CISCO;
} else if (! strcmp ("fr", (char*)data)) {
d->ifp->if_flags &= ~(PP_CISCO);
IFP2SP(d->ifp)->pp_flags |= PP_FR | PP_KEEPALIVE;
} else if (! strcmp ("ppp", (char*)data)) {
IFP2SP(d->ifp)->pp_flags &= ~(PP_FR | PP_KEEPALIVE);
d->ifp->if_flags &= ~(PP_CISCO);
} else
return EINVAL;
return 0;
case SERIAL_GETKEEPALIVE:
CX_DEBUG2 (d, ("ioctl: getkeepalive\n"));
if ((IFP2SP(d->ifp)->pp_flags & PP_FR) ||
(d->ifp->if_flags & PP_CISCO) ||
(c->mode == M_ASYNC))
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
*(int*)data = (IFP2SP(d->ifp)->pp_flags & PP_KEEPALIVE) ? 1 : 0;
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_SETKEEPALIVE:
CX_DEBUG2 (d, ("ioctl: setkeepalive\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if ((IFP2SP(d->ifp)->pp_flags & PP_FR) ||
(d->ifp->if_flags & PP_CISCO))
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
if (*(int*)data)
IFP2SP(d->ifp)->pp_flags |= PP_KEEPALIVE;
else
IFP2SP(d->ifp)->pp_flags &= ~PP_KEEPALIVE;
CX_UNLOCK (bd);
splx (s);
return 0;
#endif /*NETGRAPH*/
case SERIAL_GETMODE:
CX_DEBUG2 (d, ("ioctl: getmode\n"));
s = splhigh ();
CX_LOCK (bd);
*(int*)data = (c->mode == M_ASYNC) ?
SERIAL_ASYNC : SERIAL_HDLC;
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_SETMODE:
CX_DEBUG2 (d, ("ioctl: setmode\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
/* Somebody is waiting for carrier? */
if (d->lock)
return EBUSY;
/* /dev/ttyXX is already opened by someone? */
if (c->mode == M_ASYNC && d->tty && (d->tty->t_state & TS_ISOPEN) &&
(d->open_dev|0x2))
return EBUSY;
/* Network interface is up?
* Cannot change to async mode. */
if (c->mode != M_ASYNC && d->running &&
(*(int*)data == SERIAL_ASYNC))
return EBUSY;
s = splhigh ();
CX_LOCK (bd);
if (c->mode == M_HDLC && *(int*)data == SERIAL_ASYNC) {
cx_set_mode (c, M_ASYNC);
cx_enable_receive (c, 0);
cx_enable_transmit (c, 0);
} else if (c->mode == M_ASYNC && *(int*)data == SERIAL_HDLC) {
if (d->ifp->if_flags & IFF_DEBUG)
c->debug = c->debug_shadow;
cx_set_mode (c, M_HDLC);
cx_enable_receive (c, 1);
cx_enable_transmit (c, 1);
}
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETSTAT:
CX_DEBUG2 (d, ("ioctl: getestat\n"));
st = (struct serial_statistics*) data;
s = splhigh ();
CX_LOCK (bd);
st->rintr = c->rintr;
st->tintr = c->tintr;
st->mintr = c->mintr;
st->ibytes = c->ibytes;
st->ipkts = c->ipkts;
st->ierrs = c->ierrs;
st->obytes = c->obytes;
st->opkts = c->opkts;
st->oerrs = c->oerrs;
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_CLRSTAT:
CX_DEBUG2 (d, ("ioctl: clrstat\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splhigh ();
CX_LOCK (bd);
c->rintr = 0;
c->tintr = 0;
c->mintr = 0;
c->ibytes = 0;
c->ipkts = 0;
c->ierrs = 0;
c->obytes = 0;
c->opkts = 0;
c->oerrs = 0;
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETBAUD:
CX_DEBUG2 (d, ("ioctl: getbaud\n"));
if (c->mode == M_ASYNC)
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
*(long*)data = cx_get_baud(c);
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_SETBAUD:
CX_DEBUG2 (d, ("ioctl: setbaud\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_ASYNC)
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
cx_set_baud (c, *(long*)data);
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETLOOP:
CX_DEBUG2 (d, ("ioctl: getloop\n"));
if (c->mode == M_ASYNC)
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
*(int*)data = cx_get_loop (c);
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_SETLOOP:
CX_DEBUG2 (d, ("ioctl: setloop\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_ASYNC)
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
cx_set_loop (c, *(int*)data);
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETDPLL:
CX_DEBUG2 (d, ("ioctl: getdpll\n"));
if (c->mode == M_ASYNC)
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
*(int*)data = cx_get_dpll (c);
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_SETDPLL:
CX_DEBUG2 (d, ("ioctl: setdpll\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_ASYNC)
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
cx_set_dpll (c, *(int*)data);
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETNRZI:
CX_DEBUG2 (d, ("ioctl: getnrzi\n"));
if (c->mode == M_ASYNC)
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
*(int*)data = cx_get_nrzi (c);
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_SETNRZI:
CX_DEBUG2 (d, ("ioctl: setnrzi\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
if (c->mode == M_ASYNC)
return EINVAL;
s = splhigh ();
CX_LOCK (bd);
cx_set_nrzi (c, *(int*)data);
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_GETDEBUG:
CX_DEBUG2 (d, ("ioctl: getdebug\n"));
s = splhigh ();
CX_LOCK (bd);
*(int*)data = c->debug;
CX_UNLOCK (bd);
splx (s);
return 0;
case SERIAL_SETDEBUG:
CX_DEBUG2 (d, ("ioctl: setdebug\n"));
/* Only for superuser! */
error = priv_check (td, PRIV_DRIVER);
if (error)
return error;
s = splhigh ();
CX_LOCK (bd);
#ifndef NETGRAPH
if (c->mode == M_ASYNC) {
c->debug = *(int*)data;
} else {
/*
* The debug_shadow is always greater than zero for
* logic simplicity. For switching debug off the
* IFF_DEBUG is responsible (for !M_ASYNC mode).
*/
c->debug_shadow = (*(int*)data) ? (*(int*)data) : 1;
if (d->ifp->if_flags & IFF_DEBUG)
c->debug = c->debug_shadow;
}
#else
c->debug = *(int*)data;
#endif
CX_UNLOCK (bd);
splx (s);
return 0;
}
switch (cmd) {
case TIOCSDTR: /* Set DTR */
CX_DEBUG2 (d, ("ioctl: tiocsdtr\n"));
s = splhigh ();
CX_LOCK (bd);
cx_set_dtr (c, 1);
CX_UNLOCK (bd);
splx (s);
return 0;
case TIOCCDTR: /* Clear DTR */
CX_DEBUG2 (d, ("ioctl: tioccdtr\n"));
s = splhigh ();
CX_LOCK (bd);
cx_set_dtr (c, 0);
CX_UNLOCK (bd);
splx (s);
return 0;
case TIOCMSET: /* Set DTR/RTS */
CX_DEBUG2 (d, ("ioctl: tiocmset\n"));
s = splhigh ();
CX_LOCK (bd);
cx_set_dtr (c, (*(int*)data & TIOCM_DTR) ? 1 : 0);
cx_set_rts (c, (*(int*)data & TIOCM_RTS) ? 1 : 0);
CX_UNLOCK (bd);
splx (s);
return 0;
case TIOCMBIS: /* Add DTR/RTS */
CX_DEBUG2 (d, ("ioctl: tiocmbis\n"));
s = splhigh ();
CX_LOCK (bd);
if (*(int*)data & TIOCM_DTR) cx_set_dtr (c, 1);
if (*(int*)data & TIOCM_RTS) cx_set_rts (c, 1);
CX_UNLOCK (bd);
splx (s);
return 0;
case TIOCMBIC: /* Clear DTR/RTS */
CX_DEBUG2 (d, ("ioctl: tiocmbic\n"));
s = splhigh ();
CX_LOCK (bd);
if (*(int*)data & TIOCM_DTR) cx_set_dtr (c, 0);
if (*(int*)data & TIOCM_RTS) cx_set_rts (c, 0);
CX_UNLOCK (bd);
splx (s);
return 0;
case TIOCMGET: /* Get modem status */
CX_DEBUG2 (d, ("ioctl: tiocmget\n"));
*(int*)data = cx_modem_status (d);
return 0;
}
CX_DEBUG2 (d, ("ioctl: 0x%lx\n", cmd));
return ENOTTY;
}
void cx_softintr (void *unused)
{
drv_t *d;
bdrv_t *bd;
async_q *q;
int i, s, ic, k;
while (MY_SOFT_INTR) {
MY_SOFT_INTR = 0;
for (i=0; i<NCX*NCHAN; ++i) {
d = channel [i];
if (!d || !d->chan || d->chan->type == T_NONE
|| d->chan->mode != M_ASYNC || !d->tty
|| !d->tty->t_dev)
continue;
bd = d->board->sys;
s = splhigh ();
CX_LOCK (bd);
if (d->intr_action & CX_READ) {
q = &(d->aqueue);
if (d->tty->t_state & TS_CAN_BYPASS_L_RINT) {
k = AQ_GSZ(q);
if (d->tty->t_rawq.c_cc + k >
d->tty->t_ihiwat
&& (d->tty->t_cflag & CRTS_IFLOW
|| d->tty->t_iflag & IXOFF)
&& !(d->tty->t_state & TS_TBLOCK))
ttyblock(d->tty);
d->tty->t_rawcc += k;
while (k>0) {
k--;
AQ_POP (q, ic);
CX_UNLOCK (bd);
splx (s);
putc (ic, &d->tty->t_rawq);
s = splhigh ();
CX_LOCK (bd);
}
ttwakeup(d->tty);
if (d->tty->t_state & TS_TTSTOP
&& (d->tty->t_iflag & IXANY
|| d->tty->t_cc[VSTART] ==
d->tty->t_cc[VSTOP])) {
d->tty->t_state &= ~TS_TTSTOP;
d->tty->t_lflag &= ~FLUSHO;
d->intr_action |= CX_WRITE;
}
} else {
while (q->end != q->beg) {
AQ_POP (q, ic);
CX_UNLOCK (bd);
splx (s);
ttyld_rint (d->tty, ic);
s = splhigh ();
CX_LOCK (bd);
}
}
d->intr_action &= ~CX_READ;
}
splx (s);
CX_UNLOCK (bd);
s = splhigh ();
CX_LOCK (bd);
if (d->intr_action & CX_WRITE) {
if (d->tty->t_line)
ttyld_start (d->tty);
else
cx_oproc (d->tty);
d->intr_action &= ~CX_WRITE;
}
CX_UNLOCK (bd);
splx (s);
}
}
}
/*
* Fill transmitter buffer with data.
*/
static void cx_oproc (struct tty *tp)
{
int s, k;
drv_t *d;
bdrv_t *bd;
static u_char buf[DMABUFSZ];
u_char *p;
u_short len = 0, sublen = 0;
d = tp->t_sc;
bd = d->board->sys;
CX_DEBUG2 (d, ("cx_oproc\n"));
s = splhigh ();
CX_LOCK (bd);
if (tp->t_cflag & CRTSCTS && (tp->t_state & TS_TBLOCK) && d->chan->rts)
cx_set_rts (d->chan, 0);
else if (tp->t_cflag & CRTSCTS && ! (tp->t_state & TS_TBLOCK) && ! d->chan->rts)
cx_set_rts (d->chan, 1);
if (! (tp->t_state & (TS_TIMEOUT | TS_TTSTOP))) {
/* Start transmitter. */
cx_enable_transmit (d->chan, 1);
/* Is it busy? */
if (! cx_buf_free (d->chan)) {
tp->t_state |= TS_BUSY;
CX_UNLOCK (bd);
splx (s);
return;
}
if (tp->t_iflag & IXOFF) {
p = (buf + (DMABUFSZ/2));
sublen = q_to_b (&tp->t_outq, p, (DMABUFSZ/2));
k = sublen;
while (k--) {
/* Send XON/XOFF out of band. */
if (*p == tp->t_cc[VSTOP]) {
cx_xflow_ctl (d->chan, 0);
p++;
continue;
}
if (*p == tp->t_cc[VSTART]) {
cx_xflow_ctl (d->chan, 1);
p++;
continue;
}
buf[len] = *p;
len++;
p++;
}
} else {
p = buf;
len = q_to_b (&tp->t_outq, p, (DMABUFSZ/2));
}
if (len) {
cx_send_packet (d->chan, buf, len, 0);
tp->t_state |= TS_BUSY;
d->atimeout = 10;
CX_DEBUG2 (d, ("out %d bytes\n", len));
}
}
ttwwakeup (tp);
CX_UNLOCK (bd);
splx (s);
}
static int cx_param (struct tty *tp, struct termios *t)
{
drv_t *d;
bdrv_t *bd;
int s, bits, parity;
d = tp->t_sc;
bd = d->board->sys;
s = splhigh ();
CX_LOCK (bd);
if (t->c_ospeed == 0) {
/* Clear DTR and RTS. */
cx_set_dtr (d->chan, 0);
CX_UNLOCK (bd);
splx (s);
CX_DEBUG2 (d, ("cx_param (hangup)\n"));
return 0;
}
CX_DEBUG2 (d, ("cx_param\n"));
/* Check requested parameters. */
if (t->c_ospeed < 300 || t->c_ospeed > 256*1024) {
CX_UNLOCK (bd);
splx (s);
return EINVAL;
}
if (t->c_ispeed && (t->c_ispeed < 300 || t->c_ispeed > 256*1024)) {
CX_UNLOCK (bd);
splx (s);
return EINVAL;
}
/* And copy them to tty and channel structures. */
tp->t_ispeed = t->c_ispeed = tp->t_ospeed = t->c_ospeed;
tp->t_cflag = t->c_cflag;
/* Set character length and parity mode. */
switch (t->c_cflag & CSIZE) {
default:
case CS8: bits = 8; break;
case CS7: bits = 7; break;
case CS6: bits = 6; break;
case CS5: bits = 5; break;
}
parity = ((t->c_cflag & PARENB) ? 1 : 0) *
(1 + ((t->c_cflag & PARODD) ? 0 : 1));
/* Set current channel number. */
if (! d->chan->dtr)
cx_set_dtr (d->chan, 1);
ttyldoptim (tp);
cx_set_async_param (d->chan, t->c_ospeed, bits, parity, (t->c_cflag & CSTOPB),
!(t->c_cflag & PARENB), (t->c_cflag & CRTSCTS),
(t->c_iflag & IXON), (t->c_iflag & IXANY),
t->c_cc[VSTART], t->c_cc[VSTOP]);
CX_UNLOCK (bd);
splx (s);
return 0;
}
/*
* Stop output on a line
*/
static void cx_stop (struct tty *tp, int flag)
{
drv_t *d;
bdrv_t *bd;
int s;
d = tp->t_sc;
bd = d->board->sys;
s = splhigh ();
CX_LOCK (bd);
if (tp->t_state & TS_BUSY) {
/* Stop transmitter */
CX_DEBUG2 (d, ("cx_stop\n"));
cx_transmitter_ctl (d->chan, 0);
}
CX_UNLOCK (bd);
splx (s);
}
/*
* Process the (delayed) carrier signal setup.
*/
static void cx_carrier (void *arg)
{
drv_t *d = arg;
bdrv_t *bd = d->board->sys;
cx_chan_t *c = d->chan;
int s, cd;
s = splhigh ();
CX_LOCK (bd);
cd = cx_get_cd (c);
if (d->cd != cd) {
if (cd) {
CX_DEBUG (d, ("carrier on\n"));
d->cd = 1;
CX_UNLOCK (bd);
splx (s);
if (d->tty)
ttyld_modem(d->tty, 1);
} else {
CX_DEBUG (d, ("carrier loss\n"));
d->cd = 0;
CX_UNLOCK (bd);
splx (s);
if (d->tty)
ttyld_modem(d->tty, 0);
}
} else {
CX_UNLOCK (bd);
splx (s);
}
}
/*
* Modem signal callback function.
*/
static void cx_modem (cx_chan_t *c)
{
drv_t *d = c->sys;
if (!d || c->mode != M_ASYNC)
return;
/* Handle carrier detect/loss. */
/* Carrier changed - delay processing DCD for a while
* to give both sides some time to initialize. */
callout_reset (&d->dcd_timeout_handle, hz/2, cx_carrier, d);
}
#ifdef NETGRAPH
static int ng_cx_constructor (node_p node)
{
drv_t *d = NG_NODE_PRIVATE (node);
CX_DEBUG (d, ("Constructor\n"));
return EINVAL;
}
static int ng_cx_newhook (node_p node, hook_p hook, const char *name)
{
int s;
drv_t *d = NG_NODE_PRIVATE (node);
bdrv_t *bd = d->board->sys;
if (d->chan->mode == M_ASYNC)
return EINVAL;
/* Attach debug hook */
if (strcmp (name, NG_CX_HOOK_DEBUG) == 0) {
NG_HOOK_SET_PRIVATE (hook, NULL);
d->debug_hook = hook;
return 0;
}
/* Check for raw hook */
if (strcmp (name, NG_CX_HOOK_RAW) != 0)
return EINVAL;
NG_HOOK_SET_PRIVATE (hook, d);
d->hook = hook;
s = splhigh ();
CX_LOCK (bd);
cx_up (d);
CX_UNLOCK (bd);
splx (s);
return 0;
}
static int print_modems (char *s, cx_chan_t *c, int need_header)
{
int status = cx_modem_status (c->sys);
int length = 0;
if (need_header)
length += sprintf (s + length, " LE DTR DSR RTS CTS CD\n");
length += sprintf (s + length, "%4s %4s %4s %4s %4s %4s\n",
status & TIOCM_LE ? "On" : "-",
status & TIOCM_DTR ? "On" : "-",
status & TIOCM_DSR ? "On" : "-",
status & TIOCM_RTS ? "On" : "-",
status & TIOCM_CTS ? "On" : "-",
status & TIOCM_CD ? "On" : "-");
return length;
}
static int print_stats (char *s, cx_chan_t *c, int need_header)
{
int length = 0;
if (need_header)
length += sprintf (s + length, " Rintr Tintr Mintr Ibytes Ipkts Ierrs Obytes Opkts Oerrs\n");
length += sprintf (s + length, "%7ld %7ld %7ld %8ld %7ld %7ld %8ld %7ld %7ld\n",
c->rintr, c->tintr, c->mintr, c->ibytes, c->ipkts,
c->ierrs, c->obytes, c->opkts, c->oerrs);
return length;
}
static int print_chan (char *s, cx_chan_t *c)
{
drv_t *d = c->sys;
int length = 0;
length += sprintf (s + length, "cx%d", c->board->num * NCHAN + c->num);
if (d->chan->debug)
length += sprintf (s + length, " debug=%d", d->chan->debug);
if (cx_get_baud (c))
length += sprintf (s + length, " %ld", cx_get_baud (c));
else
length += sprintf (s + length, " extclock");
if (c->mode == M_HDLC) {
length += sprintf (s + length, " dpll=%s", cx_get_dpll (c) ? "on" : "off");
length += sprintf (s + length, " nrzi=%s", cx_get_nrzi (c) ? "on" : "off");
}
length += sprintf (s + length, " loop=%s", cx_get_loop (c) ? "on\n" : "off\n");
return length;
}
static int ng_cx_rcvmsg (node_p node, item_p item, hook_p lasthook)
{
drv_t *d = NG_NODE_PRIVATE (node);
struct ng_mesg *msg;
struct ng_mesg *resp = NULL;
int error = 0;
if (!d)
return EINVAL;
CX_DEBUG (d, ("Rcvmsg\n"));
NGI_GET_MSG (item, msg);
switch (msg->header.typecookie) {
default:
error = EINVAL;
break;
case NGM_CX_COOKIE:
printf ("Don't forget to implement\n");
error = EINVAL;
break;
case NGM_GENERIC_COOKIE:
switch (msg->header.cmd) {
default:
error = EINVAL;
break;
case NGM_TEXT_STATUS: {
char *s;
int l = 0;
int dl = sizeof (struct ng_mesg) + 730;
NG_MKRESPONSE (resp, msg, dl, M_NOWAIT);
if (! resp) {
error = ENOMEM;
break;
}
bzero (resp, dl);
s = (resp)->data;
l += print_chan (s + l, d->chan);
l += print_stats (s + l, d->chan, 1);
l += print_modems (s + l, d->chan, 1);
strncpy ((resp)->header.cmdstr, "status", NG_CMDSTRSIZ);
}
break;
}
break;
}
NG_RESPOND_MSG (error, node, item, resp);
NG_FREE_MSG (msg);
return error;
}
static int ng_cx_rcvdata (hook_p hook, item_p item)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE(hook));
struct mbuf *m;
struct ng_tag_prio *ptag;
bdrv_t *bd;
struct ifqueue *q;
int s;
NGI_GET_M (item, m);
NG_FREE_ITEM (item);
if (! NG_HOOK_PRIVATE (hook) || ! d) {
NG_FREE_M (m);
return ENETDOWN;
}
bd = d->board->sys;
/* Check for high priority data */
if ((ptag = (struct ng_tag_prio *)m_tag_locate(m, NGM_GENERIC_COOKIE,
NG_TAG_PRIO, NULL)) != NULL && (ptag->priority > NG_PRIO_CUTOFF) )
q = &d->hi_queue;
else
q = &d->lo_queue;
s = splhigh ();
CX_LOCK (bd);
IF_LOCK (q);
if (_IF_QFULL (q)) {
IF_UNLOCK (q);
CX_UNLOCK (bd);
splx (s);
NG_FREE_M (m);
return ENOBUFS;
}
_IF_ENQUEUE (q, m);
IF_UNLOCK (q);
cx_start (d);
CX_UNLOCK (bd);
splx (s);
return 0;
}
static int ng_cx_rmnode (node_p node)
{
drv_t *d = NG_NODE_PRIVATE (node);
bdrv_t *bd;
CX_DEBUG (d, ("Rmnode\n"));
if (d && d->running) {
int s = splhigh ();
bd = d->board->sys;
CX_LOCK (bd);
cx_down (d);
CX_UNLOCK (bd);
splx (s);
}
#ifdef KLD_MODULE
if (node->nd_flags & NGF_REALLY_DIE) {
NG_NODE_SET_PRIVATE (node, NULL);
NG_NODE_UNREF (node);
}
NG_NODE_REVIVE(node); /* Persistant node */
#endif
return 0;
}
static int ng_cx_connect (hook_p hook)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE (hook));
callout_reset (&d->timeout_handle, hz, cx_watchdog_timer, d);
return 0;
}
static int ng_cx_disconnect (hook_p hook)
{
drv_t *d = NG_NODE_PRIVATE (NG_HOOK_NODE (hook));
bdrv_t *bd = d->board->sys;
int s;
s = splhigh ();
CX_LOCK (bd);
if (NG_HOOK_PRIVATE (hook))
cx_down (d);
CX_UNLOCK (bd);
splx (s);
/* If we were wait it than it reasserted now, just stop it. */
if (!callout_drain (&d->timeout_handle))
callout_stop (&d->timeout_handle);
return 0;
}
#endif /*NETGRAPH*/
static int cx_modevent (module_t mod, int type, void *unused)
{
static int load_count = 0;
switch (type) {
case MOD_LOAD:
#ifdef NETGRAPH
if (ng_newtype (&typestruct))
printf ("Failed to register ng_cx\n");
#endif
++load_count;
- callout_init (&timeout_handle, CALLOUT_MPSAFE);
+ callout_init (&timeout_handle, 1);
callout_reset (&timeout_handle, hz*5, cx_timeout, 0);
/* Software interrupt. */
swi_add(&tty_intr_event, "cx", cx_softintr, NULL, SWI_TTY,
INTR_MPSAFE, &cx_fast_ih);
break;
case MOD_UNLOAD:
if (load_count == 1) {
printf ("Removing device entry for Sigma\n");
#ifdef NETGRAPH
ng_rmtype (&typestruct);
#endif
}
/* If we were wait it than it reasserted now, just stop it. */
if (!callout_drain (&timeout_handle))
callout_stop (&timeout_handle);
swi_remove (cx_fast_ih);
--load_count;
break;
case MOD_SHUTDOWN:
break;
}
return 0;
}
#ifdef NETGRAPH
static struct ng_type typestruct = {
.version = NG_ABI_VERSION,
.name = NG_CX_NODE_TYPE,
.constructor = ng_cx_constructor,
.rcvmsg = ng_cx_rcvmsg,
.shutdown = ng_cx_rmnode,
.newhook = ng_cx_newhook,
.connect = ng_cx_connect,
.rcvdata = ng_cx_rcvdata,
.disconnect = ng_cx_disconnect,
};
#endif /*NETGRAPH*/
#ifdef NETGRAPH
MODULE_DEPEND (ng_cx, netgraph, NG_ABI_VERSION, NG_ABI_VERSION, NG_ABI_VERSION);
#else
MODULE_DEPEND (isa_cx, sppp, 1, 1, 1);
#endif
DRIVER_MODULE (cx, isa, cx_isa_driver, cx_devclass, cx_modevent, NULL);
MODULE_VERSION (cx, 1);
Index: head/sys/dev/cxgb/cxgb_main.c
===================================================================
--- head/sys/dev/cxgb/cxgb_main.c (revision 283290)
+++ head/sys/dev/cxgb/cxgb_main.c (revision 283291)
@@ -1,3574 +1,3574 @@
/**************************************************************************
Copyright (c) 2007-2009, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***************************************************************************/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/bus.h>
#include <sys/module.h>
#include <sys/pciio.h>
#include <sys/conf.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus_dma.h>
#include <sys/ktr.h>
#include <sys/rman.h>
#include <sys/ioccom.h>
#include <sys/mbuf.h>
#include <sys/linker.h>
#include <sys/firmware.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/queue.h>
#include <sys/taskqueue.h>
#include <sys/proc.h>
#include <net/bpf.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_arp.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/if_types.h>
#include <net/if_vlan_var.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/if_ether.h>
#include <netinet/ip.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <netinet/udp.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/pci_private.h>
#include <cxgb_include.h>
#ifdef PRIV_SUPPORTED
#include <sys/priv.h>
#endif
static int cxgb_setup_interrupts(adapter_t *);
static void cxgb_teardown_interrupts(adapter_t *);
static void cxgb_init(void *);
static int cxgb_init_locked(struct port_info *);
static int cxgb_uninit_locked(struct port_info *);
static int cxgb_uninit_synchronized(struct port_info *);
static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
static int cxgb_media_change(struct ifnet *);
static int cxgb_ifm_type(int);
static void cxgb_build_medialist(struct port_info *);
static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
static uint64_t cxgb_get_counter(struct ifnet *, ift_counter);
static int setup_sge_qsets(adapter_t *);
static void cxgb_async_intr(void *);
static void cxgb_tick_handler(void *, int);
static void cxgb_tick(void *);
static void link_check_callout(void *);
static void check_link_status(void *, int);
static void setup_rss(adapter_t *sc);
static int alloc_filters(struct adapter *);
static int setup_hw_filters(struct adapter *);
static int set_filter(struct adapter *, int, const struct filter_info *);
static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
unsigned int, u64, u64);
static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
unsigned int, u64, u64);
#ifdef TCP_OFFLOAD
static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
#endif
/* Attachment glue for the PCI controller end of the device. Each port of
* the device is attached separately, as defined later.
*/
static int cxgb_controller_probe(device_t);
static int cxgb_controller_attach(device_t);
static int cxgb_controller_detach(device_t);
static void cxgb_free(struct adapter *);
static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
unsigned int end);
static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
static int cxgb_get_regs_len(void);
static void touch_bars(device_t dev);
static void cxgb_update_mac_settings(struct port_info *p);
#ifdef TCP_OFFLOAD
static int toe_capability(struct port_info *, int);
#endif
static device_method_t cxgb_controller_methods[] = {
DEVMETHOD(device_probe, cxgb_controller_probe),
DEVMETHOD(device_attach, cxgb_controller_attach),
DEVMETHOD(device_detach, cxgb_controller_detach),
DEVMETHOD_END
};
static driver_t cxgb_controller_driver = {
"cxgbc",
cxgb_controller_methods,
sizeof(struct adapter)
};
static int cxgbc_mod_event(module_t, int, void *);
static devclass_t cxgb_controller_devclass;
DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
cxgbc_mod_event, 0);
MODULE_VERSION(cxgbc, 1);
MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
/*
* Attachment glue for the ports. Attachment is done directly to the
* controller device.
*/
static int cxgb_port_probe(device_t);
static int cxgb_port_attach(device_t);
static int cxgb_port_detach(device_t);
static device_method_t cxgb_port_methods[] = {
DEVMETHOD(device_probe, cxgb_port_probe),
DEVMETHOD(device_attach, cxgb_port_attach),
DEVMETHOD(device_detach, cxgb_port_detach),
{ 0, 0 }
};
static driver_t cxgb_port_driver = {
"cxgb",
cxgb_port_methods,
0
};
static d_ioctl_t cxgb_extension_ioctl;
static d_open_t cxgb_extension_open;
static d_close_t cxgb_extension_close;
static struct cdevsw cxgb_cdevsw = {
.d_version = D_VERSION,
.d_flags = 0,
.d_open = cxgb_extension_open,
.d_close = cxgb_extension_close,
.d_ioctl = cxgb_extension_ioctl,
.d_name = "cxgb",
};
static devclass_t cxgb_port_devclass;
DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
MODULE_VERSION(cxgb, 1);
static struct mtx t3_list_lock;
static SLIST_HEAD(, adapter) t3_list;
#ifdef TCP_OFFLOAD
static struct mtx t3_uld_list_lock;
static SLIST_HEAD(, uld_info) t3_uld_list;
#endif
/*
* The driver uses the best interrupt scheme available on a platform in the
* order MSI-X, MSI, legacy pin interrupts. This parameter determines which
* of these schemes the driver may consider as follows:
*
* msi = 2: choose from among all three options
* msi = 1 : only consider MSI and pin interrupts
* msi = 0: force pin interrupts
*/
static int msi_allowed = 2;
SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
"MSI-X, MSI, INTx selector");
/*
* The driver uses an auto-queue algorithm by default.
* To disable it and force a single queue-set per port, use multiq = 0
*/
static int multiq = 1;
SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
"use min(ncpus/ports, 8) queue-sets per port");
/*
* By default the driver will not update the firmware unless
* it was compiled against a newer version
*
*/
static int force_fw_update = 0;
SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
"update firmware even if up to date");
int cxgb_use_16k_clusters = -1;
SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
&cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
static int nfilters = -1;
SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
&nfilters, 0, "max number of entries in the filter table");
enum {
MAX_TXQ_ENTRIES = 16384,
MAX_CTRL_TXQ_ENTRIES = 1024,
MAX_RSPQ_ENTRIES = 16384,
MAX_RX_BUFFERS = 16384,
MAX_RX_JUMBO_BUFFERS = 16384,
MIN_TXQ_ENTRIES = 4,
MIN_CTRL_TXQ_ENTRIES = 4,
MIN_RSPQ_ENTRIES = 32,
MIN_FL_ENTRIES = 32,
MIN_FL_JUMBO_ENTRIES = 32
};
struct filter_info {
u32 sip;
u32 sip_mask;
u32 dip;
u16 sport;
u16 dport;
u32 vlan:12;
u32 vlan_prio:3;
u32 mac_hit:1;
u32 mac_idx:4;
u32 mac_vld:1;
u32 pkt_type:2;
u32 report_filter_id:1;
u32 pass:1;
u32 rss:1;
u32 qset:3;
u32 locked:1;
u32 valid:1;
};
enum { FILTER_NO_VLAN_PRI = 7 };
#define EEPROM_MAGIC 0x38E2F10C
#define PORT_MASK ((1 << MAX_NPORTS) - 1)
/* Table for probing the cards. The desc field isn't actually used */
struct cxgb_ident {
uint16_t vendor;
uint16_t device;
int index;
char *desc;
} cxgb_identifiers[] = {
{PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
{PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
{PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
{PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
{PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
{PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
{PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
{PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
{PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
{PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
{PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
{PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
{PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
{PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
{0, 0, 0, NULL}
};
static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
static __inline char
t3rev2char(struct adapter *adapter)
{
char rev = 'z';
switch(adapter->params.rev) {
case T3_REV_A:
rev = 'a';
break;
case T3_REV_B:
case T3_REV_B2:
rev = 'b';
break;
case T3_REV_C:
rev = 'c';
break;
}
return rev;
}
static struct cxgb_ident *
cxgb_get_ident(device_t dev)
{
struct cxgb_ident *id;
for (id = cxgb_identifiers; id->desc != NULL; id++) {
if ((id->vendor == pci_get_vendor(dev)) &&
(id->device == pci_get_device(dev))) {
return (id);
}
}
return (NULL);
}
static const struct adapter_info *
cxgb_get_adapter_info(device_t dev)
{
struct cxgb_ident *id;
const struct adapter_info *ai;
id = cxgb_get_ident(dev);
if (id == NULL)
return (NULL);
ai = t3_get_adapter_info(id->index);
return (ai);
}
static int
cxgb_controller_probe(device_t dev)
{
const struct adapter_info *ai;
char *ports, buf[80];
int nports;
ai = cxgb_get_adapter_info(dev);
if (ai == NULL)
return (ENXIO);
nports = ai->nports0 + ai->nports1;
if (nports == 1)
ports = "port";
else
ports = "ports";
snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
device_set_desc_copy(dev, buf);
return (BUS_PROBE_DEFAULT);
}
#define FW_FNAME "cxgb_t3fw"
#define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
#define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
static int
upgrade_fw(adapter_t *sc)
{
const struct firmware *fw;
int status;
u32 vers;
if ((fw = firmware_get(FW_FNAME)) == NULL) {
device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
return (ENOENT);
} else
device_printf(sc->dev, "installing firmware on card\n");
status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
if (status != 0) {
device_printf(sc->dev, "failed to install firmware: %d\n",
status);
} else {
t3_get_fw_version(sc, &vers);
snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
G_FW_VERSION_MICRO(vers));
}
firmware_put(fw, FIRMWARE_UNLOAD);
return (status);
}
/*
* The cxgb_controller_attach function is responsible for the initial
* bringup of the device. Its responsibilities include:
*
* 1. Determine if the device supports MSI or MSI-X.
* 2. Allocate bus resources so that we can access the Base Address Register
* 3. Create and initialize mutexes for the controller and its control
* logic such as SGE and MDIO.
* 4. Call hardware specific setup routine for the adapter as a whole.
* 5. Allocate the BAR for doing MSI-X.
* 6. Setup the line interrupt iff MSI-X is not supported.
* 7. Create the driver's taskq.
* 8. Start one task queue service thread.
* 9. Check if the firmware and SRAM are up-to-date. They will be
* auto-updated later (before FULL_INIT_DONE), if required.
* 10. Create a child device for each MAC (port)
* 11. Initialize T3 private state.
* 12. Trigger the LED
* 13. Setup offload iff supported.
* 14. Reset/restart the tick callout.
* 15. Attach sysctls
*
* NOTE: Any modification or deviation from this list MUST be reflected in
* the above comment. Failure to do so will result in problems on various
* error conditions including link flapping.
*/
static int
cxgb_controller_attach(device_t dev)
{
device_t child;
const struct adapter_info *ai;
struct adapter *sc;
int i, error = 0;
uint32_t vers;
int port_qsets = 1;
int msi_needed, reg;
char buf[80];
sc = device_get_softc(dev);
sc->dev = dev;
sc->msi_count = 0;
ai = cxgb_get_adapter_info(dev);
snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
device_get_unit(dev));
ADAPTER_LOCK_INIT(sc, sc->lockbuf);
snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
device_get_unit(dev));
snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
device_get_unit(dev));
snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
device_get_unit(dev));
MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
mtx_lock(&t3_list_lock);
SLIST_INSERT_HEAD(&t3_list, sc, link);
mtx_unlock(&t3_list_lock);
/* find the PCIe link width and set max read request to 4KB*/
if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
uint16_t lnk;
lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
if (sc->link_width < 8 &&
(ai->caps & SUPPORTED_10000baseT_Full)) {
device_printf(sc->dev,
"PCIe x%d Link, expect reduced performance\n",
sc->link_width);
}
pci_set_max_read_req(dev, 4096);
}
touch_bars(dev);
pci_enable_busmaster(dev);
/*
* Allocate the registers and make them available to the driver.
* The registers that we care about for NIC mode are in BAR 0
*/
sc->regs_rid = PCIR_BAR(0);
if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
&sc->regs_rid, RF_ACTIVE)) == NULL) {
device_printf(dev, "Cannot allocate BAR region 0\n");
error = ENXIO;
goto out;
}
sc->bt = rman_get_bustag(sc->regs_res);
sc->bh = rman_get_bushandle(sc->regs_res);
sc->mmio_len = rman_get_size(sc->regs_res);
for (i = 0; i < MAX_NPORTS; i++)
sc->port[i].adapter = sc;
if (t3_prep_adapter(sc, ai, 1) < 0) {
printf("prep adapter failed\n");
error = ENODEV;
goto out;
}
sc->udbs_rid = PCIR_BAR(2);
sc->udbs_res = NULL;
if (is_offload(sc) &&
((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
&sc->udbs_rid, RF_ACTIVE)) == NULL)) {
device_printf(dev, "Cannot allocate BAR region 1\n");
error = ENXIO;
goto out;
}
/* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
* enough messages for the queue sets. If that fails, try falling
* back to MSI. If that fails, then try falling back to the legacy
* interrupt pin model.
*/
sc->msix_regs_rid = 0x20;
if ((msi_allowed >= 2) &&
(sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
&sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
if (multiq)
port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
if (pci_msix_count(dev) == 0 ||
(error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
sc->msi_count != msi_needed) {
device_printf(dev, "alloc msix failed - "
"msi_count=%d, msi_needed=%d, err=%d; "
"will try MSI\n", sc->msi_count,
msi_needed, error);
sc->msi_count = 0;
port_qsets = 1;
pci_release_msi(dev);
bus_release_resource(dev, SYS_RES_MEMORY,
sc->msix_regs_rid, sc->msix_regs_res);
sc->msix_regs_res = NULL;
} else {
sc->flags |= USING_MSIX;
sc->cxgb_intr = cxgb_async_intr;
device_printf(dev,
"using MSI-X interrupts (%u vectors)\n",
sc->msi_count);
}
}
if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
sc->msi_count = 1;
if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
device_printf(dev, "alloc msi failed - "
"err=%d; will try INTx\n", error);
sc->msi_count = 0;
port_qsets = 1;
pci_release_msi(dev);
} else {
sc->flags |= USING_MSI;
sc->cxgb_intr = t3_intr_msi;
device_printf(dev, "using MSI interrupts\n");
}
}
if (sc->msi_count == 0) {
device_printf(dev, "using line interrupts\n");
sc->cxgb_intr = t3b_intr;
}
/* Create a private taskqueue thread for handling driver events */
sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
taskqueue_thread_enqueue, &sc->tq);
if (sc->tq == NULL) {
device_printf(dev, "failed to allocate controller task queue\n");
goto out;
}
taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
device_get_nameunit(dev));
TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
/* Create a periodic callout for checking adapter status */
- callout_init(&sc->cxgb_tick_ch, TRUE);
+ callout_init(&sc->cxgb_tick_ch, 1);
if (t3_check_fw_version(sc) < 0 || force_fw_update) {
/*
* Warn user that a firmware update will be attempted in init.
*/
device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
sc->flags &= ~FW_UPTODATE;
} else {
sc->flags |= FW_UPTODATE;
}
if (t3_check_tpsram_version(sc) < 0) {
/*
* Warn user that a firmware update will be attempted in init.
*/
device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
sc->flags &= ~TPS_UPTODATE;
} else {
sc->flags |= TPS_UPTODATE;
}
/*
* Create a child device for each MAC. The ethernet attachment
* will be done in these children.
*/
for (i = 0; i < (sc)->params.nports; i++) {
struct port_info *pi;
if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
device_printf(dev, "failed to add child port\n");
error = EINVAL;
goto out;
}
pi = &sc->port[i];
pi->adapter = sc;
pi->nqsets = port_qsets;
pi->first_qset = i*port_qsets;
pi->port_id = i;
pi->tx_chan = i >= ai->nports0;
pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
sc->rxpkt_map[pi->txpkt_intf] = i;
sc->port[i].tx_chan = i >= ai->nports0;
sc->portdev[i] = child;
device_set_softc(child, pi);
}
if ((error = bus_generic_attach(dev)) != 0)
goto out;
/* initialize sge private state */
t3_sge_init_adapter(sc);
t3_led_ready(sc);
error = t3_get_fw_version(sc, &vers);
if (error)
goto out;
snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
G_FW_VERSION_MICRO(vers));
snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
ai->desc, is_offload(sc) ? "R" : "",
sc->params.vpd.ec, sc->params.vpd.sn);
device_set_desc_copy(dev, buf);
snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
t3_add_attach_sysctls(sc);
#ifdef TCP_OFFLOAD
for (i = 0; i < NUM_CPL_HANDLERS; i++)
sc->cpl_handler[i] = cpl_not_handled;
#endif
t3_intr_clear(sc);
error = cxgb_setup_interrupts(sc);
out:
if (error)
cxgb_free(sc);
return (error);
}
/*
* The cxgb_controller_detach routine is called with the device is
* unloaded from the system.
*/
static int
cxgb_controller_detach(device_t dev)
{
struct adapter *sc;
sc = device_get_softc(dev);
cxgb_free(sc);
return (0);
}
/*
* The cxgb_free() is called by the cxgb_controller_detach() routine
* to tear down the structures that were built up in
* cxgb_controller_attach(), and should be the final piece of work
* done when fully unloading the driver.
*
*
* 1. Shutting down the threads started by the cxgb_controller_attach()
* routine.
* 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
* 3. Detaching all of the port devices created during the
* cxgb_controller_attach() routine.
* 4. Removing the device children created via cxgb_controller_attach().
* 5. Releasing PCI resources associated with the device.
* 6. Turning off the offload support, iff it was turned on.
* 7. Destroying the mutexes created in cxgb_controller_attach().
*
*/
static void
cxgb_free(struct adapter *sc)
{
int i, nqsets = 0;
ADAPTER_LOCK(sc);
sc->flags |= CXGB_SHUTDOWN;
ADAPTER_UNLOCK(sc);
/*
* Make sure all child devices are gone.
*/
bus_generic_detach(sc->dev);
for (i = 0; i < (sc)->params.nports; i++) {
if (sc->portdev[i] &&
device_delete_child(sc->dev, sc->portdev[i]) != 0)
device_printf(sc->dev, "failed to delete child port\n");
nqsets += sc->port[i].nqsets;
}
/*
* At this point, it is as if cxgb_port_detach has run on all ports, and
* cxgb_down has run on the adapter. All interrupts have been silenced,
* all open devices have been closed.
*/
KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
__func__, sc->open_device_map));
for (i = 0; i < sc->params.nports; i++) {
KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
__func__, i));
}
/*
* Finish off the adapter's callouts.
*/
callout_drain(&sc->cxgb_tick_ch);
callout_drain(&sc->sge_timer_ch);
/*
* Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
* sysctls are cleaned up by the kernel linker.
*/
if (sc->flags & FULL_INIT_DONE) {
t3_free_sge_resources(sc, nqsets);
sc->flags &= ~FULL_INIT_DONE;
}
/*
* Release all interrupt resources.
*/
cxgb_teardown_interrupts(sc);
if (sc->flags & (USING_MSI | USING_MSIX)) {
device_printf(sc->dev, "releasing msi message(s)\n");
pci_release_msi(sc->dev);
} else {
device_printf(sc->dev, "no msi message to release\n");
}
if (sc->msix_regs_res != NULL) {
bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
sc->msix_regs_res);
}
/*
* Free the adapter's taskqueue.
*/
if (sc->tq != NULL) {
taskqueue_free(sc->tq);
sc->tq = NULL;
}
free(sc->filters, M_DEVBUF);
t3_sge_free(sc);
if (sc->udbs_res != NULL)
bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
sc->udbs_res);
if (sc->regs_res != NULL)
bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
sc->regs_res);
MTX_DESTROY(&sc->mdio_lock);
MTX_DESTROY(&sc->sge.reg_lock);
MTX_DESTROY(&sc->elmer_lock);
mtx_lock(&t3_list_lock);
SLIST_REMOVE(&t3_list, sc, adapter, link);
mtx_unlock(&t3_list_lock);
ADAPTER_LOCK_DEINIT(sc);
}
/**
* setup_sge_qsets - configure SGE Tx/Rx/response queues
* @sc: the controller softc
*
* Determines how many sets of SGE queues to use and initializes them.
* We support multiple queue sets per port if we have MSI-X, otherwise
* just one queue set per port.
*/
static int
setup_sge_qsets(adapter_t *sc)
{
int i, j, err, irq_idx = 0, qset_idx = 0;
u_int ntxq = SGE_TXQ_PER_SET;
if ((err = t3_sge_alloc(sc)) != 0) {
device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
return (err);
}
if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
irq_idx = -1;
for (i = 0; i < (sc)->params.nports; i++) {
struct port_info *pi = &sc->port[i];
for (j = 0; j < pi->nqsets; j++, qset_idx++) {
err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
(sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
&sc->params.sge.qset[qset_idx], ntxq, pi);
if (err) {
t3_free_sge_resources(sc, qset_idx);
device_printf(sc->dev,
"t3_sge_alloc_qset failed with %d\n", err);
return (err);
}
}
}
return (0);
}
static void
cxgb_teardown_interrupts(adapter_t *sc)
{
int i;
for (i = 0; i < SGE_QSETS; i++) {
if (sc->msix_intr_tag[i] == NULL) {
/* Should have been setup fully or not at all */
KASSERT(sc->msix_irq_res[i] == NULL &&
sc->msix_irq_rid[i] == 0,
("%s: half-done interrupt (%d).", __func__, i));
continue;
}
bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
sc->msix_intr_tag[i]);
bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
sc->msix_irq_res[i]);
sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
sc->msix_irq_rid[i] = 0;
}
if (sc->intr_tag) {
KASSERT(sc->irq_res != NULL,
("%s: half-done interrupt.", __func__));
bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
sc->irq_res);
sc->irq_res = sc->intr_tag = NULL;
sc->irq_rid = 0;
}
}
static int
cxgb_setup_interrupts(adapter_t *sc)
{
struct resource *res;
void *tag;
int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
sc->irq_rid = intr_flag ? 1 : 0;
sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
RF_SHAREABLE | RF_ACTIVE);
if (sc->irq_res == NULL) {
device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
intr_flag, sc->irq_rid);
err = EINVAL;
sc->irq_rid = 0;
} else {
err = bus_setup_intr(sc->dev, sc->irq_res,
INTR_MPSAFE | INTR_TYPE_NET, NULL,
sc->cxgb_intr, sc, &sc->intr_tag);
if (err) {
device_printf(sc->dev,
"Cannot set up interrupt (%x, %u, %d)\n",
intr_flag, sc->irq_rid, err);
bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
sc->irq_res);
sc->irq_res = sc->intr_tag = NULL;
sc->irq_rid = 0;
}
}
/* That's all for INTx or MSI */
if (!(intr_flag & USING_MSIX) || err)
return (err);
bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
for (i = 0; i < sc->msi_count - 1; i++) {
rid = i + 2;
res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
RF_SHAREABLE | RF_ACTIVE);
if (res == NULL) {
device_printf(sc->dev, "Cannot allocate interrupt "
"for message %d\n", rid);
err = EINVAL;
break;
}
err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
if (err) {
device_printf(sc->dev, "Cannot set up interrupt "
"for message %d (%d)\n", rid, err);
bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
break;
}
sc->msix_irq_rid[i] = rid;
sc->msix_irq_res[i] = res;
sc->msix_intr_tag[i] = tag;
bus_describe_intr(sc->dev, res, tag, "qs%d", i);
}
if (err)
cxgb_teardown_interrupts(sc);
return (err);
}
static int
cxgb_port_probe(device_t dev)
{
struct port_info *p;
char buf[80];
const char *desc;
p = device_get_softc(dev);
desc = p->phy.desc;
snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
device_set_desc_copy(dev, buf);
return (0);
}
static int
cxgb_makedev(struct port_info *pi)
{
pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
if (pi->port_cdev == NULL)
return (ENOMEM);
pi->port_cdev->si_drv1 = (void *)pi;
return (0);
}
#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
#define CXGB_CAP_ENABLE CXGB_CAP
static int
cxgb_port_attach(device_t dev)
{
struct port_info *p;
struct ifnet *ifp;
int err;
struct adapter *sc;
p = device_get_softc(dev);
sc = p->adapter;
snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
device_get_unit(device_get_parent(dev)), p->port_id);
PORT_LOCK_INIT(p, p->lockbuf);
- callout_init(&p->link_check_ch, CALLOUT_MPSAFE);
+ callout_init(&p->link_check_ch, 1);
TASK_INIT(&p->link_check_task, 0, check_link_status, p);
/* Allocate an ifnet object and set it up */
ifp = p->ifp = if_alloc(IFT_ETHER);
if (ifp == NULL) {
device_printf(dev, "Cannot allocate ifnet\n");
return (ENOMEM);
}
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_init = cxgb_init;
ifp->if_softc = p;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = cxgb_ioctl;
ifp->if_transmit = cxgb_transmit;
ifp->if_qflush = cxgb_qflush;
ifp->if_get_counter = cxgb_get_counter;
ifp->if_capabilities = CXGB_CAP;
#ifdef TCP_OFFLOAD
if (is_offload(sc))
ifp->if_capabilities |= IFCAP_TOE4;
#endif
ifp->if_capenable = CXGB_CAP_ENABLE;
ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
/*
* Disable TSO on 4-port - it isn't supported by the firmware.
*/
if (sc->params.nports > 2) {
ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
ifp->if_hwassist &= ~CSUM_TSO;
}
ether_ifattach(ifp, p->hw_addr);
#ifdef DEFAULT_JUMBO
if (sc->params.nports <= 2)
ifp->if_mtu = ETHERMTU_JUMBO;
#endif
if ((err = cxgb_makedev(p)) != 0) {
printf("makedev failed %d\n", err);
return (err);
}
/* Create a list of media supported by this port */
ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
cxgb_media_status);
cxgb_build_medialist(p);
t3_sge_init_port(p);
return (err);
}
/*
* cxgb_port_detach() is called via the device_detach methods when
* cxgb_free() calls the bus_generic_detach. It is responsible for
* removing the device from the view of the kernel, i.e. from all
* interfaces lists etc. This routine is only called when the driver is
* being unloaded, not when the link goes down.
*/
static int
cxgb_port_detach(device_t dev)
{
struct port_info *p;
struct adapter *sc;
int i;
p = device_get_softc(dev);
sc = p->adapter;
/* Tell cxgb_ioctl and if_init that the port is going away */
ADAPTER_LOCK(sc);
SET_DOOMED(p);
wakeup(&sc->flags);
while (IS_BUSY(sc))
mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
SET_BUSY(sc);
ADAPTER_UNLOCK(sc);
if (p->port_cdev != NULL)
destroy_dev(p->port_cdev);
cxgb_uninit_synchronized(p);
ether_ifdetach(p->ifp);
for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
struct sge_qset *qs = &sc->sge.qs[i];
struct sge_txq *txq = &qs->txq[TXQ_ETH];
callout_drain(&txq->txq_watchdog);
callout_drain(&txq->txq_timer);
}
PORT_LOCK_DEINIT(p);
if_free(p->ifp);
p->ifp = NULL;
ADAPTER_LOCK(sc);
CLR_BUSY(sc);
wakeup_one(&sc->flags);
ADAPTER_UNLOCK(sc);
return (0);
}
void
t3_fatal_err(struct adapter *sc)
{
u_int fw_status[4];
if (sc->flags & FULL_INIT_DONE) {
t3_sge_stop(sc);
t3_write_reg(sc, A_XGM_TX_CTRL, 0);
t3_write_reg(sc, A_XGM_RX_CTRL, 0);
t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
t3_intr_disable(sc);
}
device_printf(sc->dev,"encountered fatal error, operation suspended\n");
if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
}
int
t3_os_find_pci_capability(adapter_t *sc, int cap)
{
device_t dev;
struct pci_devinfo *dinfo;
pcicfgregs *cfg;
uint32_t status;
uint8_t ptr;
dev = sc->dev;
dinfo = device_get_ivars(dev);
cfg = &dinfo->cfg;
status = pci_read_config(dev, PCIR_STATUS, 2);
if (!(status & PCIM_STATUS_CAPPRESENT))
return (0);
switch (cfg->hdrtype & PCIM_HDRTYPE) {
case 0:
case 1:
ptr = PCIR_CAP_PTR;
break;
case 2:
ptr = PCIR_CAP_PTR_2;
break;
default:
return (0);
break;
}
ptr = pci_read_config(dev, ptr, 1);
while (ptr != 0) {
if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
return (ptr);
ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
}
return (0);
}
int
t3_os_pci_save_state(struct adapter *sc)
{
device_t dev;
struct pci_devinfo *dinfo;
dev = sc->dev;
dinfo = device_get_ivars(dev);
pci_cfg_save(dev, dinfo, 0);
return (0);
}
int
t3_os_pci_restore_state(struct adapter *sc)
{
device_t dev;
struct pci_devinfo *dinfo;
dev = sc->dev;
dinfo = device_get_ivars(dev);
pci_cfg_restore(dev, dinfo);
return (0);
}
/**
* t3_os_link_changed - handle link status changes
* @sc: the adapter associated with the link change
* @port_id: the port index whose link status has changed
* @link_status: the new status of the link
* @speed: the new speed setting
* @duplex: the new duplex setting
* @fc: the new flow-control setting
*
* This is the OS-dependent handler for link status changes. The OS
* neutral handler takes care of most of the processing for these events,
* then calls this handler for any OS-specific processing.
*/
void
t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
int duplex, int fc, int mac_was_reset)
{
struct port_info *pi = &adapter->port[port_id];
struct ifnet *ifp = pi->ifp;
/* no race with detach, so ifp should always be good */
KASSERT(ifp, ("%s: if detached.", __func__));
/* Reapply mac settings if they were lost due to a reset */
if (mac_was_reset) {
PORT_LOCK(pi);
cxgb_update_mac_settings(pi);
PORT_UNLOCK(pi);
}
if (link_status) {
ifp->if_baudrate = IF_Mbps(speed);
if_link_state_change(ifp, LINK_STATE_UP);
} else
if_link_state_change(ifp, LINK_STATE_DOWN);
}
/**
* t3_os_phymod_changed - handle PHY module changes
* @phy: the PHY reporting the module change
* @mod_type: new module type
*
* This is the OS-dependent handler for PHY module changes. It is
* invoked when a PHY module is removed or inserted for any OS-specific
* processing.
*/
void t3_os_phymod_changed(struct adapter *adap, int port_id)
{
static const char *mod_str[] = {
NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
};
struct port_info *pi = &adap->port[port_id];
int mod = pi->phy.modtype;
if (mod != pi->media.ifm_cur->ifm_data)
cxgb_build_medialist(pi);
if (mod == phy_modtype_none)
if_printf(pi->ifp, "PHY module unplugged\n");
else {
KASSERT(mod < ARRAY_SIZE(mod_str),
("invalid PHY module type %d", mod));
if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
}
}
void
t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
{
/*
* The ifnet might not be allocated before this gets called,
* as this is called early on in attach by t3_prep_adapter
* save the address off in the port structure
*/
if (cxgb_debug)
printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
}
/*
* Programs the XGMAC based on the settings in the ifnet. These settings
* include MTU, MAC address, mcast addresses, etc.
*/
static void
cxgb_update_mac_settings(struct port_info *p)
{
struct ifnet *ifp = p->ifp;
struct t3_rx_mode rm;
struct cmac *mac = &p->mac;
int mtu, hwtagging;
PORT_LOCK_ASSERT_OWNED(p);
bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
mtu = ifp->if_mtu;
if (ifp->if_capenable & IFCAP_VLAN_MTU)
mtu += ETHER_VLAN_ENCAP_LEN;
hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
t3_mac_set_mtu(mac, mtu);
t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
t3_mac_set_address(mac, 0, p->hw_addr);
t3_init_rx_mode(&rm, p);
t3_mac_set_rx_mode(mac, &rm);
}
static int
await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
unsigned long n)
{
int attempts = 5;
while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
if (!--attempts)
return (ETIMEDOUT);
t3_os_sleep(10);
}
return 0;
}
static int
init_tp_parity(struct adapter *adap)
{
int i;
struct mbuf *m;
struct cpl_set_tcb_field *greq;
unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
t3_tp_set_offload_mode(adap, 1);
for (i = 0; i < 16; i++) {
struct cpl_smt_write_req *req;
m = m_gethdr(M_WAITOK, MT_DATA);
req = mtod(m, struct cpl_smt_write_req *);
m->m_len = m->m_pkthdr.len = sizeof(*req);
memset(req, 0, sizeof(*req));
req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
req->iff = i;
t3_mgmt_tx(adap, m);
}
for (i = 0; i < 2048; i++) {
struct cpl_l2t_write_req *req;
m = m_gethdr(M_WAITOK, MT_DATA);
req = mtod(m, struct cpl_l2t_write_req *);
m->m_len = m->m_pkthdr.len = sizeof(*req);
memset(req, 0, sizeof(*req));
req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
req->params = htonl(V_L2T_W_IDX(i));
t3_mgmt_tx(adap, m);
}
for (i = 0; i < 2048; i++) {
struct cpl_rte_write_req *req;
m = m_gethdr(M_WAITOK, MT_DATA);
req = mtod(m, struct cpl_rte_write_req *);
m->m_len = m->m_pkthdr.len = sizeof(*req);
memset(req, 0, sizeof(*req));
req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
req->l2t_idx = htonl(V_L2T_W_IDX(i));
t3_mgmt_tx(adap, m);
}
m = m_gethdr(M_WAITOK, MT_DATA);
greq = mtod(m, struct cpl_set_tcb_field *);
m->m_len = m->m_pkthdr.len = sizeof(*greq);
memset(greq, 0, sizeof(*greq));
greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
greq->mask = htobe64(1);
t3_mgmt_tx(adap, m);
i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
t3_tp_set_offload_mode(adap, 0);
return (i);
}
/**
* setup_rss - configure Receive Side Steering (per-queue connection demux)
* @adap: the adapter
*
* Sets up RSS to distribute packets to multiple receive queues. We
* configure the RSS CPU lookup table to distribute to the number of HW
* receive queues, and the response queue lookup table to narrow that
* down to the response queues actually configured for each port.
* We always configure the RSS mapping for two ports since the mapping
* table has plenty of entries.
*/
static void
setup_rss(adapter_t *adap)
{
int i;
u_int nq[2];
uint8_t cpus[SGE_QSETS + 1];
uint16_t rspq_map[RSS_TABLE_SIZE];
for (i = 0; i < SGE_QSETS; ++i)
cpus[i] = i;
cpus[SGE_QSETS] = 0xff;
nq[0] = nq[1] = 0;
for_each_port(adap, i) {
const struct port_info *pi = adap2pinfo(adap, i);
nq[pi->tx_chan] += pi->nqsets;
}
for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
rspq_map[i] = nq[0] ? i % nq[0] : 0;
rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
}
/* Calculate the reverse RSS map table */
for (i = 0; i < SGE_QSETS; ++i)
adap->rrss_map[i] = 0xff;
for (i = 0; i < RSS_TABLE_SIZE; ++i)
if (adap->rrss_map[rspq_map[i]] == 0xff)
adap->rrss_map[rspq_map[i]] = i;
t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
cpus, rspq_map);
}
static void
send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
int hi, int port)
{
struct mbuf *m;
struct mngt_pktsched_wr *req;
m = m_gethdr(M_NOWAIT, MT_DATA);
if (m) {
req = mtod(m, struct mngt_pktsched_wr *);
req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
req->sched = sched;
req->idx = qidx;
req->min = lo;
req->max = hi;
req->binding = port;
m->m_len = m->m_pkthdr.len = sizeof(*req);
t3_mgmt_tx(adap, m);
}
}
static void
bind_qsets(adapter_t *sc)
{
int i, j;
for (i = 0; i < (sc)->params.nports; ++i) {
const struct port_info *pi = adap2pinfo(sc, i);
for (j = 0; j < pi->nqsets; ++j) {
send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
-1, pi->tx_chan);
}
}
}
static void
update_tpeeprom(struct adapter *adap)
{
const struct firmware *tpeeprom;
uint32_t version;
unsigned int major, minor;
int ret, len;
char rev, name[32];
t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
major = G_TP_VERSION_MAJOR(version);
minor = G_TP_VERSION_MINOR(version);
if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
return;
rev = t3rev2char(adap);
snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
tpeeprom = firmware_get(name);
if (tpeeprom == NULL) {
device_printf(adap->dev,
"could not load TP EEPROM: unable to load %s\n",
name);
return;
}
len = tpeeprom->datasize - 4;
ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
if (ret)
goto release_tpeeprom;
if (len != TP_SRAM_LEN) {
device_printf(adap->dev,
"%s length is wrong len=%d expected=%d\n", name,
len, TP_SRAM_LEN);
return;
}
ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
TP_SRAM_OFFSET);
if (!ret) {
device_printf(adap->dev,
"Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
} else
device_printf(adap->dev,
"Protocol SRAM image update in EEPROM failed\n");
release_tpeeprom:
firmware_put(tpeeprom, FIRMWARE_UNLOAD);
return;
}
static int
update_tpsram(struct adapter *adap)
{
const struct firmware *tpsram;
int ret;
char rev, name[32];
rev = t3rev2char(adap);
snprintf(name, sizeof(name), TPSRAM_NAME, rev);
update_tpeeprom(adap);
tpsram = firmware_get(name);
if (tpsram == NULL){
device_printf(adap->dev, "could not load TP SRAM\n");
return (EINVAL);
} else
device_printf(adap->dev, "updating TP SRAM\n");
ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
if (ret)
goto release_tpsram;
ret = t3_set_proto_sram(adap, tpsram->data);
if (ret)
device_printf(adap->dev, "loading protocol SRAM failed\n");
release_tpsram:
firmware_put(tpsram, FIRMWARE_UNLOAD);
return ret;
}
/**
* cxgb_up - enable the adapter
* @adap: adapter being enabled
*
* Called when the first port is enabled, this function performs the
* actions necessary to make an adapter operational, such as completing
* the initialization of HW modules, and enabling interrupts.
*/
static int
cxgb_up(struct adapter *sc)
{
int err = 0;
unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
__func__, sc->open_device_map));
if ((sc->flags & FULL_INIT_DONE) == 0) {
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
if ((sc->flags & FW_UPTODATE) == 0)
if ((err = upgrade_fw(sc)))
goto out;
if ((sc->flags & TPS_UPTODATE) == 0)
if ((err = update_tpsram(sc)))
goto out;
if (is_offload(sc) && nfilters != 0) {
sc->params.mc5.nservers = 0;
if (nfilters < 0)
sc->params.mc5.nfilters = mxf;
else
sc->params.mc5.nfilters = min(nfilters, mxf);
}
err = t3_init_hw(sc, 0);
if (err)
goto out;
t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
err = setup_sge_qsets(sc);
if (err)
goto out;
alloc_filters(sc);
setup_rss(sc);
t3_add_configured_sysctls(sc);
sc->flags |= FULL_INIT_DONE;
}
t3_intr_clear(sc);
t3_sge_start(sc);
t3_intr_enable(sc);
if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
is_offload(sc) && init_tp_parity(sc) == 0)
sc->flags |= TP_PARITY_INIT;
if (sc->flags & TP_PARITY_INIT) {
t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
}
if (!(sc->flags & QUEUES_BOUND)) {
bind_qsets(sc);
setup_hw_filters(sc);
sc->flags |= QUEUES_BOUND;
}
t3_sge_reset_adapter(sc);
out:
return (err);
}
/*
* Called when the last open device is closed. Does NOT undo all of cxgb_up's
* work. Specifically, the resources grabbed under FULL_INIT_DONE are released
* during controller_detach, not here.
*/
static void
cxgb_down(struct adapter *sc)
{
t3_sge_stop(sc);
t3_intr_disable(sc);
}
/*
* if_init for cxgb ports.
*/
static void
cxgb_init(void *arg)
{
struct port_info *p = arg;
struct adapter *sc = p->adapter;
ADAPTER_LOCK(sc);
cxgb_init_locked(p); /* releases adapter lock */
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
}
static int
cxgb_init_locked(struct port_info *p)
{
struct adapter *sc = p->adapter;
struct ifnet *ifp = p->ifp;
struct cmac *mac = &p->mac;
int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
ADAPTER_LOCK_ASSERT_OWNED(sc);
while (!IS_DOOMED(p) && IS_BUSY(sc)) {
gave_up_lock = 1;
if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
rc = EINTR;
goto done;
}
}
if (IS_DOOMED(p)) {
rc = ENXIO;
goto done;
}
KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
/*
* The code that runs during one-time adapter initialization can sleep
* so it's important not to hold any locks across it.
*/
may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
if (may_sleep) {
SET_BUSY(sc);
gave_up_lock = 1;
ADAPTER_UNLOCK(sc);
}
if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
goto done;
PORT_LOCK(p);
if (isset(&sc->open_device_map, p->port_id) &&
(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
PORT_UNLOCK(p);
goto done;
}
t3_port_intr_enable(sc, p->port_id);
if (!mac->multiport)
t3_mac_init(mac);
cxgb_update_mac_settings(p);
t3_link_start(&p->phy, mac, &p->link_config);
t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
PORT_UNLOCK(p);
for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
struct sge_qset *qs = &sc->sge.qs[i];
struct sge_txq *txq = &qs->txq[TXQ_ETH];
callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
txq->txq_watchdog.c_cpu);
}
/* all ok */
setbit(&sc->open_device_map, p->port_id);
callout_reset(&p->link_check_ch,
p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4,
link_check_callout, p);
done:
if (may_sleep) {
ADAPTER_LOCK(sc);
KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
CLR_BUSY(sc);
}
if (gave_up_lock)
wakeup_one(&sc->flags);
ADAPTER_UNLOCK(sc);
return (rc);
}
static int
cxgb_uninit_locked(struct port_info *p)
{
struct adapter *sc = p->adapter;
int rc;
ADAPTER_LOCK_ASSERT_OWNED(sc);
while (!IS_DOOMED(p) && IS_BUSY(sc)) {
if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
rc = EINTR;
goto done;
}
}
if (IS_DOOMED(p)) {
rc = ENXIO;
goto done;
}
KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
SET_BUSY(sc);
ADAPTER_UNLOCK(sc);
rc = cxgb_uninit_synchronized(p);
ADAPTER_LOCK(sc);
KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
CLR_BUSY(sc);
wakeup_one(&sc->flags);
done:
ADAPTER_UNLOCK(sc);
return (rc);
}
/*
* Called on "ifconfig down", and from port_detach
*/
static int
cxgb_uninit_synchronized(struct port_info *pi)
{
struct adapter *sc = pi->adapter;
struct ifnet *ifp = pi->ifp;
/*
* taskqueue_drain may cause a deadlock if the adapter lock is held.
*/
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
/*
* Clear this port's bit from the open device map, and then drain all
* the tasks that can access/manipulate this port's port_info or ifp.
* We disable this port's interrupts here and so the slow/ext
* interrupt tasks won't be enqueued. The tick task will continue to
* be enqueued every second but the runs after this drain will not see
* this port in the open device map.
*
* A well behaved task must take open_device_map into account and ignore
* ports that are not open.
*/
clrbit(&sc->open_device_map, pi->port_id);
t3_port_intr_disable(sc, pi->port_id);
taskqueue_drain(sc->tq, &sc->slow_intr_task);
taskqueue_drain(sc->tq, &sc->tick_task);
callout_drain(&pi->link_check_ch);
taskqueue_drain(sc->tq, &pi->link_check_task);
PORT_LOCK(pi);
ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
/* disable pause frames */
t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
/* Reset RX FIFO HWM */
t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
DELAY(100 * 1000);
/* Wait for TXFIFO empty */
t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
F_TXFIFO_EMPTY, 1, 20, 5);
DELAY(100 * 1000);
t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
pi->phy.ops->power_down(&pi->phy, 1);
PORT_UNLOCK(pi);
pi->link_config.link_ok = 0;
t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
if (sc->open_device_map == 0)
cxgb_down(pi->adapter);
return (0);
}
/*
* Mark lro enabled or disabled in all qsets for this port
*/
static int
cxgb_set_lro(struct port_info *p, int enabled)
{
int i;
struct adapter *adp = p->adapter;
struct sge_qset *q;
for (i = 0; i < p->nqsets; i++) {
q = &adp->sge.qs[p->first_qset + i];
q->lro.enabled = (enabled != 0);
}
return (0);
}
static int
cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
{
struct port_info *p = ifp->if_softc;
struct adapter *sc = p->adapter;
struct ifreq *ifr = (struct ifreq *)data;
int flags, error = 0, mtu;
uint32_t mask;
switch (command) {
case SIOCSIFMTU:
ADAPTER_LOCK(sc);
error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
if (error) {
fail:
ADAPTER_UNLOCK(sc);
return (error);
}
mtu = ifr->ifr_mtu;
if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
error = EINVAL;
} else {
ifp->if_mtu = mtu;
PORT_LOCK(p);
cxgb_update_mac_settings(p);
PORT_UNLOCK(p);
}
ADAPTER_UNLOCK(sc);
break;
case SIOCSIFFLAGS:
ADAPTER_LOCK(sc);
if (IS_DOOMED(p)) {
error = ENXIO;
goto fail;
}
if (ifp->if_flags & IFF_UP) {
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
flags = p->if_flags;
if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
if (IS_BUSY(sc)) {
error = EBUSY;
goto fail;
}
PORT_LOCK(p);
cxgb_update_mac_settings(p);
PORT_UNLOCK(p);
}
ADAPTER_UNLOCK(sc);
} else
error = cxgb_init_locked(p);
p->if_flags = ifp->if_flags;
} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
error = cxgb_uninit_locked(p);
else
ADAPTER_UNLOCK(sc);
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
break;
case SIOCADDMULTI:
case SIOCDELMULTI:
ADAPTER_LOCK(sc);
error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
if (error)
goto fail;
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
PORT_LOCK(p);
cxgb_update_mac_settings(p);
PORT_UNLOCK(p);
}
ADAPTER_UNLOCK(sc);
break;
case SIOCSIFCAP:
ADAPTER_LOCK(sc);
error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
if (error)
goto fail;
mask = ifr->ifr_reqcap ^ ifp->if_capenable;
if (mask & IFCAP_TXCSUM) {
ifp->if_capenable ^= IFCAP_TXCSUM;
ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
if (IFCAP_TSO4 & ifp->if_capenable &&
!(IFCAP_TXCSUM & ifp->if_capenable)) {
ifp->if_capenable &= ~IFCAP_TSO4;
if_printf(ifp,
"tso4 disabled due to -txcsum.\n");
}
}
if (mask & IFCAP_TXCSUM_IPV6) {
ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
if (IFCAP_TSO6 & ifp->if_capenable &&
!(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
ifp->if_capenable &= ~IFCAP_TSO6;
if_printf(ifp,
"tso6 disabled due to -txcsum6.\n");
}
}
if (mask & IFCAP_RXCSUM)
ifp->if_capenable ^= IFCAP_RXCSUM;
if (mask & IFCAP_RXCSUM_IPV6)
ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
/*
* Note that we leave CSUM_TSO alone (it is always set). The
* kernel takes both IFCAP_TSOx and CSUM_TSO into account before
* sending a TSO request our way, so it's sufficient to toggle
* IFCAP_TSOx only.
*/
if (mask & IFCAP_TSO4) {
if (!(IFCAP_TSO4 & ifp->if_capenable) &&
!(IFCAP_TXCSUM & ifp->if_capenable)) {
if_printf(ifp, "enable txcsum first.\n");
error = EAGAIN;
goto fail;
}
ifp->if_capenable ^= IFCAP_TSO4;
}
if (mask & IFCAP_TSO6) {
if (!(IFCAP_TSO6 & ifp->if_capenable) &&
!(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
if_printf(ifp, "enable txcsum6 first.\n");
error = EAGAIN;
goto fail;
}
ifp->if_capenable ^= IFCAP_TSO6;
}
if (mask & IFCAP_LRO) {
ifp->if_capenable ^= IFCAP_LRO;
/* Safe to do this even if cxgb_up not called yet */
cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
}
#ifdef TCP_OFFLOAD
if (mask & IFCAP_TOE4) {
int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
error = toe_capability(p, enable);
if (error == 0)
ifp->if_capenable ^= mask;
}
#endif
if (mask & IFCAP_VLAN_HWTAGGING) {
ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
PORT_LOCK(p);
cxgb_update_mac_settings(p);
PORT_UNLOCK(p);
}
}
if (mask & IFCAP_VLAN_MTU) {
ifp->if_capenable ^= IFCAP_VLAN_MTU;
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
PORT_LOCK(p);
cxgb_update_mac_settings(p);
PORT_UNLOCK(p);
}
}
if (mask & IFCAP_VLAN_HWTSO)
ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
if (mask & IFCAP_VLAN_HWCSUM)
ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
#ifdef VLAN_CAPABILITIES
VLAN_CAPABILITIES(ifp);
#endif
ADAPTER_UNLOCK(sc);
break;
case SIOCSIFMEDIA:
case SIOCGIFMEDIA:
error = ifmedia_ioctl(ifp, ifr, &p->media, command);
break;
default:
error = ether_ioctl(ifp, command, data);
}
return (error);
}
static int
cxgb_media_change(struct ifnet *ifp)
{
return (EOPNOTSUPP);
}
/*
* Translates phy->modtype to the correct Ethernet media subtype.
*/
static int
cxgb_ifm_type(int mod)
{
switch (mod) {
case phy_modtype_sr:
return (IFM_10G_SR);
case phy_modtype_lr:
return (IFM_10G_LR);
case phy_modtype_lrm:
return (IFM_10G_LRM);
case phy_modtype_twinax:
return (IFM_10G_TWINAX);
case phy_modtype_twinax_long:
return (IFM_10G_TWINAX_LONG);
case phy_modtype_none:
return (IFM_NONE);
case phy_modtype_unknown:
return (IFM_UNKNOWN);
}
KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
return (IFM_UNKNOWN);
}
/*
* Rebuilds the ifmedia list for this port, and sets the current media.
*/
static void
cxgb_build_medialist(struct port_info *p)
{
struct cphy *phy = &p->phy;
struct ifmedia *media = &p->media;
int mod = phy->modtype;
int m = IFM_ETHER | IFM_FDX;
PORT_LOCK(p);
ifmedia_removeall(media);
if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
/* Copper (RJ45) */
if (phy->caps & SUPPORTED_10000baseT_Full)
ifmedia_add(media, m | IFM_10G_T, mod, NULL);
if (phy->caps & SUPPORTED_1000baseT_Full)
ifmedia_add(media, m | IFM_1000_T, mod, NULL);
if (phy->caps & SUPPORTED_100baseT_Full)
ifmedia_add(media, m | IFM_100_TX, mod, NULL);
if (phy->caps & SUPPORTED_10baseT_Full)
ifmedia_add(media, m | IFM_10_T, mod, NULL);
ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
ifmedia_set(media, IFM_ETHER | IFM_AUTO);
} else if (phy->caps & SUPPORTED_TP) {
/* Copper (CX4) */
KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
("%s: unexpected cap 0x%x", __func__, phy->caps));
ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
ifmedia_set(media, m | IFM_10G_CX4);
} else if (phy->caps & SUPPORTED_FIBRE &&
phy->caps & SUPPORTED_10000baseT_Full) {
/* 10G optical (but includes SFP+ twinax) */
m |= cxgb_ifm_type(mod);
if (IFM_SUBTYPE(m) == IFM_NONE)
m &= ~IFM_FDX;
ifmedia_add(media, m, mod, NULL);
ifmedia_set(media, m);
} else if (phy->caps & SUPPORTED_FIBRE &&
phy->caps & SUPPORTED_1000baseT_Full) {
/* 1G optical */
/* XXX: Lie and claim to be SX, could actually be any 1G-X */
ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
ifmedia_set(media, m | IFM_1000_SX);
} else {
KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
phy->caps));
}
PORT_UNLOCK(p);
}
static void
cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
struct port_info *p = ifp->if_softc;
struct ifmedia_entry *cur = p->media.ifm_cur;
int speed = p->link_config.speed;
if (cur->ifm_data != p->phy.modtype) {
cxgb_build_medialist(p);
cur = p->media.ifm_cur;
}
ifmr->ifm_status = IFM_AVALID;
if (!p->link_config.link_ok)
return;
ifmr->ifm_status |= IFM_ACTIVE;
/*
* active and current will differ iff current media is autoselect. That
* can happen only for copper RJ45.
*/
if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
return;
KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
ifmr->ifm_active = IFM_ETHER | IFM_FDX;
if (speed == SPEED_10000)
ifmr->ifm_active |= IFM_10G_T;
else if (speed == SPEED_1000)
ifmr->ifm_active |= IFM_1000_T;
else if (speed == SPEED_100)
ifmr->ifm_active |= IFM_100_TX;
else if (speed == SPEED_10)
ifmr->ifm_active |= IFM_10_T;
else
KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
speed));
}
static uint64_t
cxgb_get_counter(struct ifnet *ifp, ift_counter c)
{
struct port_info *pi = ifp->if_softc;
struct adapter *sc = pi->adapter;
struct cmac *mac = &pi->mac;
struct mac_stats *mstats = &mac->stats;
cxgb_refresh_stats(pi);
switch (c) {
case IFCOUNTER_IPACKETS:
return (mstats->rx_frames);
case IFCOUNTER_IERRORS:
return (mstats->rx_jabber + mstats->rx_data_errs +
mstats->rx_sequence_errs + mstats->rx_runt +
mstats->rx_too_long + mstats->rx_mac_internal_errs +
mstats->rx_short + mstats->rx_fcs_errs);
case IFCOUNTER_OPACKETS:
return (mstats->tx_frames);
case IFCOUNTER_OERRORS:
return (mstats->tx_excess_collisions + mstats->tx_underrun +
mstats->tx_len_errs + mstats->tx_mac_internal_errs +
mstats->tx_excess_deferral + mstats->tx_fcs_errs);
case IFCOUNTER_COLLISIONS:
return (mstats->tx_total_collisions);
case IFCOUNTER_IBYTES:
return (mstats->rx_octets);
case IFCOUNTER_OBYTES:
return (mstats->tx_octets);
case IFCOUNTER_IMCASTS:
return (mstats->rx_mcast_frames);
case IFCOUNTER_OMCASTS:
return (mstats->tx_mcast_frames);
case IFCOUNTER_IQDROPS:
return (mstats->rx_cong_drops);
case IFCOUNTER_OQDROPS: {
int i;
uint64_t drops;
drops = 0;
if (sc->flags & FULL_INIT_DONE) {
for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
}
return (drops);
}
default:
return (if_get_counter_default(ifp, c));
}
}
static void
cxgb_async_intr(void *data)
{
adapter_t *sc = data;
t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
(void) t3_read_reg(sc, A_PL_INT_ENABLE0);
taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
}
static void
link_check_callout(void *arg)
{
struct port_info *pi = arg;
struct adapter *sc = pi->adapter;
if (!isset(&sc->open_device_map, pi->port_id))
return;
taskqueue_enqueue(sc->tq, &pi->link_check_task);
}
static void
check_link_status(void *arg, int pending)
{
struct port_info *pi = arg;
struct adapter *sc = pi->adapter;
if (!isset(&sc->open_device_map, pi->port_id))
return;
t3_link_changed(sc, pi->port_id);
if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
pi->link_config.link_ok == 0)
callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
}
void
t3_os_link_intr(struct port_info *pi)
{
/*
* Schedule a link check in the near future. If the link is flapping
* rapidly we'll keep resetting the callout and delaying the check until
* things stabilize a bit.
*/
callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
}
static void
check_t3b2_mac(struct adapter *sc)
{
int i;
if (sc->flags & CXGB_SHUTDOWN)
return;
for_each_port(sc, i) {
struct port_info *p = &sc->port[i];
int status;
#ifdef INVARIANTS
struct ifnet *ifp = p->ifp;
#endif
if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
!p->link_config.link_ok)
continue;
KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
("%s: state mismatch (drv_flags %x, device_map %x)",
__func__, ifp->if_drv_flags, sc->open_device_map));
PORT_LOCK(p);
status = t3b2_mac_watchdog_task(&p->mac);
if (status == 1)
p->mac.stats.num_toggled++;
else if (status == 2) {
struct cmac *mac = &p->mac;
cxgb_update_mac_settings(p);
t3_link_start(&p->phy, mac, &p->link_config);
t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
t3_port_intr_enable(sc, p->port_id);
p->mac.stats.num_resets++;
}
PORT_UNLOCK(p);
}
}
static void
cxgb_tick(void *arg)
{
adapter_t *sc = (adapter_t *)arg;
if (sc->flags & CXGB_SHUTDOWN)
return;
taskqueue_enqueue(sc->tq, &sc->tick_task);
callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
}
void
cxgb_refresh_stats(struct port_info *pi)
{
struct timeval tv;
const struct timeval interval = {0, 250000}; /* 250ms */
getmicrotime(&tv);
timevalsub(&tv, &interval);
if (timevalcmp(&tv, &pi->last_refreshed, <))
return;
PORT_LOCK(pi);
t3_mac_update_stats(&pi->mac);
PORT_UNLOCK(pi);
getmicrotime(&pi->last_refreshed);
}
static void
cxgb_tick_handler(void *arg, int count)
{
adapter_t *sc = (adapter_t *)arg;
const struct adapter_params *p = &sc->params;
int i;
uint32_t cause, reset;
if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
return;
if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
check_t3b2_mac(sc);
cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
if (cause) {
struct sge_qset *qs = &sc->sge.qs[0];
uint32_t mask, v;
v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
mask = 1;
for (i = 0; i < SGE_QSETS; i++) {
if (v & mask)
qs[i].rspq.starved++;
mask <<= 1;
}
mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
for (i = 0; i < SGE_QSETS * 2; i++) {
if (v & mask) {
qs[i / 2].fl[i % 2].empty++;
}
mask <<= 1;
}
/* clear */
t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
t3_write_reg(sc, A_SG_INT_CAUSE, cause);
}
for (i = 0; i < sc->params.nports; i++) {
struct port_info *pi = &sc->port[i];
struct cmac *mac = &pi->mac;
if (!isset(&sc->open_device_map, pi->port_id))
continue;
cxgb_refresh_stats(pi);
if (mac->multiport)
continue;
/* Count rx fifo overflows, once per second */
cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
reset = 0;
if (cause & F_RXFIFO_OVERFLOW) {
mac->stats.rx_fifo_ovfl++;
reset |= F_RXFIFO_OVERFLOW;
}
t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
}
}
static void
touch_bars(device_t dev)
{
/*
* Don't enable yet
*/
#if !defined(__LP64__) && 0
u32 v;
pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
#endif
}
static int
set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
{
uint8_t *buf;
int err = 0;
u32 aligned_offset, aligned_len, *p;
struct adapter *adapter = pi->adapter;
aligned_offset = offset & ~3;
aligned_len = (len + (offset & 3) + 3) & ~3;
if (aligned_offset != offset || aligned_len != len) {
buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
if (!buf)
return (ENOMEM);
err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
if (!err && aligned_len > 4)
err = t3_seeprom_read(adapter,
aligned_offset + aligned_len - 4,
(u32 *)&buf[aligned_len - 4]);
if (err)
goto out;
memcpy(buf + (offset & 3), data, len);
} else
buf = (uint8_t *)(uintptr_t)data;
err = t3_seeprom_wp(adapter, 0);
if (err)
goto out;
for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
err = t3_seeprom_write(adapter, aligned_offset, *p);
aligned_offset += 4;
}
if (!err)
err = t3_seeprom_wp(adapter, 1);
out:
if (buf != data)
free(buf, M_DEVBUF);
return err;
}
static int
in_range(int val, int lo, int hi)
{
return val < 0 || (val <= hi && val >= lo);
}
static int
cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
{
return (0);
}
static int
cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
{
return (0);
}
static int
cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
int fflag, struct thread *td)
{
int mmd, error = 0;
struct port_info *pi = dev->si_drv1;
adapter_t *sc = pi->adapter;
#ifdef PRIV_SUPPORTED
if (priv_check(td, PRIV_DRIVER)) {
if (cxgb_debug)
printf("user does not have access to privileged ioctls\n");
return (EPERM);
}
#else
if (suser(td)) {
if (cxgb_debug)
printf("user does not have access to privileged ioctls\n");
return (EPERM);
}
#endif
switch (cmd) {
case CHELSIO_GET_MIIREG: {
uint32_t val;
struct cphy *phy = &pi->phy;
struct ch_mii_data *mid = (struct ch_mii_data *)data;
if (!phy->mdio_read)
return (EOPNOTSUPP);
if (is_10G(sc)) {
mmd = mid->phy_id >> 8;
if (!mmd)
mmd = MDIO_DEV_PCS;
else if (mmd > MDIO_DEV_VEND2)
return (EINVAL);
error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
mid->reg_num, &val);
} else
error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
mid->reg_num & 0x1f, &val);
if (error == 0)
mid->val_out = val;
break;
}
case CHELSIO_SET_MIIREG: {
struct cphy *phy = &pi->phy;
struct ch_mii_data *mid = (struct ch_mii_data *)data;
if (!phy->mdio_write)
return (EOPNOTSUPP);
if (is_10G(sc)) {
mmd = mid->phy_id >> 8;
if (!mmd)
mmd = MDIO_DEV_PCS;
else if (mmd > MDIO_DEV_VEND2)
return (EINVAL);
error = phy->mdio_write(sc, mid->phy_id & 0x1f,
mmd, mid->reg_num, mid->val_in);
} else
error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
mid->reg_num & 0x1f,
mid->val_in);
break;
}
case CHELSIO_SETREG: {
struct ch_reg *edata = (struct ch_reg *)data;
if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
return (EFAULT);
t3_write_reg(sc, edata->addr, edata->val);
break;
}
case CHELSIO_GETREG: {
struct ch_reg *edata = (struct ch_reg *)data;
if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
return (EFAULT);
edata->val = t3_read_reg(sc, edata->addr);
break;
}
case CHELSIO_GET_SGE_CONTEXT: {
struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
mtx_lock_spin(&sc->sge.reg_lock);
switch (ecntxt->cntxt_type) {
case CNTXT_TYPE_EGRESS:
error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
ecntxt->data);
break;
case CNTXT_TYPE_FL:
error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
ecntxt->data);
break;
case CNTXT_TYPE_RSP:
error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
ecntxt->data);
break;
case CNTXT_TYPE_CQ:
error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
ecntxt->data);
break;
default:
error = EINVAL;
break;
}
mtx_unlock_spin(&sc->sge.reg_lock);
break;
}
case CHELSIO_GET_SGE_DESC: {
struct ch_desc *edesc = (struct ch_desc *)data;
int ret;
if (edesc->queue_num >= SGE_QSETS * 6)
return (EINVAL);
ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
edesc->queue_num % 6, edesc->idx, edesc->data);
if (ret < 0)
return (EINVAL);
edesc->size = ret;
break;
}
case CHELSIO_GET_QSET_PARAMS: {
struct qset_params *q;
struct ch_qset_params *t = (struct ch_qset_params *)data;
int q1 = pi->first_qset;
int nqsets = pi->nqsets;
int i;
if (t->qset_idx >= nqsets)
return EINVAL;
i = q1 + t->qset_idx;
q = &sc->params.sge.qset[i];
t->rspq_size = q->rspq_size;
t->txq_size[0] = q->txq_size[0];
t->txq_size[1] = q->txq_size[1];
t->txq_size[2] = q->txq_size[2];
t->fl_size[0] = q->fl_size;
t->fl_size[1] = q->jumbo_size;
t->polling = q->polling;
t->lro = q->lro;
t->intr_lat = q->coalesce_usecs;
t->cong_thres = q->cong_thres;
t->qnum = i;
if ((sc->flags & FULL_INIT_DONE) == 0)
t->vector = 0;
else if (sc->flags & USING_MSIX)
t->vector = rman_get_start(sc->msix_irq_res[i]);
else
t->vector = rman_get_start(sc->irq_res);
break;
}
case CHELSIO_GET_QSET_NUM: {
struct ch_reg *edata = (struct ch_reg *)data;
edata->val = pi->nqsets;
break;
}
case CHELSIO_LOAD_FW: {
uint8_t *fw_data;
uint32_t vers;
struct ch_mem_range *t = (struct ch_mem_range *)data;
/*
* You're allowed to load a firmware only before FULL_INIT_DONE
*
* FW_UPTODATE is also set so the rest of the initialization
* will not overwrite what was loaded here. This gives you the
* flexibility to load any firmware (and maybe shoot yourself in
* the foot).
*/
ADAPTER_LOCK(sc);
if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
ADAPTER_UNLOCK(sc);
return (EBUSY);
}
fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
if (!fw_data)
error = ENOMEM;
else
error = copyin(t->buf, fw_data, t->len);
if (!error)
error = -t3_load_fw(sc, fw_data, t->len);
if (t3_get_fw_version(sc, &vers) == 0) {
snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
"%d.%d.%d", G_FW_VERSION_MAJOR(vers),
G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
}
if (!error)
sc->flags |= FW_UPTODATE;
free(fw_data, M_DEVBUF);
ADAPTER_UNLOCK(sc);
break;
}
case CHELSIO_LOAD_BOOT: {
uint8_t *boot_data;
struct ch_mem_range *t = (struct ch_mem_range *)data;
boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
if (!boot_data)
return ENOMEM;
error = copyin(t->buf, boot_data, t->len);
if (!error)
error = -t3_load_boot(sc, boot_data, t->len);
free(boot_data, M_DEVBUF);
break;
}
case CHELSIO_GET_PM: {
struct ch_pm *m = (struct ch_pm *)data;
struct tp_params *p = &sc->params.tp;
if (!is_offload(sc))
return (EOPNOTSUPP);
m->tx_pg_sz = p->tx_pg_size;
m->tx_num_pg = p->tx_num_pgs;
m->rx_pg_sz = p->rx_pg_size;
m->rx_num_pg = p->rx_num_pgs;
m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
break;
}
case CHELSIO_SET_PM: {
struct ch_pm *m = (struct ch_pm *)data;
struct tp_params *p = &sc->params.tp;
if (!is_offload(sc))
return (EOPNOTSUPP);
if (sc->flags & FULL_INIT_DONE)
return (EBUSY);
if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
!m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
return (EINVAL); /* not power of 2 */
if (!(m->rx_pg_sz & 0x14000))
return (EINVAL); /* not 16KB or 64KB */
if (!(m->tx_pg_sz & 0x1554000))
return (EINVAL);
if (m->tx_num_pg == -1)
m->tx_num_pg = p->tx_num_pgs;
if (m->rx_num_pg == -1)
m->rx_num_pg = p->rx_num_pgs;
if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
return (EINVAL);
if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
return (EINVAL);
p->rx_pg_size = m->rx_pg_sz;
p->tx_pg_size = m->tx_pg_sz;
p->rx_num_pgs = m->rx_num_pg;
p->tx_num_pgs = m->tx_num_pg;
break;
}
case CHELSIO_SETMTUTAB: {
struct ch_mtus *m = (struct ch_mtus *)data;
int i;
if (!is_offload(sc))
return (EOPNOTSUPP);
if (offload_running(sc))
return (EBUSY);
if (m->nmtus != NMTUS)
return (EINVAL);
if (m->mtus[0] < 81) /* accommodate SACK */
return (EINVAL);
/*
* MTUs must be in ascending order
*/
for (i = 1; i < NMTUS; ++i)
if (m->mtus[i] < m->mtus[i - 1])
return (EINVAL);
memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
break;
}
case CHELSIO_GETMTUTAB: {
struct ch_mtus *m = (struct ch_mtus *)data;
if (!is_offload(sc))
return (EOPNOTSUPP);
memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
m->nmtus = NMTUS;
break;
}
case CHELSIO_GET_MEM: {
struct ch_mem_range *t = (struct ch_mem_range *)data;
struct mc7 *mem;
uint8_t *useraddr;
u64 buf[32];
/*
* Use these to avoid modifying len/addr in the return
* struct
*/
uint32_t len = t->len, addr = t->addr;
if (!is_offload(sc))
return (EOPNOTSUPP);
if (!(sc->flags & FULL_INIT_DONE))
return (EIO); /* need the memory controllers */
if ((addr & 0x7) || (len & 0x7))
return (EINVAL);
if (t->mem_id == MEM_CM)
mem = &sc->cm;
else if (t->mem_id == MEM_PMRX)
mem = &sc->pmrx;
else if (t->mem_id == MEM_PMTX)
mem = &sc->pmtx;
else
return (EINVAL);
/*
* Version scheme:
* bits 0..9: chip version
* bits 10..15: chip revision
*/
t->version = 3 | (sc->params.rev << 10);
/*
* Read 256 bytes at a time as len can be large and we don't
* want to use huge intermediate buffers.
*/
useraddr = (uint8_t *)t->buf;
while (len) {
unsigned int chunk = min(len, sizeof(buf));
error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
if (error)
return (-error);
if (copyout(buf, useraddr, chunk))
return (EFAULT);
useraddr += chunk;
addr += chunk;
len -= chunk;
}
break;
}
case CHELSIO_READ_TCAM_WORD: {
struct ch_tcam_word *t = (struct ch_tcam_word *)data;
if (!is_offload(sc))
return (EOPNOTSUPP);
if (!(sc->flags & FULL_INIT_DONE))
return (EIO); /* need MC5 */
return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
break;
}
case CHELSIO_SET_TRACE_FILTER: {
struct ch_trace *t = (struct ch_trace *)data;
const struct trace_params *tp;
tp = (const struct trace_params *)&t->sip;
if (t->config_tx)
t3_config_trace_filter(sc, tp, 0, t->invert_match,
t->trace_tx);
if (t->config_rx)
t3_config_trace_filter(sc, tp, 1, t->invert_match,
t->trace_rx);
break;
}
case CHELSIO_SET_PKTSCHED: {
struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
if (sc->open_device_map == 0)
return (EAGAIN);
send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
p->binding);
break;
}
case CHELSIO_IFCONF_GETREGS: {
struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
int reglen = cxgb_get_regs_len();
uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
if (buf == NULL) {
return (ENOMEM);
}
if (regs->len > reglen)
regs->len = reglen;
else if (regs->len < reglen)
error = ENOBUFS;
if (!error) {
cxgb_get_regs(sc, regs, buf);
error = copyout(buf, regs->data, reglen);
}
free(buf, M_DEVBUF);
break;
}
case CHELSIO_SET_HW_SCHED: {
struct ch_hw_sched *t = (struct ch_hw_sched *)data;
unsigned int ticks_per_usec = core_ticks_per_usec(sc);
if ((sc->flags & FULL_INIT_DONE) == 0)
return (EAGAIN); /* need TP to be initialized */
if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
!in_range(t->channel, 0, 1) ||
!in_range(t->kbps, 0, 10000000) ||
!in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
!in_range(t->flow_ipg, 0,
dack_ticks_to_usec(sc, 0x7ff)))
return (EINVAL);
if (t->kbps >= 0) {
error = t3_config_sched(sc, t->kbps, t->sched);
if (error < 0)
return (-error);
}
if (t->class_ipg >= 0)
t3_set_sched_ipg(sc, t->sched, t->class_ipg);
if (t->flow_ipg >= 0) {
t->flow_ipg *= 1000; /* us -> ns */
t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
}
if (t->mode >= 0) {
int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
bit, t->mode ? bit : 0);
}
if (t->channel >= 0)
t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
1 << t->sched, t->channel << t->sched);
break;
}
case CHELSIO_GET_EEPROM: {
int i;
struct ch_eeprom *e = (struct ch_eeprom *)data;
uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
if (buf == NULL) {
return (ENOMEM);
}
e->magic = EEPROM_MAGIC;
for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
if (!error)
error = copyout(buf + e->offset, e->data, e->len);
free(buf, M_DEVBUF);
break;
}
case CHELSIO_CLEAR_STATS: {
if (!(sc->flags & FULL_INIT_DONE))
return EAGAIN;
PORT_LOCK(pi);
t3_mac_update_stats(&pi->mac);
memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
PORT_UNLOCK(pi);
break;
}
case CHELSIO_GET_UP_LA: {
struct ch_up_la *la = (struct ch_up_la *)data;
uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
if (buf == NULL) {
return (ENOMEM);
}
if (la->bufsize < LA_BUFSIZE)
error = ENOBUFS;
if (!error)
error = -t3_get_up_la(sc, &la->stopped, &la->idx,
&la->bufsize, buf);
if (!error)
error = copyout(buf, la->data, la->bufsize);
free(buf, M_DEVBUF);
break;
}
case CHELSIO_GET_UP_IOQS: {
struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
uint32_t *v;
if (buf == NULL) {
return (ENOMEM);
}
if (ioqs->bufsize < IOQS_BUFSIZE)
error = ENOBUFS;
if (!error)
error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
if (!error) {
v = (uint32_t *)buf;
ioqs->ioq_rx_enable = *v++;
ioqs->ioq_tx_enable = *v++;
ioqs->ioq_rx_status = *v++;
ioqs->ioq_tx_status = *v++;
error = copyout(v, ioqs->data, ioqs->bufsize);
}
free(buf, M_DEVBUF);
break;
}
case CHELSIO_SET_FILTER: {
struct ch_filter *f = (struct ch_filter *)data;
struct filter_info *p;
unsigned int nfilters = sc->params.mc5.nfilters;
if (!is_offload(sc))
return (EOPNOTSUPP); /* No TCAM */
if (!(sc->flags & FULL_INIT_DONE))
return (EAGAIN); /* mc5 not setup yet */
if (nfilters == 0)
return (EBUSY); /* TOE will use TCAM */
/* sanity checks */
if (f->filter_id >= nfilters ||
(f->val.dip && f->mask.dip != 0xffffffff) ||
(f->val.sport && f->mask.sport != 0xffff) ||
(f->val.dport && f->mask.dport != 0xffff) ||
(f->val.vlan && f->mask.vlan != 0xfff) ||
(f->val.vlan_prio &&
f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
(f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
f->qset >= SGE_QSETS ||
sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
return (EINVAL);
/* Was allocated with M_WAITOK */
KASSERT(sc->filters, ("filter table NULL\n"));
p = &sc->filters[f->filter_id];
if (p->locked)
return (EPERM);
bzero(p, sizeof(*p));
p->sip = f->val.sip;
p->sip_mask = f->mask.sip;
p->dip = f->val.dip;
p->sport = f->val.sport;
p->dport = f->val.dport;
p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
FILTER_NO_VLAN_PRI;
p->mac_hit = f->mac_hit;
p->mac_vld = f->mac_addr_idx != 0xffff;
p->mac_idx = f->mac_addr_idx;
p->pkt_type = f->proto;
p->report_filter_id = f->want_filter_id;
p->pass = f->pass;
p->rss = f->rss;
p->qset = f->qset;
error = set_filter(sc, f->filter_id, p);
if (error == 0)
p->valid = 1;
break;
}
case CHELSIO_DEL_FILTER: {
struct ch_filter *f = (struct ch_filter *)data;
struct filter_info *p;
unsigned int nfilters = sc->params.mc5.nfilters;
if (!is_offload(sc))
return (EOPNOTSUPP);
if (!(sc->flags & FULL_INIT_DONE))
return (EAGAIN);
if (nfilters == 0 || sc->filters == NULL)
return (EINVAL);
if (f->filter_id >= nfilters)
return (EINVAL);
p = &sc->filters[f->filter_id];
if (p->locked)
return (EPERM);
if (!p->valid)
return (EFAULT); /* Read "Bad address" as "Bad index" */
bzero(p, sizeof(*p));
p->sip = p->sip_mask = 0xffffffff;
p->vlan = 0xfff;
p->vlan_prio = FILTER_NO_VLAN_PRI;
p->pkt_type = 1;
error = set_filter(sc, f->filter_id, p);
break;
}
case CHELSIO_GET_FILTER: {
struct ch_filter *f = (struct ch_filter *)data;
struct filter_info *p;
unsigned int i, nfilters = sc->params.mc5.nfilters;
if (!is_offload(sc))
return (EOPNOTSUPP);
if (!(sc->flags & FULL_INIT_DONE))
return (EAGAIN);
if (nfilters == 0 || sc->filters == NULL)
return (EINVAL);
i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
for (; i < nfilters; i++) {
p = &sc->filters[i];
if (!p->valid)
continue;
bzero(f, sizeof(*f));
f->filter_id = i;
f->val.sip = p->sip;
f->mask.sip = p->sip_mask;
f->val.dip = p->dip;
f->mask.dip = p->dip ? 0xffffffff : 0;
f->val.sport = p->sport;
f->mask.sport = p->sport ? 0xffff : 0;
f->val.dport = p->dport;
f->mask.dport = p->dport ? 0xffff : 0;
f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
0 : p->vlan_prio;
f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
0 : FILTER_NO_VLAN_PRI;
f->mac_hit = p->mac_hit;
f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
f->proto = p->pkt_type;
f->want_filter_id = p->report_filter_id;
f->pass = p->pass;
f->rss = p->rss;
f->qset = p->qset;
break;
}
if (i == nfilters)
f->filter_id = 0xffffffff;
break;
}
default:
return (EOPNOTSUPP);
break;
}
return (error);
}
static __inline void
reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
unsigned int end)
{
uint32_t *p = (uint32_t *)(buf + start);
for ( ; start <= end; start += sizeof(uint32_t))
*p++ = t3_read_reg(ap, start);
}
#define T3_REGMAP_SIZE (3 * 1024)
static int
cxgb_get_regs_len(void)
{
return T3_REGMAP_SIZE;
}
static void
cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
{
/*
* Version scheme:
* bits 0..9: chip version
* bits 10..15: chip revision
* bit 31: set for PCIe cards
*/
regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
/*
* We skip the MAC statistics registers because they are clear-on-read.
* Also reading multi-register stats would need to synchronize with the
* periodic mac stats accumulation. Hard to justify the complexity.
*/
memset(buf, 0, cxgb_get_regs_len());
reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
XGM_REG(A_XGM_SERDES_STAT3, 1));
reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
}
static int
alloc_filters(struct adapter *sc)
{
struct filter_info *p;
unsigned int nfilters = sc->params.mc5.nfilters;
if (nfilters == 0)
return (0);
p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
sc->filters = p;
p = &sc->filters[nfilters - 1];
p->vlan = 0xfff;
p->vlan_prio = FILTER_NO_VLAN_PRI;
p->pass = p->rss = p->valid = p->locked = 1;
return (0);
}
static int
setup_hw_filters(struct adapter *sc)
{
int i, rc;
unsigned int nfilters = sc->params.mc5.nfilters;
if (!sc->filters)
return (0);
t3_enable_filters(sc);
for (i = rc = 0; i < nfilters && !rc; i++) {
if (sc->filters[i].locked)
rc = set_filter(sc, i, &sc->filters[i]);
}
return (rc);
}
static int
set_filter(struct adapter *sc, int id, const struct filter_info *f)
{
int len;
struct mbuf *m;
struct ulp_txpkt *txpkt;
struct work_request_hdr *wr;
struct cpl_pass_open_req *oreq;
struct cpl_set_tcb_field *sreq;
len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
sc->params.mc5.nfilters;
m = m_gethdr(M_WAITOK, MT_DATA);
m->m_len = m->m_pkthdr.len = len;
bzero(mtod(m, char *), len);
wr = mtod(m, struct work_request_hdr *);
wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
oreq = (struct cpl_pass_open_req *)(wr + 1);
txpkt = (struct ulp_txpkt *)oreq;
txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
oreq->local_port = htons(f->dport);
oreq->peer_port = htons(f->sport);
oreq->local_ip = htonl(f->dip);
oreq->peer_ip = htonl(f->sip);
oreq->peer_netmask = htonl(f->sip_mask);
oreq->opt0h = 0;
oreq->opt0l = htonl(F_NO_OFFLOAD);
oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
V_VLAN_PRI(f->vlan_prio >> 1) |
V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
sreq = (struct cpl_set_tcb_field *)(oreq + 1);
set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
(f->report_filter_id << 15) | (1 << 23) |
((u64)f->pass << 35) | ((u64)!f->rss << 36));
set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
t3_mgmt_tx(sc, m);
if (f->pass && !f->rss) {
len = sizeof(*sreq);
m = m_gethdr(M_WAITOK, MT_DATA);
m->m_len = m->m_pkthdr.len = len;
bzero(mtod(m, char *), len);
sreq = mtod(m, struct cpl_set_tcb_field *);
sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
mk_set_tcb_field(sreq, id, 25, 0x3f80000,
(u64)sc->rrss_map[f->qset] << 19);
t3_mgmt_tx(sc, m);
}
return 0;
}
static inline void
mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
unsigned int word, u64 mask, u64 val)
{
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
req->reply = V_NO_REPLY(1);
req->cpu_idx = 0;
req->word = htons(word);
req->mask = htobe64(mask);
req->val = htobe64(val);
}
static inline void
set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
unsigned int word, u64 mask, u64 val)
{
struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
mk_set_tcb_field(req, tid, word, mask, val);
}
void
t3_iterate(void (*func)(struct adapter *, void *), void *arg)
{
struct adapter *sc;
mtx_lock(&t3_list_lock);
SLIST_FOREACH(sc, &t3_list, link) {
/*
* func should not make any assumptions about what state sc is
* in - the only guarantee is that sc->sc_lock is a valid lock.
*/
func(sc, arg);
}
mtx_unlock(&t3_list_lock);
}
#ifdef TCP_OFFLOAD
static int
toe_capability(struct port_info *pi, int enable)
{
int rc;
struct adapter *sc = pi->adapter;
ADAPTER_LOCK_ASSERT_OWNED(sc);
if (!is_offload(sc))
return (ENODEV);
if (enable) {
if (!(sc->flags & FULL_INIT_DONE)) {
log(LOG_WARNING,
"You must enable a cxgb interface first\n");
return (EAGAIN);
}
if (isset(&sc->offload_map, pi->port_id))
return (0);
if (!(sc->flags & TOM_INIT_DONE)) {
rc = t3_activate_uld(sc, ULD_TOM);
if (rc == EAGAIN) {
log(LOG_WARNING,
"You must kldload t3_tom.ko before trying "
"to enable TOE on a cxgb interface.\n");
}
if (rc != 0)
return (rc);
KASSERT(sc->tom_softc != NULL,
("%s: TOM activated but softc NULL", __func__));
KASSERT(sc->flags & TOM_INIT_DONE,
("%s: TOM activated but flag not set", __func__));
}
setbit(&sc->offload_map, pi->port_id);
/*
* XXX: Temporary code to allow iWARP to be enabled when TOE is
* enabled on any port. Need to figure out how to enable,
* disable, load, and unload iWARP cleanly.
*/
if (!isset(&sc->offload_map, MAX_NPORTS) &&
t3_activate_uld(sc, ULD_IWARP) == 0)
setbit(&sc->offload_map, MAX_NPORTS);
} else {
if (!isset(&sc->offload_map, pi->port_id))
return (0);
KASSERT(sc->flags & TOM_INIT_DONE,
("%s: TOM never initialized?", __func__));
clrbit(&sc->offload_map, pi->port_id);
}
return (0);
}
/*
* Add an upper layer driver to the global list.
*/
int
t3_register_uld(struct uld_info *ui)
{
int rc = 0;
struct uld_info *u;
mtx_lock(&t3_uld_list_lock);
SLIST_FOREACH(u, &t3_uld_list, link) {
if (u->uld_id == ui->uld_id) {
rc = EEXIST;
goto done;
}
}
SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
ui->refcount = 0;
done:
mtx_unlock(&t3_uld_list_lock);
return (rc);
}
int
t3_unregister_uld(struct uld_info *ui)
{
int rc = EINVAL;
struct uld_info *u;
mtx_lock(&t3_uld_list_lock);
SLIST_FOREACH(u, &t3_uld_list, link) {
if (u == ui) {
if (ui->refcount > 0) {
rc = EBUSY;
goto done;
}
SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
rc = 0;
goto done;
}
}
done:
mtx_unlock(&t3_uld_list_lock);
return (rc);
}
int
t3_activate_uld(struct adapter *sc, int id)
{
int rc = EAGAIN;
struct uld_info *ui;
mtx_lock(&t3_uld_list_lock);
SLIST_FOREACH(ui, &t3_uld_list, link) {
if (ui->uld_id == id) {
rc = ui->activate(sc);
if (rc == 0)
ui->refcount++;
goto done;
}
}
done:
mtx_unlock(&t3_uld_list_lock);
return (rc);
}
int
t3_deactivate_uld(struct adapter *sc, int id)
{
int rc = EINVAL;
struct uld_info *ui;
mtx_lock(&t3_uld_list_lock);
SLIST_FOREACH(ui, &t3_uld_list, link) {
if (ui->uld_id == id) {
rc = ui->deactivate(sc);
if (rc == 0)
ui->refcount--;
goto done;
}
}
done:
mtx_unlock(&t3_uld_list_lock);
return (rc);
}
static int
cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
struct mbuf *m)
{
m_freem(m);
return (EDOOFUS);
}
int
t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
{
uintptr_t *loc, new;
if (opcode >= NUM_CPL_HANDLERS)
return (EINVAL);
new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
loc = (uintptr_t *) &sc->cpl_handler[opcode];
atomic_store_rel_ptr(loc, new);
return (0);
}
#endif
static int
cxgbc_mod_event(module_t mod, int cmd, void *arg)
{
int rc = 0;
switch (cmd) {
case MOD_LOAD:
mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
SLIST_INIT(&t3_list);
#ifdef TCP_OFFLOAD
mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
SLIST_INIT(&t3_uld_list);
#endif
break;
case MOD_UNLOAD:
#ifdef TCP_OFFLOAD
mtx_lock(&t3_uld_list_lock);
if (!SLIST_EMPTY(&t3_uld_list)) {
rc = EBUSY;
mtx_unlock(&t3_uld_list_lock);
break;
}
mtx_unlock(&t3_uld_list_lock);
mtx_destroy(&t3_uld_list_lock);
#endif
mtx_lock(&t3_list_lock);
if (!SLIST_EMPTY(&t3_list)) {
rc = EBUSY;
mtx_unlock(&t3_list_lock);
break;
}
mtx_unlock(&t3_list_lock);
mtx_destroy(&t3_list_lock);
break;
}
return (rc);
}
Index: head/sys/dev/cxgb/cxgb_sge.c
===================================================================
--- head/sys/dev/cxgb/cxgb_sge.c (revision 283290)
+++ head/sys/dev/cxgb/cxgb_sge.c (revision 283291)
@@ -1,3712 +1,3712 @@
/**************************************************************************
Copyright (c) 2007-2009, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***************************************************************************/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet6.h"
#include "opt_inet.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus_dma.h>
#include <sys/rman.h>
#include <sys/queue.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/proc.h>
#include <sys/sbuf.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/systm.h>
#include <sys/syslog.h>
#include <sys/socket.h>
#include <sys/sglist.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/bpf.h>
#include <net/ethernet.h>
#include <net/if_vlan_var.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>
#include <netinet/tcp.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <cxgb_include.h>
#include <sys/mvec.h>
int txq_fills = 0;
int multiq_tx_enable = 1;
#ifdef TCP_OFFLOAD
CTASSERT(NUM_CPL_HANDLERS >= NUM_CPL_CMDS);
#endif
extern struct sysctl_oid_list sysctl__hw_cxgb_children;
int cxgb_txq_buf_ring_size = TX_ETH_Q_SIZE;
SYSCTL_INT(_hw_cxgb, OID_AUTO, txq_mr_size, CTLFLAG_RDTUN, &cxgb_txq_buf_ring_size, 0,
"size of per-queue mbuf ring");
static int cxgb_tx_coalesce_force = 0;
SYSCTL_INT(_hw_cxgb, OID_AUTO, tx_coalesce_force, CTLFLAG_RWTUN,
&cxgb_tx_coalesce_force, 0,
"coalesce small packets into a single work request regardless of ring state");
#define COALESCE_START_DEFAULT TX_ETH_Q_SIZE>>1
#define COALESCE_START_MAX (TX_ETH_Q_SIZE-(TX_ETH_Q_SIZE>>3))
#define COALESCE_STOP_DEFAULT TX_ETH_Q_SIZE>>2
#define COALESCE_STOP_MIN TX_ETH_Q_SIZE>>5
#define TX_RECLAIM_DEFAULT TX_ETH_Q_SIZE>>5
#define TX_RECLAIM_MAX TX_ETH_Q_SIZE>>2
#define TX_RECLAIM_MIN TX_ETH_Q_SIZE>>6
static int cxgb_tx_coalesce_enable_start = COALESCE_START_DEFAULT;
SYSCTL_INT(_hw_cxgb, OID_AUTO, tx_coalesce_enable_start, CTLFLAG_RWTUN,
&cxgb_tx_coalesce_enable_start, 0,
"coalesce enable threshold");
static int cxgb_tx_coalesce_enable_stop = COALESCE_STOP_DEFAULT;
SYSCTL_INT(_hw_cxgb, OID_AUTO, tx_coalesce_enable_stop, CTLFLAG_RWTUN,
&cxgb_tx_coalesce_enable_stop, 0,
"coalesce disable threshold");
static int cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT;
SYSCTL_INT(_hw_cxgb, OID_AUTO, tx_reclaim_threshold, CTLFLAG_RWTUN,
&cxgb_tx_reclaim_threshold, 0,
"tx cleaning minimum threshold");
/*
* XXX don't re-enable this until TOE stops assuming
* we have an m_ext
*/
static int recycle_enable = 0;
extern int cxgb_use_16k_clusters;
extern int nmbjumbop;
extern int nmbjumbo9;
extern int nmbjumbo16;
#define USE_GTS 0
#define SGE_RX_SM_BUF_SIZE 1536
#define SGE_RX_DROP_THRES 16
#define SGE_RX_COPY_THRES 128
/*
* Period of the Tx buffer reclaim timer. This timer does not need to run
* frequently as Tx buffers are usually reclaimed by new Tx packets.
*/
#define TX_RECLAIM_PERIOD (hz >> 1)
/*
* Values for sge_txq.flags
*/
enum {
TXQ_RUNNING = 1 << 0, /* fetch engine is running */
TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */
};
struct tx_desc {
uint64_t flit[TX_DESC_FLITS];
} __packed;
struct rx_desc {
uint32_t addr_lo;
uint32_t len_gen;
uint32_t gen2;
uint32_t addr_hi;
} __packed;
struct rsp_desc { /* response queue descriptor */
struct rss_header rss_hdr;
uint32_t flags;
uint32_t len_cq;
uint8_t imm_data[47];
uint8_t intr_gen;
} __packed;
#define RX_SW_DESC_MAP_CREATED (1 << 0)
#define TX_SW_DESC_MAP_CREATED (1 << 1)
#define RX_SW_DESC_INUSE (1 << 3)
#define TX_SW_DESC_MAPPED (1 << 4)
#define RSPQ_NSOP_NEOP G_RSPD_SOP_EOP(0)
#define RSPQ_EOP G_RSPD_SOP_EOP(F_RSPD_EOP)
#define RSPQ_SOP G_RSPD_SOP_EOP(F_RSPD_SOP)
#define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP)
struct tx_sw_desc { /* SW state per Tx descriptor */
struct mbuf *m;
bus_dmamap_t map;
int flags;
};
struct rx_sw_desc { /* SW state per Rx descriptor */
caddr_t rxsd_cl;
struct mbuf *m;
bus_dmamap_t map;
int flags;
};
struct txq_state {
unsigned int compl;
unsigned int gen;
unsigned int pidx;
};
struct refill_fl_cb_arg {
int error;
bus_dma_segment_t seg;
int nseg;
};
/*
* Maps a number of flits to the number of Tx descriptors that can hold them.
* The formula is
*
* desc = 1 + (flits - 2) / (WR_FLITS - 1).
*
* HW allows up to 4 descriptors to be combined into a WR.
*/
static uint8_t flit_desc_map[] = {
0,
#if SGE_NUM_GENBITS == 1
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
#elif SGE_NUM_GENBITS == 2
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
#else
# error "SGE_NUM_GENBITS must be 1 or 2"
#endif
};
#define TXQ_LOCK_ASSERT(qs) mtx_assert(&(qs)->lock, MA_OWNED)
#define TXQ_TRYLOCK(qs) mtx_trylock(&(qs)->lock)
#define TXQ_LOCK(qs) mtx_lock(&(qs)->lock)
#define TXQ_UNLOCK(qs) mtx_unlock(&(qs)->lock)
#define TXQ_RING_EMPTY(qs) drbr_empty((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
#define TXQ_RING_NEEDS_ENQUEUE(qs) \
drbr_needs_enqueue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
#define TXQ_RING_FLUSH(qs) drbr_flush((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
#define TXQ_RING_DEQUEUE_COND(qs, func, arg) \
drbr_dequeue_cond((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr, func, arg)
#define TXQ_RING_DEQUEUE(qs) \
drbr_dequeue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
int cxgb_debug = 0;
static void sge_timer_cb(void *arg);
static void sge_timer_reclaim(void *arg, int ncount);
static void sge_txq_reclaim_handler(void *arg, int ncount);
static void cxgb_start_locked(struct sge_qset *qs);
/*
* XXX need to cope with bursty scheduling by looking at a wider
* window than we are now for determining the need for coalescing
*
*/
static __inline uint64_t
check_pkt_coalesce(struct sge_qset *qs)
{
struct adapter *sc;
struct sge_txq *txq;
uint8_t *fill;
if (__predict_false(cxgb_tx_coalesce_force))
return (1);
txq = &qs->txq[TXQ_ETH];
sc = qs->port->adapter;
fill = &sc->tunq_fill[qs->idx];
if (cxgb_tx_coalesce_enable_start > COALESCE_START_MAX)
cxgb_tx_coalesce_enable_start = COALESCE_START_MAX;
if (cxgb_tx_coalesce_enable_stop < COALESCE_STOP_MIN)
cxgb_tx_coalesce_enable_start = COALESCE_STOP_MIN;
/*
* if the hardware transmit queue is more than 1/8 full
* we mark it as coalescing - we drop back from coalescing
* when we go below 1/32 full and there are no packets enqueued,
* this provides us with some degree of hysteresis
*/
if (*fill != 0 && (txq->in_use <= cxgb_tx_coalesce_enable_stop) &&
TXQ_RING_EMPTY(qs) && (qs->coalescing == 0))
*fill = 0;
else if (*fill == 0 && (txq->in_use >= cxgb_tx_coalesce_enable_start))
*fill = 1;
return (sc->tunq_coalesce);
}
#ifdef __LP64__
static void
set_wr_hdr(struct work_request_hdr *wrp, uint32_t wr_hi, uint32_t wr_lo)
{
uint64_t wr_hilo;
#if _BYTE_ORDER == _LITTLE_ENDIAN
wr_hilo = wr_hi;
wr_hilo |= (((uint64_t)wr_lo)<<32);
#else
wr_hilo = wr_lo;
wr_hilo |= (((uint64_t)wr_hi)<<32);
#endif
wrp->wrh_hilo = wr_hilo;
}
#else
static void
set_wr_hdr(struct work_request_hdr *wrp, uint32_t wr_hi, uint32_t wr_lo)
{
wrp->wrh_hi = wr_hi;
wmb();
wrp->wrh_lo = wr_lo;
}
#endif
struct coalesce_info {
int count;
int nbytes;
};
static int
coalesce_check(struct mbuf *m, void *arg)
{
struct coalesce_info *ci = arg;
int *count = &ci->count;
int *nbytes = &ci->nbytes;
if ((*nbytes == 0) || ((*nbytes + m->m_len <= 10500) &&
(*count < 7) && (m->m_next == NULL))) {
*count += 1;
*nbytes += m->m_len;
return (1);
}
return (0);
}
static struct mbuf *
cxgb_dequeue(struct sge_qset *qs)
{
struct mbuf *m, *m_head, *m_tail;
struct coalesce_info ci;
if (check_pkt_coalesce(qs) == 0)
return TXQ_RING_DEQUEUE(qs);
m_head = m_tail = NULL;
ci.count = ci.nbytes = 0;
do {
m = TXQ_RING_DEQUEUE_COND(qs, coalesce_check, &ci);
if (m_head == NULL) {
m_tail = m_head = m;
} else if (m != NULL) {
m_tail->m_nextpkt = m;
m_tail = m;
}
} while (m != NULL);
if (ci.count > 7)
panic("trying to coalesce %d packets in to one WR", ci.count);
return (m_head);
}
/**
* reclaim_completed_tx - reclaims completed Tx descriptors
* @adapter: the adapter
* @q: the Tx queue to reclaim completed descriptors from
*
* Reclaims Tx descriptors that the SGE has indicated it has processed,
* and frees the associated buffers if possible. Called with the Tx
* queue's lock held.
*/
static __inline int
reclaim_completed_tx(struct sge_qset *qs, int reclaim_min, int queue)
{
struct sge_txq *q = &qs->txq[queue];
int reclaim = desc_reclaimable(q);
if ((cxgb_tx_reclaim_threshold > TX_RECLAIM_MAX) ||
(cxgb_tx_reclaim_threshold < TX_RECLAIM_MIN))
cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT;
if (reclaim < reclaim_min)
return (0);
mtx_assert(&qs->lock, MA_OWNED);
if (reclaim > 0) {
t3_free_tx_desc(qs, reclaim, queue);
q->cleaned += reclaim;
q->in_use -= reclaim;
}
if (isset(&qs->txq_stopped, TXQ_ETH))
clrbit(&qs->txq_stopped, TXQ_ETH);
return (reclaim);
}
/**
* should_restart_tx - are there enough resources to restart a Tx queue?
* @q: the Tx queue
*
* Checks if there are enough descriptors to restart a suspended Tx queue.
*/
static __inline int
should_restart_tx(const struct sge_txq *q)
{
unsigned int r = q->processed - q->cleaned;
return q->in_use - r < (q->size >> 1);
}
/**
* t3_sge_init - initialize SGE
* @adap: the adapter
* @p: the SGE parameters
*
* Performs SGE initialization needed every time after a chip reset.
* We do not initialize any of the queue sets here, instead the driver
* top-level must request those individually. We also do not enable DMA
* here, that should be done after the queues have been set up.
*/
void
t3_sge_init(adapter_t *adap, struct sge_params *p)
{
u_int ctrl, ups;
ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */
ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
#if SGE_NUM_GENBITS == 1
ctrl |= F_EGRGENCTRL;
#endif
if (adap->params.rev > 0) {
if (!(adap->flags & (USING_MSIX | USING_MSI)))
ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
}
t3_write_reg(adap, A_SG_CONTROL, ctrl);
t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
V_LORCQDRBTHRSH(512));
t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
V_TIMEOUT(200 * core_ticks_per_usec(adap)));
t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
adap->params.rev < T3_REV_C ? 1000 : 500);
t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff));
t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024);
}
/**
* sgl_len - calculates the size of an SGL of the given capacity
* @n: the number of SGL entries
*
* Calculates the number of flits needed for a scatter/gather list that
* can hold the given number of entries.
*/
static __inline unsigned int
sgl_len(unsigned int n)
{
return ((3 * n) / 2 + (n & 1));
}
/**
* get_imm_packet - return the next ingress packet buffer from a response
* @resp: the response descriptor containing the packet data
*
* Return a packet containing the immediate data of the given response.
*/
static int
get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m)
{
if (resp->rss_hdr.opcode == CPL_RX_DATA) {
const struct cpl_rx_data *cpl = (const void *)&resp->imm_data[0];
m->m_len = sizeof(*cpl) + ntohs(cpl->len);
} else if (resp->rss_hdr.opcode == CPL_RX_PKT) {
const struct cpl_rx_pkt *cpl = (const void *)&resp->imm_data[0];
m->m_len = sizeof(*cpl) + ntohs(cpl->len);
} else
m->m_len = IMMED_PKT_SIZE;
m->m_ext.ext_buf = NULL;
m->m_ext.ext_type = 0;
memcpy(mtod(m, uint8_t *), resp->imm_data, m->m_len);
return (0);
}
static __inline u_int
flits_to_desc(u_int n)
{
return (flit_desc_map[n]);
}
#define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
F_HIRCQPARITYERROR)
#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
#define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
F_RSPQDISABLED)
/**
* t3_sge_err_intr_handler - SGE async event interrupt handler
* @adapter: the adapter
*
* Interrupt handler for SGE asynchronous (non-data) events.
*/
void
t3_sge_err_intr_handler(adapter_t *adapter)
{
unsigned int v, status;
status = t3_read_reg(adapter, A_SG_INT_CAUSE);
if (status & SGE_PARERR)
CH_ALERT(adapter, "SGE parity error (0x%x)\n",
status & SGE_PARERR);
if (status & SGE_FRAMINGERR)
CH_ALERT(adapter, "SGE framing error (0x%x)\n",
status & SGE_FRAMINGERR);
if (status & F_RSPQCREDITOVERFOW)
CH_ALERT(adapter, "SGE response queue credit overflow\n");
if (status & F_RSPQDISABLED) {
v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS);
CH_ALERT(adapter,
"packet delivered to disabled response queue (0x%x)\n",
(v >> S_RSPQ0DISABLED) & 0xff);
}
t3_write_reg(adapter, A_SG_INT_CAUSE, status);
if (status & SGE_FATALERR)
t3_fatal_err(adapter);
}
void
t3_sge_prep(adapter_t *adap, struct sge_params *p)
{
int i, nqsets, fl_q_size, jumbo_q_size, use_16k, jumbo_buf_size;
nqsets = min(SGE_QSETS / adap->params.nports, mp_ncpus);
nqsets *= adap->params.nports;
fl_q_size = min(nmbclusters/(3*nqsets), FL_Q_SIZE);
while (!powerof2(fl_q_size))
fl_q_size--;
use_16k = cxgb_use_16k_clusters != -1 ? cxgb_use_16k_clusters :
is_offload(adap);
#if __FreeBSD_version >= 700111
if (use_16k) {
jumbo_q_size = min(nmbjumbo16/(3*nqsets), JUMBO_Q_SIZE);
jumbo_buf_size = MJUM16BYTES;
} else {
jumbo_q_size = min(nmbjumbo9/(3*nqsets), JUMBO_Q_SIZE);
jumbo_buf_size = MJUM9BYTES;
}
#else
jumbo_q_size = min(nmbjumbop/(3*nqsets), JUMBO_Q_SIZE);
jumbo_buf_size = MJUMPAGESIZE;
#endif
while (!powerof2(jumbo_q_size))
jumbo_q_size--;
if (fl_q_size < (FL_Q_SIZE / 4) || jumbo_q_size < (JUMBO_Q_SIZE / 2))
device_printf(adap->dev,
"Insufficient clusters and/or jumbo buffers.\n");
p->max_pkt_size = jumbo_buf_size - sizeof(struct cpl_rx_data);
for (i = 0; i < SGE_QSETS; ++i) {
struct qset_params *q = p->qset + i;
if (adap->params.nports > 2) {
q->coalesce_usecs = 50;
} else {
#ifdef INVARIANTS
q->coalesce_usecs = 10;
#else
q->coalesce_usecs = 5;
#endif
}
q->polling = 0;
q->rspq_size = RSPQ_Q_SIZE;
q->fl_size = fl_q_size;
q->jumbo_size = jumbo_q_size;
q->jumbo_buf_size = jumbo_buf_size;
q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE;
q->txq_size[TXQ_OFLD] = is_offload(adap) ? TX_OFLD_Q_SIZE : 16;
q->txq_size[TXQ_CTRL] = TX_CTRL_Q_SIZE;
q->cong_thres = 0;
}
}
int
t3_sge_alloc(adapter_t *sc)
{
/* The parent tag. */
if (bus_dma_tag_create( bus_get_dma_tag(sc->dev),/* PCI parent */
1, 0, /* algnmnt, boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
BUS_SPACE_UNRESTRICTED, /* nsegments */
BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
0, /* flags */
NULL, NULL, /* lock, lockarg */
&sc->parent_dmat)) {
device_printf(sc->dev, "Cannot allocate parent DMA tag\n");
return (ENOMEM);
}
/*
* DMA tag for normal sized RX frames
*/
if (bus_dma_tag_create(sc->parent_dmat, MCLBYTES, 0, BUS_SPACE_MAXADDR,
BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1,
MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_dmat)) {
device_printf(sc->dev, "Cannot allocate RX DMA tag\n");
return (ENOMEM);
}
/*
* DMA tag for jumbo sized RX frames.
*/
if (bus_dma_tag_create(sc->parent_dmat, MJUM16BYTES, 0, BUS_SPACE_MAXADDR,
BUS_SPACE_MAXADDR, NULL, NULL, MJUM16BYTES, 1, MJUM16BYTES,
BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) {
device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n");
return (ENOMEM);
}
/*
* DMA tag for TX frames.
*/
if (bus_dma_tag_create(sc->parent_dmat, 1, 0, BUS_SPACE_MAXADDR,
BUS_SPACE_MAXADDR, NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS,
TX_MAX_SIZE, BUS_DMA_ALLOCNOW,
NULL, NULL, &sc->tx_dmat)) {
device_printf(sc->dev, "Cannot allocate TX DMA tag\n");
return (ENOMEM);
}
return (0);
}
int
t3_sge_free(struct adapter * sc)
{
if (sc->tx_dmat != NULL)
bus_dma_tag_destroy(sc->tx_dmat);
if (sc->rx_jumbo_dmat != NULL)
bus_dma_tag_destroy(sc->rx_jumbo_dmat);
if (sc->rx_dmat != NULL)
bus_dma_tag_destroy(sc->rx_dmat);
if (sc->parent_dmat != NULL)
bus_dma_tag_destroy(sc->parent_dmat);
return (0);
}
void
t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)
{
qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);
qs->rspq.polling = 0 /* p->polling */;
}
#if !defined(__i386__) && !defined(__amd64__)
static void
refill_fl_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
struct refill_fl_cb_arg *cb_arg = arg;
cb_arg->error = error;
cb_arg->seg = segs[0];
cb_arg->nseg = nseg;
}
#endif
/**
* refill_fl - refill an SGE free-buffer list
* @sc: the controller softc
* @q: the free-list to refill
* @n: the number of new buffers to allocate
*
* (Re)populate an SGE free-buffer list with up to @n new packet buffers.
* The caller must assure that @n does not exceed the queue's capacity.
*/
static void
refill_fl(adapter_t *sc, struct sge_fl *q, int n)
{
struct rx_sw_desc *sd = &q->sdesc[q->pidx];
struct rx_desc *d = &q->desc[q->pidx];
struct refill_fl_cb_arg cb_arg;
struct mbuf *m;
caddr_t cl;
int err;
cb_arg.error = 0;
while (n--) {
/*
* We allocate an uninitialized mbuf + cluster, mbuf is
* initialized after rx.
*/
if (q->zone == zone_pack) {
if ((m = m_getcl(M_NOWAIT, MT_NOINIT, M_PKTHDR)) == NULL)
break;
cl = m->m_ext.ext_buf;
} else {
if ((cl = m_cljget(NULL, M_NOWAIT, q->buf_size)) == NULL)
break;
if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) {
uma_zfree(q->zone, cl);
break;
}
}
if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) {
if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) {
log(LOG_WARNING, "bus_dmamap_create failed %d\n", err);
uma_zfree(q->zone, cl);
goto done;
}
sd->flags |= RX_SW_DESC_MAP_CREATED;
}
#if !defined(__i386__) && !defined(__amd64__)
err = bus_dmamap_load(q->entry_tag, sd->map,
cl, q->buf_size, refill_fl_cb, &cb_arg, 0);
if (err != 0 || cb_arg.error) {
if (q->zone == zone_pack)
uma_zfree(q->zone, cl);
m_free(m);
goto done;
}
#else
cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)cl);
#endif
sd->flags |= RX_SW_DESC_INUSE;
sd->rxsd_cl = cl;
sd->m = m;
d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff);
d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff);
d->len_gen = htobe32(V_FLD_GEN1(q->gen));
d->gen2 = htobe32(V_FLD_GEN2(q->gen));
d++;
sd++;
if (++q->pidx == q->size) {
q->pidx = 0;
q->gen ^= 1;
sd = q->sdesc;
d = q->desc;
}
q->credits++;
q->db_pending++;
}
done:
if (q->db_pending >= 32) {
q->db_pending = 0;
t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
}
}
/**
* free_rx_bufs - free the Rx buffers on an SGE free list
* @sc: the controle softc
* @q: the SGE free list to clean up
*
* Release the buffers on an SGE free-buffer Rx queue. HW fetching from
* this queue should be stopped before calling this function.
*/
static void
free_rx_bufs(adapter_t *sc, struct sge_fl *q)
{
u_int cidx = q->cidx;
while (q->credits--) {
struct rx_sw_desc *d = &q->sdesc[cidx];
if (d->flags & RX_SW_DESC_INUSE) {
bus_dmamap_unload(q->entry_tag, d->map);
bus_dmamap_destroy(q->entry_tag, d->map);
if (q->zone == zone_pack) {
m_init(d->m, zone_pack, MCLBYTES,
M_NOWAIT, MT_DATA, M_EXT);
uma_zfree(zone_pack, d->m);
} else {
m_init(d->m, zone_mbuf, MLEN,
M_NOWAIT, MT_DATA, 0);
uma_zfree(zone_mbuf, d->m);
uma_zfree(q->zone, d->rxsd_cl);
}
}
d->rxsd_cl = NULL;
d->m = NULL;
if (++cidx == q->size)
cidx = 0;
}
}
static __inline void
__refill_fl(adapter_t *adap, struct sge_fl *fl)
{
refill_fl(adap, fl, min(16U, fl->size - fl->credits));
}
static __inline void
__refill_fl_lt(adapter_t *adap, struct sge_fl *fl, int max)
{
uint32_t reclaimable = fl->size - fl->credits;
if (reclaimable > 0)
refill_fl(adap, fl, min(max, reclaimable));
}
/**
* recycle_rx_buf - recycle a receive buffer
* @adapter: the adapter
* @q: the SGE free list
* @idx: index of buffer to recycle
*
* Recycles the specified buffer on the given free list by adding it at
* the next available slot on the list.
*/
static void
recycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx)
{
struct rx_desc *from = &q->desc[idx];
struct rx_desc *to = &q->desc[q->pidx];
q->sdesc[q->pidx] = q->sdesc[idx];
to->addr_lo = from->addr_lo; // already big endian
to->addr_hi = from->addr_hi; // likewise
wmb(); /* necessary ? */
to->len_gen = htobe32(V_FLD_GEN1(q->gen));
to->gen2 = htobe32(V_FLD_GEN2(q->gen));
q->credits++;
if (++q->pidx == q->size) {
q->pidx = 0;
q->gen ^= 1;
}
t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
}
static void
alloc_ring_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
{
uint32_t *addr;
addr = arg;
*addr = segs[0].ds_addr;
}
static int
alloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size,
bus_addr_t *phys, void *desc, void *sdesc, bus_dma_tag_t *tag,
bus_dmamap_t *map, bus_dma_tag_t parent_entry_tag, bus_dma_tag_t *entry_tag)
{
size_t len = nelem * elem_size;
void *s = NULL;
void *p = NULL;
int err;
if ((err = bus_dma_tag_create(sc->parent_dmat, PAGE_SIZE, 0,
BUS_SPACE_MAXADDR_32BIT,
BUS_SPACE_MAXADDR, NULL, NULL, len, 1,
len, 0, NULL, NULL, tag)) != 0) {
device_printf(sc->dev, "Cannot allocate descriptor tag\n");
return (ENOMEM);
}
if ((err = bus_dmamem_alloc(*tag, (void **)&p, BUS_DMA_NOWAIT,
map)) != 0) {
device_printf(sc->dev, "Cannot allocate descriptor memory\n");
return (ENOMEM);
}
bus_dmamap_load(*tag, *map, p, len, alloc_ring_cb, phys, 0);
bzero(p, len);
*(void **)desc = p;
if (sw_size) {
len = nelem * sw_size;
s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO);
*(void **)sdesc = s;
}
if (parent_entry_tag == NULL)
return (0);
if ((err = bus_dma_tag_create(parent_entry_tag, 1, 0,
BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS,
TX_MAX_SIZE, BUS_DMA_ALLOCNOW,
NULL, NULL, entry_tag)) != 0) {
device_printf(sc->dev, "Cannot allocate descriptor entry tag\n");
return (ENOMEM);
}
return (0);
}
static void
sge_slow_intr_handler(void *arg, int ncount)
{
adapter_t *sc = arg;
t3_slow_intr_handler(sc);
t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
(void) t3_read_reg(sc, A_PL_INT_ENABLE0);
}
/**
* sge_timer_cb - perform periodic maintenance of an SGE qset
* @data: the SGE queue set to maintain
*
* Runs periodically from a timer to perform maintenance of an SGE queue
* set. It performs two tasks:
*
* a) Cleans up any completed Tx descriptors that may still be pending.
* Normal descriptor cleanup happens when new packets are added to a Tx
* queue so this timer is relatively infrequent and does any cleanup only
* if the Tx queue has not seen any new packets in a while. We make a
* best effort attempt to reclaim descriptors, in that we don't wait
* around if we cannot get a queue's lock (which most likely is because
* someone else is queueing new packets and so will also handle the clean
* up). Since control queues use immediate data exclusively we don't
* bother cleaning them up here.
*
* b) Replenishes Rx queues that have run out due to memory shortage.
* Normally new Rx buffers are added when existing ones are consumed but
* when out of memory a queue can become empty. We try to add only a few
* buffers here, the queue will be replenished fully as these new buffers
* are used up if memory shortage has subsided.
*
* c) Return coalesced response queue credits in case a response queue is
* starved.
*
* d) Ring doorbells for T304 tunnel queues since we have seen doorbell
* fifo overflows and the FW doesn't implement any recovery scheme yet.
*/
static void
sge_timer_cb(void *arg)
{
adapter_t *sc = arg;
if ((sc->flags & USING_MSIX) == 0) {
struct port_info *pi;
struct sge_qset *qs;
struct sge_txq *txq;
int i, j;
int reclaim_ofl, refill_rx;
if (sc->open_device_map == 0)
return;
for (i = 0; i < sc->params.nports; i++) {
pi = &sc->port[i];
for (j = 0; j < pi->nqsets; j++) {
qs = &sc->sge.qs[pi->first_qset + j];
txq = &qs->txq[0];
reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned;
refill_rx = ((qs->fl[0].credits < qs->fl[0].size) ||
(qs->fl[1].credits < qs->fl[1].size));
if (reclaim_ofl || refill_rx) {
taskqueue_enqueue(sc->tq, &pi->timer_reclaim_task);
break;
}
}
}
}
if (sc->params.nports > 2) {
int i;
for_each_port(sc, i) {
struct port_info *pi = &sc->port[i];
t3_write_reg(sc, A_SG_KDOORBELL,
F_SELEGRCNTX |
(FW_TUNNEL_SGEEC_START + pi->first_qset));
}
}
if (((sc->flags & USING_MSIX) == 0 || sc->params.nports > 2) &&
sc->open_device_map != 0)
callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
}
/*
* This is meant to be a catch-all function to keep sge state private
* to sge.c
*
*/
int
t3_sge_init_adapter(adapter_t *sc)
{
- callout_init(&sc->sge_timer_ch, CALLOUT_MPSAFE);
+ callout_init(&sc->sge_timer_ch, 1);
callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc);
return (0);
}
int
t3_sge_reset_adapter(adapter_t *sc)
{
callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
return (0);
}
int
t3_sge_init_port(struct port_info *pi)
{
TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi);
return (0);
}
/**
* refill_rspq - replenish an SGE response queue
* @adapter: the adapter
* @q: the response queue to replenish
* @credits: how many new responses to make available
*
* Replenishes a response queue by making the supplied number of responses
* available to HW.
*/
static __inline void
refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits)
{
/* mbufs are allocated on demand when a rspq entry is processed. */
t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN,
V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
}
static void
sge_txq_reclaim_handler(void *arg, int ncount)
{
struct sge_qset *qs = arg;
int i;
for (i = 0; i < 3; i++)
reclaim_completed_tx(qs, 16, i);
}
static void
sge_timer_reclaim(void *arg, int ncount)
{
struct port_info *pi = arg;
int i, nqsets = pi->nqsets;
adapter_t *sc = pi->adapter;
struct sge_qset *qs;
struct mtx *lock;
KASSERT((sc->flags & USING_MSIX) == 0,
("can't call timer reclaim for msi-x"));
for (i = 0; i < nqsets; i++) {
qs = &sc->sge.qs[pi->first_qset + i];
reclaim_completed_tx(qs, 16, TXQ_OFLD);
lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock :
&sc->sge.qs[0].rspq.lock;
if (mtx_trylock(lock)) {
/* XXX currently assume that we are *NOT* polling */
uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS);
if (qs->fl[0].credits < qs->fl[0].size - 16)
__refill_fl(sc, &qs->fl[0]);
if (qs->fl[1].credits < qs->fl[1].size - 16)
__refill_fl(sc, &qs->fl[1]);
if (status & (1 << qs->rspq.cntxt_id)) {
if (qs->rspq.credits) {
refill_rspq(sc, &qs->rspq, 1);
qs->rspq.credits--;
t3_write_reg(sc, A_SG_RSPQ_FL_STATUS,
1 << qs->rspq.cntxt_id);
}
}
mtx_unlock(lock);
}
}
}
/**
* init_qset_cntxt - initialize an SGE queue set context info
* @qs: the queue set
* @id: the queue set id
*
* Initializes the TIDs and context ids for the queues of a queue set.
*/
static void
init_qset_cntxt(struct sge_qset *qs, u_int id)
{
qs->rspq.cntxt_id = id;
qs->fl[0].cntxt_id = 2 * id;
qs->fl[1].cntxt_id = 2 * id + 1;
qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
/* XXX: a sane limit is needed instead of INT_MAX */
mbufq_init(&qs->txq[TXQ_ETH].sendq, INT_MAX);
mbufq_init(&qs->txq[TXQ_OFLD].sendq, INT_MAX);
mbufq_init(&qs->txq[TXQ_CTRL].sendq, INT_MAX);
}
static void
txq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs)
{
txq->in_use += ndesc;
/*
* XXX we don't handle stopping of queue
* presumably start handles this when we bump against the end
*/
txqs->gen = txq->gen;
txq->unacked += ndesc;
txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5);
txq->unacked &= 31;
txqs->pidx = txq->pidx;
txq->pidx += ndesc;
#ifdef INVARIANTS
if (((txqs->pidx > txq->cidx) &&
(txq->pidx < txqs->pidx) &&
(txq->pidx >= txq->cidx)) ||
((txqs->pidx < txq->cidx) &&
(txq->pidx >= txq-> cidx)) ||
((txqs->pidx < txq->cidx) &&
(txq->cidx < txqs->pidx)))
panic("txqs->pidx=%d txq->pidx=%d txq->cidx=%d",
txqs->pidx, txq->pidx, txq->cidx);
#endif
if (txq->pidx >= txq->size) {
txq->pidx -= txq->size;
txq->gen ^= 1;
}
}
/**
* calc_tx_descs - calculate the number of Tx descriptors for a packet
* @m: the packet mbufs
* @nsegs: the number of segments
*
* Returns the number of Tx descriptors needed for the given Ethernet
* packet. Ethernet packets require addition of WR and CPL headers.
*/
static __inline unsigned int
calc_tx_descs(const struct mbuf *m, int nsegs)
{
unsigned int flits;
if (m->m_pkthdr.len <= PIO_LEN)
return 1;
flits = sgl_len(nsegs) + 2;
if (m->m_pkthdr.csum_flags & CSUM_TSO)
flits++;
return flits_to_desc(flits);
}
/**
* make_sgl - populate a scatter/gather list for a packet
* @sgp: the SGL to populate
* @segs: the packet dma segments
* @nsegs: the number of segments
*
* Generates a scatter/gather list for the buffers that make up a packet
* and returns the SGL size in 8-byte words. The caller must size the SGL
* appropriately.
*/
static __inline void
make_sgl(struct sg_ent *sgp, bus_dma_segment_t *segs, int nsegs)
{
int i, idx;
for (idx = 0, i = 0; i < nsegs; i++) {
/*
* firmware doesn't like empty segments
*/
if (segs[i].ds_len == 0)
continue;
if (i && idx == 0)
++sgp;
sgp->len[idx] = htobe32(segs[i].ds_len);
sgp->addr[idx] = htobe64(segs[i].ds_addr);
idx ^= 1;
}
if (idx) {
sgp->len[idx] = 0;
sgp->addr[idx] = 0;
}
}
/**
* check_ring_tx_db - check and potentially ring a Tx queue's doorbell
* @adap: the adapter
* @q: the Tx queue
*
* Ring the doorbell if a Tx queue is asleep. There is a natural race,
* where the HW is going to sleep just after we checked, however,
* then the interrupt handler will detect the outstanding TX packet
* and ring the doorbell for us.
*
* When GTS is disabled we unconditionally ring the doorbell.
*/
static __inline void
check_ring_tx_db(adapter_t *adap, struct sge_txq *q, int mustring)
{
#if USE_GTS
clear_bit(TXQ_LAST_PKT_DB, &q->flags);
if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
set_bit(TXQ_LAST_PKT_DB, &q->flags);
#ifdef T3_TRACE
T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d",
q->cntxt_id);
#endif
t3_write_reg(adap, A_SG_KDOORBELL,
F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
}
#else
if (mustring || ++q->db_pending >= 32) {
wmb(); /* write descriptors before telling HW */
t3_write_reg(adap, A_SG_KDOORBELL,
F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
q->db_pending = 0;
}
#endif
}
static __inline void
wr_gen2(struct tx_desc *d, unsigned int gen)
{
#if SGE_NUM_GENBITS == 2
d->flit[TX_DESC_FLITS - 1] = htobe64(gen);
#endif
}
/**
* write_wr_hdr_sgl - write a WR header and, optionally, SGL
* @ndesc: number of Tx descriptors spanned by the SGL
* @txd: first Tx descriptor to be written
* @txqs: txq state (generation and producer index)
* @txq: the SGE Tx queue
* @sgl: the SGL
* @flits: number of flits to the start of the SGL in the first descriptor
* @sgl_flits: the SGL size in flits
* @wr_hi: top 32 bits of WR header based on WR type (big endian)
* @wr_lo: low 32 bits of WR header based on WR type (big endian)
*
* Write a work request header and an associated SGL. If the SGL is
* small enough to fit into one Tx descriptor it has already been written
* and we just need to write the WR header. Otherwise we distribute the
* SGL across the number of descriptors it spans.
*/
static void
write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs,
const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits,
unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo)
{
struct work_request_hdr *wrp = (struct work_request_hdr *)txd;
struct tx_sw_desc *txsd = &txq->sdesc[txqs->pidx];
if (__predict_true(ndesc == 1)) {
set_wr_hdr(wrp, htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
V_WR_SGLSFLT(flits)) | wr_hi,
htonl(V_WR_LEN(flits + sgl_flits) | V_WR_GEN(txqs->gen)) |
wr_lo);
wr_gen2(txd, txqs->gen);
} else {
unsigned int ogen = txqs->gen;
const uint64_t *fp = (const uint64_t *)sgl;
struct work_request_hdr *wp = wrp;
wrp->wrh_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
V_WR_SGLSFLT(flits)) | wr_hi;
while (sgl_flits) {
unsigned int avail = WR_FLITS - flits;
if (avail > sgl_flits)
avail = sgl_flits;
memcpy(&txd->flit[flits], fp, avail * sizeof(*fp));
sgl_flits -= avail;
ndesc--;
if (!sgl_flits)
break;
fp += avail;
txd++;
txsd++;
if (++txqs->pidx == txq->size) {
txqs->pidx = 0;
txqs->gen ^= 1;
txd = txq->desc;
txsd = txq->sdesc;
}
/*
* when the head of the mbuf chain
* is freed all clusters will be freed
* with it
*/
wrp = (struct work_request_hdr *)txd;
wrp->wrh_hi = htonl(V_WR_DATATYPE(1) |
V_WR_SGLSFLT(1)) | wr_hi;
wrp->wrh_lo = htonl(V_WR_LEN(min(WR_FLITS,
sgl_flits + 1)) |
V_WR_GEN(txqs->gen)) | wr_lo;
wr_gen2(txd, txqs->gen);
flits = 1;
}
wrp->wrh_hi |= htonl(F_WR_EOP);
wmb();
wp->wrh_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
wr_gen2((struct tx_desc *)wp, ogen);
}
}
/* sizeof(*eh) + sizeof(*ip) + sizeof(*tcp) */
#define TCPPKTHDRSIZE (ETHER_HDR_LEN + 20 + 20)
#define GET_VTAG(cntrl, m) \
do { \
if ((m)->m_flags & M_VLANTAG) \
cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \
} while (0)
static int
t3_encap(struct sge_qset *qs, struct mbuf **m)
{
adapter_t *sc;
struct mbuf *m0;
struct sge_txq *txq;
struct txq_state txqs;
struct port_info *pi;
unsigned int ndesc, flits, cntrl, mlen;
int err, nsegs, tso_info = 0;
struct work_request_hdr *wrp;
struct tx_sw_desc *txsd;
struct sg_ent *sgp, *sgl;
uint32_t wr_hi, wr_lo, sgl_flits;
bus_dma_segment_t segs[TX_MAX_SEGS];
struct tx_desc *txd;
pi = qs->port;
sc = pi->adapter;
txq = &qs->txq[TXQ_ETH];
txd = &txq->desc[txq->pidx];
txsd = &txq->sdesc[txq->pidx];
sgl = txq->txq_sgl;
prefetch(txd);
m0 = *m;
mtx_assert(&qs->lock, MA_OWNED);
cntrl = V_TXPKT_INTF(pi->txpkt_intf);
KASSERT(m0->m_flags & M_PKTHDR, ("not packet header\n"));
if (m0->m_nextpkt == NULL && m0->m_next != NULL &&
m0->m_pkthdr.csum_flags & (CSUM_TSO))
tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz);
if (m0->m_nextpkt != NULL) {
busdma_map_sg_vec(txq->entry_tag, txsd->map, m0, segs, &nsegs);
ndesc = 1;
mlen = 0;
} else {
if ((err = busdma_map_sg_collapse(txq->entry_tag, txsd->map,
&m0, segs, &nsegs))) {
if (cxgb_debug)
printf("failed ... err=%d\n", err);
return (err);
}
mlen = m0->m_pkthdr.len;
ndesc = calc_tx_descs(m0, nsegs);
}
txq_prod(txq, ndesc, &txqs);
KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d", nsegs));
txsd->m = m0;
if (m0->m_nextpkt != NULL) {
struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd;
int i, fidx;
if (nsegs > 7)
panic("trying to coalesce %d packets in to one WR", nsegs);
txq->txq_coalesced += nsegs;
wrp = (struct work_request_hdr *)txd;
flits = nsegs*2 + 1;
for (fidx = 1, i = 0; i < nsegs; i++, fidx += 2) {
struct cpl_tx_pkt_batch_entry *cbe;
uint64_t flit;
uint32_t *hflit = (uint32_t *)&flit;
int cflags = m0->m_pkthdr.csum_flags;
cntrl = V_TXPKT_INTF(pi->txpkt_intf);
GET_VTAG(cntrl, m0);
cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
if (__predict_false(!(cflags & CSUM_IP)))
cntrl |= F_TXPKT_IPCSUM_DIS;
if (__predict_false(!(cflags & (CSUM_TCP | CSUM_UDP |
CSUM_UDP_IPV6 | CSUM_TCP_IPV6))))
cntrl |= F_TXPKT_L4CSUM_DIS;
hflit[0] = htonl(cntrl);
hflit[1] = htonl(segs[i].ds_len | 0x80000000);
flit |= htobe64(1 << 24);
cbe = &cpl_batch->pkt_entry[i];
cbe->cntrl = hflit[0];
cbe->len = hflit[1];
cbe->addr = htobe64(segs[i].ds_addr);
}
wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
V_WR_SGLSFLT(flits)) |
htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl);
wr_lo = htonl(V_WR_LEN(flits) |
V_WR_GEN(txqs.gen)) | htonl(V_WR_TID(txq->token));
set_wr_hdr(wrp, wr_hi, wr_lo);
wmb();
ETHER_BPF_MTAP(pi->ifp, m0);
wr_gen2(txd, txqs.gen);
check_ring_tx_db(sc, txq, 0);
return (0);
} else if (tso_info) {
uint16_t eth_type;
struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd;
struct ether_header *eh;
void *l3hdr;
struct tcphdr *tcp;
txd->flit[2] = 0;
GET_VTAG(cntrl, m0);
cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
hdr->cntrl = htonl(cntrl);
hdr->len = htonl(mlen | 0x80000000);
if (__predict_false(mlen < TCPPKTHDRSIZE)) {
printf("mbuf=%p,len=%d,tso_segsz=%d,csum_flags=%b,flags=%#x",
m0, mlen, m0->m_pkthdr.tso_segsz,
(int)m0->m_pkthdr.csum_flags, CSUM_BITS, m0->m_flags);
panic("tx tso packet too small");
}
/* Make sure that ether, ip, tcp headers are all in m0 */
if (__predict_false(m0->m_len < TCPPKTHDRSIZE)) {
m0 = m_pullup(m0, TCPPKTHDRSIZE);
if (__predict_false(m0 == NULL)) {
/* XXX panic probably an overreaction */
panic("couldn't fit header into mbuf");
}
}
eh = mtod(m0, struct ether_header *);
eth_type = eh->ether_type;
if (eth_type == htons(ETHERTYPE_VLAN)) {
struct ether_vlan_header *evh = (void *)eh;
tso_info |= V_LSO_ETH_TYPE(CPL_ETH_II_VLAN);
l3hdr = evh + 1;
eth_type = evh->evl_proto;
} else {
tso_info |= V_LSO_ETH_TYPE(CPL_ETH_II);
l3hdr = eh + 1;
}
if (eth_type == htons(ETHERTYPE_IP)) {
struct ip *ip = l3hdr;
tso_info |= V_LSO_IPHDR_WORDS(ip->ip_hl);
tcp = (struct tcphdr *)(ip + 1);
} else if (eth_type == htons(ETHERTYPE_IPV6)) {
struct ip6_hdr *ip6 = l3hdr;
KASSERT(ip6->ip6_nxt == IPPROTO_TCP,
("%s: CSUM_TSO with ip6_nxt %d",
__func__, ip6->ip6_nxt));
tso_info |= F_LSO_IPV6;
tso_info |= V_LSO_IPHDR_WORDS(sizeof(*ip6) >> 2);
tcp = (struct tcphdr *)(ip6 + 1);
} else
panic("%s: CSUM_TSO but neither ip nor ip6", __func__);
tso_info |= V_LSO_TCPHDR_WORDS(tcp->th_off);
hdr->lso_info = htonl(tso_info);
if (__predict_false(mlen <= PIO_LEN)) {
/*
* pkt not undersized but fits in PIO_LEN
* Indicates a TSO bug at the higher levels.
*/
txsd->m = NULL;
m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[3]);
flits = (mlen + 7) / 8 + 3;
wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) |
V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) |
F_WR_SOP | F_WR_EOP | txqs.compl);
wr_lo = htonl(V_WR_LEN(flits) |
V_WR_GEN(txqs.gen) | V_WR_TID(txq->token));
set_wr_hdr(&hdr->wr, wr_hi, wr_lo);
wmb();
ETHER_BPF_MTAP(pi->ifp, m0);
wr_gen2(txd, txqs.gen);
check_ring_tx_db(sc, txq, 0);
m_freem(m0);
return (0);
}
flits = 3;
} else {
struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd;
GET_VTAG(cntrl, m0);
cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
if (__predict_false(!(m0->m_pkthdr.csum_flags & CSUM_IP)))
cntrl |= F_TXPKT_IPCSUM_DIS;
if (__predict_false(!(m0->m_pkthdr.csum_flags & (CSUM_TCP |
CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6))))
cntrl |= F_TXPKT_L4CSUM_DIS;
cpl->cntrl = htonl(cntrl);
cpl->len = htonl(mlen | 0x80000000);
if (mlen <= PIO_LEN) {
txsd->m = NULL;
m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]);
flits = (mlen + 7) / 8 + 2;
wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) |
V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) |
F_WR_SOP | F_WR_EOP | txqs.compl);
wr_lo = htonl(V_WR_LEN(flits) |
V_WR_GEN(txqs.gen) | V_WR_TID(txq->token));
set_wr_hdr(&cpl->wr, wr_hi, wr_lo);
wmb();
ETHER_BPF_MTAP(pi->ifp, m0);
wr_gen2(txd, txqs.gen);
check_ring_tx_db(sc, txq, 0);
m_freem(m0);
return (0);
}
flits = 2;
}
wrp = (struct work_request_hdr *)txd;
sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl;
make_sgl(sgp, segs, nsegs);
sgl_flits = sgl_len(nsegs);
ETHER_BPF_MTAP(pi->ifp, m0);
KASSERT(ndesc <= 4, ("ndesc too large %d", ndesc));
wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl);
wr_lo = htonl(V_WR_TID(txq->token));
write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits,
sgl_flits, wr_hi, wr_lo);
check_ring_tx_db(sc, txq, 0);
return (0);
}
void
cxgb_tx_watchdog(void *arg)
{
struct sge_qset *qs = arg;
struct sge_txq *txq = &qs->txq[TXQ_ETH];
if (qs->coalescing != 0 &&
(txq->in_use <= cxgb_tx_coalesce_enable_stop) &&
TXQ_RING_EMPTY(qs))
qs->coalescing = 0;
else if (qs->coalescing == 0 &&
(txq->in_use >= cxgb_tx_coalesce_enable_start))
qs->coalescing = 1;
if (TXQ_TRYLOCK(qs)) {
qs->qs_flags |= QS_FLUSHING;
cxgb_start_locked(qs);
qs->qs_flags &= ~QS_FLUSHING;
TXQ_UNLOCK(qs);
}
if (qs->port->ifp->if_drv_flags & IFF_DRV_RUNNING)
callout_reset_on(&txq->txq_watchdog, hz/4, cxgb_tx_watchdog,
qs, txq->txq_watchdog.c_cpu);
}
static void
cxgb_tx_timeout(void *arg)
{
struct sge_qset *qs = arg;
struct sge_txq *txq = &qs->txq[TXQ_ETH];
if (qs->coalescing == 0 && (txq->in_use >= (txq->size>>3)))
qs->coalescing = 1;
if (TXQ_TRYLOCK(qs)) {
qs->qs_flags |= QS_TIMEOUT;
cxgb_start_locked(qs);
qs->qs_flags &= ~QS_TIMEOUT;
TXQ_UNLOCK(qs);
}
}
static void
cxgb_start_locked(struct sge_qset *qs)
{
struct mbuf *m_head = NULL;
struct sge_txq *txq = &qs->txq[TXQ_ETH];
struct port_info *pi = qs->port;
struct ifnet *ifp = pi->ifp;
if (qs->qs_flags & (QS_FLUSHING|QS_TIMEOUT))
reclaim_completed_tx(qs, 0, TXQ_ETH);
if (!pi->link_config.link_ok) {
TXQ_RING_FLUSH(qs);
return;
}
TXQ_LOCK_ASSERT(qs);
while (!TXQ_RING_EMPTY(qs) && (ifp->if_drv_flags & IFF_DRV_RUNNING) &&
pi->link_config.link_ok) {
reclaim_completed_tx(qs, cxgb_tx_reclaim_threshold, TXQ_ETH);
if (txq->size - txq->in_use <= TX_MAX_DESC)
break;
if ((m_head = cxgb_dequeue(qs)) == NULL)
break;
/*
* Encapsulation can modify our pointer, and or make it
* NULL on failure. In that event, we can't requeue.
*/
if (t3_encap(qs, &m_head) || m_head == NULL)
break;
m_head = NULL;
}
if (txq->db_pending)
check_ring_tx_db(pi->adapter, txq, 1);
if (!TXQ_RING_EMPTY(qs) && callout_pending(&txq->txq_timer) == 0 &&
pi->link_config.link_ok)
callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout,
qs, txq->txq_timer.c_cpu);
if (m_head != NULL)
m_freem(m_head);
}
static int
cxgb_transmit_locked(struct ifnet *ifp, struct sge_qset *qs, struct mbuf *m)
{
struct port_info *pi = qs->port;
struct sge_txq *txq = &qs->txq[TXQ_ETH];
struct buf_ring *br = txq->txq_mr;
int error, avail;
avail = txq->size - txq->in_use;
TXQ_LOCK_ASSERT(qs);
/*
* We can only do a direct transmit if the following are true:
* - we aren't coalescing (ring < 3/4 full)
* - the link is up -- checked in caller
* - there are no packets enqueued already
* - there is space in hardware transmit queue
*/
if (check_pkt_coalesce(qs) == 0 &&
!TXQ_RING_NEEDS_ENQUEUE(qs) && avail > TX_MAX_DESC) {
if (t3_encap(qs, &m)) {
if (m != NULL &&
(error = drbr_enqueue(ifp, br, m)) != 0)
return (error);
} else {
if (txq->db_pending)
check_ring_tx_db(pi->adapter, txq, 1);
/*
* We've bypassed the buf ring so we need to update
* the stats directly
*/
txq->txq_direct_packets++;
txq->txq_direct_bytes += m->m_pkthdr.len;
}
} else if ((error = drbr_enqueue(ifp, br, m)) != 0)
return (error);
reclaim_completed_tx(qs, cxgb_tx_reclaim_threshold, TXQ_ETH);
if (!TXQ_RING_EMPTY(qs) && pi->link_config.link_ok &&
(!check_pkt_coalesce(qs) || (drbr_inuse(ifp, br) >= 7)))
cxgb_start_locked(qs);
else if (!TXQ_RING_EMPTY(qs) && !callout_pending(&txq->txq_timer))
callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout,
qs, txq->txq_timer.c_cpu);
return (0);
}
int
cxgb_transmit(struct ifnet *ifp, struct mbuf *m)
{
struct sge_qset *qs;
struct port_info *pi = ifp->if_softc;
int error, qidx = pi->first_qset;
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0
||(!pi->link_config.link_ok)) {
m_freem(m);
return (0);
}
/* check if flowid is set */
if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
qidx = (m->m_pkthdr.flowid % pi->nqsets) + pi->first_qset;
qs = &pi->adapter->sge.qs[qidx];
if (TXQ_TRYLOCK(qs)) {
/* XXX running */
error = cxgb_transmit_locked(ifp, qs, m);
TXQ_UNLOCK(qs);
} else
error = drbr_enqueue(ifp, qs->txq[TXQ_ETH].txq_mr, m);
return (error);
}
void
cxgb_qflush(struct ifnet *ifp)
{
/*
* flush any enqueued mbufs in the buf_rings
* and in the transmit queues
* no-op for now
*/
return;
}
/**
* write_imm - write a packet into a Tx descriptor as immediate data
* @d: the Tx descriptor to write
* @m: the packet
* @len: the length of packet data to write as immediate data
* @gen: the generation bit value to write
*
* Writes a packet as immediate data into a Tx descriptor. The packet
* contains a work request at its beginning. We must write the packet
* carefully so the SGE doesn't read accidentally before it's written in
* its entirety.
*/
static __inline void
write_imm(struct tx_desc *d, caddr_t src,
unsigned int len, unsigned int gen)
{
struct work_request_hdr *from = (struct work_request_hdr *)src;
struct work_request_hdr *to = (struct work_request_hdr *)d;
uint32_t wr_hi, wr_lo;
KASSERT(len <= WR_LEN && len >= sizeof(*from),
("%s: invalid len %d", __func__, len));
memcpy(&to[1], &from[1], len - sizeof(*from));
wr_hi = from->wrh_hi | htonl(F_WR_SOP | F_WR_EOP |
V_WR_BCNTLFLT(len & 7));
wr_lo = from->wrh_lo | htonl(V_WR_GEN(gen) | V_WR_LEN((len + 7) / 8));
set_wr_hdr(to, wr_hi, wr_lo);
wmb();
wr_gen2(d, gen);
}
/**
* check_desc_avail - check descriptor availability on a send queue
* @adap: the adapter
* @q: the TX queue
* @m: the packet needing the descriptors
* @ndesc: the number of Tx descriptors needed
* @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
*
* Checks if the requested number of Tx descriptors is available on an
* SGE send queue. If the queue is already suspended or not enough
* descriptors are available the packet is queued for later transmission.
* Must be called with the Tx queue locked.
*
* Returns 0 if enough descriptors are available, 1 if there aren't
* enough descriptors and the packet has been queued, and 2 if the caller
* needs to retry because there weren't enough descriptors at the
* beginning of the call but some freed up in the mean time.
*/
static __inline int
check_desc_avail(adapter_t *adap, struct sge_txq *q,
struct mbuf *m, unsigned int ndesc,
unsigned int qid)
{
/*
* XXX We currently only use this for checking the control queue
* the control queue is only used for binding qsets which happens
* at init time so we are guaranteed enough descriptors
*/
if (__predict_false(mbufq_len(&q->sendq))) {
addq_exit: (void )mbufq_enqueue(&q->sendq, m);
return 1;
}
if (__predict_false(q->size - q->in_use < ndesc)) {
struct sge_qset *qs = txq_to_qset(q, qid);
setbit(&qs->txq_stopped, qid);
if (should_restart_tx(q) &&
test_and_clear_bit(qid, &qs->txq_stopped))
return 2;
q->stops++;
goto addq_exit;
}
return 0;
}
/**
* reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
* @q: the SGE control Tx queue
*
* This is a variant of reclaim_completed_tx() that is used for Tx queues
* that send only immediate data (presently just the control queues) and
* thus do not have any mbufs
*/
static __inline void
reclaim_completed_tx_imm(struct sge_txq *q)
{
unsigned int reclaim = q->processed - q->cleaned;
q->in_use -= reclaim;
q->cleaned += reclaim;
}
/**
* ctrl_xmit - send a packet through an SGE control Tx queue
* @adap: the adapter
* @q: the control queue
* @m: the packet
*
* Send a packet through an SGE control Tx queue. Packets sent through
* a control queue must fit entirely as immediate data in a single Tx
* descriptor and have no page fragments.
*/
static int
ctrl_xmit(adapter_t *adap, struct sge_qset *qs, struct mbuf *m)
{
int ret;
struct work_request_hdr *wrp = mtod(m, struct work_request_hdr *);
struct sge_txq *q = &qs->txq[TXQ_CTRL];
KASSERT(m->m_len <= WR_LEN, ("%s: bad tx data", __func__));
wrp->wrh_hi |= htonl(F_WR_SOP | F_WR_EOP);
wrp->wrh_lo = htonl(V_WR_TID(q->token));
TXQ_LOCK(qs);
again: reclaim_completed_tx_imm(q);
ret = check_desc_avail(adap, q, m, 1, TXQ_CTRL);
if (__predict_false(ret)) {
if (ret == 1) {
TXQ_UNLOCK(qs);
return (ENOSPC);
}
goto again;
}
write_imm(&q->desc[q->pidx], m->m_data, m->m_len, q->gen);
q->in_use++;
if (++q->pidx >= q->size) {
q->pidx = 0;
q->gen ^= 1;
}
TXQ_UNLOCK(qs);
wmb();
t3_write_reg(adap, A_SG_KDOORBELL,
F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
m_free(m);
return (0);
}
/**
* restart_ctrlq - restart a suspended control queue
* @qs: the queue set cotaining the control queue
*
* Resumes transmission on a suspended Tx control queue.
*/
static void
restart_ctrlq(void *data, int npending)
{
struct mbuf *m;
struct sge_qset *qs = (struct sge_qset *)data;
struct sge_txq *q = &qs->txq[TXQ_CTRL];
adapter_t *adap = qs->port->adapter;
TXQ_LOCK(qs);
again: reclaim_completed_tx_imm(q);
while (q->in_use < q->size &&
(m = mbufq_dequeue(&q->sendq)) != NULL) {
write_imm(&q->desc[q->pidx], m->m_data, m->m_len, q->gen);
m_free(m);
if (++q->pidx >= q->size) {
q->pidx = 0;
q->gen ^= 1;
}
q->in_use++;
}
if (mbufq_len(&q->sendq)) {
setbit(&qs->txq_stopped, TXQ_CTRL);
if (should_restart_tx(q) &&
test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
goto again;
q->stops++;
}
TXQ_UNLOCK(qs);
t3_write_reg(adap, A_SG_KDOORBELL,
F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
}
/*
* Send a management message through control queue 0
*/
int
t3_mgmt_tx(struct adapter *adap, struct mbuf *m)
{
return ctrl_xmit(adap, &adap->sge.qs[0], m);
}
/**
* free_qset - free the resources of an SGE queue set
* @sc: the controller owning the queue set
* @q: the queue set
*
* Release the HW and SW resources associated with an SGE queue set, such
* as HW contexts, packet buffers, and descriptor rings. Traffic to the
* queue set must be quiesced prior to calling this.
*/
static void
t3_free_qset(adapter_t *sc, struct sge_qset *q)
{
int i;
reclaim_completed_tx(q, 0, TXQ_ETH);
if (q->txq[TXQ_ETH].txq_mr != NULL)
buf_ring_free(q->txq[TXQ_ETH].txq_mr, M_DEVBUF);
if (q->txq[TXQ_ETH].txq_ifq != NULL) {
ifq_delete(q->txq[TXQ_ETH].txq_ifq);
free(q->txq[TXQ_ETH].txq_ifq, M_DEVBUF);
}
for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
if (q->fl[i].desc) {
mtx_lock_spin(&sc->sge.reg_lock);
t3_sge_disable_fl(sc, q->fl[i].cntxt_id);
mtx_unlock_spin(&sc->sge.reg_lock);
bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map);
bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc,
q->fl[i].desc_map);
bus_dma_tag_destroy(q->fl[i].desc_tag);
bus_dma_tag_destroy(q->fl[i].entry_tag);
}
if (q->fl[i].sdesc) {
free_rx_bufs(sc, &q->fl[i]);
free(q->fl[i].sdesc, M_DEVBUF);
}
}
mtx_unlock(&q->lock);
MTX_DESTROY(&q->lock);
for (i = 0; i < SGE_TXQ_PER_SET; i++) {
if (q->txq[i].desc) {
mtx_lock_spin(&sc->sge.reg_lock);
t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0);
mtx_unlock_spin(&sc->sge.reg_lock);
bus_dmamap_unload(q->txq[i].desc_tag,
q->txq[i].desc_map);
bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc,
q->txq[i].desc_map);
bus_dma_tag_destroy(q->txq[i].desc_tag);
bus_dma_tag_destroy(q->txq[i].entry_tag);
}
if (q->txq[i].sdesc) {
free(q->txq[i].sdesc, M_DEVBUF);
}
}
if (q->rspq.desc) {
mtx_lock_spin(&sc->sge.reg_lock);
t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id);
mtx_unlock_spin(&sc->sge.reg_lock);
bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map);
bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc,
q->rspq.desc_map);
bus_dma_tag_destroy(q->rspq.desc_tag);
MTX_DESTROY(&q->rspq.lock);
}
#if defined(INET6) || defined(INET)
tcp_lro_free(&q->lro.ctrl);
#endif
bzero(q, sizeof(*q));
}
/**
* t3_free_sge_resources - free SGE resources
* @sc: the adapter softc
*
* Frees resources used by the SGE queue sets.
*/
void
t3_free_sge_resources(adapter_t *sc, int nqsets)
{
int i;
for (i = 0; i < nqsets; ++i) {
TXQ_LOCK(&sc->sge.qs[i]);
t3_free_qset(sc, &sc->sge.qs[i]);
}
}
/**
* t3_sge_start - enable SGE
* @sc: the controller softc
*
* Enables the SGE for DMAs. This is the last step in starting packet
* transfers.
*/
void
t3_sge_start(adapter_t *sc)
{
t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
}
/**
* t3_sge_stop - disable SGE operation
* @sc: the adapter
*
* Disables the DMA engine. This can be called in emeregencies (e.g.,
* from error interrupts) or from normal process context. In the latter
* case it also disables any pending queue restart tasklets. Note that
* if it is called in interrupt context it cannot disable the restart
* tasklets as it cannot wait, however the tasklets will have no effect
* since the doorbells are disabled and the driver will call this again
* later from process context, at which time the tasklets will be stopped
* if they are still running.
*/
void
t3_sge_stop(adapter_t *sc)
{
int i, nqsets;
t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, 0);
if (sc->tq == NULL)
return;
for (nqsets = i = 0; i < (sc)->params.nports; i++)
nqsets += sc->port[i].nqsets;
#ifdef notyet
/*
*
* XXX
*/
for (i = 0; i < nqsets; ++i) {
struct sge_qset *qs = &sc->sge.qs[i];
taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_task);
taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_task);
}
#endif
}
/**
* t3_free_tx_desc - reclaims Tx descriptors and their buffers
* @adapter: the adapter
* @q: the Tx queue to reclaim descriptors from
* @reclaimable: the number of descriptors to reclaim
* @m_vec_size: maximum number of buffers to reclaim
* @desc_reclaimed: returns the number of descriptors reclaimed
*
* Reclaims Tx descriptors from an SGE Tx queue and frees the associated
* Tx buffers. Called with the Tx queue lock held.
*
* Returns number of buffers of reclaimed
*/
void
t3_free_tx_desc(struct sge_qset *qs, int reclaimable, int queue)
{
struct tx_sw_desc *txsd;
unsigned int cidx, mask;
struct sge_txq *q = &qs->txq[queue];
#ifdef T3_TRACE
T3_TRACE2(sc->tb[q->cntxt_id & 7],
"reclaiming %u Tx descriptors at cidx %u", reclaimable, cidx);
#endif
cidx = q->cidx;
mask = q->size - 1;
txsd = &q->sdesc[cidx];
mtx_assert(&qs->lock, MA_OWNED);
while (reclaimable--) {
prefetch(q->sdesc[(cidx + 1) & mask].m);
prefetch(q->sdesc[(cidx + 2) & mask].m);
if (txsd->m != NULL) {
if (txsd->flags & TX_SW_DESC_MAPPED) {
bus_dmamap_unload(q->entry_tag, txsd->map);
txsd->flags &= ~TX_SW_DESC_MAPPED;
}
m_freem_list(txsd->m);
txsd->m = NULL;
} else
q->txq_skipped++;
++txsd;
if (++cidx == q->size) {
cidx = 0;
txsd = q->sdesc;
}
}
q->cidx = cidx;
}
/**
* is_new_response - check if a response is newly written
* @r: the response descriptor
* @q: the response queue
*
* Returns true if a response descriptor contains a yet unprocessed
* response.
*/
static __inline int
is_new_response(const struct rsp_desc *r,
const struct sge_rspq *q)
{
return (r->intr_gen & F_RSPD_GEN2) == q->gen;
}
#define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
#define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
/* How long to delay the next interrupt in case of memory shortage, in 0.1us. */
#define NOMEM_INTR_DELAY 2500
#ifdef TCP_OFFLOAD
/**
* write_ofld_wr - write an offload work request
* @adap: the adapter
* @m: the packet to send
* @q: the Tx queue
* @pidx: index of the first Tx descriptor to write
* @gen: the generation value to use
* @ndesc: number of descriptors the packet will occupy
*
* Write an offload work request to send the supplied packet. The packet
* data already carry the work request with most fields populated.
*/
static void
write_ofld_wr(adapter_t *adap, struct mbuf *m, struct sge_txq *q,
unsigned int pidx, unsigned int gen, unsigned int ndesc)
{
unsigned int sgl_flits, flits;
int i, idx, nsegs, wrlen;
struct work_request_hdr *from;
struct sg_ent *sgp, t3sgl[TX_MAX_SEGS / 2 + 1];
struct tx_desc *d = &q->desc[pidx];
struct txq_state txqs;
struct sglist_seg *segs;
struct ofld_hdr *oh = mtod(m, struct ofld_hdr *);
struct sglist *sgl;
from = (void *)(oh + 1); /* Start of WR within mbuf */
wrlen = m->m_len - sizeof(*oh);
if (!(oh->flags & F_HDR_SGL)) {
write_imm(d, (caddr_t)from, wrlen, gen);
/*
* mbuf with "real" immediate tx data will be enqueue_wr'd by
* t3_push_frames and freed in wr_ack. Others, like those sent
* down by close_conn, t3_send_reset, etc. should be freed here.
*/
if (!(oh->flags & F_HDR_DF))
m_free(m);
return;
}
memcpy(&d->flit[1], &from[1], wrlen - sizeof(*from));
sgl = oh->sgl;
flits = wrlen / 8;
sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : t3sgl;
nsegs = sgl->sg_nseg;
segs = sgl->sg_segs;
for (idx = 0, i = 0; i < nsegs; i++) {
KASSERT(segs[i].ss_len, ("%s: 0 len in sgl", __func__));
if (i && idx == 0)
++sgp;
sgp->len[idx] = htobe32(segs[i].ss_len);
sgp->addr[idx] = htobe64(segs[i].ss_paddr);
idx ^= 1;
}
if (idx) {
sgp->len[idx] = 0;
sgp->addr[idx] = 0;
}
sgl_flits = sgl_len(nsegs);
txqs.gen = gen;
txqs.pidx = pidx;
txqs.compl = 0;
write_wr_hdr_sgl(ndesc, d, &txqs, q, t3sgl, flits, sgl_flits,
from->wrh_hi, from->wrh_lo);
}
/**
* ofld_xmit - send a packet through an offload queue
* @adap: the adapter
* @q: the Tx offload queue
* @m: the packet
*
* Send an offload packet through an SGE offload queue.
*/
static int
ofld_xmit(adapter_t *adap, struct sge_qset *qs, struct mbuf *m)
{
int ret;
unsigned int ndesc;
unsigned int pidx, gen;
struct sge_txq *q = &qs->txq[TXQ_OFLD];
struct ofld_hdr *oh = mtod(m, struct ofld_hdr *);
ndesc = G_HDR_NDESC(oh->flags);
TXQ_LOCK(qs);
again: reclaim_completed_tx(qs, 16, TXQ_OFLD);
ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD);
if (__predict_false(ret)) {
if (ret == 1) {
TXQ_UNLOCK(qs);
return (EINTR);
}
goto again;
}
gen = q->gen;
q->in_use += ndesc;
pidx = q->pidx;
q->pidx += ndesc;
if (q->pidx >= q->size) {
q->pidx -= q->size;
q->gen ^= 1;
}
write_ofld_wr(adap, m, q, pidx, gen, ndesc);
check_ring_tx_db(adap, q, 1);
TXQ_UNLOCK(qs);
return (0);
}
/**
* restart_offloadq - restart a suspended offload queue
* @qs: the queue set cotaining the offload queue
*
* Resumes transmission on a suspended Tx offload queue.
*/
static void
restart_offloadq(void *data, int npending)
{
struct mbuf *m;
struct sge_qset *qs = data;
struct sge_txq *q = &qs->txq[TXQ_OFLD];
adapter_t *adap = qs->port->adapter;
int cleaned;
TXQ_LOCK(qs);
again: cleaned = reclaim_completed_tx(qs, 16, TXQ_OFLD);
while ((m = mbufq_first(&q->sendq)) != NULL) {
unsigned int gen, pidx;
struct ofld_hdr *oh = mtod(m, struct ofld_hdr *);
unsigned int ndesc = G_HDR_NDESC(oh->flags);
if (__predict_false(q->size - q->in_use < ndesc)) {
setbit(&qs->txq_stopped, TXQ_OFLD);
if (should_restart_tx(q) &&
test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
goto again;
q->stops++;
break;
}
gen = q->gen;
q->in_use += ndesc;
pidx = q->pidx;
q->pidx += ndesc;
if (q->pidx >= q->size) {
q->pidx -= q->size;
q->gen ^= 1;
}
(void)mbufq_dequeue(&q->sendq);
TXQ_UNLOCK(qs);
write_ofld_wr(adap, m, q, pidx, gen, ndesc);
TXQ_LOCK(qs);
}
#if USE_GTS
set_bit(TXQ_RUNNING, &q->flags);
set_bit(TXQ_LAST_PKT_DB, &q->flags);
#endif
TXQ_UNLOCK(qs);
wmb();
t3_write_reg(adap, A_SG_KDOORBELL,
F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
}
/**
* t3_offload_tx - send an offload packet
* @m: the packet
*
* Sends an offload packet. We use the packet priority to select the
* appropriate Tx queue as follows: bit 0 indicates whether the packet
* should be sent as regular or control, bits 1-3 select the queue set.
*/
int
t3_offload_tx(struct adapter *sc, struct mbuf *m)
{
struct ofld_hdr *oh = mtod(m, struct ofld_hdr *);
struct sge_qset *qs = &sc->sge.qs[G_HDR_QSET(oh->flags)];
if (oh->flags & F_HDR_CTRL) {
m_adj(m, sizeof (*oh)); /* trim ofld_hdr off */
return (ctrl_xmit(sc, qs, m));
} else
return (ofld_xmit(sc, qs, m));
}
#endif
static void
restart_tx(struct sge_qset *qs)
{
struct adapter *sc = qs->port->adapter;
if (isset(&qs->txq_stopped, TXQ_OFLD) &&
should_restart_tx(&qs->txq[TXQ_OFLD]) &&
test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
qs->txq[TXQ_OFLD].restarts++;
taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task);
}
if (isset(&qs->txq_stopped, TXQ_CTRL) &&
should_restart_tx(&qs->txq[TXQ_CTRL]) &&
test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
qs->txq[TXQ_CTRL].restarts++;
taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task);
}
}
/**
* t3_sge_alloc_qset - initialize an SGE queue set
* @sc: the controller softc
* @id: the queue set id
* @nports: how many Ethernet ports will be using this queue set
* @irq_vec_idx: the IRQ vector index for response queue interrupts
* @p: configuration parameters for this queue set
* @ntxq: number of Tx queues for the queue set
* @pi: port info for queue set
*
* Allocate resources and initialize an SGE queue set. A queue set
* comprises a response queue, two Rx free-buffer queues, and up to 3
* Tx queues. The Tx queues are assigned roles in the order Ethernet
* queue, offload queue, and control queue.
*/
int
t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
const struct qset_params *p, int ntxq, struct port_info *pi)
{
struct sge_qset *q = &sc->sge.qs[id];
int i, ret = 0;
MTX_INIT(&q->lock, q->namebuf, NULL, MTX_DEF);
q->port = pi;
q->adap = sc;
if ((q->txq[TXQ_ETH].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size,
M_DEVBUF, M_WAITOK, &q->lock)) == NULL) {
device_printf(sc->dev, "failed to allocate mbuf ring\n");
goto err;
}
if ((q->txq[TXQ_ETH].txq_ifq = malloc(sizeof(struct ifaltq), M_DEVBUF,
M_NOWAIT | M_ZERO)) == NULL) {
device_printf(sc->dev, "failed to allocate ifq\n");
goto err;
}
ifq_init(q->txq[TXQ_ETH].txq_ifq, pi->ifp);
callout_init(&q->txq[TXQ_ETH].txq_timer, 1);
callout_init(&q->txq[TXQ_ETH].txq_watchdog, 1);
q->txq[TXQ_ETH].txq_timer.c_cpu = id % mp_ncpus;
q->txq[TXQ_ETH].txq_watchdog.c_cpu = id % mp_ncpus;
init_qset_cntxt(q, id);
q->idx = id;
if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc),
sizeof(struct rx_sw_desc), &q->fl[0].phys_addr,
&q->fl[0].desc, &q->fl[0].sdesc,
&q->fl[0].desc_tag, &q->fl[0].desc_map,
sc->rx_dmat, &q->fl[0].entry_tag)) != 0) {
printf("error %d from alloc ring fl0\n", ret);
goto err;
}
if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc),
sizeof(struct rx_sw_desc), &q->fl[1].phys_addr,
&q->fl[1].desc, &q->fl[1].sdesc,
&q->fl[1].desc_tag, &q->fl[1].desc_map,
sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) {
printf("error %d from alloc ring fl1\n", ret);
goto err;
}
if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0,
&q->rspq.phys_addr, &q->rspq.desc, NULL,
&q->rspq.desc_tag, &q->rspq.desc_map,
NULL, NULL)) != 0) {
printf("error %d from alloc ring rspq\n", ret);
goto err;
}
snprintf(q->rspq.lockbuf, RSPQ_NAME_LEN, "t3 rspq lock %d:%d",
device_get_unit(sc->dev), irq_vec_idx);
MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF);
for (i = 0; i < ntxq; ++i) {
size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
if ((ret = alloc_ring(sc, p->txq_size[i],
sizeof(struct tx_desc), sz,
&q->txq[i].phys_addr, &q->txq[i].desc,
&q->txq[i].sdesc, &q->txq[i].desc_tag,
&q->txq[i].desc_map,
sc->tx_dmat, &q->txq[i].entry_tag)) != 0) {
printf("error %d from alloc ring tx %i\n", ret, i);
goto err;
}
mbufq_init(&q->txq[i].sendq, INT_MAX);
q->txq[i].gen = 1;
q->txq[i].size = p->txq_size[i];
}
#ifdef TCP_OFFLOAD
TASK_INIT(&q->txq[TXQ_OFLD].qresume_task, 0, restart_offloadq, q);
#endif
TASK_INIT(&q->txq[TXQ_CTRL].qresume_task, 0, restart_ctrlq, q);
TASK_INIT(&q->txq[TXQ_ETH].qreclaim_task, 0, sge_txq_reclaim_handler, q);
TASK_INIT(&q->txq[TXQ_OFLD].qreclaim_task, 0, sge_txq_reclaim_handler, q);
q->fl[0].gen = q->fl[1].gen = 1;
q->fl[0].size = p->fl_size;
q->fl[1].size = p->jumbo_size;
q->rspq.gen = 1;
q->rspq.cidx = 0;
q->rspq.size = p->rspq_size;
q->txq[TXQ_ETH].stop_thres = nports *
flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3);
q->fl[0].buf_size = MCLBYTES;
q->fl[0].zone = zone_pack;
q->fl[0].type = EXT_PACKET;
if (p->jumbo_buf_size == MJUM16BYTES) {
q->fl[1].zone = zone_jumbo16;
q->fl[1].type = EXT_JUMBO16;
} else if (p->jumbo_buf_size == MJUM9BYTES) {
q->fl[1].zone = zone_jumbo9;
q->fl[1].type = EXT_JUMBO9;
} else if (p->jumbo_buf_size == MJUMPAGESIZE) {
q->fl[1].zone = zone_jumbop;
q->fl[1].type = EXT_JUMBOP;
} else {
KASSERT(0, ("can't deal with jumbo_buf_size %d.", p->jumbo_buf_size));
ret = EDOOFUS;
goto err;
}
q->fl[1].buf_size = p->jumbo_buf_size;
/* Allocate and setup the lro_ctrl structure */
q->lro.enabled = !!(pi->ifp->if_capenable & IFCAP_LRO);
#if defined(INET6) || defined(INET)
ret = tcp_lro_init(&q->lro.ctrl);
if (ret) {
printf("error %d from tcp_lro_init\n", ret);
goto err;
}
#endif
q->lro.ctrl.ifp = pi->ifp;
mtx_lock_spin(&sc->sge.reg_lock);
ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx,
q->rspq.phys_addr, q->rspq.size,
q->fl[0].buf_size, 1, 0);
if (ret) {
printf("error %d from t3_sge_init_rspcntxt\n", ret);
goto err_unlock;
}
for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0,
q->fl[i].phys_addr, q->fl[i].size,
q->fl[i].buf_size, p->cong_thres, 1,
0);
if (ret) {
printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i);
goto err_unlock;
}
}
ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
1, 0);
if (ret) {
printf("error %d from t3_sge_init_ecntxt\n", ret);
goto err_unlock;
}
if (ntxq > 1) {
ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id,
USE_GTS, SGE_CNTXT_OFLD, id,
q->txq[TXQ_OFLD].phys_addr,
q->txq[TXQ_OFLD].size, 0, 1, 0);
if (ret) {
printf("error %d from t3_sge_init_ecntxt\n", ret);
goto err_unlock;
}
}
if (ntxq > 2) {
ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0,
SGE_CNTXT_CTRL, id,
q->txq[TXQ_CTRL].phys_addr,
q->txq[TXQ_CTRL].size,
q->txq[TXQ_CTRL].token, 1, 0);
if (ret) {
printf("error %d from t3_sge_init_ecntxt\n", ret);
goto err_unlock;
}
}
mtx_unlock_spin(&sc->sge.reg_lock);
t3_update_qset_coalesce(q, p);
refill_fl(sc, &q->fl[0], q->fl[0].size);
refill_fl(sc, &q->fl[1], q->fl[1].size);
refill_rspq(sc, &q->rspq, q->rspq.size - 1);
t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
V_NEWTIMER(q->rspq.holdoff_tmr));
return (0);
err_unlock:
mtx_unlock_spin(&sc->sge.reg_lock);
err:
TXQ_LOCK(q);
t3_free_qset(sc, q);
return (ret);
}
/*
* Remove CPL_RX_PKT headers from the mbuf and reduce it to a regular mbuf with
* ethernet data. Hardware assistance with various checksums and any vlan tag
* will also be taken into account here.
*/
void
t3_rx_eth(struct adapter *adap, struct mbuf *m, int ethpad)
{
struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad);
struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]];
struct ifnet *ifp = pi->ifp;
if (cpl->vlan_valid) {
m->m_pkthdr.ether_vtag = ntohs(cpl->vlan);
m->m_flags |= M_VLANTAG;
}
m->m_pkthdr.rcvif = ifp;
/*
* adjust after conversion to mbuf chain
*/
m->m_pkthdr.len -= (sizeof(*cpl) + ethpad);
m->m_len -= (sizeof(*cpl) + ethpad);
m->m_data += (sizeof(*cpl) + ethpad);
if (!cpl->fragment && cpl->csum_valid && cpl->csum == 0xffff) {
struct ether_header *eh = mtod(m, void *);
uint16_t eh_type;
if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
struct ether_vlan_header *evh = mtod(m, void *);
eh_type = evh->evl_proto;
} else
eh_type = eh->ether_type;
if (ifp->if_capenable & IFCAP_RXCSUM &&
eh_type == htons(ETHERTYPE_IP)) {
m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED |
CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
m->m_pkthdr.csum_data = 0xffff;
} else if (ifp->if_capenable & IFCAP_RXCSUM_IPV6 &&
eh_type == htons(ETHERTYPE_IPV6)) {
m->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 |
CSUM_PSEUDO_HDR);
m->m_pkthdr.csum_data = 0xffff;
}
}
}
/**
* get_packet - return the next ingress packet buffer from a free list
* @adap: the adapter that received the packet
* @drop_thres: # of remaining buffers before we start dropping packets
* @qs: the qset that the SGE free list holding the packet belongs to
* @mh: the mbuf header, contains a pointer to the head and tail of the mbuf chain
* @r: response descriptor
*
* Get the next packet from a free list and complete setup of the
* sk_buff. If the packet is small we make a copy and recycle the
* original buffer, otherwise we use the original buffer itself. If a
* positive drop threshold is supplied packets are dropped and their
* buffers recycled if (a) the number of remaining buffers is under the
* threshold and the packet is too big to copy, or (b) the packet should
* be copied but there is no memory for the copy.
*/
static int
get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
struct t3_mbuf_hdr *mh, struct rsp_desc *r)
{
unsigned int len_cq = ntohl(r->len_cq);
struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];
int mask, cidx = fl->cidx;
struct rx_sw_desc *sd = &fl->sdesc[cidx];
uint32_t len = G_RSPD_LEN(len_cq);
uint32_t flags = M_EXT;
uint8_t sopeop = G_RSPD_SOP_EOP(ntohl(r->flags));
caddr_t cl;
struct mbuf *m;
int ret = 0;
mask = fl->size - 1;
prefetch(fl->sdesc[(cidx + 1) & mask].m);
prefetch(fl->sdesc[(cidx + 2) & mask].m);
prefetch(fl->sdesc[(cidx + 1) & mask].rxsd_cl);
prefetch(fl->sdesc[(cidx + 2) & mask].rxsd_cl);
fl->credits--;
bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD);
if (recycle_enable && len <= SGE_RX_COPY_THRES &&
sopeop == RSPQ_SOP_EOP) {
if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
goto skip_recycle;
cl = mtod(m, void *);
memcpy(cl, sd->rxsd_cl, len);
recycle_rx_buf(adap, fl, fl->cidx);
m->m_pkthdr.len = m->m_len = len;
m->m_flags = 0;
mh->mh_head = mh->mh_tail = m;
ret = 1;
goto done;
} else {
skip_recycle:
bus_dmamap_unload(fl->entry_tag, sd->map);
cl = sd->rxsd_cl;
m = sd->m;
if ((sopeop == RSPQ_SOP_EOP) ||
(sopeop == RSPQ_SOP))
flags |= M_PKTHDR;
m_init(m, fl->zone, fl->buf_size, M_NOWAIT, MT_DATA, flags);
if (fl->zone == zone_pack) {
/*
* restore clobbered data pointer
*/
m->m_data = m->m_ext.ext_buf;
} else {
m_cljset(m, cl, fl->type);
}
m->m_len = len;
}
switch(sopeop) {
case RSPQ_SOP_EOP:
ret = 1;
/* FALLTHROUGH */
case RSPQ_SOP:
mh->mh_head = mh->mh_tail = m;
m->m_pkthdr.len = len;
break;
case RSPQ_EOP:
ret = 1;
/* FALLTHROUGH */
case RSPQ_NSOP_NEOP:
if (mh->mh_tail == NULL) {
log(LOG_ERR, "discarding intermediate descriptor entry\n");
m_freem(m);
break;
}
mh->mh_tail->m_next = m;
mh->mh_tail = m;
mh->mh_head->m_pkthdr.len += len;
break;
}
if (cxgb_debug)
printf("len=%d pktlen=%d\n", m->m_len, m->m_pkthdr.len);
done:
if (++fl->cidx == fl->size)
fl->cidx = 0;
return (ret);
}
/**
* handle_rsp_cntrl_info - handles control information in a response
* @qs: the queue set corresponding to the response
* @flags: the response control flags
*
* Handles the control information of an SGE response, such as GTS
* indications and completion credits for the queue set's Tx queues.
* HW coalesces credits, we don't do any extra SW coalescing.
*/
static __inline void
handle_rsp_cntrl_info(struct sge_qset *qs, uint32_t flags)
{
unsigned int credits;
#if USE_GTS
if (flags & F_RSPD_TXQ0_GTS)
clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
#endif
credits = G_RSPD_TXQ0_CR(flags);
if (credits)
qs->txq[TXQ_ETH].processed += credits;
credits = G_RSPD_TXQ2_CR(flags);
if (credits)
qs->txq[TXQ_CTRL].processed += credits;
# if USE_GTS
if (flags & F_RSPD_TXQ1_GTS)
clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);
# endif
credits = G_RSPD_TXQ1_CR(flags);
if (credits)
qs->txq[TXQ_OFLD].processed += credits;
}
static void
check_ring_db(adapter_t *adap, struct sge_qset *qs,
unsigned int sleeping)
{
;
}
/**
* process_responses - process responses from an SGE response queue
* @adap: the adapter
* @qs: the queue set to which the response queue belongs
* @budget: how many responses can be processed in this round
*
* Process responses from an SGE response queue up to the supplied budget.
* Responses include received packets as well as credits and other events
* for the queues that belong to the response queue's queue set.
* A negative budget is effectively unlimited.
*
* Additionally choose the interrupt holdoff time for the next interrupt
* on this queue. If the system is under memory shortage use a fairly
* long delay to help recovery.
*/
static int
process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
{
struct sge_rspq *rspq = &qs->rspq;
struct rsp_desc *r = &rspq->desc[rspq->cidx];
int budget_left = budget;
unsigned int sleeping = 0;
#if defined(INET6) || defined(INET)
int lro_enabled = qs->lro.enabled;
int skip_lro;
struct lro_ctrl *lro_ctrl = &qs->lro.ctrl;
#endif
struct t3_mbuf_hdr *mh = &rspq->rspq_mh;
#ifdef DEBUG
static int last_holdoff = 0;
if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) {
printf("next_holdoff=%d\n", rspq->holdoff_tmr);
last_holdoff = rspq->holdoff_tmr;
}
#endif
rspq->next_holdoff = rspq->holdoff_tmr;
while (__predict_true(budget_left && is_new_response(r, rspq))) {
int eth, eop = 0, ethpad = 0;
uint32_t flags = ntohl(r->flags);
uint32_t rss_hash = be32toh(r->rss_hdr.rss_hash_val);
uint8_t opcode = r->rss_hdr.opcode;
eth = (opcode == CPL_RX_PKT);
if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) {
struct mbuf *m;
if (cxgb_debug)
printf("async notification\n");
if (mh->mh_head == NULL) {
mh->mh_head = m_gethdr(M_NOWAIT, MT_DATA);
m = mh->mh_head;
} else {
m = m_gethdr(M_NOWAIT, MT_DATA);
}
if (m == NULL)
goto no_mem;
memcpy(mtod(m, char *), r, AN_PKT_SIZE);
m->m_len = m->m_pkthdr.len = AN_PKT_SIZE;
*mtod(m, char *) = CPL_ASYNC_NOTIF;
opcode = CPL_ASYNC_NOTIF;
eop = 1;
rspq->async_notif++;
goto skip;
} else if (flags & F_RSPD_IMM_DATA_VALID) {
struct mbuf *m = m_gethdr(M_NOWAIT, MT_DATA);
if (m == NULL) {
no_mem:
rspq->next_holdoff = NOMEM_INTR_DELAY;
budget_left--;
break;
}
if (mh->mh_head == NULL)
mh->mh_head = m;
else
mh->mh_tail->m_next = m;
mh->mh_tail = m;
get_imm_packet(adap, r, m);
mh->mh_head->m_pkthdr.len += m->m_len;
eop = 1;
rspq->imm_data++;
} else if (r->len_cq) {
int drop_thresh = eth ? SGE_RX_DROP_THRES : 0;
eop = get_packet(adap, drop_thresh, qs, mh, r);
if (eop) {
if (r->rss_hdr.hash_type && !adap->timestamp) {
M_HASHTYPE_SET(mh->mh_head, M_HASHTYPE_OPAQUE);
mh->mh_head->m_pkthdr.flowid = rss_hash;
}
}
ethpad = 2;
} else {
rspq->pure_rsps++;
}
skip:
if (flags & RSPD_CTRL_MASK) {
sleeping |= flags & RSPD_GTS_MASK;
handle_rsp_cntrl_info(qs, flags);
}
if (!eth && eop) {
rspq->offload_pkts++;
#ifdef TCP_OFFLOAD
adap->cpl_handler[opcode](qs, r, mh->mh_head);
#else
m_freem(mh->mh_head);
#endif
mh->mh_head = NULL;
} else if (eth && eop) {
struct mbuf *m = mh->mh_head;
t3_rx_eth(adap, m, ethpad);
/*
* The T304 sends incoming packets on any qset. If LRO
* is also enabled, we could end up sending packet up
* lro_ctrl->ifp's input. That is incorrect.
*
* The mbuf's rcvif was derived from the cpl header and
* is accurate. Skip LRO and just use that.
*/
#if defined(INET6) || defined(INET)
skip_lro = __predict_false(qs->port->ifp != m->m_pkthdr.rcvif);
if (lro_enabled && lro_ctrl->lro_cnt && !skip_lro
&& (tcp_lro_rx(lro_ctrl, m, 0) == 0)
) {
/* successfully queue'd for LRO */
} else
#endif
{
/*
* LRO not enabled, packet unsuitable for LRO,
* or unable to queue. Pass it up right now in
* either case.
*/
struct ifnet *ifp = m->m_pkthdr.rcvif;
(*ifp->if_input)(ifp, m);
}
mh->mh_head = NULL;
}
r++;
if (__predict_false(++rspq->cidx == rspq->size)) {
rspq->cidx = 0;
rspq->gen ^= 1;
r = rspq->desc;
}
if (++rspq->credits >= 64) {
refill_rspq(adap, rspq, rspq->credits);
rspq->credits = 0;
}
__refill_fl_lt(adap, &qs->fl[0], 32);
__refill_fl_lt(adap, &qs->fl[1], 32);
--budget_left;
}
#if defined(INET6) || defined(INET)
/* Flush LRO */
while (!SLIST_EMPTY(&lro_ctrl->lro_active)) {
struct lro_entry *queued = SLIST_FIRST(&lro_ctrl->lro_active);
SLIST_REMOVE_HEAD(&lro_ctrl->lro_active, next);
tcp_lro_flush(lro_ctrl, queued);
}
#endif
if (sleeping)
check_ring_db(adap, qs, sleeping);
mb(); /* commit Tx queue processed updates */
if (__predict_false(qs->txq_stopped > 1))
restart_tx(qs);
__refill_fl_lt(adap, &qs->fl[0], 512);
__refill_fl_lt(adap, &qs->fl[1], 512);
budget -= budget_left;
return (budget);
}
/*
* A helper function that processes responses and issues GTS.
*/
static __inline int
process_responses_gts(adapter_t *adap, struct sge_rspq *rq)
{
int work;
static int last_holdoff = 0;
work = process_responses(adap, rspq_to_qset(rq), -1);
if (cxgb_debug && (rq->next_holdoff != last_holdoff)) {
printf("next_holdoff=%d\n", rq->next_holdoff);
last_holdoff = rq->next_holdoff;
}
t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) |
V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx));
return (work);
}
/*
* Interrupt handler for legacy INTx interrupts for T3B-based cards.
* Handles data events from SGE response queues as well as error and other
* async events as they all use the same interrupt pin. We use one SGE
* response queue per port in this mode and protect all response queues with
* queue 0's lock.
*/
void
t3b_intr(void *data)
{
uint32_t i, map;
adapter_t *adap = data;
struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
t3_write_reg(adap, A_PL_CLI, 0);
map = t3_read_reg(adap, A_SG_DATA_INTR);
if (!map)
return;
if (__predict_false(map & F_ERRINTR)) {
t3_write_reg(adap, A_PL_INT_ENABLE0, 0);
(void) t3_read_reg(adap, A_PL_INT_ENABLE0);
taskqueue_enqueue(adap->tq, &adap->slow_intr_task);
}
mtx_lock(&q0->lock);
for_each_port(adap, i)
if (map & (1 << i))
process_responses_gts(adap, &adap->sge.qs[i].rspq);
mtx_unlock(&q0->lock);
}
/*
* The MSI interrupt handler. This needs to handle data events from SGE
* response queues as well as error and other async events as they all use
* the same MSI vector. We use one SGE response queue per port in this mode
* and protect all response queues with queue 0's lock.
*/
void
t3_intr_msi(void *data)
{
adapter_t *adap = data;
struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
int i, new_packets = 0;
mtx_lock(&q0->lock);
for_each_port(adap, i)
if (process_responses_gts(adap, &adap->sge.qs[i].rspq))
new_packets = 1;
mtx_unlock(&q0->lock);
if (new_packets == 0) {
t3_write_reg(adap, A_PL_INT_ENABLE0, 0);
(void) t3_read_reg(adap, A_PL_INT_ENABLE0);
taskqueue_enqueue(adap->tq, &adap->slow_intr_task);
}
}
void
t3_intr_msix(void *data)
{
struct sge_qset *qs = data;
adapter_t *adap = qs->port->adapter;
struct sge_rspq *rspq = &qs->rspq;
if (process_responses_gts(adap, rspq) == 0)
rspq->unhandled_irqs++;
}
#define QDUMP_SBUF_SIZE 32 * 400
static int
t3_dump_rspq(SYSCTL_HANDLER_ARGS)
{
struct sge_rspq *rspq;
struct sge_qset *qs;
int i, err, dump_end, idx;
struct sbuf *sb;
struct rsp_desc *rspd;
uint32_t data[4];
rspq = arg1;
qs = rspq_to_qset(rspq);
if (rspq->rspq_dump_count == 0)
return (0);
if (rspq->rspq_dump_count > RSPQ_Q_SIZE) {
log(LOG_WARNING,
"dump count is too large %d\n", rspq->rspq_dump_count);
rspq->rspq_dump_count = 0;
return (EINVAL);
}
if (rspq->rspq_dump_start > (RSPQ_Q_SIZE-1)) {
log(LOG_WARNING,
"dump start of %d is greater than queue size\n",
rspq->rspq_dump_start);
rspq->rspq_dump_start = 0;
return (EINVAL);
}
err = t3_sge_read_rspq(qs->port->adapter, rspq->cntxt_id, data);
if (err)
return (err);
err = sysctl_wire_old_buffer(req, 0);
if (err)
return (err);
sb = sbuf_new_for_sysctl(NULL, NULL, QDUMP_SBUF_SIZE, req);
sbuf_printf(sb, " \n index=%u size=%u MSI-X/RspQ=%u intr enable=%u intr armed=%u\n",
(data[0] & 0xffff), data[0] >> 16, ((data[2] >> 20) & 0x3f),
((data[2] >> 26) & 1), ((data[2] >> 27) & 1));
sbuf_printf(sb, " generation=%u CQ mode=%u FL threshold=%u\n",
((data[2] >> 28) & 1), ((data[2] >> 31) & 1), data[3]);
sbuf_printf(sb, " start=%d -> end=%d\n", rspq->rspq_dump_start,
(rspq->rspq_dump_start + rspq->rspq_dump_count) & (RSPQ_Q_SIZE-1));
dump_end = rspq->rspq_dump_start + rspq->rspq_dump_count;
for (i = rspq->rspq_dump_start; i < dump_end; i++) {
idx = i & (RSPQ_Q_SIZE-1);
rspd = &rspq->desc[idx];
sbuf_printf(sb, "\tidx=%04d opcode=%02x cpu_idx=%x hash_type=%x cq_idx=%x\n",
idx, rspd->rss_hdr.opcode, rspd->rss_hdr.cpu_idx,
rspd->rss_hdr.hash_type, be16toh(rspd->rss_hdr.cq_idx));
sbuf_printf(sb, "\trss_hash_val=%x flags=%08x len_cq=%x intr_gen=%x\n",
rspd->rss_hdr.rss_hash_val, be32toh(rspd->flags),
be32toh(rspd->len_cq), rspd->intr_gen);
}
err = sbuf_finish(sb);
sbuf_delete(sb);
return (err);
}
static int
t3_dump_txq_eth(SYSCTL_HANDLER_ARGS)
{
struct sge_txq *txq;
struct sge_qset *qs;
int i, j, err, dump_end;
struct sbuf *sb;
struct tx_desc *txd;
uint32_t *WR, wr_hi, wr_lo, gen;
uint32_t data[4];
txq = arg1;
qs = txq_to_qset(txq, TXQ_ETH);
if (txq->txq_dump_count == 0) {
return (0);
}
if (txq->txq_dump_count > TX_ETH_Q_SIZE) {
log(LOG_WARNING,
"dump count is too large %d\n", txq->txq_dump_count);
txq->txq_dump_count = 1;
return (EINVAL);
}
if (txq->txq_dump_start > (TX_ETH_Q_SIZE-1)) {
log(LOG_WARNING,
"dump start of %d is greater than queue size\n",
txq->txq_dump_start);
txq->txq_dump_start = 0;
return (EINVAL);
}
err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data);
if (err)
return (err);
err = sysctl_wire_old_buffer(req, 0);
if (err)
return (err);
sb = sbuf_new_for_sysctl(NULL, NULL, QDUMP_SBUF_SIZE, req);
sbuf_printf(sb, " \n credits=%u GTS=%u index=%u size=%u rspq#=%u cmdq#=%u\n",
(data[0] & 0x7fff), ((data[0] >> 15) & 1), (data[0] >> 16),
(data[1] & 0xffff), ((data[3] >> 4) & 7), ((data[3] >> 7) & 1));
sbuf_printf(sb, " TUN=%u TOE=%u generation%u uP token=%u valid=%u\n",
((data[3] >> 8) & 1), ((data[3] >> 9) & 1), ((data[3] >> 10) & 1),
((data[3] >> 11) & 0xfffff), ((data[3] >> 31) & 1));
sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx,
txq->txq_dump_start,
(txq->txq_dump_start + txq->txq_dump_count) & (TX_ETH_Q_SIZE-1));
dump_end = txq->txq_dump_start + txq->txq_dump_count;
for (i = txq->txq_dump_start; i < dump_end; i++) {
txd = &txq->desc[i & (TX_ETH_Q_SIZE-1)];
WR = (uint32_t *)txd->flit;
wr_hi = ntohl(WR[0]);
wr_lo = ntohl(WR[1]);
gen = G_WR_GEN(wr_lo);
sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n",
wr_hi, wr_lo, gen);
for (j = 2; j < 30; j += 4)
sbuf_printf(sb, "\t%08x %08x %08x %08x \n",
WR[j], WR[j + 1], WR[j + 2], WR[j + 3]);
}
err = sbuf_finish(sb);
sbuf_delete(sb);
return (err);
}
static int
t3_dump_txq_ctrl(SYSCTL_HANDLER_ARGS)
{
struct sge_txq *txq;
struct sge_qset *qs;
int i, j, err, dump_end;
struct sbuf *sb;
struct tx_desc *txd;
uint32_t *WR, wr_hi, wr_lo, gen;
txq = arg1;
qs = txq_to_qset(txq, TXQ_CTRL);
if (txq->txq_dump_count == 0) {
return (0);
}
if (txq->txq_dump_count > 256) {
log(LOG_WARNING,
"dump count is too large %d\n", txq->txq_dump_count);
txq->txq_dump_count = 1;
return (EINVAL);
}
if (txq->txq_dump_start > 255) {
log(LOG_WARNING,
"dump start of %d is greater than queue size\n",
txq->txq_dump_start);
txq->txq_dump_start = 0;
return (EINVAL);
}
err = sysctl_wire_old_buffer(req, 0);
if (err != 0)
return (err);
sb = sbuf_new_for_sysctl(NULL, NULL, QDUMP_SBUF_SIZE, req);
sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx,
txq->txq_dump_start,
(txq->txq_dump_start + txq->txq_dump_count) & 255);
dump_end = txq->txq_dump_start + txq->txq_dump_count;
for (i = txq->txq_dump_start; i < dump_end; i++) {
txd = &txq->desc[i & (255)];
WR = (uint32_t *)txd->flit;
wr_hi = ntohl(WR[0]);
wr_lo = ntohl(WR[1]);
gen = G_WR_GEN(wr_lo);
sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n",
wr_hi, wr_lo, gen);
for (j = 2; j < 30; j += 4)
sbuf_printf(sb, "\t%08x %08x %08x %08x \n",
WR[j], WR[j + 1], WR[j + 2], WR[j + 3]);
}
err = sbuf_finish(sb);
sbuf_delete(sb);
return (err);
}
static int
t3_set_coalesce_usecs(SYSCTL_HANDLER_ARGS)
{
adapter_t *sc = arg1;
struct qset_params *qsp = &sc->params.sge.qset[0];
int coalesce_usecs;
struct sge_qset *qs;
int i, j, err, nqsets = 0;
struct mtx *lock;
if ((sc->flags & FULL_INIT_DONE) == 0)
return (ENXIO);
coalesce_usecs = qsp->coalesce_usecs;
err = sysctl_handle_int(oidp, &coalesce_usecs, arg2, req);
if (err != 0) {
return (err);
}
if (coalesce_usecs == qsp->coalesce_usecs)
return (0);
for (i = 0; i < sc->params.nports; i++)
for (j = 0; j < sc->port[i].nqsets; j++)
nqsets++;
coalesce_usecs = max(1, coalesce_usecs);
for (i = 0; i < nqsets; i++) {
qs = &sc->sge.qs[i];
qsp = &sc->params.sge.qset[i];
qsp->coalesce_usecs = coalesce_usecs;
lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock :
&sc->sge.qs[0].rspq.lock;
mtx_lock(lock);
t3_update_qset_coalesce(qs, qsp);
t3_write_reg(sc, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |
V_NEWTIMER(qs->rspq.holdoff_tmr));
mtx_unlock(lock);
}
return (0);
}
static int
t3_pkt_timestamp(SYSCTL_HANDLER_ARGS)
{
adapter_t *sc = arg1;
int rc, timestamp;
if ((sc->flags & FULL_INIT_DONE) == 0)
return (ENXIO);
timestamp = sc->timestamp;
rc = sysctl_handle_int(oidp, &timestamp, arg2, req);
if (rc != 0)
return (rc);
if (timestamp != sc->timestamp) {
t3_set_reg_field(sc, A_TP_PC_CONFIG2, F_ENABLERXPKTTMSTPRSS,
timestamp ? F_ENABLERXPKTTMSTPRSS : 0);
sc->timestamp = timestamp;
}
return (0);
}
void
t3_add_attach_sysctls(adapter_t *sc)
{
struct sysctl_ctx_list *ctx;
struct sysctl_oid_list *children;
ctx = device_get_sysctl_ctx(sc->dev);
children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
/* random information */
SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
"firmware_version",
CTLFLAG_RD, sc->fw_version,
0, "firmware version");
SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
"hw_revision",
CTLFLAG_RD, &sc->params.rev,
0, "chip model");
SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
"port_types",
CTLFLAG_RD, sc->port_types,
0, "type of ports");
SYSCTL_ADD_INT(ctx, children, OID_AUTO,
"enable_debug",
CTLFLAG_RW, &cxgb_debug,
0, "enable verbose debugging output");
SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tunq_coalesce",
CTLFLAG_RD, &sc->tunq_coalesce,
"#tunneled packets freed");
SYSCTL_ADD_INT(ctx, children, OID_AUTO,
"txq_overrun",
CTLFLAG_RD, &txq_fills,
0, "#times txq overrun");
SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
"core_clock",
CTLFLAG_RD, &sc->params.vpd.cclk,
0, "core clock frequency (in KHz)");
}
static const char *rspq_name = "rspq";
static const char *txq_names[] =
{
"txq_eth",
"txq_ofld",
"txq_ctrl"
};
static int
sysctl_handle_macstat(SYSCTL_HANDLER_ARGS)
{
struct port_info *p = arg1;
uint64_t *parg;
if (!p)
return (EINVAL);
cxgb_refresh_stats(p);
parg = (uint64_t *) ((uint8_t *)&p->mac.stats + arg2);
return (sysctl_handle_64(oidp, parg, 0, req));
}
void
t3_add_configured_sysctls(adapter_t *sc)
{
struct sysctl_ctx_list *ctx;
struct sysctl_oid_list *children;
int i, j;
ctx = device_get_sysctl_ctx(sc->dev);
children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
"intr_coal",
CTLTYPE_INT|CTLFLAG_RW, sc,
0, t3_set_coalesce_usecs,
"I", "interrupt coalescing timer (us)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
"pkt_timestamp",
CTLTYPE_INT | CTLFLAG_RW, sc,
0, t3_pkt_timestamp,
"I", "provide packet timestamp instead of connection hash");
for (i = 0; i < sc->params.nports; i++) {
struct port_info *pi = &sc->port[i];
struct sysctl_oid *poid;
struct sysctl_oid_list *poidlist;
struct mac_stats *mstats = &pi->mac.stats;
snprintf(pi->namebuf, PORT_NAME_LEN, "port%d", i);
poid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO,
pi->namebuf, CTLFLAG_RD, NULL, "port statistics");
poidlist = SYSCTL_CHILDREN(poid);
SYSCTL_ADD_UINT(ctx, poidlist, OID_AUTO,
"nqsets", CTLFLAG_RD, &pi->nqsets,
0, "#queue sets");
for (j = 0; j < pi->nqsets; j++) {
struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j];
struct sysctl_oid *qspoid, *rspqpoid, *txqpoid,
*ctrlqpoid, *lropoid;
struct sysctl_oid_list *qspoidlist, *rspqpoidlist,
*txqpoidlist, *ctrlqpoidlist,
*lropoidlist;
struct sge_txq *txq = &qs->txq[TXQ_ETH];
snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j);
qspoid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO,
qs->namebuf, CTLFLAG_RD, NULL, "qset statistics");
qspoidlist = SYSCTL_CHILDREN(qspoid);
SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "fl0_empty",
CTLFLAG_RD, &qs->fl[0].empty, 0,
"freelist #0 empty");
SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "fl1_empty",
CTLFLAG_RD, &qs->fl[1].empty, 0,
"freelist #1 empty");
rspqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO,
rspq_name, CTLFLAG_RD, NULL, "rspq statistics");
rspqpoidlist = SYSCTL_CHILDREN(rspqpoid);
txqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO,
txq_names[0], CTLFLAG_RD, NULL, "txq statistics");
txqpoidlist = SYSCTL_CHILDREN(txqpoid);
ctrlqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO,
txq_names[2], CTLFLAG_RD, NULL, "ctrlq statistics");
ctrlqpoidlist = SYSCTL_CHILDREN(ctrlqpoid);
lropoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO,
"lro_stats", CTLFLAG_RD, NULL, "LRO statistics");
lropoidlist = SYSCTL_CHILDREN(lropoid);
SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "size",
CTLFLAG_RD, &qs->rspq.size,
0, "#entries in response queue");
SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "cidx",
CTLFLAG_RD, &qs->rspq.cidx,
0, "consumer index");
SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "credits",
CTLFLAG_RD, &qs->rspq.credits,
0, "#credits");
SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "starved",
CTLFLAG_RD, &qs->rspq.starved,
0, "#times starved");
SYSCTL_ADD_UAUTO(ctx, rspqpoidlist, OID_AUTO, "phys_addr",
CTLFLAG_RD, &qs->rspq.phys_addr,
"physical_address_of the queue");
SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_start",
CTLFLAG_RW, &qs->rspq.rspq_dump_start,
0, "start rspq dump entry");
SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_count",
CTLFLAG_RW, &qs->rspq.rspq_dump_count,
0, "#rspq entries to dump");
SYSCTL_ADD_PROC(ctx, rspqpoidlist, OID_AUTO, "qdump",
CTLTYPE_STRING | CTLFLAG_RD, &qs->rspq,
0, t3_dump_rspq, "A", "dump of the response queue");
SYSCTL_ADD_UQUAD(ctx, txqpoidlist, OID_AUTO, "dropped",
CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_mr->br_drops,
"#tunneled packets dropped");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "sendqlen",
CTLFLAG_RD, &qs->txq[TXQ_ETH].sendq.mq_len,
0, "#tunneled packets waiting to be sent");
#if 0
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_pidx",
CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_prod,
0, "#tunneled packets queue producer index");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_cidx",
CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_cons,
0, "#tunneled packets queue consumer index");
#endif
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "processed",
CTLFLAG_RD, &qs->txq[TXQ_ETH].processed,
0, "#tunneled packets processed by the card");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "cleaned",
CTLFLAG_RD, &txq->cleaned,
0, "#tunneled packets cleaned");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "in_use",
CTLFLAG_RD, &txq->in_use,
0, "#tunneled packet slots in use");
SYSCTL_ADD_UQUAD(ctx, txqpoidlist, OID_AUTO, "frees",
CTLFLAG_RD, &txq->txq_frees,
"#tunneled packets freed");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "skipped",
CTLFLAG_RD, &txq->txq_skipped,
0, "#tunneled packet descriptors skipped");
SYSCTL_ADD_UQUAD(ctx, txqpoidlist, OID_AUTO, "coalesced",
CTLFLAG_RD, &txq->txq_coalesced,
"#tunneled packets coalesced");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "enqueued",
CTLFLAG_RD, &txq->txq_enqueued,
0, "#tunneled packets enqueued to hardware");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "stopped_flags",
CTLFLAG_RD, &qs->txq_stopped,
0, "tx queues stopped");
SYSCTL_ADD_UAUTO(ctx, txqpoidlist, OID_AUTO, "phys_addr",
CTLFLAG_RD, &txq->phys_addr,
"physical_address_of the queue");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "qgen",
CTLFLAG_RW, &qs->txq[TXQ_ETH].gen,
0, "txq generation");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_cidx",
CTLFLAG_RD, &txq->cidx,
0, "hardware queue cidx");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_pidx",
CTLFLAG_RD, &txq->pidx,
0, "hardware queue pidx");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_start",
CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_start,
0, "txq start idx for dump");
SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_count",
CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_count,
0, "txq #entries to dump");
SYSCTL_ADD_PROC(ctx, txqpoidlist, OID_AUTO, "qdump",
CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_ETH],
0, t3_dump_txq_eth, "A", "dump of the transmit queue");
SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_start",
CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start,
0, "ctrlq start idx for dump");
SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_count",
CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count,
0, "ctrl #entries to dump");
SYSCTL_ADD_PROC(ctx, ctrlqpoidlist, OID_AUTO, "qdump",
CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_CTRL],
0, t3_dump_txq_ctrl, "A", "dump of the transmit queue");
SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_queued",
CTLFLAG_RD, &qs->lro.ctrl.lro_queued, 0, NULL);
SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_flushed",
CTLFLAG_RD, &qs->lro.ctrl.lro_flushed, 0, NULL);
SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_bad_csum",
CTLFLAG_RD, &qs->lro.ctrl.lro_bad_csum, 0, NULL);
SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_cnt",
CTLFLAG_RD, &qs->lro.ctrl.lro_cnt, 0, NULL);
}
/* Now add a node for mac stats. */
poid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO, "mac_stats",
CTLFLAG_RD, NULL, "MAC statistics");
poidlist = SYSCTL_CHILDREN(poid);
/*
* We (ab)use the length argument (arg2) to pass on the offset
* of the data that we are interested in. This is only required
* for the quad counters that are updated from the hardware (we
* make sure that we return the latest value).
* sysctl_handle_macstat first updates *all* the counters from
* the hardware, and then returns the latest value of the
* requested counter. Best would be to update only the
* requested counter from hardware, but t3_mac_update_stats()
* hides all the register details and we don't want to dive into
* all that here.
*/
#define CXGB_SYSCTL_ADD_QUAD(a) SYSCTL_ADD_OID(ctx, poidlist, OID_AUTO, #a, \
(CTLTYPE_U64 | CTLFLAG_RD), pi, offsetof(struct mac_stats, a), \
sysctl_handle_macstat, "QU", 0)
CXGB_SYSCTL_ADD_QUAD(tx_octets);
CXGB_SYSCTL_ADD_QUAD(tx_octets_bad);
CXGB_SYSCTL_ADD_QUAD(tx_frames);
CXGB_SYSCTL_ADD_QUAD(tx_mcast_frames);
CXGB_SYSCTL_ADD_QUAD(tx_bcast_frames);
CXGB_SYSCTL_ADD_QUAD(tx_pause);
CXGB_SYSCTL_ADD_QUAD(tx_deferred);
CXGB_SYSCTL_ADD_QUAD(tx_late_collisions);
CXGB_SYSCTL_ADD_QUAD(tx_total_collisions);
CXGB_SYSCTL_ADD_QUAD(tx_excess_collisions);
CXGB_SYSCTL_ADD_QUAD(tx_underrun);
CXGB_SYSCTL_ADD_QUAD(tx_len_errs);
CXGB_SYSCTL_ADD_QUAD(tx_mac_internal_errs);
CXGB_SYSCTL_ADD_QUAD(tx_excess_deferral);
CXGB_SYSCTL_ADD_QUAD(tx_fcs_errs);
CXGB_SYSCTL_ADD_QUAD(tx_frames_64);
CXGB_SYSCTL_ADD_QUAD(tx_frames_65_127);
CXGB_SYSCTL_ADD_QUAD(tx_frames_128_255);
CXGB_SYSCTL_ADD_QUAD(tx_frames_256_511);
CXGB_SYSCTL_ADD_QUAD(tx_frames_512_1023);
CXGB_SYSCTL_ADD_QUAD(tx_frames_1024_1518);
CXGB_SYSCTL_ADD_QUAD(tx_frames_1519_max);
CXGB_SYSCTL_ADD_QUAD(rx_octets);
CXGB_SYSCTL_ADD_QUAD(rx_octets_bad);
CXGB_SYSCTL_ADD_QUAD(rx_frames);
CXGB_SYSCTL_ADD_QUAD(rx_mcast_frames);
CXGB_SYSCTL_ADD_QUAD(rx_bcast_frames);
CXGB_SYSCTL_ADD_QUAD(rx_pause);
CXGB_SYSCTL_ADD_QUAD(rx_fcs_errs);
CXGB_SYSCTL_ADD_QUAD(rx_align_errs);
CXGB_SYSCTL_ADD_QUAD(rx_symbol_errs);
CXGB_SYSCTL_ADD_QUAD(rx_data_errs);
CXGB_SYSCTL_ADD_QUAD(rx_sequence_errs);
CXGB_SYSCTL_ADD_QUAD(rx_runt);
CXGB_SYSCTL_ADD_QUAD(rx_jabber);
CXGB_SYSCTL_ADD_QUAD(rx_short);
CXGB_SYSCTL_ADD_QUAD(rx_too_long);
CXGB_SYSCTL_ADD_QUAD(rx_mac_internal_errs);
CXGB_SYSCTL_ADD_QUAD(rx_cong_drops);
CXGB_SYSCTL_ADD_QUAD(rx_frames_64);
CXGB_SYSCTL_ADD_QUAD(rx_frames_65_127);
CXGB_SYSCTL_ADD_QUAD(rx_frames_128_255);
CXGB_SYSCTL_ADD_QUAD(rx_frames_256_511);
CXGB_SYSCTL_ADD_QUAD(rx_frames_512_1023);
CXGB_SYSCTL_ADD_QUAD(rx_frames_1024_1518);
CXGB_SYSCTL_ADD_QUAD(rx_frames_1519_max);
#undef CXGB_SYSCTL_ADD_QUAD
#define CXGB_SYSCTL_ADD_ULONG(a) SYSCTL_ADD_ULONG(ctx, poidlist, OID_AUTO, #a, \
CTLFLAG_RD, &mstats->a, 0)
CXGB_SYSCTL_ADD_ULONG(tx_fifo_parity_err);
CXGB_SYSCTL_ADD_ULONG(rx_fifo_parity_err);
CXGB_SYSCTL_ADD_ULONG(tx_fifo_urun);
CXGB_SYSCTL_ADD_ULONG(rx_fifo_ovfl);
CXGB_SYSCTL_ADD_ULONG(serdes_signal_loss);
CXGB_SYSCTL_ADD_ULONG(xaui_pcs_ctc_err);
CXGB_SYSCTL_ADD_ULONG(xaui_pcs_align_change);
CXGB_SYSCTL_ADD_ULONG(num_toggled);
CXGB_SYSCTL_ADD_ULONG(num_resets);
CXGB_SYSCTL_ADD_ULONG(link_faults);
#undef CXGB_SYSCTL_ADD_ULONG
}
}
/**
* t3_get_desc - dump an SGE descriptor for debugging purposes
* @qs: the queue set
* @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
* @idx: the descriptor index in the queue
* @data: where to dump the descriptor contents
*
* Dumps the contents of a HW descriptor of an SGE queue. Returns the
* size of the descriptor.
*/
int
t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx,
unsigned char *data)
{
if (qnum >= 6)
return (EINVAL);
if (qnum < 3) {
if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size)
return -EINVAL;
memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc));
return sizeof(struct tx_desc);
}
if (qnum == 3) {
if (!qs->rspq.desc || idx >= qs->rspq.size)
return (EINVAL);
memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc));
return sizeof(struct rsp_desc);
}
qnum -= 4;
if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size)
return (EINVAL);
memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc));
return sizeof(struct rx_desc);
}
Index: head/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c
===================================================================
--- head/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c (revision 283290)
+++ head/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c (revision 283291)
@@ -1,1724 +1,1724 @@
/**************************************************************************
Copyright (c) 2007, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***************************************************************************/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet.h"
#ifdef TCP_OFFLOAD
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/bus.h>
#include <sys/pciio.h>
#include <sys/conf.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus_dma.h>
#include <sys/rman.h>
#include <sys/ioccom.h>
#include <sys/mbuf.h>
#include <sys/rwlock.h>
#include <sys/linker.h>
#include <sys/firmware.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sockio.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/queue.h>
#include <sys/taskqueue.h>
#include <sys/proc.h>
#include <sys/uio.h>
#include <net/route.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/in_pcb.h>
#include <netinet/ip.h>
#include <netinet/ip_var.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp.h>
#include <netinet/tcpip.h>
#include <rdma/ib_verbs.h>
#include <linux/idr.h>
#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h>
#include <cxgb_include.h>
#include <ulp/tom/cxgb_tom.h>
#include <ulp/tom/cxgb_toepcb.h>
#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h>
#include <rdma/ib_verbs.h>
#include <linux/idr.h>
#include <ulp/iw_cxgb/iw_cxgb_wr.h>
#include <ulp/iw_cxgb/iw_cxgb_hal.h>
#include <ulp/iw_cxgb/iw_cxgb_provider.h>
#include <ulp/iw_cxgb/iw_cxgb_cm.h>
#include <ulp/iw_cxgb/iw_cxgb.h>
#ifdef KTR
static char *states[] = {
"idle",
"listen",
"connecting",
"mpa_wait_req",
"mpa_req_sent",
"mpa_req_rcvd",
"mpa_rep_sent",
"fpdu_mode",
"aborting",
"closing",
"moribund",
"dead",
NULL,
};
#endif
SYSCTL_NODE(_hw, OID_AUTO, iw_cxgb, CTLFLAG_RD, 0, "iw_cxgb driver parameters");
static int ep_timeout_secs = 60;
SYSCTL_INT(_hw_iw_cxgb, OID_AUTO, ep_timeout_secs, CTLFLAG_RWTUN, &ep_timeout_secs, 0,
"CM Endpoint operation timeout in seconds (default=60)");
static int mpa_rev = 1;
SYSCTL_INT(_hw_iw_cxgb, OID_AUTO, mpa_rev, CTLFLAG_RWTUN, &mpa_rev, 0,
"MPA Revision, 0 supports amso1100, 1 is spec compliant. (default=1)");
static int markers_enabled = 0;
SYSCTL_INT(_hw_iw_cxgb, OID_AUTO, markers_enabled, CTLFLAG_RWTUN, &markers_enabled, 0,
"Enable MPA MARKERS (default(0)=disabled)");
static int crc_enabled = 1;
SYSCTL_INT(_hw_iw_cxgb, OID_AUTO, crc_enabled, CTLFLAG_RWTUN, &crc_enabled, 0,
"Enable MPA CRC (default(1)=enabled)");
static int rcv_win = 256 * 1024;
SYSCTL_INT(_hw_iw_cxgb, OID_AUTO, rcv_win, CTLFLAG_RWTUN, &rcv_win, 0,
"TCP receive window in bytes (default=256KB)");
static int snd_win = 32 * 1024;
SYSCTL_INT(_hw_iw_cxgb, OID_AUTO, snd_win, CTLFLAG_RWTUN, &snd_win, 0,
"TCP send window in bytes (default=32KB)");
static unsigned int nocong = 0;
SYSCTL_UINT(_hw_iw_cxgb, OID_AUTO, nocong, CTLFLAG_RWTUN, &nocong, 0,
"Turn off congestion control (default=0)");
static unsigned int cong_flavor = 1;
SYSCTL_UINT(_hw_iw_cxgb, OID_AUTO, cong_flavor, CTLFLAG_RWTUN, &cong_flavor, 0,
"TCP Congestion control flavor (default=1)");
static void ep_timeout(void *arg);
static void connect_reply_upcall(struct iwch_ep *ep, int status);
static int iwch_so_upcall(struct socket *so, void *arg, int waitflag);
/*
* Cruft to offload socket upcalls onto thread.
*/
static struct mtx req_lock;
static TAILQ_HEAD(iwch_ep_list, iwch_ep_common) req_list;
static struct task iw_cxgb_task;
static struct taskqueue *iw_cxgb_taskq;
static void process_req(void *ctx, int pending);
static void
start_ep_timer(struct iwch_ep *ep)
{
CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
if (callout_pending(&ep->timer)) {
CTR2(KTR_IW_CXGB, "%s stopped / restarted timer ep %p", __FUNCTION__, ep);
callout_deactivate(&ep->timer);
callout_drain(&ep->timer);
} else {
/*
* XXX this looks racy
*/
get_ep(&ep->com);
- callout_init(&ep->timer, TRUE);
+ callout_init(&ep->timer, 1);
}
callout_reset(&ep->timer, ep_timeout_secs * hz, ep_timeout, ep);
}
static void
stop_ep_timer(struct iwch_ep *ep)
{
CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
if (!callout_pending(&ep->timer)) {
CTR3(KTR_IW_CXGB, "%s timer stopped when its not running! ep %p state %u\n",
__func__, ep, ep->com.state);
return;
}
callout_drain(&ep->timer);
put_ep(&ep->com);
}
static int
set_tcpinfo(struct iwch_ep *ep)
{
struct socket *so = ep->com.so;
struct inpcb *inp = sotoinpcb(so);
struct tcpcb *tp;
struct toepcb *toep;
int rc = 0;
INP_WLOCK(inp);
tp = intotcpcb(inp);
if ((tp->t_flags & TF_TOE) == 0) {
rc = EINVAL;
printf("%s: connection NOT OFFLOADED!\n", __func__);
goto done;
}
toep = tp->t_toe;
ep->hwtid = toep->tp_tid;
ep->snd_seq = tp->snd_nxt;
ep->rcv_seq = tp->rcv_nxt;
ep->emss = tp->t_maxseg;
if (ep->emss < 128)
ep->emss = 128;
done:
INP_WUNLOCK(inp);
return (rc);
}
static enum iwch_ep_state
state_read(struct iwch_ep_common *epc)
{
enum iwch_ep_state state;
mtx_lock(&epc->lock);
state = epc->state;
mtx_unlock(&epc->lock);
return state;
}
static void
__state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
{
epc->state = new;
}
static void
state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
{
mtx_lock(&epc->lock);
CTR3(KTR_IW_CXGB, "%s - %s -> %s", __FUNCTION__, states[epc->state], states[new]);
__state_set(epc, new);
mtx_unlock(&epc->lock);
return;
}
static void *
alloc_ep(int size, int flags)
{
struct iwch_ep_common *epc;
epc = malloc(size, M_DEVBUF, flags);
if (epc) {
memset(epc, 0, size);
refcount_init(&epc->refcount, 1);
mtx_init(&epc->lock, "iwch_epc lock", NULL, MTX_DEF|MTX_DUPOK);
cv_init(&epc->waitq, "iwch_epc cv");
}
CTR2(KTR_IW_CXGB, "%s alloc ep %p", __FUNCTION__, epc);
return epc;
}
void __free_ep(struct iwch_ep_common *epc)
{
CTR3(KTR_IW_CXGB, "%s ep %p state %s", __FUNCTION__, epc, states[state_read(epc)]);
KASSERT(!epc->so, ("%s warning ep->so %p \n", __FUNCTION__, epc->so));
KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list!\n", __FUNCTION__, epc));
free(epc, M_DEVBUF);
}
static struct rtentry *
find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port,
__be16 peer_port, u8 tos)
{
struct route iproute;
struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst;
bzero(&iproute, sizeof iproute);
dst->sin_family = AF_INET;
dst->sin_len = sizeof *dst;
dst->sin_addr.s_addr = peer_ip;
rtalloc(&iproute);
return iproute.ro_rt;
}
static void
close_socket(struct iwch_ep_common *epc, int close)
{
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]);
SOCK_LOCK(epc->so);
soupcall_clear(epc->so, SO_RCV);
SOCK_UNLOCK(epc->so);
if (close)
soclose(epc->so);
else
soshutdown(epc->so, SHUT_WR|SHUT_RD);
epc->so = NULL;
}
static void
shutdown_socket(struct iwch_ep_common *epc)
{
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]);
soshutdown(epc->so, SHUT_WR);
}
static void
abort_socket(struct iwch_ep *ep)
{
struct sockopt sopt;
int err;
struct linger l;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
l.l_onoff = 1;
l.l_linger = 0;
/* linger_time of 0 forces RST to be sent */
sopt.sopt_dir = SOPT_SET;
sopt.sopt_level = SOL_SOCKET;
sopt.sopt_name = SO_LINGER;
sopt.sopt_val = (caddr_t)&l;
sopt.sopt_valsize = sizeof l;
sopt.sopt_td = NULL;
err = sosetopt(ep->com.so, &sopt);
if (err)
printf("%s can't set linger to 0, no RST! err %d\n", __FUNCTION__, err);
}
static void
send_mpa_req(struct iwch_ep *ep)
{
int mpalen;
struct mpa_message *mpa;
struct mbuf *m;
int err;
CTR3(KTR_IW_CXGB, "%s ep %p pd_len %d", __FUNCTION__, ep, ep->plen);
mpalen = sizeof(*mpa) + ep->plen;
m = m_gethdr(mpalen, M_NOWAIT);
if (m == NULL) {
connect_reply_upcall(ep, -ENOMEM);
return;
}
mpa = mtod(m, struct mpa_message *);
m->m_len = mpalen;
m->m_pkthdr.len = mpalen;
memset(mpa, 0, sizeof(*mpa));
memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key));
mpa->flags = (crc_enabled ? MPA_CRC : 0) |
(markers_enabled ? MPA_MARKERS : 0);
mpa->private_data_size = htons(ep->plen);
mpa->revision = mpa_rev;
if (ep->plen)
memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen);
err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread);
if (err) {
m_freem(m);
connect_reply_upcall(ep, -ENOMEM);
return;
}
start_ep_timer(ep);
state_set(&ep->com, MPA_REQ_SENT);
return;
}
static int
send_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen)
{
int mpalen;
struct mpa_message *mpa;
struct mbuf *m;
int err;
CTR3(KTR_IW_CXGB, "%s ep %p plen %d", __FUNCTION__, ep, plen);
mpalen = sizeof(*mpa) + plen;
m = m_gethdr(mpalen, M_NOWAIT);
if (m == NULL) {
printf("%s - cannot alloc mbuf!\n", __FUNCTION__);
return (-ENOMEM);
}
mpa = mtod(m, struct mpa_message *);
m->m_len = mpalen;
m->m_pkthdr.len = mpalen;
memset(mpa, 0, sizeof(*mpa));
memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
mpa->flags = MPA_REJECT;
mpa->revision = mpa_rev;
mpa->private_data_size = htons(plen);
if (plen)
memcpy(mpa->private_data, pdata, plen);
err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread);
PANIC_IF(err);
return 0;
}
static int
send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen)
{
int mpalen;
struct mpa_message *mpa;
struct mbuf *m;
CTR4(KTR_IW_CXGB, "%s ep %p so %p plen %d", __FUNCTION__, ep, ep->com.so, plen);
mpalen = sizeof(*mpa) + plen;
m = m_gethdr(mpalen, M_NOWAIT);
if (m == NULL) {
printf("%s - cannot alloc mbuf!\n", __FUNCTION__);
return (-ENOMEM);
}
mpa = mtod(m, struct mpa_message *);
m->m_len = mpalen;
m->m_pkthdr.len = mpalen;
memset(mpa, 0, sizeof(*mpa));
memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) |
(markers_enabled ? MPA_MARKERS : 0);
mpa->revision = mpa_rev;
mpa->private_data_size = htons(plen);
if (plen)
memcpy(mpa->private_data, pdata, plen);
state_set(&ep->com, MPA_REP_SENT);
return sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT,
ep->com.thread);
}
static void
close_complete_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CLOSE;
if (ep->com.cm_id) {
CTR3(KTR_IW_CXGB, "close complete delivered ep %p cm_id %p tid %d",
ep, ep->com.cm_id, ep->hwtid);
ep->com.cm_id->event_handler(ep->com.cm_id, &event);
ep->com.cm_id->rem_ref(ep->com.cm_id);
ep->com.cm_id = NULL;
ep->com.qp = NULL;
}
}
static void
abort_connection(struct iwch_ep *ep)
{
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
state_set(&ep->com, ABORTING);
abort_socket(ep);
close_socket(&ep->com, 0);
close_complete_upcall(ep);
state_set(&ep->com, DEAD);
put_ep(&ep->com);
}
static void
peer_close_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_DISCONNECT;
if (ep->com.cm_id) {
CTR3(KTR_IW_CXGB, "peer close delivered ep %p cm_id %p tid %d",
ep, ep->com.cm_id, ep->hwtid);
ep->com.cm_id->event_handler(ep->com.cm_id, &event);
}
}
static void
peer_abort_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CLOSE;
event.status = ECONNRESET;
if (ep->com.cm_id) {
CTR3(KTR_IW_CXGB, "abort delivered ep %p cm_id %p tid %d", ep,
ep->com.cm_id, ep->hwtid);
ep->com.cm_id->event_handler(ep->com.cm_id, &event);
ep->com.cm_id->rem_ref(ep->com.cm_id);
ep->com.cm_id = NULL;
ep->com.qp = NULL;
}
}
static void
connect_reply_upcall(struct iwch_ep *ep, int status)
{
struct iw_cm_event event;
CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s status %d", __FUNCTION__, ep, ep->com.so, states[ep->com.state], status);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CONNECT_REPLY;
event.status = status;
event.local_addr = ep->com.local_addr;
event.remote_addr = ep->com.remote_addr;
if ((status == 0) || (status == ECONNREFUSED)) {
event.private_data_len = ep->plen;
event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
}
if (ep->com.cm_id) {
CTR4(KTR_IW_CXGB, "%s ep %p tid %d status %d", __FUNCTION__, ep,
ep->hwtid, status);
ep->com.cm_id->event_handler(ep->com.cm_id, &event);
}
if (status < 0) {
ep->com.cm_id->rem_ref(ep->com.cm_id);
ep->com.cm_id = NULL;
ep->com.qp = NULL;
}
}
static void
connect_request_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CONNECT_REQUEST;
event.local_addr = ep->com.local_addr;
event.remote_addr = ep->com.remote_addr;
event.private_data_len = ep->plen;
event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
event.provider_data = ep;
event.so = ep->com.so;
if (state_read(&ep->parent_ep->com) != DEAD) {
get_ep(&ep->com);
ep->parent_ep->com.cm_id->event_handler(
ep->parent_ep->com.cm_id,
&event);
}
put_ep(&ep->parent_ep->com);
}
static void
established_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_ESTABLISHED;
if (ep->com.cm_id) {
CTR3(KTR_IW_CXGB, "%s ep %p tid %d", __FUNCTION__, ep, ep->hwtid);
ep->com.cm_id->event_handler(ep->com.cm_id, &event);
}
}
static void
process_mpa_reply(struct iwch_ep *ep)
{
struct mpa_message *mpa;
u16 plen;
struct iwch_qp_attributes attrs;
enum iwch_qp_attr_mask mask;
int err;
struct mbuf *top, *m;
int flags = MSG_DONTWAIT;
struct uio uio;
int len;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
/*
* Stop mpa timer. If it expired, then the state has
* changed and we bail since ep_timeout already aborted
* the connection.
*/
stop_ep_timer(ep);
if (state_read(&ep->com) != MPA_REQ_SENT)
return;
uio.uio_resid = len = 1000000;
uio.uio_td = ep->com.thread;
err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags);
if (err) {
if (err == EWOULDBLOCK) {
start_ep_timer(ep);
return;
}
err = -err;
goto err;
}
if (ep->com.so->so_rcv.sb_mb) {
printf("%s data after soreceive called! so %p sb_mb %p top %p\n",
__FUNCTION__, ep->com.so, ep->com.so->so_rcv.sb_mb, top);
}
m = top;
do {
/*
* If we get more than the supported amount of private data
* then we must fail this connection.
*/
if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) {
err = (-EINVAL);
goto err;
}
/*
* copy the new data into our accumulation buffer.
*/
m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len]));
ep->mpa_pkt_len += m->m_len;
if (!m->m_next)
m = m->m_nextpkt;
else
m = m->m_next;
} while (m);
m_freem(top);
/*
* if we don't even have the mpa message, then bail.
*/
if (ep->mpa_pkt_len < sizeof(*mpa))
return;
mpa = (struct mpa_message *)ep->mpa_pkt;
/* Validate MPA header. */
if (mpa->revision != mpa_rev) {
CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision);
err = EPROTO;
goto err;
}
if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) {
CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key);
err = EPROTO;
goto err;
}
plen = ntohs(mpa->private_data_size);
/*
* Fail if there's too much private data.
*/
if (plen > MPA_MAX_PRIVATE_DATA) {
CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen);
err = EPROTO;
goto err;
}
/*
* If plen does not account for pkt size
*/
if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
CTR2(KTR_IW_CXGB, "%s pkt too big %d", __FUNCTION__, ep->mpa_pkt_len);
err = EPROTO;
goto err;
}
ep->plen = (u8) plen;
/*
* If we don't have all the pdata yet, then bail.
* We'll continue process when more data arrives.
*/
if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
return;
if (mpa->flags & MPA_REJECT) {
err = ECONNREFUSED;
goto err;
}
/*
* If we get here we have accumulated the entire mpa
* start reply message including private data. And
* the MPA header is valid.
*/
CTR1(KTR_IW_CXGB, "%s mpa rpl looks good!", __FUNCTION__);
state_set(&ep->com, FPDU_MODE);
ep->mpa_attr.initiator = 1;
ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
ep->mpa_attr.recv_marker_enabled = markers_enabled;
ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
ep->mpa_attr.version = mpa_rev;
if (set_tcpinfo(ep)) {
printf("%s set_tcpinfo error\n", __FUNCTION__);
goto err;
}
CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, "
"xmit_marker_enabled=%d, version=%d", __FUNCTION__,
ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
attrs.mpa_attr = ep->mpa_attr;
attrs.max_ird = ep->ird;
attrs.max_ord = ep->ord;
attrs.llp_stream_handle = ep;
attrs.next_state = IWCH_QP_STATE_RTS;
mask = IWCH_QP_ATTR_NEXT_STATE |
IWCH_QP_ATTR_LLP_STREAM_HANDLE | IWCH_QP_ATTR_MPA_ATTR |
IWCH_QP_ATTR_MAX_IRD | IWCH_QP_ATTR_MAX_ORD;
/* bind QP and TID with INIT_WR */
err = iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp, mask, &attrs, 1);
if (!err)
goto out;
err:
abort_connection(ep);
out:
connect_reply_upcall(ep, err);
return;
}
static void
process_mpa_request(struct iwch_ep *ep)
{
struct mpa_message *mpa;
u16 plen;
int flags = MSG_DONTWAIT;
struct mbuf *top, *m;
int err;
struct uio uio;
int len;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
/*
* Stop mpa timer. If it expired, then the state has
* changed and we bail since ep_timeout already aborted
* the connection.
*/
stop_ep_timer(ep);
if (state_read(&ep->com) != MPA_REQ_WAIT)
return;
uio.uio_resid = len = 1000000;
uio.uio_td = ep->com.thread;
err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags);
if (err) {
if (err == EWOULDBLOCK) {
start_ep_timer(ep);
return;
}
err = -err;
goto err;
}
m = top;
do {
/*
* If we get more than the supported amount of private data
* then we must fail this connection.
*/
if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) {
CTR2(KTR_IW_CXGB, "%s mpa message too big %d", __FUNCTION__,
ep->mpa_pkt_len + m->m_len);
goto err;
}
/*
* Copy the new data into our accumulation buffer.
*/
m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len]));
ep->mpa_pkt_len += m->m_len;
if (!m->m_next)
m = m->m_nextpkt;
else
m = m->m_next;
} while (m);
m_freem(top);
/*
* If we don't even have the mpa message, then bail.
* We'll continue process when more data arrives.
*/
if (ep->mpa_pkt_len < sizeof(*mpa)) {
start_ep_timer(ep);
CTR2(KTR_IW_CXGB, "%s not enough header %d...waiting...", __FUNCTION__,
ep->mpa_pkt_len);
return;
}
mpa = (struct mpa_message *) ep->mpa_pkt;
/*
* Validate MPA Header.
*/
if (mpa->revision != mpa_rev) {
CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision);
goto err;
}
if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) {
CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key);
goto err;
}
plen = ntohs(mpa->private_data_size);
/*
* Fail if there's too much private data.
*/
if (plen > MPA_MAX_PRIVATE_DATA) {
CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen);
goto err;
}
/*
* If plen does not account for pkt size
*/
if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
CTR2(KTR_IW_CXGB, "%s more data after private data %d", __FUNCTION__,
ep->mpa_pkt_len);
goto err;
}
ep->plen = (u8) plen;
/*
* If we don't have all the pdata yet, then bail.
*/
if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) {
start_ep_timer(ep);
CTR2(KTR_IW_CXGB, "%s more mpa msg to come %d", __FUNCTION__,
ep->mpa_pkt_len);
return;
}
/*
* If we get here we have accumulated the entire mpa
* start reply message including private data.
*/
ep->mpa_attr.initiator = 0;
ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
ep->mpa_attr.recv_marker_enabled = markers_enabled;
ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
ep->mpa_attr.version = mpa_rev;
if (set_tcpinfo(ep)) {
printf("%s set_tcpinfo error\n", __FUNCTION__);
goto err;
}
CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, "
"xmit_marker_enabled=%d, version=%d", __FUNCTION__,
ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
state_set(&ep->com, MPA_REQ_RCVD);
/* drive upcall */
connect_request_upcall(ep);
return;
err:
abort_connection(ep);
return;
}
static void
process_peer_close(struct iwch_ep *ep)
{
struct iwch_qp_attributes attrs;
int disconnect = 1;
int release = 0;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
mtx_lock(&ep->com.lock);
switch (ep->com.state) {
case MPA_REQ_WAIT:
__state_set(&ep->com, CLOSING);
break;
case MPA_REQ_SENT:
__state_set(&ep->com, CLOSING);
connect_reply_upcall(ep, -ECONNRESET);
break;
case MPA_REQ_RCVD:
/*
* We're gonna mark this puppy DEAD, but keep
* the reference on it until the ULP accepts or
* rejects the CR.
*/
__state_set(&ep->com, CLOSING);
break;
case MPA_REP_SENT:
__state_set(&ep->com, CLOSING);
break;
case FPDU_MODE:
start_ep_timer(ep);
__state_set(&ep->com, CLOSING);
attrs.next_state = IWCH_QP_STATE_CLOSING;
iwch_modify_qp(ep->com.qp->rhp, ep->com.qp,
IWCH_QP_ATTR_NEXT_STATE, &attrs, 1);
peer_close_upcall(ep);
break;
case ABORTING:
disconnect = 0;
break;
case CLOSING:
__state_set(&ep->com, MORIBUND);
disconnect = 0;
break;
case MORIBUND:
stop_ep_timer(ep);
if (ep->com.cm_id && ep->com.qp) {
attrs.next_state = IWCH_QP_STATE_IDLE;
iwch_modify_qp(ep->com.qp->rhp, ep->com.qp,
IWCH_QP_ATTR_NEXT_STATE, &attrs, 1);
}
close_socket(&ep->com, 0);
close_complete_upcall(ep);
__state_set(&ep->com, DEAD);
release = 1;
disconnect = 0;
break;
case DEAD:
disconnect = 0;
break;
default:
PANIC_IF(1);
}
mtx_unlock(&ep->com.lock);
if (disconnect)
iwch_ep_disconnect(ep, 0, M_NOWAIT);
if (release)
put_ep(&ep->com);
return;
}
static void
process_conn_error(struct iwch_ep *ep)
{
struct iwch_qp_attributes attrs;
int ret;
mtx_lock(&ep->com.lock);
CTR3(KTR_IW_CXGB, "%s ep %p state %u", __func__, ep, ep->com.state);
switch (ep->com.state) {
case MPA_REQ_WAIT:
stop_ep_timer(ep);
break;
case MPA_REQ_SENT:
stop_ep_timer(ep);
connect_reply_upcall(ep, -ECONNRESET);
break;
case MPA_REP_SENT:
ep->com.rpl_err = ECONNRESET;
CTR1(KTR_IW_CXGB, "waking up ep %p", ep);
break;
case MPA_REQ_RCVD:
/*
* We're gonna mark this puppy DEAD, but keep
* the reference on it until the ULP accepts or
* rejects the CR.
*/
break;
case MORIBUND:
case CLOSING:
stop_ep_timer(ep);
/*FALLTHROUGH*/
case FPDU_MODE:
if (ep->com.cm_id && ep->com.qp) {
attrs.next_state = IWCH_QP_STATE_ERROR;
ret = iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
&attrs, 1);
if (ret)
log(LOG_ERR,
"%s - qp <- error failed!\n",
__FUNCTION__);
}
peer_abort_upcall(ep);
break;
case ABORTING:
break;
case DEAD:
mtx_unlock(&ep->com.lock);
CTR2(KTR_IW_CXGB, "%s so_error %d IN DEAD STATE!!!!", __FUNCTION__,
ep->com.so->so_error);
return;
default:
PANIC_IF(1);
break;
}
if (ep->com.state != ABORTING) {
close_socket(&ep->com, 0);
__state_set(&ep->com, DEAD);
put_ep(&ep->com);
}
mtx_unlock(&ep->com.lock);
return;
}
static void
process_close_complete(struct iwch_ep *ep)
{
struct iwch_qp_attributes attrs;
int release = 0;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
PANIC_IF(!ep);
/* The cm_id may be null if we failed to connect */
mtx_lock(&ep->com.lock);
switch (ep->com.state) {
case CLOSING:
__state_set(&ep->com, MORIBUND);
break;
case MORIBUND:
stop_ep_timer(ep);
if ((ep->com.cm_id) && (ep->com.qp)) {
attrs.next_state = IWCH_QP_STATE_IDLE;
iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp,
IWCH_QP_ATTR_NEXT_STATE,
&attrs, 1);
}
if (ep->parent_ep)
close_socket(&ep->com, 1);
else
close_socket(&ep->com, 0);
close_complete_upcall(ep);
__state_set(&ep->com, DEAD);
release = 1;
break;
case ABORTING:
break;
case DEAD:
default:
PANIC_IF(1);
break;
}
mtx_unlock(&ep->com.lock);
if (release)
put_ep(&ep->com);
return;
}
/*
* T3A does 3 things when a TERM is received:
* 1) send up a CPL_RDMA_TERMINATE message with the TERM packet
* 2) generate an async event on the QP with the TERMINATE opcode
* 3) post a TERMINATE opcde cqe into the associated CQ.
*
* For (1), we save the message in the qp for later consumer consumption.
* For (2), we move the QP into TERMINATE, post a QP event and disconnect.
* For (3), we toss the CQE in cxio_poll_cq().
*
* terminate() handles case (1)...
*/
static int
terminate(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m)
{
struct adapter *sc = qs->adap;
struct tom_data *td = sc->tom_softc;
uint32_t hash = *((uint32_t *)r + 1);
unsigned int tid = ntohl(hash) >> 8 & 0xfffff;
struct toepcb *toep = lookup_tid(&td->tid_maps, tid);
struct socket *so = toep->tp_inp->inp_socket;
struct iwch_ep *ep = so->so_rcv.sb_upcallarg;
if (state_read(&ep->com) != FPDU_MODE)
goto done;
m_adj(m, sizeof(struct cpl_rdma_terminate));
CTR4(KTR_IW_CXGB, "%s: tid %u, ep %p, saved %d bytes",
__func__, tid, ep, m->m_len);
m_copydata(m, 0, m->m_len, ep->com.qp->attr.terminate_buffer);
ep->com.qp->attr.terminate_msg_len = m->m_len;
ep->com.qp->attr.is_terminate_local = 0;
done:
m_freem(m);
return (0);
}
static int
ec_status(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m)
{
struct adapter *sc = qs->adap;
struct tom_data *td = sc->tom_softc;
struct cpl_rdma_ec_status *rep = mtod(m, void *);
unsigned int tid = GET_TID(rep);
struct toepcb *toep = lookup_tid(&td->tid_maps, tid);
struct socket *so = toep->tp_inp->inp_socket;
struct iwch_ep *ep = so->so_rcv.sb_upcallarg;
if (rep->status) {
struct iwch_qp_attributes attrs;
CTR1(KTR_IW_CXGB, "%s BAD CLOSE - Aborting", __FUNCTION__);
stop_ep_timer(ep);
attrs.next_state = IWCH_QP_STATE_ERROR;
iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp,
IWCH_QP_ATTR_NEXT_STATE,
&attrs, 1);
abort_connection(ep);
}
m_freem(m);
return (0);
}
static void
ep_timeout(void *arg)
{
struct iwch_ep *ep = (struct iwch_ep *)arg;
struct iwch_qp_attributes attrs;
int err = 0;
int abort = 1;
mtx_lock(&ep->com.lock);
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
switch (ep->com.state) {
case MPA_REQ_SENT:
__state_set(&ep->com, ABORTING);
connect_reply_upcall(ep, -ETIMEDOUT);
break;
case MPA_REQ_WAIT:
__state_set(&ep->com, ABORTING);
break;
case CLOSING:
case MORIBUND:
if (ep->com.cm_id && ep->com.qp)
err = 1;
__state_set(&ep->com, ABORTING);
break;
default:
CTR3(KTR_IW_CXGB, "%s unexpected state ep %p state %u\n",
__func__, ep, ep->com.state);
abort = 0;
}
mtx_unlock(&ep->com.lock);
if (err){
attrs.next_state = IWCH_QP_STATE_ERROR;
iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
&attrs, 1);
}
if (abort)
abort_connection(ep);
put_ep(&ep->com);
}
int
iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
{
int err;
struct iwch_ep *ep = to_ep(cm_id);
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
if (state_read(&ep->com) == DEAD) {
put_ep(&ep->com);
return (-ECONNRESET);
}
PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD);
if (mpa_rev == 0) {
abort_connection(ep);
} else {
err = send_mpa_reject(ep, pdata, pdata_len);
err = soshutdown(ep->com.so, 3);
}
put_ep(&ep->com);
return 0;
}
int
iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
{
int err;
struct iwch_qp_attributes attrs;
enum iwch_qp_attr_mask mask;
struct iwch_ep *ep = to_ep(cm_id);
struct iwch_dev *h = to_iwch_dev(cm_id->device);
struct iwch_qp *qp = get_qhp(h, conn_param->qpn);
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
if (state_read(&ep->com) == DEAD) {
err = -ECONNRESET;
goto err;
}
PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD);
PANIC_IF(!qp);
if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) ||
(conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) {
abort_connection(ep);
err = -EINVAL;
goto err;
}
cm_id->add_ref(cm_id);
ep->com.cm_id = cm_id;
ep->com.qp = qp;
ep->com.rpl_err = 0;
ep->com.rpl_done = 0;
ep->ird = conn_param->ird;
ep->ord = conn_param->ord;
CTR3(KTR_IW_CXGB, "%s ird %d ord %d", __FUNCTION__, ep->ird, ep->ord);
/* bind QP to EP and move to RTS */
attrs.mpa_attr = ep->mpa_attr;
attrs.max_ird = ep->ird;
attrs.max_ord = ep->ord;
attrs.llp_stream_handle = ep;
attrs.next_state = IWCH_QP_STATE_RTS;
/* bind QP and TID with INIT_WR */
mask = IWCH_QP_ATTR_NEXT_STATE |
IWCH_QP_ATTR_LLP_STREAM_HANDLE |
IWCH_QP_ATTR_MPA_ATTR |
IWCH_QP_ATTR_MAX_IRD |
IWCH_QP_ATTR_MAX_ORD;
err = iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp, mask, &attrs, 1);
if (err)
goto err1;
err = send_mpa_reply(ep, conn_param->private_data,
conn_param->private_data_len);
if (err)
goto err1;
state_set(&ep->com, FPDU_MODE);
established_upcall(ep);
put_ep(&ep->com);
return 0;
err1:
ep->com.cm_id = NULL;
ep->com.qp = NULL;
cm_id->rem_ref(cm_id);
err:
put_ep(&ep->com);
return err;
}
static int init_sock(struct iwch_ep_common *epc)
{
int err;
struct sockopt sopt;
int on=1;
SOCK_LOCK(epc->so);
soupcall_set(epc->so, SO_RCV, iwch_so_upcall, epc);
epc->so->so_state |= SS_NBIO;
SOCK_UNLOCK(epc->so);
sopt.sopt_dir = SOPT_SET;
sopt.sopt_level = IPPROTO_TCP;
sopt.sopt_name = TCP_NODELAY;
sopt.sopt_val = (caddr_t)&on;
sopt.sopt_valsize = sizeof on;
sopt.sopt_td = NULL;
err = sosetopt(epc->so, &sopt);
if (err)
printf("%s can't set TCP_NODELAY err %d\n", __FUNCTION__, err);
return 0;
}
static int
is_loopback_dst(struct iw_cm_id *cm_id)
{
uint16_t port = cm_id->remote_addr.sin_port;
int ifa_present;
cm_id->remote_addr.sin_port = 0;
ifa_present = ifa_ifwithaddr_check(
(struct sockaddr *)&cm_id->remote_addr);
cm_id->remote_addr.sin_port = port;
return (ifa_present);
}
int
iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
{
int err = 0;
struct iwch_dev *h = to_iwch_dev(cm_id->device);
struct iwch_ep *ep;
struct rtentry *rt;
struct toedev *tdev;
if (is_loopback_dst(cm_id)) {
err = -ENOSYS;
goto out;
}
ep = alloc_ep(sizeof(*ep), M_NOWAIT);
if (!ep) {
printf("%s - cannot alloc ep.\n", __FUNCTION__);
err = (-ENOMEM);
goto out;
}
- callout_init(&ep->timer, TRUE);
+ callout_init(&ep->timer, 1);
ep->plen = conn_param->private_data_len;
if (ep->plen)
memcpy(ep->mpa_pkt + sizeof(struct mpa_message),
conn_param->private_data, ep->plen);
ep->ird = conn_param->ird;
ep->ord = conn_param->ord;
cm_id->add_ref(cm_id);
ep->com.cm_id = cm_id;
ep->com.qp = get_qhp(h, conn_param->qpn);
ep->com.thread = curthread;
PANIC_IF(!ep->com.qp);
CTR4(KTR_IW_CXGB, "%s qpn 0x%x qp %p cm_id %p", __FUNCTION__, conn_param->qpn,
ep->com.qp, cm_id);
ep->com.so = cm_id->so;
err = init_sock(&ep->com);
if (err)
goto fail2;
/* find a route */
rt = find_route(cm_id->local_addr.sin_addr.s_addr,
cm_id->remote_addr.sin_addr.s_addr,
cm_id->local_addr.sin_port,
cm_id->remote_addr.sin_port, IPTOS_LOWDELAY);
if (!rt) {
printf("%s - cannot find route.\n", __FUNCTION__);
err = EHOSTUNREACH;
goto fail2;
}
if (!(rt->rt_ifp->if_flags & IFCAP_TOE)) {
printf("%s - interface not TOE capable.\n", __FUNCTION__);
RTFREE(rt);
goto fail2;
}
tdev = TOEDEV(rt->rt_ifp);
if (tdev == NULL) {
printf("%s - No toedev for interface.\n", __FUNCTION__);
RTFREE(rt);
goto fail2;
}
RTFREE(rt);
state_set(&ep->com, CONNECTING);
ep->com.local_addr = cm_id->local_addr;
ep->com.remote_addr = cm_id->remote_addr;
err = soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr,
ep->com.thread);
if (!err)
goto out;
fail2:
put_ep(&ep->com);
out:
return err;
}
int
iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
{
int err = 0;
struct iwch_listen_ep *ep;
ep = alloc_ep(sizeof(*ep), M_NOWAIT);
if (!ep) {
printf("%s - cannot alloc ep.\n", __FUNCTION__);
err = ENOMEM;
goto out;
}
CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
cm_id->add_ref(cm_id);
ep->com.cm_id = cm_id;
ep->backlog = backlog;
ep->com.local_addr = cm_id->local_addr;
ep->com.thread = curthread;
state_set(&ep->com, LISTEN);
ep->com.so = cm_id->so;
err = init_sock(&ep->com);
if (err)
goto fail;
err = solisten(ep->com.so, ep->backlog, ep->com.thread);
if (!err) {
cm_id->provider_data = ep;
goto out;
}
close_socket(&ep->com, 0);
fail:
cm_id->rem_ref(cm_id);
put_ep(&ep->com);
out:
return err;
}
int
iwch_destroy_listen(struct iw_cm_id *cm_id)
{
struct iwch_listen_ep *ep = to_listen_ep(cm_id);
CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
state_set(&ep->com, DEAD);
close_socket(&ep->com, 0);
cm_id->rem_ref(cm_id);
put_ep(&ep->com);
return 0;
}
int
iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, int flags)
{
int close = 0;
mtx_lock(&ep->com.lock);
PANIC_IF(!ep);
PANIC_IF(!ep->com.so);
CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s, abrupt %d", __FUNCTION__, ep,
ep->com.so, states[ep->com.state], abrupt);
switch (ep->com.state) {
case MPA_REQ_WAIT:
case MPA_REQ_SENT:
case MPA_REQ_RCVD:
case MPA_REP_SENT:
case FPDU_MODE:
close = 1;
if (abrupt)
ep->com.state = ABORTING;
else {
ep->com.state = CLOSING;
start_ep_timer(ep);
}
break;
case CLOSING:
close = 1;
if (abrupt) {
stop_ep_timer(ep);
ep->com.state = ABORTING;
} else
ep->com.state = MORIBUND;
break;
case MORIBUND:
case ABORTING:
case DEAD:
CTR3(KTR_IW_CXGB, "%s ignoring disconnect ep %p state %u\n",
__func__, ep, ep->com.state);
break;
default:
panic("unknown state: %d\n", ep->com.state);
break;
}
mtx_unlock(&ep->com.lock);
if (close) {
if (abrupt)
abort_connection(ep);
else {
if (!ep->parent_ep)
__state_set(&ep->com, MORIBUND);
shutdown_socket(&ep->com);
}
}
return 0;
}
static void
process_data(struct iwch_ep *ep)
{
struct sockaddr_in *local, *remote;
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
switch (state_read(&ep->com)) {
case MPA_REQ_SENT:
process_mpa_reply(ep);
break;
case MPA_REQ_WAIT:
/*
* XXX
* Set local and remote addrs here because when we
* dequeue the newly accepted socket, they aren't set
* yet in the pcb!
*/
in_getsockaddr(ep->com.so, (struct sockaddr **)&local);
in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote);
CTR3(KTR_IW_CXGB, "%s local %s remote %s", __FUNCTION__,
inet_ntoa(local->sin_addr),
inet_ntoa(remote->sin_addr));
ep->com.local_addr = *local;
ep->com.remote_addr = *remote;
free(local, M_SONAME);
free(remote, M_SONAME);
process_mpa_request(ep);
break;
default:
if (sbavail(&ep->com.so->so_rcv))
printf("%s Unexpected streaming data."
" ep %p state %d so %p so_state %x so_rcv.sb_cc %u so_rcv.sb_mb %p\n",
__FUNCTION__, ep, state_read(&ep->com), ep->com.so, ep->com.so->so_state,
sbavail(&ep->com.so->so_rcv), ep->com.so->so_rcv.sb_mb);
break;
}
return;
}
static void
process_connected(struct iwch_ep *ep)
{
CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
if ((ep->com.so->so_state & SS_ISCONNECTED) && !ep->com.so->so_error) {
send_mpa_req(ep);
} else {
connect_reply_upcall(ep, -ep->com.so->so_error);
close_socket(&ep->com, 0);
state_set(&ep->com, DEAD);
put_ep(&ep->com);
}
}
static struct socket *
dequeue_socket(struct socket *head, struct sockaddr_in **remote, struct iwch_ep *child_ep)
{
struct socket *so;
ACCEPT_LOCK();
so = TAILQ_FIRST(&head->so_comp);
if (!so) {
ACCEPT_UNLOCK();
return NULL;
}
TAILQ_REMOVE(&head->so_comp, so, so_list);
head->so_qlen--;
SOCK_LOCK(so);
so->so_qstate &= ~SQ_COMP;
so->so_head = NULL;
soref(so);
soupcall_set(so, SO_RCV, iwch_so_upcall, child_ep);
so->so_state |= SS_NBIO;
PANIC_IF(!(so->so_state & SS_ISCONNECTED));
PANIC_IF(so->so_error);
SOCK_UNLOCK(so);
ACCEPT_UNLOCK();
soaccept(so, (struct sockaddr **)remote);
return so;
}
static void
process_newconn(struct iwch_ep *parent_ep)
{
struct socket *child_so;
struct iwch_ep *child_ep;
struct sockaddr_in *remote;
CTR3(KTR_IW_CXGB, "%s parent ep %p so %p", __FUNCTION__, parent_ep, parent_ep->com.so);
child_ep = alloc_ep(sizeof(*child_ep), M_NOWAIT);
if (!child_ep) {
log(LOG_ERR, "%s - failed to allocate ep entry!\n",
__FUNCTION__);
return;
}
child_so = dequeue_socket(parent_ep->com.so, &remote, child_ep);
if (!child_so) {
log(LOG_ERR, "%s - failed to dequeue child socket!\n",
__FUNCTION__);
__free_ep(&child_ep->com);
return;
}
CTR3(KTR_IW_CXGB, "%s remote addr %s port %d", __FUNCTION__,
inet_ntoa(remote->sin_addr), ntohs(remote->sin_port));
child_ep->com.tdev = parent_ep->com.tdev;
child_ep->com.local_addr.sin_family = parent_ep->com.local_addr.sin_family;
child_ep->com.local_addr.sin_port = parent_ep->com.local_addr.sin_port;
child_ep->com.local_addr.sin_addr.s_addr = parent_ep->com.local_addr.sin_addr.s_addr;
child_ep->com.local_addr.sin_len = parent_ep->com.local_addr.sin_len;
child_ep->com.remote_addr.sin_family = remote->sin_family;
child_ep->com.remote_addr.sin_port = remote->sin_port;
child_ep->com.remote_addr.sin_addr.s_addr = remote->sin_addr.s_addr;
child_ep->com.remote_addr.sin_len = remote->sin_len;
child_ep->com.so = child_so;
child_ep->com.cm_id = NULL;
child_ep->com.thread = parent_ep->com.thread;
child_ep->parent_ep = parent_ep;
free(remote, M_SONAME);
get_ep(&parent_ep->com);
child_ep->parent_ep = parent_ep;
- callout_init(&child_ep->timer, TRUE);
+ callout_init(&child_ep->timer, 1);
state_set(&child_ep->com, MPA_REQ_WAIT);
start_ep_timer(child_ep);
/* maybe the request has already been queued up on the socket... */
process_mpa_request(child_ep);
}
static int
iwch_so_upcall(struct socket *so, void *arg, int waitflag)
{
struct iwch_ep *ep = arg;
CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]);
mtx_lock(&req_lock);
if (ep && ep->com.so && !ep->com.entry.tqe_prev) {
get_ep(&ep->com);
TAILQ_INSERT_TAIL(&req_list, &ep->com, entry);
taskqueue_enqueue(iw_cxgb_taskq, &iw_cxgb_task);
}
mtx_unlock(&req_lock);
return (SU_OK);
}
static void
process_socket_event(struct iwch_ep *ep)
{
int state = state_read(&ep->com);
struct socket *so = ep->com.so;
CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]);
if (state == CONNECTING) {
process_connected(ep);
return;
}
if (state == LISTEN) {
process_newconn(ep);
return;
}
/* connection error */
if (so->so_error) {
process_conn_error(ep);
return;
}
/* peer close */
if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state < CLOSING) {
process_peer_close(ep);
return;
}
/* close complete */
if (so->so_state & (SS_ISDISCONNECTED)) {
process_close_complete(ep);
return;
}
/* rx data */
process_data(ep);
return;
}
static void
process_req(void *ctx, int pending)
{
struct iwch_ep_common *epc;
CTR1(KTR_IW_CXGB, "%s enter", __FUNCTION__);
mtx_lock(&req_lock);
while (!TAILQ_EMPTY(&req_list)) {
epc = TAILQ_FIRST(&req_list);
TAILQ_REMOVE(&req_list, epc, entry);
epc->entry.tqe_prev = NULL;
mtx_unlock(&req_lock);
if (epc->so)
process_socket_event((struct iwch_ep *)epc);
put_ep(epc);
mtx_lock(&req_lock);
}
mtx_unlock(&req_lock);
}
int
iwch_cm_init(void)
{
TAILQ_INIT(&req_list);
mtx_init(&req_lock, "iw_cxgb req_list lock", NULL, MTX_DEF);
iw_cxgb_taskq = taskqueue_create("iw_cxgb_taskq", M_NOWAIT,
taskqueue_thread_enqueue, &iw_cxgb_taskq);
if (iw_cxgb_taskq == NULL) {
printf("failed to allocate iw_cxgb taskqueue\n");
return (ENOMEM);
}
taskqueue_start_threads(&iw_cxgb_taskq, 1, PI_NET, "iw_cxgb taskq");
TASK_INIT(&iw_cxgb_task, 0, process_req, NULL);
return (0);
}
void
iwch_cm_term(void)
{
taskqueue_drain(iw_cxgb_taskq, &iw_cxgb_task);
taskqueue_free(iw_cxgb_taskq);
}
void
iwch_cm_init_cpl(struct adapter *sc)
{
t3_register_cpl_handler(sc, CPL_RDMA_TERMINATE, terminate);
t3_register_cpl_handler(sc, CPL_RDMA_EC_STATUS, ec_status);
}
void
iwch_cm_term_cpl(struct adapter *sc)
{
t3_register_cpl_handler(sc, CPL_RDMA_TERMINATE, NULL);
t3_register_cpl_handler(sc, CPL_RDMA_EC_STATUS, NULL);
}
#endif
Index: head/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c
===================================================================
--- head/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c (revision 283290)
+++ head/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c (revision 283291)
@@ -1,1167 +1,1167 @@
/**************************************************************************
Copyright (c) 2007, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***************************************************************************/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet.h"
#ifdef TCP_OFFLOAD
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/bus.h>
#include <sys/pciio.h>
#include <sys/conf.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus_dma.h>
#include <sys/rman.h>
#include <sys/ioccom.h>
#include <sys/mbuf.h>
#include <sys/mutex.h>
#include <sys/rwlock.h>
#include <sys/linker.h>
#include <sys/firmware.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/queue.h>
#include <sys/taskqueue.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <netinet/in.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>
#include <linux/idr.h>
#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h>
#include <cxgb_include.h>
#include <ulp/iw_cxgb/iw_cxgb_wr.h>
#include <ulp/iw_cxgb/iw_cxgb_hal.h>
#include <ulp/iw_cxgb/iw_cxgb_provider.h>
#include <ulp/iw_cxgb/iw_cxgb_cm.h>
#include <ulp/iw_cxgb/iw_cxgb.h>
#include <ulp/iw_cxgb/iw_cxgb_resource.h>
#include <ulp/iw_cxgb/iw_cxgb_user.h>
static int
iwch_modify_port(struct ib_device *ibdev,
u8 port, int port_modify_mask,
struct ib_port_modify *props)
{
return (-ENOSYS);
}
static struct ib_ah *
iwch_ah_create(struct ib_pd *pd,
struct ib_ah_attr *ah_attr)
{
return ERR_PTR(-ENOSYS);
}
static int
iwch_ah_destroy(struct ib_ah *ah)
{
return (-ENOSYS);
}
static int iwch_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
{
return (-ENOSYS);
}
static int
iwch_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
{
return (-ENOSYS);
}
static int
iwch_process_mad(struct ib_device *ibdev,
int mad_flags,
u8 port_num,
struct ib_wc *in_wc,
struct ib_grh *in_grh,
struct ib_mad *in_mad, struct ib_mad *out_mad)
{
return (-ENOSYS);
}
static int
iwch_dealloc_ucontext(struct ib_ucontext *context)
{
struct iwch_dev *rhp = to_iwch_dev(context->device);
struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
struct iwch_mm_entry *mm, *tmp;
CTR2(KTR_IW_CXGB, "%s context %p", __FUNCTION__, context);
TAILQ_FOREACH_SAFE(mm, &ucontext->mmaps, entry, tmp) {
TAILQ_REMOVE(&ucontext->mmaps, mm, entry);
cxfree(mm);
}
cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
cxfree(ucontext);
return 0;
}
static struct ib_ucontext *
iwch_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata)
{
struct iwch_ucontext *context;
struct iwch_dev *rhp = to_iwch_dev(ibdev);
CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
context = malloc(sizeof(*context), M_DEVBUF, M_ZERO|M_NOWAIT);
if (!context)
return ERR_PTR(-ENOMEM);
cxio_init_ucontext(&rhp->rdev, &context->uctx);
TAILQ_INIT(&context->mmaps);
mtx_init(&context->mmap_lock, "ucontext mmap", NULL, MTX_DEF);
return &context->ibucontext;
}
static int
iwch_destroy_cq(struct ib_cq *ib_cq)
{
struct iwch_cq *chp;
CTR2(KTR_IW_CXGB, "%s ib_cq %p", __FUNCTION__, ib_cq);
chp = to_iwch_cq(ib_cq);
remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
mtx_lock(&chp->lock);
if (--chp->refcnt)
msleep(chp, &chp->lock, 0, "iwch_destroy_cq", 0);
mtx_unlock(&chp->lock);
cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
cxfree(chp);
return 0;
}
static struct ib_cq *
iwch_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_context,
struct ib_udata *udata)
{
struct iwch_dev *rhp;
struct iwch_cq *chp;
struct iwch_create_cq_resp uresp;
struct iwch_create_cq_req ureq;
struct iwch_ucontext *ucontext = NULL;
static int warned;
size_t resplen;
int entries = attr->cqe;
CTR3(KTR_IW_CXGB, "%s ib_dev %p entries %d", __FUNCTION__, ibdev, entries);
rhp = to_iwch_dev(ibdev);
chp = malloc(sizeof(*chp), M_DEVBUF, M_NOWAIT|M_ZERO);
if (!chp) {
return ERR_PTR(-ENOMEM);
}
if (ib_context) {
ucontext = to_iwch_ucontext(ib_context);
if (!t3a_device(rhp)) {
if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
cxfree(chp);
return ERR_PTR(-EFAULT);
}
chp->user_rptr_addr = (u32 /*__user */*)(unsigned long)ureq.user_rptr_addr;
}
}
if (t3a_device(rhp)) {
/*
* T3A: Add some fluff to handle extra CQEs inserted
* for various errors.
* Additional CQE possibilities:
* TERMINATE,
* incoming RDMA WRITE Failures
* incoming RDMA READ REQUEST FAILUREs
* NOTE: We cannot ensure the CQ won't overflow.
*/
entries += 16;
}
entries = roundup_pow_of_two(entries);
chp->cq.size_log2 = ilog2(entries);
if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) {
cxfree(chp);
return ERR_PTR(-ENOMEM);
}
chp->rhp = rhp;
chp->ibcq.cqe = 1 << chp->cq.size_log2;
mtx_init(&chp->lock, "cxgb cq", NULL, MTX_DEF|MTX_DUPOK);
chp->refcnt = 1;
if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) {
cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
cxfree(chp);
return ERR_PTR(-ENOMEM);
}
if (ucontext) {
struct iwch_mm_entry *mm;
mm = kmalloc(sizeof *mm, M_NOWAIT);
if (!mm) {
iwch_destroy_cq(&chp->ibcq);
return ERR_PTR(-ENOMEM);
}
uresp.cqid = chp->cq.cqid;
uresp.size_log2 = chp->cq.size_log2;
mtx_lock(&ucontext->mmap_lock);
uresp.key = ucontext->key;
ucontext->key += PAGE_SIZE;
mtx_unlock(&ucontext->mmap_lock);
mm->key = uresp.key;
mm->addr = vtophys(chp->cq.queue);
if (udata->outlen < sizeof uresp) {
if (!warned++)
CTR1(KTR_IW_CXGB, "%s Warning - "
"downlevel libcxgb3 (non-fatal).\n",
__func__);
mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
sizeof(struct t3_cqe));
resplen = sizeof(struct iwch_create_cq_resp_v0);
} else {
mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) *
sizeof(struct t3_cqe));
uresp.memsize = mm->len;
resplen = sizeof uresp;
}
if (ib_copy_to_udata(udata, &uresp, resplen)) {
cxfree(mm);
iwch_destroy_cq(&chp->ibcq);
return ERR_PTR(-EFAULT);
}
insert_mmap(ucontext, mm);
}
CTR4(KTR_IW_CXGB, "created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx",
chp->cq.cqid, chp, (1 << chp->cq.size_log2),
(unsigned long long) chp->cq.dma_addr);
return &chp->ibcq;
}
static int
iwch_resize_cq(struct ib_cq *cq __unused, int cqe __unused,
struct ib_udata *udata __unused)
{
return (-ENOSYS);
}
static int
iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
{
struct iwch_dev *rhp;
struct iwch_cq *chp;
enum t3_cq_opcode cq_op;
int err;
u32 rptr;
chp = to_iwch_cq(ibcq);
rhp = chp->rhp;
if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
cq_op = CQ_ARM_SE;
else
cq_op = CQ_ARM_AN;
if (chp->user_rptr_addr) {
if (copyin(&rptr, chp->user_rptr_addr, 4))
return (-EFAULT);
mtx_lock(&chp->lock);
chp->cq.rptr = rptr;
} else
mtx_lock(&chp->lock);
CTR2(KTR_IW_CXGB, "%s rptr 0x%x", __FUNCTION__, chp->cq.rptr);
err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
mtx_unlock(&chp->lock);
if (err < 0)
log(LOG_ERR, "Error %d rearming CQID 0x%x\n", err,
chp->cq.cqid);
if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
err = 0;
return err;
}
static int
iwch_mmap(struct ib_ucontext *context __unused, struct vm_area_struct *vma __unused)
{
return (-ENOSYS);
}
static int iwch_deallocate_pd(struct ib_pd *pd)
{
struct iwch_dev *rhp;
struct iwch_pd *php;
php = to_iwch_pd(pd);
rhp = php->rhp;
CTR3(KTR_IW_CXGB, "%s ibpd %p pdid 0x%x", __FUNCTION__, pd, php->pdid);
cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
cxfree(php);
return 0;
}
static struct ib_pd *iwch_allocate_pd(struct ib_device *ibdev,
struct ib_ucontext *context,
struct ib_udata *udata)
{
struct iwch_pd *php;
u32 pdid;
struct iwch_dev *rhp;
CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
rhp = (struct iwch_dev *) ibdev;
pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
if (!pdid)
return ERR_PTR(-EINVAL);
php = malloc(sizeof(*php), M_DEVBUF, M_ZERO|M_NOWAIT);
if (!php) {
cxio_hal_put_pdid(rhp->rdev.rscp, pdid);
return ERR_PTR(-ENOMEM);
}
php->pdid = pdid;
php->rhp = rhp;
if (context) {
if (ib_copy_to_udata(udata, &php->pdid, sizeof (__u32))) {
iwch_deallocate_pd(&php->ibpd);
return ERR_PTR(-EFAULT);
}
}
CTR3(KTR_IW_CXGB, "%s pdid 0x%0x ptr 0x%p", __FUNCTION__, pdid, php);
return &php->ibpd;
}
static int iwch_dereg_mr(struct ib_mr *ib_mr)
{
struct iwch_dev *rhp;
struct iwch_mr *mhp;
u32 mmid;
CTR2(KTR_IW_CXGB, "%s ib_mr %p", __FUNCTION__, ib_mr);
/* There can be no memory windows */
if (atomic_load_acq_int(&ib_mr->usecnt.counter))
return (-EINVAL);
mhp = to_iwch_mr(ib_mr);
rhp = mhp->rhp;
mmid = mhp->attr.stag >> 8;
cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
mhp->attr.pbl_addr);
iwch_free_pbl(mhp);
remove_handle(rhp, &rhp->mmidr, mmid);
if (mhp->kva)
cxfree((void *) (unsigned long) mhp->kva);
if (mhp->umem)
ib_umem_release(mhp->umem);
CTR3(KTR_IW_CXGB, "%s mmid 0x%x ptr %p", __FUNCTION__, mmid, mhp);
cxfree(mhp);
return 0;
}
static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
struct ib_phys_buf *buffer_list,
int num_phys_buf,
int acc,
u64 *iova_start)
{
__be64 *page_list;
int shift;
u64 total_size;
int npages;
struct iwch_dev *rhp;
struct iwch_pd *php;
struct iwch_mr *mhp;
int ret;
CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
php = to_iwch_pd(pd);
rhp = php->rhp;
mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT);
if (!mhp)
return ERR_PTR(-ENOMEM);
mhp->rhp = rhp;
/* First check that we have enough alignment */
if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) {
ret = -EINVAL;
goto err;
}
if (num_phys_buf > 1 &&
((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) {
ret = -EINVAL;
goto err;
}
ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start,
&total_size, &npages, &shift, &page_list);
if (ret)
goto err;
ret = iwch_alloc_pbl(mhp, npages);
if (ret) {
cxfree(page_list);
goto err_pbl;
}
ret = iwch_write_pbl(mhp, page_list, npages, 0);
cxfree(page_list);
if (ret)
goto err;
mhp->attr.pdid = php->pdid;
mhp->attr.zbva = 0;
mhp->attr.perms = iwch_ib_to_tpt_access(acc);
mhp->attr.va_fbo = *iova_start;
mhp->attr.page_size = shift - 12;
mhp->attr.len = (u32) total_size;
mhp->attr.pbl_size = npages;
ret = iwch_register_mem(rhp, php, mhp, shift);
if (ret)
goto err_pbl;
return &mhp->ibmr;
err_pbl:
iwch_free_pbl(mhp);
err:
cxfree(mhp);
return ERR_PTR(ret);
}
static int iwch_reregister_phys_mem(struct ib_mr *mr,
int mr_rereg_mask,
struct ib_pd *pd,
struct ib_phys_buf *buffer_list,
int num_phys_buf,
int acc, u64 * iova_start)
{
struct iwch_mr mh, *mhp;
struct iwch_pd *php;
struct iwch_dev *rhp;
__be64 *page_list = NULL;
int shift = 0;
u64 total_size;
int npages = 0;
int ret;
CTR3(KTR_IW_CXGB, "%s ib_mr %p ib_pd %p", __FUNCTION__, mr, pd);
/* There can be no memory windows */
if (atomic_load_acq_int(&mr->usecnt.counter))
return (-EINVAL);
mhp = to_iwch_mr(mr);
rhp = mhp->rhp;
php = to_iwch_pd(mr->pd);
/* make sure we are on the same adapter */
if (rhp != php->rhp)
return (-EINVAL);
memcpy(&mh, mhp, sizeof *mhp);
if (mr_rereg_mask & IB_MR_REREG_PD)
php = to_iwch_pd(pd);
if (mr_rereg_mask & IB_MR_REREG_ACCESS)
mh.attr.perms = iwch_ib_to_tpt_access(acc);
if (mr_rereg_mask & IB_MR_REREG_TRANS) {
ret = build_phys_page_list(buffer_list, num_phys_buf,
iova_start,
&total_size, &npages,
&shift, &page_list);
if (ret)
return ret;
}
ret = iwch_reregister_mem(rhp, php, &mh, shift, npages);
cxfree(page_list);
if (ret) {
return ret;
}
if (mr_rereg_mask & IB_MR_REREG_PD)
mhp->attr.pdid = php->pdid;
if (mr_rereg_mask & IB_MR_REREG_ACCESS)
mhp->attr.perms = iwch_ib_to_tpt_access(acc);
if (mr_rereg_mask & IB_MR_REREG_TRANS) {
mhp->attr.zbva = 0;
mhp->attr.va_fbo = *iova_start;
mhp->attr.page_size = shift - 12;
mhp->attr.len = (u32) total_size;
mhp->attr.pbl_size = npages;
}
return 0;
}
static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt, int acc, struct ib_udata *udata,
int mr_id)
{
__be64 *pages;
int shift, n, len;
int i, k, entry;
int err = 0;
struct iwch_dev *rhp;
struct iwch_pd *php;
struct iwch_mr *mhp;
struct iwch_reg_user_mr_resp uresp;
struct scatterlist *sg;
CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
php = to_iwch_pd(pd);
rhp = php->rhp;
mhp = malloc(sizeof(*mhp), M_DEVBUF, M_NOWAIT|M_ZERO);
if (!mhp)
return ERR_PTR(-ENOMEM);
mhp->rhp = rhp;
mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
if (IS_ERR(mhp->umem)) {
err = PTR_ERR(mhp->umem);
cxfree(mhp);
return ERR_PTR(-err);
}
shift = ffs(mhp->umem->page_size) - 1;
n = mhp->umem->nmap;
err = iwch_alloc_pbl(mhp, n);
if (err)
goto err;
pages = (__be64 *) kmalloc(n * sizeof(u64), M_NOWAIT);
if (!pages) {
err = -ENOMEM;
goto err_pbl;
}
i = n = 0;
for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {
len = sg_dma_len(sg) >> shift;
for (k = 0; k < len; ++k) {
pages[i++] = cpu_to_be64(sg_dma_address(sg) +
mhp->umem->page_size * k);
if (i == PAGE_SIZE / sizeof *pages) {
err = iwch_write_pbl(mhp, pages, i, n);
if (err)
goto pbl_done;
n += i;
i = 0;
}
}
}
#if 0
TAILQ_FOREACH(chunk, &mhp->umem->chunk_list, entry)
for (j = 0; j < chunk->nmap; ++j) {
len = sg_dma_len(&chunk->page_list[j]) >> shift;
for (k = 0; k < len; ++k) {
pages[i++] = htobe64(sg_dma_address(
&chunk->page_list[j]) +
mhp->umem->page_size * k);
if (i == PAGE_SIZE / sizeof *pages) {
err = iwch_write_pbl(mhp, pages, i, n);
if (err)
goto pbl_done;
n += i;
i = 0;
}
}
}
#endif
if (i)
err = iwch_write_pbl(mhp, pages, i, n);
pbl_done:
cxfree(pages);
if (err)
goto err_pbl;
mhp->attr.pdid = php->pdid;
mhp->attr.zbva = 0;
mhp->attr.perms = iwch_ib_to_tpt_access(acc);
mhp->attr.va_fbo = virt;
mhp->attr.page_size = shift - 12;
mhp->attr.len = (u32) length;
err = iwch_register_mem(rhp, php, mhp, shift);
if (err)
goto err_pbl;
if (udata && !t3a_device(rhp)) {
uresp.pbl_addr = (mhp->attr.pbl_addr -
rhp->rdev.rnic_info.pbl_base) >> 3;
CTR2(KTR_IW_CXGB, "%s user resp pbl_addr 0x%x", __FUNCTION__,
uresp.pbl_addr);
if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
iwch_dereg_mr(&mhp->ibmr);
err = EFAULT;
goto err;
}
}
return &mhp->ibmr;
err_pbl:
iwch_free_pbl(mhp);
err:
ib_umem_release(mhp->umem);
cxfree(mhp);
return ERR_PTR(-err);
}
static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
{
struct ib_phys_buf bl;
u64 kva;
struct ib_mr *ibmr;
CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
/*
* T3 only supports 32 bits of size.
*/
bl.size = 0xffffffff;
bl.addr = 0;
kva = 0;
ibmr = iwch_register_phys_mem(pd, &bl, 1, acc, &kva);
return ibmr;
}
static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
{
struct iwch_dev *rhp;
struct iwch_pd *php;
struct iwch_mw *mhp;
u32 mmid;
u32 stag = 0;
int ret;
php = to_iwch_pd(pd);
rhp = php->rhp;
mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT);
if (!mhp)
return ERR_PTR(-ENOMEM);
ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
if (ret) {
cxfree(mhp);
return ERR_PTR(-ret);
}
mhp->rhp = rhp;
mhp->attr.pdid = php->pdid;
mhp->attr.type = TPT_MW;
mhp->attr.stag = stag;
mmid = (stag) >> 8;
mhp->ibmw.rkey = stag;
if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
cxfree(mhp);
return ERR_PTR(-ENOMEM);
}
CTR4(KTR_IW_CXGB, "%s mmid 0x%x mhp %p stag 0x%x", __FUNCTION__, mmid, mhp, stag);
return &(mhp->ibmw);
}
static int iwch_dealloc_mw(struct ib_mw *mw)
{
struct iwch_dev *rhp;
struct iwch_mw *mhp;
u32 mmid;
mhp = to_iwch_mw(mw);
rhp = mhp->rhp;
mmid = (mw->rkey) >> 8;
cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
remove_handle(rhp, &rhp->mmidr, mmid);
cxfree(mhp);
CTR4(KTR_IW_CXGB, "%s ib_mw %p mmid 0x%x ptr %p", __FUNCTION__, mw, mmid, mhp);
return 0;
}
static int iwch_destroy_qp(struct ib_qp *ib_qp)
{
struct iwch_dev *rhp;
struct iwch_qp *qhp;
struct iwch_qp_attributes attrs;
struct iwch_ucontext *ucontext;
qhp = to_iwch_qp(ib_qp);
rhp = qhp->rhp;
attrs.next_state = IWCH_QP_STATE_ERROR;
iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
mtx_lock(&qhp->lock);
if (qhp->ep)
msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp1", 0);
mtx_unlock(&qhp->lock);
remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
mtx_lock(&qhp->lock);
if (--qhp->refcnt)
msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp2", 0);
mtx_unlock(&qhp->lock);
ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
: NULL;
cxio_destroy_qp(&rhp->rdev, &qhp->wq,
ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
CTR4(KTR_IW_CXGB, "%s ib_qp %p qpid 0x%0x qhp %p", __FUNCTION__,
ib_qp, qhp->wq.qpid, qhp);
cxfree(qhp);
return 0;
}
static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
struct ib_qp_init_attr *attrs,
struct ib_udata *udata)
{
struct iwch_dev *rhp;
struct iwch_qp *qhp;
struct iwch_pd *php;
struct iwch_cq *schp;
struct iwch_cq *rchp;
struct iwch_create_qp_resp uresp;
int wqsize, sqsize, rqsize;
struct iwch_ucontext *ucontext;
CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
if (attrs->qp_type != IB_QPT_RC)
return ERR_PTR(-EINVAL);
php = to_iwch_pd(pd);
rhp = php->rhp;
schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
if (!schp || !rchp)
return ERR_PTR(-EINVAL);
/* The RQT size must be # of entries + 1 rounded up to a power of two */
rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
if (rqsize == attrs->cap.max_recv_wr)
rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
/* T3 doesn't support RQT depth < 16 */
if (rqsize < 16)
rqsize = 16;
if (rqsize > T3_MAX_RQ_SIZE)
return ERR_PTR(-EINVAL);
if (attrs->cap.max_inline_data > T3_MAX_INLINE)
return ERR_PTR(-EINVAL);
/*
* NOTE: The SQ and total WQ sizes don't need to be
* a power of two. However, all the code assumes
* they are. EG: Q_FREECNT() and friends.
*/
sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
wqsize = roundup_pow_of_two(rqsize + sqsize);
CTR4(KTR_IW_CXGB, "%s wqsize %d sqsize %d rqsize %d", __FUNCTION__,
wqsize, sqsize, rqsize);
qhp = malloc(sizeof(*qhp), M_DEVBUF, M_ZERO|M_NOWAIT);
if (!qhp)
return ERR_PTR(-ENOMEM);
qhp->wq.size_log2 = ilog2(wqsize);
qhp->wq.rq_size_log2 = ilog2(rqsize);
qhp->wq.sq_size_log2 = ilog2(sqsize);
ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
cxfree(qhp);
return ERR_PTR(-ENOMEM);
}
attrs->cap.max_recv_wr = rqsize - 1;
attrs->cap.max_send_wr = sqsize;
attrs->cap.max_inline_data = T3_MAX_INLINE;
qhp->rhp = rhp;
qhp->attr.pd = php->pdid;
qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
qhp->attr.state = IWCH_QP_STATE_IDLE;
qhp->attr.next_state = IWCH_QP_STATE_IDLE;
/*
* XXX - These don't get passed in from the openib user
* at create time. The CM sets them via a QP modify.
* Need to fix... I think the CM should
*/
qhp->attr.enable_rdma_read = 1;
qhp->attr.enable_rdma_write = 1;
qhp->attr.enable_bind = 1;
qhp->attr.max_ord = 1;
qhp->attr.max_ird = 1;
mtx_init(&qhp->lock, "cxgb qp", NULL, MTX_DEF|MTX_DUPOK);
qhp->refcnt = 1;
if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) {
cxio_destroy_qp(&rhp->rdev, &qhp->wq,
ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
cxfree(qhp);
return ERR_PTR(-ENOMEM);
}
if (udata) {
struct iwch_mm_entry *mm1, *mm2;
mm1 = kmalloc(sizeof *mm1, M_NOWAIT);
if (!mm1) {
iwch_destroy_qp(&qhp->ibqp);
return ERR_PTR(-ENOMEM);
}
mm2 = kmalloc(sizeof *mm2, M_NOWAIT);
if (!mm2) {
cxfree(mm1);
iwch_destroy_qp(&qhp->ibqp);
return ERR_PTR(-ENOMEM);
}
uresp.qpid = qhp->wq.qpid;
uresp.size_log2 = qhp->wq.size_log2;
uresp.sq_size_log2 = qhp->wq.sq_size_log2;
uresp.rq_size_log2 = qhp->wq.rq_size_log2;
mtx_lock(&ucontext->mmap_lock);
uresp.key = ucontext->key;
ucontext->key += PAGE_SIZE;
uresp.db_key = ucontext->key;
ucontext->key += PAGE_SIZE;
mtx_unlock(&ucontext->mmap_lock);
if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
cxfree(mm1);
cxfree(mm2);
iwch_destroy_qp(&qhp->ibqp);
return ERR_PTR(-EFAULT);
}
mm1->key = uresp.key;
mm1->addr = vtophys(qhp->wq.queue);
mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
insert_mmap(ucontext, mm1);
mm2->key = uresp.db_key;
mm2->addr = qhp->wq.udb & PAGE_MASK;
mm2->len = PAGE_SIZE;
insert_mmap(ucontext, mm2);
}
qhp->ibqp.qp_num = qhp->wq.qpid;
- callout_init(&(qhp->timer), TRUE);
+ callout_init(&(qhp->timer), 1);
CTR6(KTR_IW_CXGB, "sq_num_entries %d, rq_num_entries %d "
"qpid 0x%0x qhp %p dma_addr 0x%llx size %d",
qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
qhp->wq.qpid, qhp, (unsigned long long) qhp->wq.dma_addr,
1 << qhp->wq.size_log2);
return &qhp->ibqp;
}
static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
struct iwch_dev *rhp;
struct iwch_qp *qhp;
enum iwch_qp_attr_mask mask = 0;
struct iwch_qp_attributes attrs;
CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, ibqp);
/* iwarp does not support the RTR state */
if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
attr_mask &= ~IB_QP_STATE;
/* Make sure we still have something left to do */
if (!attr_mask)
return 0;
memset(&attrs, 0, sizeof attrs);
qhp = to_iwch_qp(ibqp);
rhp = qhp->rhp;
attrs.next_state = iwch_convert_state(attr->qp_state);
attrs.enable_rdma_read = (attr->qp_access_flags &
IB_ACCESS_REMOTE_READ) ? 1 : 0;
attrs.enable_rdma_write = (attr->qp_access_flags &
IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
(IWCH_QP_ATTR_ENABLE_RDMA_READ |
IWCH_QP_ATTR_ENABLE_RDMA_WRITE |
IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;
return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
}
void iwch_qp_add_ref(struct ib_qp *qp)
{
CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp);
mtx_lock(&to_iwch_qp(qp)->lock);
to_iwch_qp(qp)->refcnt++;
mtx_unlock(&to_iwch_qp(qp)->lock);
}
void iwch_qp_rem_ref(struct ib_qp *qp)
{
CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp);
mtx_lock(&to_iwch_qp(qp)->lock);
if (--to_iwch_qp(qp)->refcnt == 0)
wakeup(to_iwch_qp(qp));
mtx_unlock(&to_iwch_qp(qp)->lock);
}
static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
{
CTR3(KTR_IW_CXGB, "%s ib_dev %p qpn 0x%x", __FUNCTION__, dev, qpn);
return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
}
static int iwch_query_pkey(struct ib_device *ibdev,
u8 port, u16 index, u16 * pkey)
{
CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
*pkey = 0;
return 0;
}
static int iwch_query_gid(struct ib_device *ibdev, u8 port,
int index, union ib_gid *gid)
{
struct iwch_dev *dev;
struct port_info *pi;
struct adapter *sc;
CTR5(KTR_IW_CXGB, "%s ibdev %p, port %d, index %d, gid %p",
__FUNCTION__, ibdev, port, index, gid);
dev = to_iwch_dev(ibdev);
sc = dev->rdev.adap;
PANIC_IF(port == 0 || port > 2);
pi = &sc->port[port - 1];
memset(&(gid->raw[0]), 0, sizeof(gid->raw));
memcpy(&(gid->raw[0]), pi->hw_addr, 6);
return 0;
}
static int iwch_query_device(struct ib_device *ibdev,
struct ib_device_attr *props)
{
struct iwch_dev *dev;
struct adapter *sc;
CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
dev = to_iwch_dev(ibdev);
sc = dev->rdev.adap;
memset(props, 0, sizeof *props);
memcpy(&props->sys_image_guid, sc->port[0].hw_addr, 6);
props->device_cap_flags = dev->device_cap_flags;
props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
props->vendor_id = pci_get_vendor(sc->dev);
props->vendor_part_id = pci_get_device(sc->dev);
props->max_mr_size = dev->attr.max_mr_size;
props->max_qp = dev->attr.max_qps;
props->max_qp_wr = dev->attr.max_wrs;
props->max_sge = dev->attr.max_sge_per_wr;
props->max_sge_rd = 1;
props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
props->max_cq = dev->attr.max_cqs;
props->max_cqe = dev->attr.max_cqes_per_cq;
props->max_mr = dev->attr.max_mem_regs;
props->max_pd = dev->attr.max_pds;
props->local_ca_ack_delay = 0;
return 0;
}
static int iwch_query_port(struct ib_device *ibdev,
u8 port, struct ib_port_attr *props)
{
CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
memset(props, 0, sizeof(struct ib_port_attr));
props->max_mtu = IB_MTU_4096;
props->active_mtu = IB_MTU_2048;
props->state = IB_PORT_ACTIVE;
props->port_cap_flags =
IB_PORT_CM_SUP |
IB_PORT_SNMP_TUNNEL_SUP |
IB_PORT_REINIT_SUP |
IB_PORT_DEVICE_MGMT_SUP |
IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
props->gid_tbl_len = 1;
props->pkey_tbl_len = 1;
props->active_width = 2;
props->active_speed = 2;
props->max_msg_sz = -1;
return 0;
}
int iwch_register_device(struct iwch_dev *dev)
{
int ret;
struct adapter *sc = dev->rdev.adap;
CTR2(KTR_IW_CXGB, "%s iwch_dev %p", __FUNCTION__, dev);
strlcpy(dev->ibdev.name, "cxgb3_%d", IB_DEVICE_NAME_MAX);
memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
memcpy(&dev->ibdev.node_guid, sc->port[0].hw_addr, 6);
dev->device_cap_flags =
(IB_DEVICE_LOCAL_DMA_LKEY |
IB_DEVICE_MEM_WINDOW);
dev->ibdev.uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
(1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
(1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
(1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
(1ull << IB_USER_VERBS_CMD_REG_MR) |
(1ull << IB_USER_VERBS_CMD_DEREG_MR) |
(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
(1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
(1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
(1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
(1ull << IB_USER_VERBS_CMD_CREATE_QP) |
(1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
(1ull << IB_USER_VERBS_CMD_POLL_CQ) |
(1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
(1ull << IB_USER_VERBS_CMD_POST_SEND) |
(1ull << IB_USER_VERBS_CMD_POST_RECV);
dev->ibdev.node_type = RDMA_NODE_RNIC;
memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
dev->ibdev.phys_port_cnt = sc->params.nports;
dev->ibdev.num_comp_vectors = 1;
dev->ibdev.dma_device = dev->rdev.adap->dev;
dev->ibdev.query_device = iwch_query_device;
dev->ibdev.query_port = iwch_query_port;
dev->ibdev.modify_port = iwch_modify_port;
dev->ibdev.query_pkey = iwch_query_pkey;
dev->ibdev.query_gid = iwch_query_gid;
dev->ibdev.alloc_ucontext = iwch_alloc_ucontext;
dev->ibdev.dealloc_ucontext = iwch_dealloc_ucontext;
dev->ibdev.mmap = iwch_mmap;
dev->ibdev.alloc_pd = iwch_allocate_pd;
dev->ibdev.dealloc_pd = iwch_deallocate_pd;
dev->ibdev.create_ah = iwch_ah_create;
dev->ibdev.destroy_ah = iwch_ah_destroy;
dev->ibdev.create_qp = iwch_create_qp;
dev->ibdev.modify_qp = iwch_ib_modify_qp;
dev->ibdev.destroy_qp = iwch_destroy_qp;
dev->ibdev.create_cq = iwch_create_cq;
dev->ibdev.destroy_cq = iwch_destroy_cq;
dev->ibdev.resize_cq = iwch_resize_cq;
dev->ibdev.poll_cq = iwch_poll_cq;
dev->ibdev.get_dma_mr = iwch_get_dma_mr;
dev->ibdev.reg_phys_mr = iwch_register_phys_mem;
dev->ibdev.rereg_phys_mr = iwch_reregister_phys_mem;
dev->ibdev.reg_user_mr = iwch_reg_user_mr;
dev->ibdev.dereg_mr = iwch_dereg_mr;
dev->ibdev.alloc_mw = iwch_alloc_mw;
dev->ibdev.bind_mw = iwch_bind_mw;
dev->ibdev.dealloc_mw = iwch_dealloc_mw;
dev->ibdev.attach_mcast = iwch_multicast_attach;
dev->ibdev.detach_mcast = iwch_multicast_detach;
dev->ibdev.process_mad = iwch_process_mad;
dev->ibdev.req_notify_cq = iwch_arm_cq;
dev->ibdev.post_send = iwch_post_send;
dev->ibdev.post_recv = iwch_post_receive;
dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
dev->ibdev.iwcm =
kmalloc(sizeof(struct iw_cm_verbs), M_NOWAIT);
if (!dev->ibdev.iwcm)
return (ENOMEM);
dev->ibdev.iwcm->connect = iwch_connect;
dev->ibdev.iwcm->accept = iwch_accept_cr;
dev->ibdev.iwcm->reject = iwch_reject_cr;
dev->ibdev.iwcm->create_listen = iwch_create_listen;
dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
dev->ibdev.iwcm->get_qp = iwch_get_qp;
ret = ib_register_device(&dev->ibdev, NULL);
if (ret)
goto bail1;
return (0);
bail1:
cxfree(dev->ibdev.iwcm);
return (ret);
}
void iwch_unregister_device(struct iwch_dev *dev)
{
ib_unregister_device(&dev->ibdev);
cxfree(dev->ibdev.iwcm);
return;
}
#endif
Index: head/sys/dev/cxgbe/t4_main.c
===================================================================
--- head/sys/dev/cxgbe/t4_main.c (revision 283290)
+++ head/sys/dev/cxgbe/t4_main.c (revision 283291)
@@ -1,8555 +1,8555 @@
/*-
* Copyright (c) 2011 Chelsio Communications, Inc.
* All rights reserved.
* Written by: Navdeep Parhar <np@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include <sys/param.h>
#include <sys/conf.h>
#include <sys/priv.h>
#include <sys/kernel.h>
#include <sys/bus.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/queue.h>
#include <sys/taskqueue.h>
#include <sys/pciio.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/pci_private.h>
#include <sys/firmware.h>
#include <sys/sbuf.h>
#include <sys/smp.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_types.h>
#include <net/if_dl.h>
#include <net/if_vlan_var.h>
#if defined(__i386__) || defined(__amd64__)
#include <vm/vm.h>
#include <vm/pmap.h>
#endif
#include "common/common.h"
#include "common/t4_msg.h"
#include "common/t4_regs.h"
#include "common/t4_regs_values.h"
#include "t4_ioctl.h"
#include "t4_l2t.h"
#include "t4_mp_ring.h"
/* T4 bus driver interface */
static int t4_probe(device_t);
static int t4_attach(device_t);
static int t4_detach(device_t);
static device_method_t t4_methods[] = {
DEVMETHOD(device_probe, t4_probe),
DEVMETHOD(device_attach, t4_attach),
DEVMETHOD(device_detach, t4_detach),
DEVMETHOD_END
};
static driver_t t4_driver = {
"t4nex",
t4_methods,
sizeof(struct adapter)
};
/* T4 port (cxgbe) interface */
static int cxgbe_probe(device_t);
static int cxgbe_attach(device_t);
static int cxgbe_detach(device_t);
static device_method_t cxgbe_methods[] = {
DEVMETHOD(device_probe, cxgbe_probe),
DEVMETHOD(device_attach, cxgbe_attach),
DEVMETHOD(device_detach, cxgbe_detach),
{ 0, 0 }
};
static driver_t cxgbe_driver = {
"cxgbe",
cxgbe_methods,
sizeof(struct port_info)
};
static d_ioctl_t t4_ioctl;
static d_open_t t4_open;
static d_close_t t4_close;
static struct cdevsw t4_cdevsw = {
.d_version = D_VERSION,
.d_flags = 0,
.d_open = t4_open,
.d_close = t4_close,
.d_ioctl = t4_ioctl,
.d_name = "t4nex",
};
/* T5 bus driver interface */
static int t5_probe(device_t);
static device_method_t t5_methods[] = {
DEVMETHOD(device_probe, t5_probe),
DEVMETHOD(device_attach, t4_attach),
DEVMETHOD(device_detach, t4_detach),
DEVMETHOD_END
};
static driver_t t5_driver = {
"t5nex",
t5_methods,
sizeof(struct adapter)
};
/* T5 port (cxl) interface */
static driver_t cxl_driver = {
"cxl",
cxgbe_methods,
sizeof(struct port_info)
};
static struct cdevsw t5_cdevsw = {
.d_version = D_VERSION,
.d_flags = 0,
.d_open = t4_open,
.d_close = t4_close,
.d_ioctl = t4_ioctl,
.d_name = "t5nex",
};
/* ifnet + media interface */
static void cxgbe_init(void *);
static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
static int cxgbe_transmit(struct ifnet *, struct mbuf *);
static void cxgbe_qflush(struct ifnet *);
static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
static int cxgbe_media_change(struct ifnet *);
static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
/*
* Correct lock order when you need to acquire multiple locks is t4_list_lock,
* then ADAPTER_LOCK, then t4_uld_list_lock.
*/
static struct sx t4_list_lock;
SLIST_HEAD(, adapter) t4_list;
#ifdef TCP_OFFLOAD
static struct sx t4_uld_list_lock;
SLIST_HEAD(, uld_info) t4_uld_list;
#endif
/*
* Tunables. See tweak_tunables() too.
*
* Each tunable is set to a default value here if it's known at compile-time.
* Otherwise it is set to -1 as an indication to tweak_tunables() that it should
* provide a reasonable default when the driver is loaded.
*
* Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to
* T5 are under hw.cxl.
*/
/*
* Number of queues for tx and rx, 10G and 1G, NIC and offload.
*/
#define NTXQ_10G 16
static int t4_ntxq10g = -1;
TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
#define NRXQ_10G 8
static int t4_nrxq10g = -1;
TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
#define NTXQ_1G 4
static int t4_ntxq1g = -1;
TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
#define NRXQ_1G 2
static int t4_nrxq1g = -1;
TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
static int t4_rsrv_noflowq = 0;
TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
#ifdef TCP_OFFLOAD
#define NOFLDTXQ_10G 8
static int t4_nofldtxq10g = -1;
TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
#define NOFLDRXQ_10G 2
static int t4_nofldrxq10g = -1;
TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
#define NOFLDTXQ_1G 2
static int t4_nofldtxq1g = -1;
TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
#define NOFLDRXQ_1G 1
static int t4_nofldrxq1g = -1;
TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
#endif
#ifdef DEV_NETMAP
#define NNMTXQ_10G 2
static int t4_nnmtxq10g = -1;
TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
#define NNMRXQ_10G 2
static int t4_nnmrxq10g = -1;
TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
#define NNMTXQ_1G 1
static int t4_nnmtxq1g = -1;
TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
#define NNMRXQ_1G 1
static int t4_nnmrxq1g = -1;
TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
#endif
/*
* Holdoff parameters for 10G and 1G ports.
*/
#define TMR_IDX_10G 1
static int t4_tmr_idx_10g = TMR_IDX_10G;
TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
#define PKTC_IDX_10G (-1)
static int t4_pktc_idx_10g = PKTC_IDX_10G;
TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
#define TMR_IDX_1G 1
static int t4_tmr_idx_1g = TMR_IDX_1G;
TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
#define PKTC_IDX_1G (-1)
static int t4_pktc_idx_1g = PKTC_IDX_1G;
TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
/*
* Size (# of entries) of each tx and rx queue.
*/
static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
/*
* Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
*/
static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
/*
* Configuration file.
*/
#define DEFAULT_CF "default"
#define FLASH_CF "flash"
#define UWIRE_CF "uwire"
#define FPGA_CF "fpga"
static char t4_cfg_file[32] = DEFAULT_CF;
TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
/*
* PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
* rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
* tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
* mark or when signalled to do so, 0 to never emit PAUSE.
*/
static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
/*
* Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
* encouraged respectively).
*/
static unsigned int t4_fw_install = 1;
TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
/*
* ASIC features that will be used. Disable the ones you don't want so that the
* chip resources aren't wasted on features that will not be used.
*/
static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */
TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
static int t4_toecaps_allowed = -1;
TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
static int t4_rdmacaps_allowed = 0;
TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
static int t4_iscsicaps_allowed = 0;
TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
static int t4_fcoecaps_allowed = 0;
TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
static int t5_write_combine = 0;
TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
struct intrs_and_queues {
uint16_t intr_type; /* INTx, MSI, or MSI-X */
uint16_t nirq; /* Total # of vectors */
uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
uint16_t ntxq10g; /* # of NIC txq's for each 10G port */
uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */
uint16_t ntxq1g; /* # of NIC txq's for each 1G port */
uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */
uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */
#ifdef TCP_OFFLOAD
uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */
uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */
uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */
uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */
#endif
#ifdef DEV_NETMAP
uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */
uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */
uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */
uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */
#endif
};
struct filter_entry {
uint32_t valid:1; /* filter allocated and valid */
uint32_t locked:1; /* filter is administratively locked */
uint32_t pending:1; /* filter action is pending firmware reply */
uint32_t smtidx:8; /* Source MAC Table index for smac */
struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
struct t4_filter_specification fs;
};
static int map_bars_0_and_4(struct adapter *);
static int map_bar_2(struct adapter *);
static void setup_memwin(struct adapter *);
static int validate_mem_range(struct adapter *, uint32_t, int);
static int fwmtype_to_hwmtype(int);
static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
uint32_t *);
static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
static uint32_t position_memwin(struct adapter *, int, uint32_t);
static int cfg_itype_and_nqueues(struct adapter *, int, int,
struct intrs_and_queues *);
static int prep_firmware(struct adapter *);
static int partition_resources(struct adapter *, const struct firmware *,
const char *);
static int get_params__pre_init(struct adapter *);
static int get_params__post_init(struct adapter *);
static int set_params__post_init(struct adapter *);
static void t4_set_desc(struct adapter *);
static void build_medialist(struct port_info *, struct ifmedia *);
static int cxgbe_init_synchronized(struct port_info *);
static int cxgbe_uninit_synchronized(struct port_info *);
static int setup_intr_handlers(struct adapter *);
static void quiesce_txq(struct adapter *, struct sge_txq *);
static void quiesce_wrq(struct adapter *, struct sge_wrq *);
static void quiesce_iq(struct adapter *, struct sge_iq *);
static void quiesce_fl(struct adapter *, struct sge_fl *);
static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
driver_intr_t *, void *, char *);
static int t4_free_irq(struct adapter *, struct irq *);
static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
unsigned int);
static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
static void cxgbe_tick(void *);
static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
struct mbuf *);
static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
static int fw_msg_not_handled(struct adapter *, const __be64 *);
static int t4_sysctls(struct adapter *);
static int cxgbe_sysctls(struct port_info *);
static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
#ifdef SBUF_DRAIN
static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_tids(SYSCTL_HANDLER_ARGS);
static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
#endif
static uint32_t fconf_to_mode(uint32_t);
static uint32_t mode_to_fconf(uint32_t);
static uint32_t fspec_to_fconf(struct t4_filter_specification *);
static int get_filter_mode(struct adapter *, uint32_t *);
static int set_filter_mode(struct adapter *, uint32_t);
static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
static int get_filter(struct adapter *, struct t4_filter *);
static int set_filter(struct adapter *, struct t4_filter *);
static int del_filter(struct adapter *, struct t4_filter *);
static void clear_filter(struct filter_entry *);
static int set_filter_wr(struct adapter *, int);
static int del_filter_wr(struct adapter *, int);
static int get_sge_context(struct adapter *, struct t4_sge_context *);
static int load_fw(struct adapter *, struct t4_data *);
static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
static int read_i2c(struct adapter *, struct t4_i2c_data *);
static int set_sched_class(struct adapter *, struct t4_sched_params *);
static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
#ifdef TCP_OFFLOAD
static int toe_capability(struct port_info *, int);
#endif
static int mod_event(module_t, int, void *);
struct {
uint16_t device;
char *desc;
} t4_pciids[] = {
{0xa000, "Chelsio Terminator 4 FPGA"},
{0x4400, "Chelsio T440-dbg"},
{0x4401, "Chelsio T420-CR"},
{0x4402, "Chelsio T422-CR"},
{0x4403, "Chelsio T440-CR"},
{0x4404, "Chelsio T420-BCH"},
{0x4405, "Chelsio T440-BCH"},
{0x4406, "Chelsio T440-CH"},
{0x4407, "Chelsio T420-SO"},
{0x4408, "Chelsio T420-CX"},
{0x4409, "Chelsio T420-BT"},
{0x440a, "Chelsio T404-BT"},
{0x440e, "Chelsio T440-LP-CR"},
}, t5_pciids[] = {
{0xb000, "Chelsio Terminator 5 FPGA"},
{0x5400, "Chelsio T580-dbg"},
{0x5401, "Chelsio T520-CR"}, /* 2 x 10G */
{0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */
{0x5403, "Chelsio T540-CR"}, /* 4 x 10G */
{0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */
{0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */
{0x540a, "Chelsio T504-BT"}, /* 4 x 1G */
{0x540d, "Chelsio T580-CR"}, /* 2 x 40G */
{0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */
{0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */
{0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */
{0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */
{0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */
{0x5415, "Chelsio T502-BT"}, /* 2 x 1G */
#ifdef notyet
{0x5404, "Chelsio T520-BCH"},
{0x5405, "Chelsio T540-BCH"},
{0x5406, "Chelsio T540-CH"},
{0x5408, "Chelsio T520-CX"},
{0x540b, "Chelsio B520-SR"},
{0x540c, "Chelsio B504-BT"},
{0x540f, "Chelsio Amsterdam"},
{0x5413, "Chelsio T580-CHR"},
#endif
};
#ifdef TCP_OFFLOAD
/*
* service_iq() has an iq and needs the fl. Offset of fl from the iq should be
* exactly the same for both rxq and ofld_rxq.
*/
CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
#endif
/* No easy way to include t4_msg.h before adapter.h so we check this way */
CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
static int
t4_probe(device_t dev)
{
int i;
uint16_t v = pci_get_vendor(dev);
uint16_t d = pci_get_device(dev);
uint8_t f = pci_get_function(dev);
if (v != PCI_VENDOR_ID_CHELSIO)
return (ENXIO);
/* Attach only to PF0 of the FPGA */
if (d == 0xa000 && f != 0)
return (ENXIO);
for (i = 0; i < nitems(t4_pciids); i++) {
if (d == t4_pciids[i].device) {
device_set_desc(dev, t4_pciids[i].desc);
return (BUS_PROBE_DEFAULT);
}
}
return (ENXIO);
}
static int
t5_probe(device_t dev)
{
int i;
uint16_t v = pci_get_vendor(dev);
uint16_t d = pci_get_device(dev);
uint8_t f = pci_get_function(dev);
if (v != PCI_VENDOR_ID_CHELSIO)
return (ENXIO);
/* Attach only to PF0 of the FPGA */
if (d == 0xb000 && f != 0)
return (ENXIO);
for (i = 0; i < nitems(t5_pciids); i++) {
if (d == t5_pciids[i].device) {
device_set_desc(dev, t5_pciids[i].desc);
return (BUS_PROBE_DEFAULT);
}
}
return (ENXIO);
}
static int
t4_attach(device_t dev)
{
struct adapter *sc;
int rc = 0, i, n10g, n1g, rqidx, tqidx;
struct intrs_and_queues iaq;
struct sge *s;
#ifdef TCP_OFFLOAD
int ofld_rqidx, ofld_tqidx;
#endif
#ifdef DEV_NETMAP
int nm_rqidx, nm_tqidx;
#endif
const char *pcie_ts;
sc = device_get_softc(dev);
sc->dev = dev;
pci_enable_busmaster(dev);
if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
uint32_t v;
pci_set_max_read_req(dev, 4096);
v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
}
sc->traceq = -1;
mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
device_get_nameunit(dev));
snprintf(sc->lockname, sizeof(sc->lockname), "%s",
device_get_nameunit(dev));
mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
sx_xlock(&t4_list_lock);
SLIST_INSERT_HEAD(&t4_list, sc, link);
sx_xunlock(&t4_list_lock);
mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
TAILQ_INIT(&sc->sfl);
- callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->sfl_callout, 1);
mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
rc = map_bars_0_and_4(sc);
if (rc != 0)
goto done; /* error message displayed already */
/*
* This is the real PF# to which we're attaching. Works from within PCI
* passthrough environments too, where pci_get_function() could return a
* different PF# depending on the passthrough configuration. We need to
* use the real PF# in all our communication with the firmware.
*/
sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
sc->mbox = sc->pf;
memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
sc->an_handler = an_not_handled;
for (i = 0; i < nitems(sc->cpl_handler); i++)
sc->cpl_handler[i] = cpl_not_handled;
for (i = 0; i < nitems(sc->fw_msg_handler); i++)
sc->fw_msg_handler[i] = fw_msg_not_handled;
t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
t4_init_sge_cpl_handlers(sc);
/* Prepare the adapter for operation */
rc = -t4_prep_adapter(sc);
if (rc != 0) {
device_printf(dev, "failed to prepare adapter: %d.\n", rc);
goto done;
}
/*
* Do this really early, with the memory windows set up even before the
* character device. The userland tool's register i/o and mem read
* will work even in "recovery mode".
*/
setup_memwin(sc);
sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
device_get_nameunit(dev));
if (sc->cdev == NULL)
device_printf(dev, "failed to create nexus char device.\n");
else
sc->cdev->si_drv1 = sc;
/* Go no further if recovery mode has been requested. */
if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
device_printf(dev, "recovery mode.\n");
goto done;
}
#if defined(__i386__)
if ((cpu_feature & CPUID_CX8) == 0) {
device_printf(dev, "64 bit atomics not available.\n");
rc = ENOTSUP;
goto done;
}
#endif
/* Prepare the firmware for operation */
rc = prep_firmware(sc);
if (rc != 0)
goto done; /* error message displayed already */
rc = get_params__post_init(sc);
if (rc != 0)
goto done; /* error message displayed already */
rc = set_params__post_init(sc);
if (rc != 0)
goto done; /* error message displayed already */
rc = map_bar_2(sc);
if (rc != 0)
goto done; /* error message displayed already */
rc = t4_create_dma_tag(sc);
if (rc != 0)
goto done; /* error message displayed already */
/*
* First pass over all the ports - allocate VIs and initialize some
* basic parameters like mac address, port type, etc. We also figure
* out whether a port is 10G or 1G and use that information when
* calculating how many interrupts to attempt to allocate.
*/
n10g = n1g = 0;
for_each_port(sc, i) {
struct port_info *pi;
pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
sc->port[i] = pi;
/* These must be set before t4_port_init */
pi->adapter = sc;
pi->port_id = i;
/* Allocate the vi and initialize parameters like mac addr */
rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
if (rc != 0) {
device_printf(dev, "unable to initialize port %d: %d\n",
i, rc);
free(pi, M_CXGBE);
sc->port[i] = NULL;
goto done;
}
pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
pi->link_cfg.requested_fc |= t4_pause_settings;
pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
pi->link_cfg.fc |= t4_pause_settings;
rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
if (rc != 0) {
device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
free(pi, M_CXGBE);
sc->port[i] = NULL;
goto done;
}
snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
device_get_nameunit(dev), i);
mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
sc->chan_map[pi->tx_chan] = i;
if (is_10G_port(pi) || is_40G_port(pi)) {
n10g++;
pi->tmr_idx = t4_tmr_idx_10g;
pi->pktc_idx = t4_pktc_idx_10g;
} else {
n1g++;
pi->tmr_idx = t4_tmr_idx_1g;
pi->pktc_idx = t4_pktc_idx_1g;
}
pi->xact_addr_filt = -1;
pi->linkdnrc = -1;
pi->qsize_rxq = t4_qsize_rxq;
pi->qsize_txq = t4_qsize_txq;
pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
if (pi->dev == NULL) {
device_printf(dev,
"failed to add device for port %d.\n", i);
rc = ENXIO;
goto done;
}
device_set_softc(pi->dev, pi);
}
/*
* Interrupt type, # of interrupts, # of rx/tx queues, etc.
*/
rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
if (rc != 0)
goto done; /* error message displayed already */
sc->intr_type = iaq.intr_type;
sc->intr_count = iaq.nirq;
s = &sc->sge;
s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */
s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
#ifdef TCP_OFFLOAD
if (is_offload(sc)) {
s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
s->neq += s->nofldtxq + s->nofldrxq;
s->niq += s->nofldrxq;
s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
M_CXGBE, M_ZERO | M_WAITOK);
s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
M_CXGBE, M_ZERO | M_WAITOK);
}
#endif
#ifdef DEV_NETMAP
s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
s->neq += s->nnmtxq + s->nnmrxq;
s->niq += s->nnmrxq;
s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
M_CXGBE, M_ZERO | M_WAITOK);
s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
M_CXGBE, M_ZERO | M_WAITOK);
#endif
s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
M_ZERO | M_WAITOK);
s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
M_ZERO | M_WAITOK);
s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
M_ZERO | M_WAITOK);
s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
M_ZERO | M_WAITOK);
s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
M_ZERO | M_WAITOK);
sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
M_ZERO | M_WAITOK);
t4_init_l2t(sc, M_WAITOK);
/*
* Second pass over the ports. This time we know the number of rx and
* tx queues that each port should get.
*/
rqidx = tqidx = 0;
#ifdef TCP_OFFLOAD
ofld_rqidx = ofld_tqidx = 0;
#endif
#ifdef DEV_NETMAP
nm_rqidx = nm_tqidx = 0;
#endif
for_each_port(sc, i) {
struct port_info *pi = sc->port[i];
if (pi == NULL)
continue;
pi->first_rxq = rqidx;
pi->first_txq = tqidx;
if (is_10G_port(pi) || is_40G_port(pi)) {
pi->flags |= iaq.intr_flags_10g;
pi->nrxq = iaq.nrxq10g;
pi->ntxq = iaq.ntxq10g;
} else {
pi->flags |= iaq.intr_flags_1g;
pi->nrxq = iaq.nrxq1g;
pi->ntxq = iaq.ntxq1g;
}
if (pi->ntxq > 1)
pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
else
pi->rsrv_noflowq = 0;
rqidx += pi->nrxq;
tqidx += pi->ntxq;
#ifdef TCP_OFFLOAD
if (is_offload(sc)) {
pi->first_ofld_rxq = ofld_rqidx;
pi->first_ofld_txq = ofld_tqidx;
if (is_10G_port(pi) || is_40G_port(pi)) {
pi->nofldrxq = iaq.nofldrxq10g;
pi->nofldtxq = iaq.nofldtxq10g;
} else {
pi->nofldrxq = iaq.nofldrxq1g;
pi->nofldtxq = iaq.nofldtxq1g;
}
ofld_rqidx += pi->nofldrxq;
ofld_tqidx += pi->nofldtxq;
}
#endif
#ifdef DEV_NETMAP
pi->first_nm_rxq = nm_rqidx;
pi->first_nm_txq = nm_tqidx;
if (is_10G_port(pi) || is_40G_port(pi)) {
pi->nnmrxq = iaq.nnmrxq10g;
pi->nnmtxq = iaq.nnmtxq10g;
} else {
pi->nnmrxq = iaq.nnmrxq1g;
pi->nnmtxq = iaq.nnmtxq1g;
}
nm_rqidx += pi->nnmrxq;
nm_tqidx += pi->nnmtxq;
#endif
}
rc = setup_intr_handlers(sc);
if (rc != 0) {
device_printf(dev,
"failed to setup interrupt handlers: %d\n", rc);
goto done;
}
rc = bus_generic_attach(dev);
if (rc != 0) {
device_printf(dev,
"failed to attach all child ports: %d\n", rc);
goto done;
}
switch (sc->params.pci.speed) {
case 0x1:
pcie_ts = "2.5";
break;
case 0x2:
pcie_ts = "5.0";
break;
case 0x3:
pcie_ts = "8.0";
break;
default:
pcie_ts = "??";
break;
}
device_printf(dev,
"PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
sc->params.pci.width, pcie_ts, sc->params.pci.speed,
sc->params.nports, sc->intr_count,
sc->intr_type == INTR_MSIX ? "MSI-X" :
(sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
t4_set_desc(sc);
done:
if (rc != 0 && sc->cdev) {
/* cdev was created and so cxgbetool works; recover that way. */
device_printf(dev,
"error during attach, adapter is now in recovery mode.\n");
rc = 0;
}
if (rc != 0)
t4_detach(dev);
else
t4_sysctls(sc);
return (rc);
}
/*
* Idempotent
*/
static int
t4_detach(device_t dev)
{
struct adapter *sc;
struct port_info *pi;
int i, rc;
sc = device_get_softc(dev);
if (sc->flags & FULL_INIT_DONE)
t4_intr_disable(sc);
if (sc->cdev) {
destroy_dev(sc->cdev);
sc->cdev = NULL;
}
rc = bus_generic_detach(dev);
if (rc) {
device_printf(dev,
"failed to detach child devices: %d\n", rc);
return (rc);
}
for (i = 0; i < sc->intr_count; i++)
t4_free_irq(sc, &sc->irq[i]);
for (i = 0; i < MAX_NPORTS; i++) {
pi = sc->port[i];
if (pi) {
t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
if (pi->dev)
device_delete_child(dev, pi->dev);
mtx_destroy(&pi->pi_lock);
free(pi, M_CXGBE);
}
}
if (sc->flags & FULL_INIT_DONE)
adapter_full_uninit(sc);
if (sc->flags & FW_OK)
t4_fw_bye(sc, sc->mbox);
if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
pci_release_msi(dev);
if (sc->regs_res)
bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
sc->regs_res);
if (sc->udbs_res)
bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
sc->udbs_res);
if (sc->msix_res)
bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
sc->msix_res);
if (sc->l2t)
t4_free_l2t(sc->l2t);
#ifdef TCP_OFFLOAD
free(sc->sge.ofld_rxq, M_CXGBE);
free(sc->sge.ofld_txq, M_CXGBE);
#endif
#ifdef DEV_NETMAP
free(sc->sge.nm_rxq, M_CXGBE);
free(sc->sge.nm_txq, M_CXGBE);
#endif
free(sc->irq, M_CXGBE);
free(sc->sge.rxq, M_CXGBE);
free(sc->sge.txq, M_CXGBE);
free(sc->sge.ctrlq, M_CXGBE);
free(sc->sge.iqmap, M_CXGBE);
free(sc->sge.eqmap, M_CXGBE);
free(sc->tids.ftid_tab, M_CXGBE);
t4_destroy_dma_tag(sc);
if (mtx_initialized(&sc->sc_lock)) {
sx_xlock(&t4_list_lock);
SLIST_REMOVE(&t4_list, sc, adapter, link);
sx_xunlock(&t4_list_lock);
mtx_destroy(&sc->sc_lock);
}
if (mtx_initialized(&sc->tids.ftid_lock))
mtx_destroy(&sc->tids.ftid_lock);
if (mtx_initialized(&sc->sfl_lock))
mtx_destroy(&sc->sfl_lock);
if (mtx_initialized(&sc->ifp_lock))
mtx_destroy(&sc->ifp_lock);
if (mtx_initialized(&sc->regwin_lock))
mtx_destroy(&sc->regwin_lock);
bzero(sc, sizeof(*sc));
return (0);
}
static int
cxgbe_probe(device_t dev)
{
char buf[128];
struct port_info *pi = device_get_softc(dev);
snprintf(buf, sizeof(buf), "port %d", pi->port_id);
device_set_desc_copy(dev, buf);
return (BUS_PROBE_DEFAULT);
}
#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
#define T4_CAP_ENABLE (T4_CAP)
static int
cxgbe_attach(device_t dev)
{
struct port_info *pi = device_get_softc(dev);
struct ifnet *ifp;
char *s;
int n, o;
/* Allocate an ifnet and set it up */
ifp = if_alloc(IFT_ETHER);
if (ifp == NULL) {
device_printf(dev, "Cannot allocate ifnet\n");
return (ENOMEM);
}
pi->ifp = ifp;
ifp->if_softc = pi;
- callout_init(&pi->tick, CALLOUT_MPSAFE);
+ callout_init(&pi->tick, 1);
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_init = cxgbe_init;
ifp->if_ioctl = cxgbe_ioctl;
ifp->if_transmit = cxgbe_transmit;
ifp->if_qflush = cxgbe_qflush;
ifp->if_get_counter = cxgbe_get_counter;
ifp->if_capabilities = T4_CAP;
#ifdef TCP_OFFLOAD
if (is_offload(pi->adapter))
ifp->if_capabilities |= IFCAP_TOE;
#endif
ifp->if_capenable = T4_CAP_ENABLE;
ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
ifp->if_hw_tsomaxsegsize = 65536;
/* Initialize ifmedia for this port */
ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
cxgbe_media_status);
build_medialist(pi, &pi->media);
pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
EVENTHANDLER_PRI_ANY);
ether_ifattach(ifp, pi->hw_addr);
n = 128;
s = malloc(n, M_CXGBE, M_WAITOK);
o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
MPASS(n > o);
#ifdef TCP_OFFLOAD
if (is_offload(pi->adapter)) {
o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
pi->nofldtxq, pi->nofldrxq);
MPASS(n > o);
}
#endif
#ifdef DEV_NETMAP
o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
pi->nnmrxq);
MPASS(n > o);
#endif
device_printf(dev, "%s\n", s);
free(s, M_CXGBE);
#ifdef DEV_NETMAP
/* nm_media handled here to keep implementation private to this file */
ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
cxgbe_media_status);
build_medialist(pi, &pi->nm_media);
create_netmap_ifnet(pi); /* logs errors it something fails */
#endif
cxgbe_sysctls(pi);
return (0);
}
static int
cxgbe_detach(device_t dev)
{
struct port_info *pi = device_get_softc(dev);
struct adapter *sc = pi->adapter;
struct ifnet *ifp = pi->ifp;
/* Tell if_ioctl and if_init that the port is going away */
ADAPTER_LOCK(sc);
SET_DOOMED(pi);
wakeup(&sc->flags);
while (IS_BUSY(sc))
mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
SET_BUSY(sc);
#ifdef INVARIANTS
sc->last_op = "t4detach";
sc->last_op_thr = curthread;
#endif
ADAPTER_UNLOCK(sc);
if (pi->flags & HAS_TRACEQ) {
sc->traceq = -1; /* cloner should not create ifnet */
t4_tracer_port_detach(sc);
}
if (pi->vlan_c)
EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
PORT_LOCK(pi);
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
callout_stop(&pi->tick);
PORT_UNLOCK(pi);
callout_drain(&pi->tick);
/* Let detach proceed even if these fail. */
cxgbe_uninit_synchronized(pi);
port_full_uninit(pi);
ifmedia_removeall(&pi->media);
ether_ifdetach(pi->ifp);
if_free(pi->ifp);
#ifdef DEV_NETMAP
/* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
destroy_netmap_ifnet(pi);
#endif
ADAPTER_LOCK(sc);
CLR_BUSY(sc);
wakeup(&sc->flags);
ADAPTER_UNLOCK(sc);
return (0);
}
static void
cxgbe_init(void *arg)
{
struct port_info *pi = arg;
struct adapter *sc = pi->adapter;
if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
return;
cxgbe_init_synchronized(pi);
end_synchronized_op(sc, 0);
}
static int
cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
{
int rc = 0, mtu, flags, can_sleep;
struct port_info *pi = ifp->if_softc;
struct adapter *sc = pi->adapter;
struct ifreq *ifr = (struct ifreq *)data;
uint32_t mask;
switch (cmd) {
case SIOCSIFMTU:
mtu = ifr->ifr_mtu;
if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
return (EINVAL);
rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
if (rc)
return (rc);
ifp->if_mtu = mtu;
if (pi->flags & PORT_INIT_DONE) {
t4_update_fl_bufsize(ifp);
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
rc = update_mac_settings(ifp, XGMAC_MTU);
}
end_synchronized_op(sc, 0);
break;
case SIOCSIFFLAGS:
can_sleep = 0;
redo_sifflags:
rc = begin_synchronized_op(sc, pi,
can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
if (rc)
return (rc);
if (ifp->if_flags & IFF_UP) {
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
flags = pi->if_flags;
if ((ifp->if_flags ^ flags) &
(IFF_PROMISC | IFF_ALLMULTI)) {
if (can_sleep == 1) {
end_synchronized_op(sc, 0);
can_sleep = 0;
goto redo_sifflags;
}
rc = update_mac_settings(ifp,
XGMAC_PROMISC | XGMAC_ALLMULTI);
}
} else {
if (can_sleep == 0) {
end_synchronized_op(sc, LOCK_HELD);
can_sleep = 1;
goto redo_sifflags;
}
rc = cxgbe_init_synchronized(pi);
}
pi->if_flags = ifp->if_flags;
} else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
if (can_sleep == 0) {
end_synchronized_op(sc, LOCK_HELD);
can_sleep = 1;
goto redo_sifflags;
}
rc = cxgbe_uninit_synchronized(pi);
}
end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
break;
case SIOCADDMULTI:
case SIOCDELMULTI: /* these two are called with a mutex held :-( */
rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
if (rc)
return (rc);
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
rc = update_mac_settings(ifp, XGMAC_MCADDRS);
end_synchronized_op(sc, LOCK_HELD);
break;
case SIOCSIFCAP:
rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
if (rc)
return (rc);
mask = ifr->ifr_reqcap ^ ifp->if_capenable;
if (mask & IFCAP_TXCSUM) {
ifp->if_capenable ^= IFCAP_TXCSUM;
ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
if (IFCAP_TSO4 & ifp->if_capenable &&
!(IFCAP_TXCSUM & ifp->if_capenable)) {
ifp->if_capenable &= ~IFCAP_TSO4;
if_printf(ifp,
"tso4 disabled due to -txcsum.\n");
}
}
if (mask & IFCAP_TXCSUM_IPV6) {
ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
if (IFCAP_TSO6 & ifp->if_capenable &&
!(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
ifp->if_capenable &= ~IFCAP_TSO6;
if_printf(ifp,
"tso6 disabled due to -txcsum6.\n");
}
}
if (mask & IFCAP_RXCSUM)
ifp->if_capenable ^= IFCAP_RXCSUM;
if (mask & IFCAP_RXCSUM_IPV6)
ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
/*
* Note that we leave CSUM_TSO alone (it is always set). The
* kernel takes both IFCAP_TSOx and CSUM_TSO into account before
* sending a TSO request our way, so it's sufficient to toggle
* IFCAP_TSOx only.
*/
if (mask & IFCAP_TSO4) {
if (!(IFCAP_TSO4 & ifp->if_capenable) &&
!(IFCAP_TXCSUM & ifp->if_capenable)) {
if_printf(ifp, "enable txcsum first.\n");
rc = EAGAIN;
goto fail;
}
ifp->if_capenable ^= IFCAP_TSO4;
}
if (mask & IFCAP_TSO6) {
if (!(IFCAP_TSO6 & ifp->if_capenable) &&
!(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
if_printf(ifp, "enable txcsum6 first.\n");
rc = EAGAIN;
goto fail;
}
ifp->if_capenable ^= IFCAP_TSO6;
}
if (mask & IFCAP_LRO) {
#if defined(INET) || defined(INET6)
int i;
struct sge_rxq *rxq;
ifp->if_capenable ^= IFCAP_LRO;
for_each_rxq(pi, i, rxq) {
if (ifp->if_capenable & IFCAP_LRO)
rxq->iq.flags |= IQ_LRO_ENABLED;
else
rxq->iq.flags &= ~IQ_LRO_ENABLED;
}
#endif
}
#ifdef TCP_OFFLOAD
if (mask & IFCAP_TOE) {
int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
rc = toe_capability(pi, enable);
if (rc != 0)
goto fail;
ifp->if_capenable ^= mask;
}
#endif
if (mask & IFCAP_VLAN_HWTAGGING) {
ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
rc = update_mac_settings(ifp, XGMAC_VLANEX);
}
if (mask & IFCAP_VLAN_MTU) {
ifp->if_capenable ^= IFCAP_VLAN_MTU;
/* Need to find out how to disable auto-mtu-inflation */
}
if (mask & IFCAP_VLAN_HWTSO)
ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
if (mask & IFCAP_VLAN_HWCSUM)
ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
#ifdef VLAN_CAPABILITIES
VLAN_CAPABILITIES(ifp);
#endif
fail:
end_synchronized_op(sc, 0);
break;
case SIOCSIFMEDIA:
case SIOCGIFMEDIA:
ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
break;
case SIOCGI2C: {
struct ifi2creq i2c;
rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
if (rc != 0)
break;
if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
rc = EPERM;
break;
}
if (i2c.len > sizeof(i2c.data)) {
rc = EINVAL;
break;
}
rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
if (rc)
return (rc);
rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
i2c.offset, i2c.len, &i2c.data[0]);
end_synchronized_op(sc, 0);
if (rc == 0)
rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
break;
}
default:
rc = ether_ioctl(ifp, cmd, data);
}
return (rc);
}
static int
cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
{
struct port_info *pi = ifp->if_softc;
struct adapter *sc = pi->adapter;
struct sge_txq *txq;
void *items[1];
int rc;
M_ASSERTPKTHDR(m);
MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */
if (__predict_false(pi->link_cfg.link_ok == 0)) {
m_freem(m);
return (ENETDOWN);
}
rc = parse_pkt(&m);
if (__predict_false(rc != 0)) {
MPASS(m == NULL); /* was freed already */
atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
return (rc);
}
/* Select a txq. */
txq = &sc->sge.txq[pi->first_txq];
if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
pi->rsrv_noflowq);
items[0] = m;
rc = mp_ring_enqueue(txq->r, items, 1, 4096);
if (__predict_false(rc != 0))
m_freem(m);
return (rc);
}
static void
cxgbe_qflush(struct ifnet *ifp)
{
struct port_info *pi = ifp->if_softc;
struct sge_txq *txq;
int i;
/* queues do not exist if !PORT_INIT_DONE. */
if (pi->flags & PORT_INIT_DONE) {
for_each_txq(pi, i, txq) {
TXQ_LOCK(txq);
txq->eq.flags &= ~EQ_ENABLED;
TXQ_UNLOCK(txq);
while (!mp_ring_is_idle(txq->r)) {
mp_ring_check_drainage(txq->r, 0);
pause("qflush", 1);
}
}
}
if_qflush(ifp);
}
static uint64_t
cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
{
struct port_info *pi = ifp->if_softc;
struct adapter *sc = pi->adapter;
struct port_stats *s = &pi->stats;
cxgbe_refresh_stats(sc, pi);
switch (c) {
case IFCOUNTER_IPACKETS:
return (s->rx_frames - s->rx_pause);
case IFCOUNTER_IERRORS:
return (s->rx_jabber + s->rx_runt + s->rx_too_long +
s->rx_fcs_err + s->rx_len_err);
case IFCOUNTER_OPACKETS:
return (s->tx_frames - s->tx_pause);
case IFCOUNTER_OERRORS:
return (s->tx_error_frames);
case IFCOUNTER_IBYTES:
return (s->rx_octets - s->rx_pause * 64);
case IFCOUNTER_OBYTES:
return (s->tx_octets - s->tx_pause * 64);
case IFCOUNTER_IMCASTS:
return (s->rx_mcast_frames - s->rx_pause);
case IFCOUNTER_OMCASTS:
return (s->tx_mcast_frames - s->tx_pause);
case IFCOUNTER_IQDROPS:
return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
s->rx_trunc3 + pi->tnl_cong_drops);
case IFCOUNTER_OQDROPS: {
uint64_t drops;
drops = s->tx_drop;
if (pi->flags & PORT_INIT_DONE) {
int i;
struct sge_txq *txq;
for_each_txq(pi, i, txq)
drops += counter_u64_fetch(txq->r->drops);
}
return (drops);
}
default:
return (if_get_counter_default(ifp, c));
}
}
static int
cxgbe_media_change(struct ifnet *ifp)
{
struct port_info *pi = ifp->if_softc;
device_printf(pi->dev, "%s unimplemented.\n", __func__);
return (EOPNOTSUPP);
}
static void
cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
struct port_info *pi = ifp->if_softc;
struct ifmedia *media = NULL;
struct ifmedia_entry *cur;
int speed = pi->link_cfg.speed;
if (ifp == pi->ifp)
media = &pi->media;
#ifdef DEV_NETMAP
else if (ifp == pi->nm_ifp)
media = &pi->nm_media;
#endif
MPASS(media != NULL);
cur = media->ifm_cur;
ifmr->ifm_status = IFM_AVALID;
if (!pi->link_cfg.link_ok)
return;
ifmr->ifm_status |= IFM_ACTIVE;
/* active and current will differ iff current media is autoselect. */
if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
return;
ifmr->ifm_active = IFM_ETHER | IFM_FDX;
if (speed == SPEED_10000)
ifmr->ifm_active |= IFM_10G_T;
else if (speed == SPEED_1000)
ifmr->ifm_active |= IFM_1000_T;
else if (speed == SPEED_100)
ifmr->ifm_active |= IFM_100_TX;
else if (speed == SPEED_10)
ifmr->ifm_active |= IFM_10_T;
else
KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
speed));
}
void
t4_fatal_err(struct adapter *sc)
{
t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
t4_intr_disable(sc);
log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
device_get_nameunit(sc->dev));
}
static int
map_bars_0_and_4(struct adapter *sc)
{
sc->regs_rid = PCIR_BAR(0);
sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
&sc->regs_rid, RF_ACTIVE);
if (sc->regs_res == NULL) {
device_printf(sc->dev, "cannot map registers.\n");
return (ENXIO);
}
sc->bt = rman_get_bustag(sc->regs_res);
sc->bh = rman_get_bushandle(sc->regs_res);
sc->mmio_len = rman_get_size(sc->regs_res);
setbit(&sc->doorbells, DOORBELL_KDB);
sc->msix_rid = PCIR_BAR(4);
sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
&sc->msix_rid, RF_ACTIVE);
if (sc->msix_res == NULL) {
device_printf(sc->dev, "cannot map MSI-X BAR.\n");
return (ENXIO);
}
return (0);
}
static int
map_bar_2(struct adapter *sc)
{
/*
* T4: only iWARP driver uses the userspace doorbells. There is no need
* to map it if RDMA is disabled.
*/
if (is_t4(sc) && sc->rdmacaps == 0)
return (0);
sc->udbs_rid = PCIR_BAR(2);
sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
&sc->udbs_rid, RF_ACTIVE);
if (sc->udbs_res == NULL) {
device_printf(sc->dev, "cannot map doorbell BAR.\n");
return (ENXIO);
}
sc->udbs_base = rman_get_virtual(sc->udbs_res);
if (is_t5(sc)) {
setbit(&sc->doorbells, DOORBELL_UDB);
#if defined(__i386__) || defined(__amd64__)
if (t5_write_combine) {
int rc;
/*
* Enable write combining on BAR2. This is the
* userspace doorbell BAR and is split into 128B
* (UDBS_SEG_SIZE) doorbell regions, each associated
* with an egress queue. The first 64B has the doorbell
* and the second 64B can be used to submit a tx work
* request with an implicit doorbell.
*/
rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
if (rc == 0) {
clrbit(&sc->doorbells, DOORBELL_UDB);
setbit(&sc->doorbells, DOORBELL_WCWR);
setbit(&sc->doorbells, DOORBELL_UDBWC);
} else {
device_printf(sc->dev,
"couldn't enable write combining: %d\n",
rc);
}
t4_write_reg(sc, A_SGE_STAT_CFG,
V_STATSOURCE_T5(7) | V_STATMODE(0));
}
#endif
}
return (0);
}
static const struct memwin t4_memwin[] = {
{ MEMWIN0_BASE, MEMWIN0_APERTURE },
{ MEMWIN1_BASE, MEMWIN1_APERTURE },
{ MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
};
static const struct memwin t5_memwin[] = {
{ MEMWIN0_BASE, MEMWIN0_APERTURE },
{ MEMWIN1_BASE, MEMWIN1_APERTURE },
{ MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
};
static void
setup_memwin(struct adapter *sc)
{
const struct memwin *mw;
int i, n;
uint32_t bar0;
if (is_t4(sc)) {
/*
* Read low 32b of bar0 indirectly via the hardware backdoor
* mechanism. Works from within PCI passthrough environments
* too, where rman_get_start() can return a different value. We
* need to program the T4 memory window decoders with the actual
* addresses that will be coming across the PCIe link.
*/
bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
mw = &t4_memwin[0];
n = nitems(t4_memwin);
} else {
/* T5 uses the relative offset inside the PCIe BAR */
bar0 = 0;
mw = &t5_memwin[0];
n = nitems(t5_memwin);
}
for (i = 0; i < n; i++, mw++) {
t4_write_reg(sc,
PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
(mw->base + bar0) | V_BIR(0) |
V_WINDOW(ilog2(mw->aperture) - 10));
}
/* flush */
t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
}
/*
* Verify that the memory range specified by the addr/len pair is valid and lies
* entirely within a single region (EDCx or MCx).
*/
static int
validate_mem_range(struct adapter *sc, uint32_t addr, int len)
{
uint32_t em, addr_len, maddr, mlen;
/* Memory can only be accessed in naturally aligned 4 byte units */
if (addr & 3 || len & 3 || len == 0)
return (EINVAL);
/* Enabled memories */
em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
if (em & F_EDRAM0_ENABLE) {
addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
maddr = G_EDRAM0_BASE(addr_len) << 20;
mlen = G_EDRAM0_SIZE(addr_len) << 20;
if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
addr + len <= maddr + mlen)
return (0);
}
if (em & F_EDRAM1_ENABLE) {
addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
maddr = G_EDRAM1_BASE(addr_len) << 20;
mlen = G_EDRAM1_SIZE(addr_len) << 20;
if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
addr + len <= maddr + mlen)
return (0);
}
if (em & F_EXT_MEM_ENABLE) {
addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
maddr = G_EXT_MEM_BASE(addr_len) << 20;
mlen = G_EXT_MEM_SIZE(addr_len) << 20;
if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
addr + len <= maddr + mlen)
return (0);
}
if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
maddr = G_EXT_MEM1_BASE(addr_len) << 20;
mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
addr + len <= maddr + mlen)
return (0);
}
return (EFAULT);
}
static int
fwmtype_to_hwmtype(int mtype)
{
switch (mtype) {
case FW_MEMTYPE_EDC0:
return (MEM_EDC0);
case FW_MEMTYPE_EDC1:
return (MEM_EDC1);
case FW_MEMTYPE_EXTMEM:
return (MEM_MC0);
case FW_MEMTYPE_EXTMEM1:
return (MEM_MC1);
default:
panic("%s: cannot translate fw mtype %d.", __func__, mtype);
}
}
/*
* Verify that the memory range specified by the memtype/offset/len pair is
* valid and lies entirely within the memtype specified. The global address of
* the start of the range is returned in addr.
*/
static int
validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
uint32_t *addr)
{
uint32_t em, addr_len, maddr, mlen;
/* Memory can only be accessed in naturally aligned 4 byte units */
if (off & 3 || len & 3 || len == 0)
return (EINVAL);
em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
switch (fwmtype_to_hwmtype(mtype)) {
case MEM_EDC0:
if (!(em & F_EDRAM0_ENABLE))
return (EINVAL);
addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
maddr = G_EDRAM0_BASE(addr_len) << 20;
mlen = G_EDRAM0_SIZE(addr_len) << 20;
break;
case MEM_EDC1:
if (!(em & F_EDRAM1_ENABLE))
return (EINVAL);
addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
maddr = G_EDRAM1_BASE(addr_len) << 20;
mlen = G_EDRAM1_SIZE(addr_len) << 20;
break;
case MEM_MC:
if (!(em & F_EXT_MEM_ENABLE))
return (EINVAL);
addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
maddr = G_EXT_MEM_BASE(addr_len) << 20;
mlen = G_EXT_MEM_SIZE(addr_len) << 20;
break;
case MEM_MC1:
if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
return (EINVAL);
addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
maddr = G_EXT_MEM1_BASE(addr_len) << 20;
mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
break;
default:
return (EINVAL);
}
if (mlen > 0 && off < mlen && off + len <= mlen) {
*addr = maddr + off; /* global address */
return (0);
}
return (EFAULT);
}
static void
memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
{
const struct memwin *mw;
if (is_t4(sc)) {
KASSERT(win >= 0 && win < nitems(t4_memwin),
("%s: incorrect memwin# (%d)", __func__, win));
mw = &t4_memwin[win];
} else {
KASSERT(win >= 0 && win < nitems(t5_memwin),
("%s: incorrect memwin# (%d)", __func__, win));
mw = &t5_memwin[win];
}
if (base != NULL)
*base = mw->base;
if (aperture != NULL)
*aperture = mw->aperture;
}
/*
* Positions the memory window such that it can be used to access the specified
* address in the chip's address space. The return value is the offset of addr
* from the start of the window.
*/
static uint32_t
position_memwin(struct adapter *sc, int n, uint32_t addr)
{
uint32_t start, pf;
uint32_t reg;
KASSERT(n >= 0 && n <= 3,
("%s: invalid window %d.", __func__, n));
KASSERT((addr & 3) == 0,
("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
if (is_t4(sc)) {
pf = 0;
start = addr & ~0xf; /* start must be 16B aligned */
} else {
pf = V_PFNUM(sc->pf);
start = addr & ~0x7f; /* start must be 128B aligned */
}
reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
t4_write_reg(sc, reg, start | pf);
t4_read_reg(sc, reg);
return (addr - start);
}
static int
cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
struct intrs_and_queues *iaq)
{
int rc, itype, navail, nrxq10g, nrxq1g, n;
int nofldrxq10g = 0, nofldrxq1g = 0;
int nnmrxq10g = 0, nnmrxq1g = 0;
bzero(iaq, sizeof(*iaq));
iaq->ntxq10g = t4_ntxq10g;
iaq->ntxq1g = t4_ntxq1g;
iaq->nrxq10g = nrxq10g = t4_nrxq10g;
iaq->nrxq1g = nrxq1g = t4_nrxq1g;
iaq->rsrv_noflowq = t4_rsrv_noflowq;
#ifdef TCP_OFFLOAD
if (is_offload(sc)) {
iaq->nofldtxq10g = t4_nofldtxq10g;
iaq->nofldtxq1g = t4_nofldtxq1g;
iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
}
#endif
#ifdef DEV_NETMAP
iaq->nnmtxq10g = t4_nnmtxq10g;
iaq->nnmtxq1g = t4_nnmtxq1g;
iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
#endif
for (itype = INTR_MSIX; itype; itype >>= 1) {
if ((itype & t4_intr_types) == 0)
continue; /* not allowed */
if (itype == INTR_MSIX)
navail = pci_msix_count(sc->dev);
else if (itype == INTR_MSI)
navail = pci_msi_count(sc->dev);
else
navail = 1;
restart:
if (navail == 0)
continue;
iaq->intr_type = itype;
iaq->intr_flags_10g = 0;
iaq->intr_flags_1g = 0;
/*
* Best option: an interrupt vector for errors, one for the
* firmware event queue, and one for every rxq (NIC, TOE, and
* netmap).
*/
iaq->nirq = T4_EXTRA_INTR;
iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
if (iaq->nirq <= navail &&
(itype != INTR_MSI || powerof2(iaq->nirq))) {
iaq->intr_flags_10g = INTR_ALL;
iaq->intr_flags_1g = INTR_ALL;
goto allocate;
}
/*
* Second best option: a vector for errors, one for the firmware
* event queue, and vectors for either all the NIC rx queues or
* all the TOE rx queues. The queues that don't get vectors
* will forward their interrupts to those that do.
*
* Note: netmap rx queues cannot be created early and so they
* can't be setup to receive forwarded interrupts for others.
*/
iaq->nirq = T4_EXTRA_INTR;
if (nrxq10g >= nofldrxq10g) {
iaq->intr_flags_10g = INTR_RXQ;
iaq->nirq += n10g * nrxq10g;
#ifdef DEV_NETMAP
iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
#endif
} else {
iaq->intr_flags_10g = INTR_OFLD_RXQ;
iaq->nirq += n10g * nofldrxq10g;
#ifdef DEV_NETMAP
iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
#endif
}
if (nrxq1g >= nofldrxq1g) {
iaq->intr_flags_1g = INTR_RXQ;
iaq->nirq += n1g * nrxq1g;
#ifdef DEV_NETMAP
iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
#endif
} else {
iaq->intr_flags_1g = INTR_OFLD_RXQ;
iaq->nirq += n1g * nofldrxq1g;
#ifdef DEV_NETMAP
iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
#endif
}
if (iaq->nirq <= navail &&
(itype != INTR_MSI || powerof2(iaq->nirq)))
goto allocate;
/*
* Next best option: an interrupt vector for errors, one for the
* firmware event queue, and at least one per port. At this
* point we know we'll have to downsize nrxq and/or nofldrxq
* and/or nnmrxq to fit what's available to us.
*/
iaq->nirq = T4_EXTRA_INTR;
iaq->nirq += n10g + n1g;
if (iaq->nirq <= navail) {
int leftover = navail - iaq->nirq;
if (n10g > 0) {
int target = max(nrxq10g, nofldrxq10g);
iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
INTR_RXQ : INTR_OFLD_RXQ;
n = 1;
while (n < target && leftover >= n10g) {
leftover -= n10g;
iaq->nirq += n10g;
n++;
}
iaq->nrxq10g = min(n, nrxq10g);
#ifdef TCP_OFFLOAD
iaq->nofldrxq10g = min(n, nofldrxq10g);
#endif
#ifdef DEV_NETMAP
iaq->nnmrxq10g = min(n, nnmrxq10g);
#endif
}
if (n1g > 0) {
int target = max(nrxq1g, nofldrxq1g);
iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
INTR_RXQ : INTR_OFLD_RXQ;
n = 1;
while (n < target && leftover >= n1g) {
leftover -= n1g;
iaq->nirq += n1g;
n++;
}
iaq->nrxq1g = min(n, nrxq1g);
#ifdef TCP_OFFLOAD
iaq->nofldrxq1g = min(n, nofldrxq1g);
#endif
#ifdef DEV_NETMAP
iaq->nnmrxq1g = min(n, nnmrxq1g);
#endif
}
if (itype != INTR_MSI || powerof2(iaq->nirq))
goto allocate;
}
/*
* Least desirable option: one interrupt vector for everything.
*/
iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
#ifdef TCP_OFFLOAD
if (is_offload(sc))
iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
#endif
#ifdef DEV_NETMAP
iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
#endif
allocate:
navail = iaq->nirq;
rc = 0;
if (itype == INTR_MSIX)
rc = pci_alloc_msix(sc->dev, &navail);
else if (itype == INTR_MSI)
rc = pci_alloc_msi(sc->dev, &navail);
if (rc == 0) {
if (navail == iaq->nirq)
return (0);
/*
* Didn't get the number requested. Use whatever number
* the kernel is willing to allocate (it's in navail).
*/
device_printf(sc->dev, "fewer vectors than requested, "
"type=%d, req=%d, rcvd=%d; will downshift req.\n",
itype, iaq->nirq, navail);
pci_release_msi(sc->dev);
goto restart;
}
device_printf(sc->dev,
"failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
itype, rc, iaq->nirq, navail);
}
device_printf(sc->dev,
"failed to find a usable interrupt type. "
"allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
pci_msix_count(sc->dev), pci_msi_count(sc->dev));
return (ENXIO);
}
#define FW_VERSION(chip) ( \
V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
struct fw_info {
uint8_t chip;
char *kld_name;
char *fw_mod_name;
struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */
} fw_info[] = {
{
.chip = CHELSIO_T4,
.kld_name = "t4fw_cfg",
.fw_mod_name = "t4fw",
.fw_hdr = {
.chip = FW_HDR_CHIP_T4,
.fw_ver = htobe32_const(FW_VERSION(T4)),
.intfver_nic = FW_INTFVER(T4, NIC),
.intfver_vnic = FW_INTFVER(T4, VNIC),
.intfver_ofld = FW_INTFVER(T4, OFLD),
.intfver_ri = FW_INTFVER(T4, RI),
.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
.intfver_iscsi = FW_INTFVER(T4, ISCSI),
.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
.intfver_fcoe = FW_INTFVER(T4, FCOE),
},
}, {
.chip = CHELSIO_T5,
.kld_name = "t5fw_cfg",
.fw_mod_name = "t5fw",
.fw_hdr = {
.chip = FW_HDR_CHIP_T5,
.fw_ver = htobe32_const(FW_VERSION(T5)),
.intfver_nic = FW_INTFVER(T5, NIC),
.intfver_vnic = FW_INTFVER(T5, VNIC),
.intfver_ofld = FW_INTFVER(T5, OFLD),
.intfver_ri = FW_INTFVER(T5, RI),
.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
.intfver_iscsi = FW_INTFVER(T5, ISCSI),
.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
.intfver_fcoe = FW_INTFVER(T5, FCOE),
},
}
};
static struct fw_info *
find_fw_info(int chip)
{
int i;
for (i = 0; i < nitems(fw_info); i++) {
if (fw_info[i].chip == chip)
return (&fw_info[i]);
}
return (NULL);
}
/*
* Is the given firmware API compatible with the one the driver was compiled
* with?
*/
static int
fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
{
/* short circuit if it's the exact same firmware version */
if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
return (1);
/*
* XXX: Is this too conservative? Perhaps I should limit this to the
* features that are supported in the driver.
*/
#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
return (1);
#undef SAME_INTF
return (0);
}
/*
* The firmware in the KLD is usable, but should it be installed? This routine
* explains itself in detail if it indicates the KLD firmware should be
* installed.
*/
static int
should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
{
const char *reason;
if (!card_fw_usable) {
reason = "incompatible or unusable";
goto install;
}
if (k > c) {
reason = "older than the version bundled with this driver";
goto install;
}
if (t4_fw_install == 2 && k != c) {
reason = "different than the version bundled with this driver";
goto install;
}
return (0);
install:
if (t4_fw_install == 0) {
device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
"but the driver is prohibited from installing a different "
"firmware on the card.\n",
G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
return (0);
}
device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
"installing firmware %u.%u.%u.%u on card.\n",
G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
return (1);
}
/*
* Establish contact with the firmware and determine if we are the master driver
* or not, and whether we are responsible for chip initialization.
*/
static int
prep_firmware(struct adapter *sc)
{
const struct firmware *fw = NULL, *default_cfg;
int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
enum dev_state state;
struct fw_info *fw_info;
struct fw_hdr *card_fw; /* fw on the card */
const struct fw_hdr *kld_fw; /* fw in the KLD */
const struct fw_hdr *drv_fw; /* fw header the driver was compiled
against */
/* Contact firmware. */
rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
if (rc < 0 || state == DEV_STATE_ERR) {
rc = -rc;
device_printf(sc->dev,
"failed to connect to the firmware: %d, %d.\n", rc, state);
return (rc);
}
pf = rc;
if (pf == sc->mbox)
sc->flags |= MASTER_PF;
else if (state == DEV_STATE_UNINIT) {
/*
* We didn't get to be the master so we definitely won't be
* configuring the chip. It's a bug if someone else hasn't
* configured it already.
*/
device_printf(sc->dev, "couldn't be master(%d), "
"device not already initialized either(%d).\n", rc, state);
return (EDOOFUS);
}
/* This is the firmware whose headers the driver was compiled against */
fw_info = find_fw_info(chip_id(sc));
if (fw_info == NULL) {
device_printf(sc->dev,
"unable to look up firmware information for chip %d.\n",
chip_id(sc));
return (EINVAL);
}
drv_fw = &fw_info->fw_hdr;
/*
* The firmware KLD contains many modules. The KLD name is also the
* name of the module that contains the default config file.
*/
default_cfg = firmware_get(fw_info->kld_name);
/* Read the header of the firmware on the card */
card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
rc = -t4_read_flash(sc, FLASH_FW_START,
sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
if (rc == 0)
card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
else {
device_printf(sc->dev,
"Unable to read card's firmware header: %d\n", rc);
card_fw_usable = 0;
}
/* This is the firmware in the KLD */
fw = firmware_get(fw_info->fw_mod_name);
if (fw != NULL) {
kld_fw = (const void *)fw->data;
kld_fw_usable = fw_compatible(drv_fw, kld_fw);
} else {
kld_fw = NULL;
kld_fw_usable = 0;
}
if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
(!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
/*
* Common case: the firmware on the card is an exact match and
* the KLD is an exact match too, or the KLD is
* absent/incompatible. Note that t4_fw_install = 2 is ignored
* here -- use cxgbetool loadfw if you want to reinstall the
* same firmware as the one on the card.
*/
} else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
be32toh(card_fw->fw_ver))) {
rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
if (rc != 0) {
device_printf(sc->dev,
"failed to install firmware: %d\n", rc);
goto done;
}
/* Installed successfully, update the cached header too. */
memcpy(card_fw, kld_fw, sizeof(*card_fw));
card_fw_usable = 1;
need_fw_reset = 0; /* already reset as part of load_fw */
}
if (!card_fw_usable) {
uint32_t d, c, k;
d = ntohl(drv_fw->fw_ver);
c = ntohl(card_fw->fw_ver);
k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
device_printf(sc->dev, "Cannot find a usable firmware: "
"fw_install %d, chip state %d, "
"driver compiled with %d.%d.%d.%d, "
"card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
t4_fw_install, state,
G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
rc = EINVAL;
goto done;
}
/* We're using whatever's on the card and it's known to be good. */
sc->params.fw_vers = ntohl(card_fw->fw_ver);
snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
t4_get_tp_version(sc, &sc->params.tp_vers);
/* Reset device */
if (need_fw_reset &&
(rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
if (rc != ETIMEDOUT && rc != EIO)
t4_fw_bye(sc, sc->mbox);
goto done;
}
sc->flags |= FW_OK;
rc = get_params__pre_init(sc);
if (rc != 0)
goto done; /* error message displayed already */
/* Partition adapter resources as specified in the config file. */
if (state == DEV_STATE_UNINIT) {
KASSERT(sc->flags & MASTER_PF,
("%s: trying to change chip settings when not master.",
__func__));
rc = partition_resources(sc, default_cfg, fw_info->kld_name);
if (rc != 0)
goto done; /* error message displayed already */
t4_tweak_chip_settings(sc);
/* get basic stuff going */
rc = -t4_fw_initialize(sc, sc->mbox);
if (rc != 0) {
device_printf(sc->dev, "fw init failed: %d.\n", rc);
goto done;
}
} else {
snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
sc->cfcsum = 0;
}
done:
free(card_fw, M_CXGBE);
if (fw != NULL)
firmware_put(fw, FIRMWARE_UNLOAD);
if (default_cfg != NULL)
firmware_put(default_cfg, FIRMWARE_UNLOAD);
return (rc);
}
#define FW_PARAM_DEV(param) \
(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
#define FW_PARAM_PFVF(param) \
(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
/*
* Partition chip resources for use between various PFs, VFs, etc.
*/
static int
partition_resources(struct adapter *sc, const struct firmware *default_cfg,
const char *name_prefix)
{
const struct firmware *cfg = NULL;
int rc = 0;
struct fw_caps_config_cmd caps;
uint32_t mtype, moff, finicsum, cfcsum;
/*
* Figure out what configuration file to use. Pick the default config
* file for the card if the user hasn't specified one explicitly.
*/
snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
/* Card specific overrides go here. */
if (pci_get_device(sc->dev) == 0x440a)
snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
if (is_fpga(sc))
snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
}
/*
* We need to load another module if the profile is anything except
* "default" or "flash".
*/
if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
char s[32];
snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
cfg = firmware_get(s);
if (cfg == NULL) {
if (default_cfg != NULL) {
device_printf(sc->dev,
"unable to load module \"%s\" for "
"configuration profile \"%s\", will use "
"the default config file instead.\n",
s, sc->cfg_file);
snprintf(sc->cfg_file, sizeof(sc->cfg_file),
"%s", DEFAULT_CF);
} else {
device_printf(sc->dev,
"unable to load module \"%s\" for "
"configuration profile \"%s\", will use "
"the config file on the card's flash "
"instead.\n", s, sc->cfg_file);
snprintf(sc->cfg_file, sizeof(sc->cfg_file),
"%s", FLASH_CF);
}
}
}
if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
default_cfg == NULL) {
device_printf(sc->dev,
"default config file not available, will use the config "
"file on the card's flash instead.\n");
snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
}
if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
u_int cflen, i, n;
const uint32_t *cfdata;
uint32_t param, val, addr, off, mw_base, mw_aperture;
KASSERT(cfg != NULL || default_cfg != NULL,
("%s: no config to upload", __func__));
/*
* Ask the firmware where it wants us to upload the config file.
*/
param = FW_PARAM_DEV(CF);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
if (rc != 0) {
/* No support for config file? Shouldn't happen. */
device_printf(sc->dev,
"failed to query config file location: %d.\n", rc);
goto done;
}
mtype = G_FW_PARAMS_PARAM_Y(val);
moff = G_FW_PARAMS_PARAM_Z(val) << 16;
/*
* XXX: sheer laziness. We deliberately added 4 bytes of
* useless stuffing/comments at the end of the config file so
* it's ok to simply throw away the last remaining bytes when
* the config file is not an exact multiple of 4. This also
* helps with the validate_mt_off_len check.
*/
if (cfg != NULL) {
cflen = cfg->datasize & ~3;
cfdata = cfg->data;
} else {
cflen = default_cfg->datasize & ~3;
cfdata = default_cfg->data;
}
if (cflen > FLASH_CFG_MAX_SIZE) {
device_printf(sc->dev,
"config file too long (%d, max allowed is %d). "
"Will try to use the config on the card, if any.\n",
cflen, FLASH_CFG_MAX_SIZE);
goto use_config_on_flash;
}
rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
if (rc != 0) {
device_printf(sc->dev,
"%s: addr (%d/0x%x) or len %d is not valid: %d. "
"Will try to use the config on the card, if any.\n",
__func__, mtype, moff, cflen, rc);
goto use_config_on_flash;
}
memwin_info(sc, 2, &mw_base, &mw_aperture);
while (cflen) {
off = position_memwin(sc, 2, addr);
n = min(cflen, mw_aperture - off);
for (i = 0; i < n; i += 4)
t4_write_reg(sc, mw_base + off + i, *cfdata++);
cflen -= n;
addr += n;
}
} else {
use_config_on_flash:
mtype = FW_MEMTYPE_FLASH;
moff = t4_flash_cfg_addr(sc);
}
bzero(&caps, sizeof(caps));
caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
F_FW_CMD_REQUEST | F_FW_CMD_READ);
caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
if (rc != 0) {
device_printf(sc->dev,
"failed to pre-process config file: %d "
"(mtype %d, moff 0x%x).\n", rc, mtype, moff);
goto done;
}
finicsum = be32toh(caps.finicsum);
cfcsum = be32toh(caps.cfcsum);
if (finicsum != cfcsum) {
device_printf(sc->dev,
"WARNING: config file checksum mismatch: %08x %08x\n",
finicsum, cfcsum);
}
sc->cfcsum = cfcsum;
#define LIMIT_CAPS(x) do { \
caps.x &= htobe16(t4_##x##_allowed); \
} while (0)
/*
* Let the firmware know what features will (not) be used so it can tune
* things accordingly.
*/
LIMIT_CAPS(linkcaps);
LIMIT_CAPS(niccaps);
LIMIT_CAPS(toecaps);
LIMIT_CAPS(rdmacaps);
LIMIT_CAPS(iscsicaps);
LIMIT_CAPS(fcoecaps);
#undef LIMIT_CAPS
caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
if (rc != 0) {
device_printf(sc->dev,
"failed to process config file: %d.\n", rc);
}
done:
if (cfg != NULL)
firmware_put(cfg, FIRMWARE_UNLOAD);
return (rc);
}
/*
* Retrieve parameters that are needed (or nice to have) very early.
*/
static int
get_params__pre_init(struct adapter *sc)
{
int rc;
uint32_t param[2], val[2];
struct fw_devlog_cmd cmd;
struct devlog_params *dlog = &sc->params.devlog;
param[0] = FW_PARAM_DEV(PORTVEC);
param[1] = FW_PARAM_DEV(CCLK);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query parameters (pre_init): %d.\n", rc);
return (rc);
}
sc->params.portvec = val[0];
sc->params.nports = bitcount32(val[0]);
sc->params.vpd.cclk = val[1];
/* Read device log parameters. */
bzero(&cmd, sizeof(cmd));
cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
F_FW_CMD_REQUEST | F_FW_CMD_READ);
cmd.retval_len16 = htobe32(FW_LEN16(cmd));
rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
if (rc != 0) {
device_printf(sc->dev,
"failed to get devlog parameters: %d.\n", rc);
bzero(dlog, sizeof (*dlog));
rc = 0; /* devlog isn't critical for device operation */
} else {
val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
dlog->size = be32toh(cmd.memsize_devlog);
}
return (rc);
}
/*
* Retrieve various parameters that are of interest to the driver. The device
* has been initialized by the firmware at this point.
*/
static int
get_params__post_init(struct adapter *sc)
{
int rc;
uint32_t param[7], val[7];
struct fw_caps_config_cmd caps;
param[0] = FW_PARAM_PFVF(IQFLINT_START);
param[1] = FW_PARAM_PFVF(EQ_START);
param[2] = FW_PARAM_PFVF(FILTER_START);
param[3] = FW_PARAM_PFVF(FILTER_END);
param[4] = FW_PARAM_PFVF(L2T_START);
param[5] = FW_PARAM_PFVF(L2T_END);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query parameters (post_init): %d.\n", rc);
return (rc);
}
sc->sge.iq_start = val[0];
sc->sge.eq_start = val[1];
sc->tids.ftid_base = val[2];
sc->tids.nftids = val[3] - val[2] + 1;
sc->params.ftid_min = val[2];
sc->params.ftid_max = val[3];
sc->vres.l2t.start = val[4];
sc->vres.l2t.size = val[5] - val[4] + 1;
KASSERT(sc->vres.l2t.size <= L2T_SIZE,
("%s: L2 table size (%u) larger than expected (%u)",
__func__, sc->vres.l2t.size, L2T_SIZE));
/* get capabilites */
bzero(&caps, sizeof(caps));
caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
F_FW_CMD_REQUEST | F_FW_CMD_READ);
caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
if (rc != 0) {
device_printf(sc->dev,
"failed to get card capabilities: %d.\n", rc);
return (rc);
}
#define READ_CAPS(x) do { \
sc->x = htobe16(caps.x); \
} while (0)
READ_CAPS(linkcaps);
READ_CAPS(niccaps);
READ_CAPS(toecaps);
READ_CAPS(rdmacaps);
READ_CAPS(iscsicaps);
READ_CAPS(fcoecaps);
if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
param[0] = FW_PARAM_PFVF(ETHOFLD_START);
param[1] = FW_PARAM_PFVF(ETHOFLD_END);
param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query NIC parameters: %d.\n", rc);
return (rc);
}
sc->tids.etid_base = val[0];
sc->params.etid_min = val[0];
sc->tids.netids = val[1] - val[0] + 1;
sc->params.netids = sc->tids.netids;
sc->params.eo_wr_cred = val[2];
sc->params.ethoffload = 1;
}
if (sc->toecaps) {
/* query offload-related parameters */
param[0] = FW_PARAM_DEV(NTID);
param[1] = FW_PARAM_PFVF(SERVER_START);
param[2] = FW_PARAM_PFVF(SERVER_END);
param[3] = FW_PARAM_PFVF(TDDP_START);
param[4] = FW_PARAM_PFVF(TDDP_END);
param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query TOE parameters: %d.\n", rc);
return (rc);
}
sc->tids.ntids = val[0];
sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
sc->tids.stid_base = val[1];
sc->tids.nstids = val[2] - val[1] + 1;
sc->vres.ddp.start = val[3];
sc->vres.ddp.size = val[4] - val[3] + 1;
sc->params.ofldq_wr_cred = val[5];
sc->params.offload = 1;
}
if (sc->rdmacaps) {
param[0] = FW_PARAM_PFVF(STAG_START);
param[1] = FW_PARAM_PFVF(STAG_END);
param[2] = FW_PARAM_PFVF(RQ_START);
param[3] = FW_PARAM_PFVF(RQ_END);
param[4] = FW_PARAM_PFVF(PBL_START);
param[5] = FW_PARAM_PFVF(PBL_END);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query RDMA parameters(1): %d.\n", rc);
return (rc);
}
sc->vres.stag.start = val[0];
sc->vres.stag.size = val[1] - val[0] + 1;
sc->vres.rq.start = val[2];
sc->vres.rq.size = val[3] - val[2] + 1;
sc->vres.pbl.start = val[4];
sc->vres.pbl.size = val[5] - val[4] + 1;
param[0] = FW_PARAM_PFVF(SQRQ_START);
param[1] = FW_PARAM_PFVF(SQRQ_END);
param[2] = FW_PARAM_PFVF(CQ_START);
param[3] = FW_PARAM_PFVF(CQ_END);
param[4] = FW_PARAM_PFVF(OCQ_START);
param[5] = FW_PARAM_PFVF(OCQ_END);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query RDMA parameters(2): %d.\n", rc);
return (rc);
}
sc->vres.qp.start = val[0];
sc->vres.qp.size = val[1] - val[0] + 1;
sc->vres.cq.start = val[2];
sc->vres.cq.size = val[3] - val[2] + 1;
sc->vres.ocq.start = val[4];
sc->vres.ocq.size = val[5] - val[4] + 1;
}
if (sc->iscsicaps) {
param[0] = FW_PARAM_PFVF(ISCSI_START);
param[1] = FW_PARAM_PFVF(ISCSI_END);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query iSCSI parameters: %d.\n", rc);
return (rc);
}
sc->vres.iscsi.start = val[0];
sc->vres.iscsi.size = val[1] - val[0] + 1;
}
/*
* We've got the params we wanted to query via the firmware. Now grab
* some others directly from the chip.
*/
rc = t4_read_chip_settings(sc);
return (rc);
}
static int
set_params__post_init(struct adapter *sc)
{
uint32_t param, val;
/* ask for encapsulated CPLs */
param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
val = 1;
(void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
return (0);
}
#undef FW_PARAM_PFVF
#undef FW_PARAM_DEV
static void
t4_set_desc(struct adapter *sc)
{
char buf[128];
struct adapter_params *p = &sc->params;
snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
"P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
device_set_desc_copy(sc->dev, buf);
}
static void
build_medialist(struct port_info *pi, struct ifmedia *media)
{
int m;
PORT_LOCK(pi);
ifmedia_removeall(media);
m = IFM_ETHER | IFM_FDX;
switch(pi->port_type) {
case FW_PORT_TYPE_BT_XFI:
case FW_PORT_TYPE_BT_XAUI:
ifmedia_add(media, m | IFM_10G_T, 0, NULL);
/* fall through */
case FW_PORT_TYPE_BT_SGMII:
ifmedia_add(media, m | IFM_1000_T, 0, NULL);
ifmedia_add(media, m | IFM_100_TX, 0, NULL);
ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
ifmedia_set(media, IFM_ETHER | IFM_AUTO);
break;
case FW_PORT_TYPE_CX4:
ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
ifmedia_set(media, m | IFM_10G_CX4);
break;
case FW_PORT_TYPE_QSFP_10G:
case FW_PORT_TYPE_SFP:
case FW_PORT_TYPE_FIBER_XFI:
case FW_PORT_TYPE_FIBER_XAUI:
switch (pi->mod_type) {
case FW_PORT_MOD_TYPE_LR:
ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
ifmedia_set(media, m | IFM_10G_LR);
break;
case FW_PORT_MOD_TYPE_SR:
ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
ifmedia_set(media, m | IFM_10G_SR);
break;
case FW_PORT_MOD_TYPE_LRM:
ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
ifmedia_set(media, m | IFM_10G_LRM);
break;
case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
ifmedia_set(media, m | IFM_10G_TWINAX);
break;
case FW_PORT_MOD_TYPE_NONE:
m &= ~IFM_FDX;
ifmedia_add(media, m | IFM_NONE, 0, NULL);
ifmedia_set(media, m | IFM_NONE);
break;
case FW_PORT_MOD_TYPE_NA:
case FW_PORT_MOD_TYPE_ER:
default:
device_printf(pi->dev,
"unknown port_type (%d), mod_type (%d)\n",
pi->port_type, pi->mod_type);
ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
ifmedia_set(media, m | IFM_UNKNOWN);
break;
}
break;
case FW_PORT_TYPE_QSFP:
switch (pi->mod_type) {
case FW_PORT_MOD_TYPE_LR:
ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
ifmedia_set(media, m | IFM_40G_LR4);
break;
case FW_PORT_MOD_TYPE_SR:
ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
ifmedia_set(media, m | IFM_40G_SR4);
break;
case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
ifmedia_set(media, m | IFM_40G_CR4);
break;
case FW_PORT_MOD_TYPE_NONE:
m &= ~IFM_FDX;
ifmedia_add(media, m | IFM_NONE, 0, NULL);
ifmedia_set(media, m | IFM_NONE);
break;
default:
device_printf(pi->dev,
"unknown port_type (%d), mod_type (%d)\n",
pi->port_type, pi->mod_type);
ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
ifmedia_set(media, m | IFM_UNKNOWN);
break;
}
break;
default:
device_printf(pi->dev,
"unknown port_type (%d), mod_type (%d)\n", pi->port_type,
pi->mod_type);
ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
ifmedia_set(media, m | IFM_UNKNOWN);
break;
}
PORT_UNLOCK(pi);
}
#define FW_MAC_EXACT_CHUNK 7
/*
* Program the port's XGMAC based on parameters in ifnet. The caller also
* indicates which parameters should be programmed (the rest are left alone).
*/
int
update_mac_settings(struct ifnet *ifp, int flags)
{
int rc = 0;
struct port_info *pi = ifp->if_softc;
struct adapter *sc = pi->adapter;
int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
uint16_t viid = 0xffff;
int16_t *xact_addr_filt = NULL;
ASSERT_SYNCHRONIZED_OP(sc);
KASSERT(flags, ("%s: not told what to update.", __func__));
if (ifp == pi->ifp) {
viid = pi->viid;
xact_addr_filt = &pi->xact_addr_filt;
}
#ifdef DEV_NETMAP
else if (ifp == pi->nm_ifp) {
viid = pi->nm_viid;
xact_addr_filt = &pi->nm_xact_addr_filt;
}
#endif
if (flags & XGMAC_MTU)
mtu = ifp->if_mtu;
if (flags & XGMAC_PROMISC)
promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
if (flags & XGMAC_ALLMULTI)
allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
if (flags & XGMAC_VLANEX)
vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
1, vlanex, false);
if (rc) {
if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
rc);
return (rc);
}
}
if (flags & XGMAC_UCADDR) {
uint8_t ucaddr[ETHER_ADDR_LEN];
bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
true, true);
if (rc < 0) {
rc = -rc;
if_printf(ifp, "change_mac failed: %d\n", rc);
return (rc);
} else {
*xact_addr_filt = rc;
rc = 0;
}
}
if (flags & XGMAC_MCADDRS) {
const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
int del = 1;
uint64_t hash = 0;
struct ifmultiaddr *ifma;
int i = 0, j;
if_maddr_rlock(ifp);
TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
if (ifma->ifma_addr->sa_family != AF_LINK)
continue;
mcaddr[i] =
LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
i++;
if (i == FW_MAC_EXACT_CHUNK) {
rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
i, mcaddr, NULL, &hash, 0);
if (rc < 0) {
rc = -rc;
for (j = 0; j < i; j++) {
if_printf(ifp,
"failed to add mc address"
" %02x:%02x:%02x:"
"%02x:%02x:%02x rc=%d\n",
mcaddr[j][0], mcaddr[j][1],
mcaddr[j][2], mcaddr[j][3],
mcaddr[j][4], mcaddr[j][5],
rc);
}
goto mcfail;
}
del = 0;
i = 0;
}
}
if (i > 0) {
rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
mcaddr, NULL, &hash, 0);
if (rc < 0) {
rc = -rc;
for (j = 0; j < i; j++) {
if_printf(ifp,
"failed to add mc address"
" %02x:%02x:%02x:"
"%02x:%02x:%02x rc=%d\n",
mcaddr[j][0], mcaddr[j][1],
mcaddr[j][2], mcaddr[j][3],
mcaddr[j][4], mcaddr[j][5],
rc);
}
goto mcfail;
}
}
rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
if (rc != 0)
if_printf(ifp, "failed to set mc address hash: %d", rc);
mcfail:
if_maddr_runlock(ifp);
}
return (rc);
}
/*
* {begin|end}_synchronized_op must be called from the same thread.
*/
int
begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
char *wmesg)
{
int rc, pri;
#ifdef WITNESS
/* the caller thinks it's ok to sleep, but is it really? */
if (flags & SLEEP_OK)
pause("t4slptst", 1);
#endif
if (INTR_OK)
pri = PCATCH;
else
pri = 0;
ADAPTER_LOCK(sc);
for (;;) {
if (pi && IS_DOOMED(pi)) {
rc = ENXIO;
goto done;
}
if (!IS_BUSY(sc)) {
rc = 0;
break;
}
if (!(flags & SLEEP_OK)) {
rc = EBUSY;
goto done;
}
if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
rc = EINTR;
goto done;
}
}
KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
SET_BUSY(sc);
#ifdef INVARIANTS
sc->last_op = wmesg;
sc->last_op_thr = curthread;
#endif
done:
if (!(flags & HOLD_LOCK) || rc)
ADAPTER_UNLOCK(sc);
return (rc);
}
/*
* {begin|end}_synchronized_op must be called from the same thread.
*/
void
end_synchronized_op(struct adapter *sc, int flags)
{
if (flags & LOCK_HELD)
ADAPTER_LOCK_ASSERT_OWNED(sc);
else
ADAPTER_LOCK(sc);
KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
CLR_BUSY(sc);
wakeup(&sc->flags);
ADAPTER_UNLOCK(sc);
}
static int
cxgbe_init_synchronized(struct port_info *pi)
{
struct adapter *sc = pi->adapter;
struct ifnet *ifp = pi->ifp;
int rc = 0, i;
struct sge_txq *txq;
ASSERT_SYNCHRONIZED_OP(sc);
if (isset(&sc->open_device_map, pi->port_id)) {
KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
("mismatch between open_device_map and if_drv_flags"));
return (0); /* already running */
}
if (!(sc->flags & FULL_INIT_DONE) &&
((rc = adapter_full_init(sc)) != 0))
return (rc); /* error message displayed already */
if (!(pi->flags & PORT_INIT_DONE) &&
((rc = port_full_init(pi)) != 0))
return (rc); /* error message displayed already */
rc = update_mac_settings(ifp, XGMAC_ALL);
if (rc)
goto done; /* error message displayed already */
rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
if (rc != 0) {
if_printf(ifp, "enable_vi failed: %d\n", rc);
goto done;
}
/*
* Can't fail from this point onwards. Review cxgbe_uninit_synchronized
* if this changes.
*/
for_each_txq(pi, i, txq) {
TXQ_LOCK(txq);
txq->eq.flags |= EQ_ENABLED;
TXQ_UNLOCK(txq);
}
/*
* The first iq of the first port to come up is used for tracing.
*/
if (sc->traceq < 0) {
sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL :
A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
V_QUEUENUMBER(sc->traceq));
pi->flags |= HAS_TRACEQ;
}
/* all ok */
setbit(&sc->open_device_map, pi->port_id);
PORT_LOCK(pi);
ifp->if_drv_flags |= IFF_DRV_RUNNING;
PORT_UNLOCK(pi);
callout_reset(&pi->tick, hz, cxgbe_tick, pi);
done:
if (rc != 0)
cxgbe_uninit_synchronized(pi);
return (rc);
}
/*
* Idempotent.
*/
static int
cxgbe_uninit_synchronized(struct port_info *pi)
{
struct adapter *sc = pi->adapter;
struct ifnet *ifp = pi->ifp;
int rc, i;
struct sge_txq *txq;
ASSERT_SYNCHRONIZED_OP(sc);
if (!(pi->flags & PORT_INIT_DONE)) {
KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
("uninited port is running"));
return (0);
}
/*
* Disable the VI so that all its data in either direction is discarded
* by the MPS. Leave everything else (the queues, interrupts, and 1Hz
* tick) intact as the TP can deliver negative advice or data that it's
* holding in its RAM (for an offloaded connection) even after the VI is
* disabled.
*/
rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
if (rc) {
if_printf(ifp, "disable_vi failed: %d\n", rc);
return (rc);
}
for_each_txq(pi, i, txq) {
TXQ_LOCK(txq);
txq->eq.flags &= ~EQ_ENABLED;
TXQ_UNLOCK(txq);
}
clrbit(&sc->open_device_map, pi->port_id);
PORT_LOCK(pi);
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
PORT_UNLOCK(pi);
pi->link_cfg.link_ok = 0;
pi->link_cfg.speed = 0;
pi->linkdnrc = -1;
t4_os_link_changed(sc, pi->port_id, 0, -1);
return (0);
}
/*
* It is ok for this function to fail midway and return right away. t4_detach
* will walk the entire sc->irq list and clean up whatever is valid.
*/
static int
setup_intr_handlers(struct adapter *sc)
{
int rc, rid, p, q;
char s[8];
struct irq *irq;
struct port_info *pi;
struct sge_rxq *rxq;
#ifdef TCP_OFFLOAD
struct sge_ofld_rxq *ofld_rxq;
#endif
#ifdef DEV_NETMAP
struct sge_nm_rxq *nm_rxq;
#endif
/*
* Setup interrupts.
*/
irq = &sc->irq[0];
rid = sc->intr_type == INTR_INTX ? 0 : 1;
if (sc->intr_count == 1)
return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
/* Multiple interrupts. */
KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
("%s: too few intr.", __func__));
/* The first one is always error intr */
rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
if (rc != 0)
return (rc);
irq++;
rid++;
/* The second one is always the firmware event queue */
rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
if (rc != 0)
return (rc);
irq++;
rid++;
for_each_port(sc, p) {
pi = sc->port[p];
if (pi->flags & INTR_RXQ) {
for_each_rxq(pi, q, rxq) {
snprintf(s, sizeof(s), "%d.%d", p, q);
rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
s);
if (rc != 0)
return (rc);
irq++;
rid++;
}
}
#ifdef TCP_OFFLOAD
if (pi->flags & INTR_OFLD_RXQ) {
for_each_ofld_rxq(pi, q, ofld_rxq) {
snprintf(s, sizeof(s), "%d,%d", p, q);
rc = t4_alloc_irq(sc, irq, rid, t4_intr,
ofld_rxq, s);
if (rc != 0)
return (rc);
irq++;
rid++;
}
}
#endif
#ifdef DEV_NETMAP
if (pi->flags & INTR_NM_RXQ) {
for_each_nm_rxq(pi, q, nm_rxq) {
snprintf(s, sizeof(s), "%d-%d", p, q);
rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
nm_rxq, s);
if (rc != 0)
return (rc);
irq++;
rid++;
}
}
#endif
}
MPASS(irq == &sc->irq[sc->intr_count]);
return (0);
}
int
adapter_full_init(struct adapter *sc)
{
int rc, i;
ASSERT_SYNCHRONIZED_OP(sc);
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
KASSERT((sc->flags & FULL_INIT_DONE) == 0,
("%s: FULL_INIT_DONE already", __func__));
/*
* queues that belong to the adapter (not any particular port).
*/
rc = t4_setup_adapter_queues(sc);
if (rc != 0)
goto done;
for (i = 0; i < nitems(sc->tq); i++) {
sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
taskqueue_thread_enqueue, &sc->tq[i]);
if (sc->tq[i] == NULL) {
device_printf(sc->dev,
"failed to allocate task queue %d\n", i);
rc = ENOMEM;
goto done;
}
taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
device_get_nameunit(sc->dev), i);
}
t4_intr_enable(sc);
sc->flags |= FULL_INIT_DONE;
done:
if (rc != 0)
adapter_full_uninit(sc);
return (rc);
}
int
adapter_full_uninit(struct adapter *sc)
{
int i;
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
t4_teardown_adapter_queues(sc);
for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
taskqueue_free(sc->tq[i]);
sc->tq[i] = NULL;
}
sc->flags &= ~FULL_INIT_DONE;
return (0);
}
int
port_full_init(struct port_info *pi)
{
struct adapter *sc = pi->adapter;
struct ifnet *ifp = pi->ifp;
uint16_t *rss;
struct sge_rxq *rxq;
int rc, i, j;
ASSERT_SYNCHRONIZED_OP(sc);
KASSERT((pi->flags & PORT_INIT_DONE) == 0,
("%s: PORT_INIT_DONE already", __func__));
sysctl_ctx_init(&pi->ctx);
pi->flags |= PORT_SYSCTL_CTX;
/*
* Allocate tx/rx/fl queues for this port.
*/
rc = t4_setup_port_queues(pi);
if (rc != 0)
goto done; /* error message displayed already */
/*
* Setup RSS for this port. Save a copy of the RSS table for later use.
*/
rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
for (i = 0; i < pi->rss_size;) {
for_each_rxq(pi, j, rxq) {
rss[i++] = rxq->iq.abs_id;
if (i == pi->rss_size)
break;
}
}
rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
pi->rss_size);
if (rc != 0) {
if_printf(ifp, "rss_config failed: %d\n", rc);
goto done;
}
pi->rss = rss;
pi->flags |= PORT_INIT_DONE;
done:
if (rc != 0)
port_full_uninit(pi);
return (rc);
}
/*
* Idempotent.
*/
int
port_full_uninit(struct port_info *pi)
{
struct adapter *sc = pi->adapter;
int i;
struct sge_rxq *rxq;
struct sge_txq *txq;
#ifdef TCP_OFFLOAD
struct sge_ofld_rxq *ofld_rxq;
struct sge_wrq *ofld_txq;
#endif
if (pi->flags & PORT_INIT_DONE) {
/* Need to quiesce queues. */
quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
for_each_txq(pi, i, txq) {
quiesce_txq(sc, txq);
}
#ifdef TCP_OFFLOAD
for_each_ofld_txq(pi, i, ofld_txq) {
quiesce_wrq(sc, ofld_txq);
}
#endif
for_each_rxq(pi, i, rxq) {
quiesce_iq(sc, &rxq->iq);
quiesce_fl(sc, &rxq->fl);
}
#ifdef TCP_OFFLOAD
for_each_ofld_rxq(pi, i, ofld_rxq) {
quiesce_iq(sc, &ofld_rxq->iq);
quiesce_fl(sc, &ofld_rxq->fl);
}
#endif
free(pi->rss, M_CXGBE);
}
t4_teardown_port_queues(pi);
pi->flags &= ~PORT_INIT_DONE;
return (0);
}
static void
quiesce_txq(struct adapter *sc, struct sge_txq *txq)
{
struct sge_eq *eq = &txq->eq;
struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
(void) sc; /* unused */
#ifdef INVARIANTS
TXQ_LOCK(txq);
MPASS((eq->flags & EQ_ENABLED) == 0);
TXQ_UNLOCK(txq);
#endif
/* Wait for the mp_ring to empty. */
while (!mp_ring_is_idle(txq->r)) {
mp_ring_check_drainage(txq->r, 0);
pause("rquiesce", 1);
}
/* Then wait for the hardware to finish. */
while (spg->cidx != htobe16(eq->pidx))
pause("equiesce", 1);
/* Finally, wait for the driver to reclaim all descriptors. */
while (eq->cidx != eq->pidx)
pause("dquiesce", 1);
}
static void
quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
{
/* XXXTX */
}
static void
quiesce_iq(struct adapter *sc, struct sge_iq *iq)
{
(void) sc; /* unused */
/* Synchronize with the interrupt handler */
while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
pause("iqfree", 1);
}
static void
quiesce_fl(struct adapter *sc, struct sge_fl *fl)
{
mtx_lock(&sc->sfl_lock);
FL_LOCK(fl);
fl->flags |= FL_DOOMED;
FL_UNLOCK(fl);
mtx_unlock(&sc->sfl_lock);
callout_drain(&sc->sfl_callout);
KASSERT((fl->flags & FL_STARVING) == 0,
("%s: still starving", __func__));
}
static int
t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
driver_intr_t *handler, void *arg, char *name)
{
int rc;
irq->rid = rid;
irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
RF_SHAREABLE | RF_ACTIVE);
if (irq->res == NULL) {
device_printf(sc->dev,
"failed to allocate IRQ for rid %d, name %s.\n", rid, name);
return (ENOMEM);
}
rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
NULL, handler, arg, &irq->tag);
if (rc != 0) {
device_printf(sc->dev,
"failed to setup interrupt for rid %d, name %s: %d\n",
rid, name, rc);
} else if (name)
bus_describe_intr(sc->dev, irq->res, irq->tag, name);
return (rc);
}
static int
t4_free_irq(struct adapter *sc, struct irq *irq)
{
if (irq->tag)
bus_teardown_intr(sc->dev, irq->res, irq->tag);
if (irq->res)
bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
bzero(irq, sizeof(*irq));
return (0);
}
static void
reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
unsigned int end)
{
uint32_t *p = (uint32_t *)(buf + start);
for ( ; start <= end; start += sizeof(uint32_t))
*p++ = t4_read_reg(sc, start);
}
static void
t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
{
int i, n;
const unsigned int *reg_ranges;
static const unsigned int t4_reg_ranges[] = {
0x1008, 0x1108,
0x1180, 0x11b4,
0x11fc, 0x123c,
0x1300, 0x173c,
0x1800, 0x18fc,
0x3000, 0x30d8,
0x30e0, 0x5924,
0x5960, 0x59d4,
0x5a00, 0x5af8,
0x6000, 0x6098,
0x6100, 0x6150,
0x6200, 0x6208,
0x6240, 0x6248,
0x6280, 0x6338,
0x6370, 0x638c,
0x6400, 0x643c,
0x6500, 0x6524,
0x6a00, 0x6a38,
0x6a60, 0x6a78,
0x6b00, 0x6b84,
0x6bf0, 0x6c84,
0x6cf0, 0x6d84,
0x6df0, 0x6e84,
0x6ef0, 0x6f84,
0x6ff0, 0x7084,
0x70f0, 0x7184,
0x71f0, 0x7284,
0x72f0, 0x7384,
0x73f0, 0x7450,
0x7500, 0x7530,
0x7600, 0x761c,
0x7680, 0x76cc,
0x7700, 0x7798,
0x77c0, 0x77fc,
0x7900, 0x79fc,
0x7b00, 0x7c38,
0x7d00, 0x7efc,
0x8dc0, 0x8e1c,
0x8e30, 0x8e78,
0x8ea0, 0x8f6c,
0x8fc0, 0x9074,
0x90fc, 0x90fc,
0x9400, 0x9458,
0x9600, 0x96bc,
0x9800, 0x9808,
0x9820, 0x983c,
0x9850, 0x9864,
0x9c00, 0x9c6c,
0x9c80, 0x9cec,
0x9d00, 0x9d6c,
0x9d80, 0x9dec,
0x9e00, 0x9e6c,
0x9e80, 0x9eec,
0x9f00, 0x9f6c,
0x9f80, 0x9fec,
0xd004, 0xd03c,
0xdfc0, 0xdfe0,
0xe000, 0xea7c,
0xf000, 0x11110,
0x11118, 0x11190,
0x19040, 0x1906c,
0x19078, 0x19080,
0x1908c, 0x19124,
0x19150, 0x191b0,
0x191d0, 0x191e8,
0x19238, 0x1924c,
0x193f8, 0x19474,
0x19490, 0x194f8,
0x19800, 0x19f30,
0x1a000, 0x1a06c,
0x1a0b0, 0x1a120,
0x1a128, 0x1a138,
0x1a190, 0x1a1c4,
0x1a1fc, 0x1a1fc,
0x1e040, 0x1e04c,
0x1e284, 0x1e28c,
0x1e2c0, 0x1e2c0,
0x1e2e0, 0x1e2e0,
0x1e300, 0x1e384,
0x1e3c0, 0x1e3c8,
0x1e440, 0x1e44c,
0x1e684, 0x1e68c,
0x1e6c0, 0x1e6c0,
0x1e6e0, 0x1e6e0,
0x1e700, 0x1e784,
0x1e7c0, 0x1e7c8,
0x1e840, 0x1e84c,
0x1ea84, 0x1ea8c,
0x1eac0, 0x1eac0,
0x1eae0, 0x1eae0,
0x1eb00, 0x1eb84,
0x1ebc0, 0x1ebc8,
0x1ec40, 0x1ec4c,
0x1ee84, 0x1ee8c,
0x1eec0, 0x1eec0,
0x1eee0, 0x1eee0,
0x1ef00, 0x1ef84,
0x1efc0, 0x1efc8,
0x1f040, 0x1f04c,
0x1f284, 0x1f28c,
0x1f2c0, 0x1f2c0,
0x1f2e0, 0x1f2e0,
0x1f300, 0x1f384,
0x1f3c0, 0x1f3c8,
0x1f440, 0x1f44c,
0x1f684, 0x1f68c,
0x1f6c0, 0x1f6c0,
0x1f6e0, 0x1f6e0,
0x1f700, 0x1f784,
0x1f7c0, 0x1f7c8,
0x1f840, 0x1f84c,
0x1fa84, 0x1fa8c,
0x1fac0, 0x1fac0,
0x1fae0, 0x1fae0,
0x1fb00, 0x1fb84,
0x1fbc0, 0x1fbc8,
0x1fc40, 0x1fc4c,
0x1fe84, 0x1fe8c,
0x1fec0, 0x1fec0,
0x1fee0, 0x1fee0,
0x1ff00, 0x1ff84,
0x1ffc0, 0x1ffc8,
0x20000, 0x2002c,
0x20100, 0x2013c,
0x20190, 0x201c8,
0x20200, 0x20318,
0x20400, 0x20528,
0x20540, 0x20614,
0x21000, 0x21040,
0x2104c, 0x21060,
0x210c0, 0x210ec,
0x21200, 0x21268,
0x21270, 0x21284,
0x212fc, 0x21388,
0x21400, 0x21404,
0x21500, 0x21518,
0x2152c, 0x2153c,
0x21550, 0x21554,
0x21600, 0x21600,
0x21608, 0x21628,
0x21630, 0x2163c,
0x21700, 0x2171c,
0x21780, 0x2178c,
0x21800, 0x21c38,
0x21c80, 0x21d7c,
0x21e00, 0x21e04,
0x22000, 0x2202c,
0x22100, 0x2213c,
0x22190, 0x221c8,
0x22200, 0x22318,
0x22400, 0x22528,
0x22540, 0x22614,
0x23000, 0x23040,
0x2304c, 0x23060,
0x230c0, 0x230ec,
0x23200, 0x23268,
0x23270, 0x23284,
0x232fc, 0x23388,
0x23400, 0x23404,
0x23500, 0x23518,
0x2352c, 0x2353c,
0x23550, 0x23554,
0x23600, 0x23600,
0x23608, 0x23628,
0x23630, 0x2363c,
0x23700, 0x2371c,
0x23780, 0x2378c,
0x23800, 0x23c38,
0x23c80, 0x23d7c,
0x23e00, 0x23e04,
0x24000, 0x2402c,
0x24100, 0x2413c,
0x24190, 0x241c8,
0x24200, 0x24318,
0x24400, 0x24528,
0x24540, 0x24614,
0x25000, 0x25040,
0x2504c, 0x25060,
0x250c0, 0x250ec,
0x25200, 0x25268,
0x25270, 0x25284,
0x252fc, 0x25388,
0x25400, 0x25404,
0x25500, 0x25518,
0x2552c, 0x2553c,
0x25550, 0x25554,
0x25600, 0x25600,
0x25608, 0x25628,
0x25630, 0x2563c,
0x25700, 0x2571c,
0x25780, 0x2578c,
0x25800, 0x25c38,
0x25c80, 0x25d7c,
0x25e00, 0x25e04,
0x26000, 0x2602c,
0x26100, 0x2613c,
0x26190, 0x261c8,
0x26200, 0x26318,
0x26400, 0x26528,
0x26540, 0x26614,
0x27000, 0x27040,
0x2704c, 0x27060,
0x270c0, 0x270ec,
0x27200, 0x27268,
0x27270, 0x27284,
0x272fc, 0x27388,
0x27400, 0x27404,
0x27500, 0x27518,
0x2752c, 0x2753c,
0x27550, 0x27554,
0x27600, 0x27600,
0x27608, 0x27628,
0x27630, 0x2763c,
0x27700, 0x2771c,
0x27780, 0x2778c,
0x27800, 0x27c38,
0x27c80, 0x27d7c,
0x27e00, 0x27e04
};
static const unsigned int t5_reg_ranges[] = {
0x1008, 0x1148,
0x1180, 0x11b4,
0x11fc, 0x123c,
0x1280, 0x173c,
0x1800, 0x18fc,
0x3000, 0x3028,
0x3060, 0x30d8,
0x30e0, 0x30fc,
0x3140, 0x357c,
0x35a8, 0x35cc,
0x35ec, 0x35ec,
0x3600, 0x5624,
0x56cc, 0x575c,
0x580c, 0x5814,
0x5890, 0x58bc,
0x5940, 0x59dc,
0x59fc, 0x5a18,
0x5a60, 0x5a9c,
0x5b94, 0x5bfc,
0x6000, 0x6040,
0x6058, 0x614c,
0x7700, 0x7798,
0x77c0, 0x78fc,
0x7b00, 0x7c54,
0x7d00, 0x7efc,
0x8dc0, 0x8de0,
0x8df8, 0x8e84,
0x8ea0, 0x8f84,
0x8fc0, 0x90f8,
0x9400, 0x9470,
0x9600, 0x96f4,
0x9800, 0x9808,
0x9820, 0x983c,
0x9850, 0x9864,
0x9c00, 0x9c6c,
0x9c80, 0x9cec,
0x9d00, 0x9d6c,
0x9d80, 0x9dec,
0x9e00, 0x9e6c,
0x9e80, 0x9eec,
0x9f00, 0x9f6c,
0x9f80, 0xa020,
0xd004, 0xd03c,
0xdfc0, 0xdfe0,
0xe000, 0x11088,
0x1109c, 0x11110,
0x11118, 0x1117c,
0x11190, 0x11204,
0x19040, 0x1906c,
0x19078, 0x19080,
0x1908c, 0x19124,
0x19150, 0x191b0,
0x191d0, 0x191e8,
0x19238, 0x19290,
0x193f8, 0x19474,
0x19490, 0x194cc,
0x194f0, 0x194f8,
0x19c00, 0x19c60,
0x19c94, 0x19e10,
0x19e50, 0x19f34,
0x19f40, 0x19f50,
0x19f90, 0x19fe4,
0x1a000, 0x1a06c,
0x1a0b0, 0x1a120,
0x1a128, 0x1a138,
0x1a190, 0x1a1c4,
0x1a1fc, 0x1a1fc,
0x1e008, 0x1e00c,
0x1e040, 0x1e04c,
0x1e284, 0x1e290,
0x1e2c0, 0x1e2c0,
0x1e2e0, 0x1e2e0,
0x1e300, 0x1e384,
0x1e3c0, 0x1e3c8,
0x1e408, 0x1e40c,
0x1e440, 0x1e44c,
0x1e684, 0x1e690,
0x1e6c0, 0x1e6c0,
0x1e6e0, 0x1e6e0,
0x1e700, 0x1e784,
0x1e7c0, 0x1e7c8,
0x1e808, 0x1e80c,
0x1e840, 0x1e84c,
0x1ea84, 0x1ea90,
0x1eac0, 0x1eac0,
0x1eae0, 0x1eae0,
0x1eb00, 0x1eb84,
0x1ebc0, 0x1ebc8,
0x1ec08, 0x1ec0c,
0x1ec40, 0x1ec4c,
0x1ee84, 0x1ee90,
0x1eec0, 0x1eec0,
0x1eee0, 0x1eee0,
0x1ef00, 0x1ef84,
0x1efc0, 0x1efc8,
0x1f008, 0x1f00c,
0x1f040, 0x1f04c,
0x1f284, 0x1f290,
0x1f2c0, 0x1f2c0,
0x1f2e0, 0x1f2e0,
0x1f300, 0x1f384,
0x1f3c0, 0x1f3c8,
0x1f408, 0x1f40c,
0x1f440, 0x1f44c,
0x1f684, 0x1f690,
0x1f6c0, 0x1f6c0,
0x1f6e0, 0x1f6e0,
0x1f700, 0x1f784,
0x1f7c0, 0x1f7c8,
0x1f808, 0x1f80c,
0x1f840, 0x1f84c,
0x1fa84, 0x1fa90,
0x1fac0, 0x1fac0,
0x1fae0, 0x1fae0,
0x1fb00, 0x1fb84,
0x1fbc0, 0x1fbc8,
0x1fc08, 0x1fc0c,
0x1fc40, 0x1fc4c,
0x1fe84, 0x1fe90,
0x1fec0, 0x1fec0,
0x1fee0, 0x1fee0,
0x1ff00, 0x1ff84,
0x1ffc0, 0x1ffc8,
0x30000, 0x30030,
0x30100, 0x30144,
0x30190, 0x301d0,
0x30200, 0x30318,
0x30400, 0x3052c,
0x30540, 0x3061c,
0x30800, 0x30834,
0x308c0, 0x30908,
0x30910, 0x309ac,
0x30a00, 0x30a2c,
0x30a44, 0x30a50,
0x30a74, 0x30c24,
0x30d00, 0x30d00,
0x30d08, 0x30d14,
0x30d1c, 0x30d20,
0x30d3c, 0x30d50,
0x31200, 0x3120c,
0x31220, 0x31220,
0x31240, 0x31240,
0x31600, 0x3160c,
0x31a00, 0x31a1c,
0x31e00, 0x31e20,
0x31e38, 0x31e3c,
0x31e80, 0x31e80,
0x31e88, 0x31ea8,
0x31eb0, 0x31eb4,
0x31ec8, 0x31ed4,
0x31fb8, 0x32004,
0x32200, 0x32200,
0x32208, 0x32240,
0x32248, 0x32280,
0x32288, 0x322c0,
0x322c8, 0x322fc,
0x32600, 0x32630,
0x32a00, 0x32abc,
0x32b00, 0x32b70,
0x33000, 0x33048,
0x33060, 0x3309c,
0x330f0, 0x33148,
0x33160, 0x3319c,
0x331f0, 0x332e4,
0x332f8, 0x333e4,
0x333f8, 0x33448,
0x33460, 0x3349c,
0x334f0, 0x33548,
0x33560, 0x3359c,
0x335f0, 0x336e4,
0x336f8, 0x337e4,
0x337f8, 0x337fc,
0x33814, 0x33814,
0x3382c, 0x3382c,
0x33880, 0x3388c,
0x338e8, 0x338ec,
0x33900, 0x33948,
0x33960, 0x3399c,
0x339f0, 0x33ae4,
0x33af8, 0x33b10,
0x33b28, 0x33b28,
0x33b3c, 0x33b50,
0x33bf0, 0x33c10,
0x33c28, 0x33c28,
0x33c3c, 0x33c50,
0x33cf0, 0x33cfc,
0x34000, 0x34030,
0x34100, 0x34144,
0x34190, 0x341d0,
0x34200, 0x34318,
0x34400, 0x3452c,
0x34540, 0x3461c,
0x34800, 0x34834,
0x348c0, 0x34908,
0x34910, 0x349ac,
0x34a00, 0x34a2c,
0x34a44, 0x34a50,
0x34a74, 0x34c24,
0x34d00, 0x34d00,
0x34d08, 0x34d14,
0x34d1c, 0x34d20,
0x34d3c, 0x34d50,
0x35200, 0x3520c,
0x35220, 0x35220,
0x35240, 0x35240,
0x35600, 0x3560c,
0x35a00, 0x35a1c,
0x35e00, 0x35e20,
0x35e38, 0x35e3c,
0x35e80, 0x35e80,
0x35e88, 0x35ea8,
0x35eb0, 0x35eb4,
0x35ec8, 0x35ed4,
0x35fb8, 0x36004,
0x36200, 0x36200,
0x36208, 0x36240,
0x36248, 0x36280,
0x36288, 0x362c0,
0x362c8, 0x362fc,
0x36600, 0x36630,
0x36a00, 0x36abc,
0x36b00, 0x36b70,
0x37000, 0x37048,
0x37060, 0x3709c,
0x370f0, 0x37148,
0x37160, 0x3719c,
0x371f0, 0x372e4,
0x372f8, 0x373e4,
0x373f8, 0x37448,
0x37460, 0x3749c,
0x374f0, 0x37548,
0x37560, 0x3759c,
0x375f0, 0x376e4,
0x376f8, 0x377e4,
0x377f8, 0x377fc,
0x37814, 0x37814,
0x3782c, 0x3782c,
0x37880, 0x3788c,
0x378e8, 0x378ec,
0x37900, 0x37948,
0x37960, 0x3799c,
0x379f0, 0x37ae4,
0x37af8, 0x37b10,
0x37b28, 0x37b28,
0x37b3c, 0x37b50,
0x37bf0, 0x37c10,
0x37c28, 0x37c28,
0x37c3c, 0x37c50,
0x37cf0, 0x37cfc,
0x38000, 0x38030,
0x38100, 0x38144,
0x38190, 0x381d0,
0x38200, 0x38318,
0x38400, 0x3852c,
0x38540, 0x3861c,
0x38800, 0x38834,
0x388c0, 0x38908,
0x38910, 0x389ac,
0x38a00, 0x38a2c,
0x38a44, 0x38a50,
0x38a74, 0x38c24,
0x38d00, 0x38d00,
0x38d08, 0x38d14,
0x38d1c, 0x38d20,
0x38d3c, 0x38d50,
0x39200, 0x3920c,
0x39220, 0x39220,
0x39240, 0x39240,
0x39600, 0x3960c,
0x39a00, 0x39a1c,
0x39e00, 0x39e20,
0x39e38, 0x39e3c,
0x39e80, 0x39e80,
0x39e88, 0x39ea8,
0x39eb0, 0x39eb4,
0x39ec8, 0x39ed4,
0x39fb8, 0x3a004,
0x3a200, 0x3a200,
0x3a208, 0x3a240,
0x3a248, 0x3a280,
0x3a288, 0x3a2c0,
0x3a2c8, 0x3a2fc,
0x3a600, 0x3a630,
0x3aa00, 0x3aabc,
0x3ab00, 0x3ab70,
0x3b000, 0x3b048,
0x3b060, 0x3b09c,
0x3b0f0, 0x3b148,
0x3b160, 0x3b19c,
0x3b1f0, 0x3b2e4,
0x3b2f8, 0x3b3e4,
0x3b3f8, 0x3b448,
0x3b460, 0x3b49c,
0x3b4f0, 0x3b548,
0x3b560, 0x3b59c,
0x3b5f0, 0x3b6e4,
0x3b6f8, 0x3b7e4,
0x3b7f8, 0x3b7fc,
0x3b814, 0x3b814,
0x3b82c, 0x3b82c,
0x3b880, 0x3b88c,
0x3b8e8, 0x3b8ec,
0x3b900, 0x3b948,
0x3b960, 0x3b99c,
0x3b9f0, 0x3bae4,
0x3baf8, 0x3bb10,
0x3bb28, 0x3bb28,
0x3bb3c, 0x3bb50,
0x3bbf0, 0x3bc10,
0x3bc28, 0x3bc28,
0x3bc3c, 0x3bc50,
0x3bcf0, 0x3bcfc,
0x3c000, 0x3c030,
0x3c100, 0x3c144,
0x3c190, 0x3c1d0,
0x3c200, 0x3c318,
0x3c400, 0x3c52c,
0x3c540, 0x3c61c,
0x3c800, 0x3c834,
0x3c8c0, 0x3c908,
0x3c910, 0x3c9ac,
0x3ca00, 0x3ca2c,
0x3ca44, 0x3ca50,
0x3ca74, 0x3cc24,
0x3cd00, 0x3cd00,
0x3cd08, 0x3cd14,
0x3cd1c, 0x3cd20,
0x3cd3c, 0x3cd50,
0x3d200, 0x3d20c,
0x3d220, 0x3d220,
0x3d240, 0x3d240,
0x3d600, 0x3d60c,
0x3da00, 0x3da1c,
0x3de00, 0x3de20,
0x3de38, 0x3de3c,
0x3de80, 0x3de80,
0x3de88, 0x3dea8,
0x3deb0, 0x3deb4,
0x3dec8, 0x3ded4,
0x3dfb8, 0x3e004,
0x3e200, 0x3e200,
0x3e208, 0x3e240,
0x3e248, 0x3e280,
0x3e288, 0x3e2c0,
0x3e2c8, 0x3e2fc,
0x3e600, 0x3e630,
0x3ea00, 0x3eabc,
0x3eb00, 0x3eb70,
0x3f000, 0x3f048,
0x3f060, 0x3f09c,
0x3f0f0, 0x3f148,
0x3f160, 0x3f19c,
0x3f1f0, 0x3f2e4,
0x3f2f8, 0x3f3e4,
0x3f3f8, 0x3f448,
0x3f460, 0x3f49c,
0x3f4f0, 0x3f548,
0x3f560, 0x3f59c,
0x3f5f0, 0x3f6e4,
0x3f6f8, 0x3f7e4,
0x3f7f8, 0x3f7fc,
0x3f814, 0x3f814,
0x3f82c, 0x3f82c,
0x3f880, 0x3f88c,
0x3f8e8, 0x3f8ec,
0x3f900, 0x3f948,
0x3f960, 0x3f99c,
0x3f9f0, 0x3fae4,
0x3faf8, 0x3fb10,
0x3fb28, 0x3fb28,
0x3fb3c, 0x3fb50,
0x3fbf0, 0x3fc10,
0x3fc28, 0x3fc28,
0x3fc3c, 0x3fc50,
0x3fcf0, 0x3fcfc,
0x40000, 0x4000c,
0x40040, 0x40068,
0x4007c, 0x40144,
0x40180, 0x4018c,
0x40200, 0x40298,
0x402ac, 0x4033c,
0x403f8, 0x403fc,
0x41304, 0x413c4,
0x41400, 0x4141c,
0x41480, 0x414d0,
0x44000, 0x44078,
0x440c0, 0x44278,
0x442c0, 0x44478,
0x444c0, 0x44678,
0x446c0, 0x44878,
0x448c0, 0x449fc,
0x45000, 0x45068,
0x45080, 0x45084,
0x450a0, 0x450b0,
0x45200, 0x45268,
0x45280, 0x45284,
0x452a0, 0x452b0,
0x460c0, 0x460e4,
0x47000, 0x4708c,
0x47200, 0x47250,
0x47400, 0x47420,
0x47600, 0x47618,
0x47800, 0x47814,
0x48000, 0x4800c,
0x48040, 0x48068,
0x4807c, 0x48144,
0x48180, 0x4818c,
0x48200, 0x48298,
0x482ac, 0x4833c,
0x483f8, 0x483fc,
0x49304, 0x493c4,
0x49400, 0x4941c,
0x49480, 0x494d0,
0x4c000, 0x4c078,
0x4c0c0, 0x4c278,
0x4c2c0, 0x4c478,
0x4c4c0, 0x4c678,
0x4c6c0, 0x4c878,
0x4c8c0, 0x4c9fc,
0x4d000, 0x4d068,
0x4d080, 0x4d084,
0x4d0a0, 0x4d0b0,
0x4d200, 0x4d268,
0x4d280, 0x4d284,
0x4d2a0, 0x4d2b0,
0x4e0c0, 0x4e0e4,
0x4f000, 0x4f08c,
0x4f200, 0x4f250,
0x4f400, 0x4f420,
0x4f600, 0x4f618,
0x4f800, 0x4f814,
0x50000, 0x500cc,
0x50400, 0x50400,
0x50800, 0x508cc,
0x50c00, 0x50c00,
0x51000, 0x5101c,
0x51300, 0x51308,
};
if (is_t4(sc)) {
reg_ranges = &t4_reg_ranges[0];
n = nitems(t4_reg_ranges);
} else {
reg_ranges = &t5_reg_ranges[0];
n = nitems(t5_reg_ranges);
}
regs->version = chip_id(sc) | chip_rev(sc) << 10;
for (i = 0; i < n; i += 2)
reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
}
static void
cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
{
int i;
u_int v, tnl_cong_drops;
struct timeval tv;
const struct timeval interval = {0, 250000}; /* 250ms */
getmicrotime(&tv);
timevalsub(&tv, &interval);
if (timevalcmp(&tv, &pi->last_refreshed, <))
return;
tnl_cong_drops = 0;
t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
for (i = 0; i < NCHAN; i++) {
if (pi->rx_chan_map & (1 << i)) {
mtx_lock(&sc->regwin_lock);
t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
1, A_TP_MIB_TNL_CNG_DROP_0 + i);
mtx_unlock(&sc->regwin_lock);
tnl_cong_drops += v;
}
}
pi->tnl_cong_drops = tnl_cong_drops;
getmicrotime(&pi->last_refreshed);
}
static void
cxgbe_tick(void *arg)
{
struct port_info *pi = arg;
struct adapter *sc = pi->adapter;
struct ifnet *ifp = pi->ifp;
PORT_LOCK(pi);
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
PORT_UNLOCK(pi);
return; /* without scheduling another callout */
}
cxgbe_refresh_stats(sc, pi);
callout_schedule(&pi->tick, hz);
PORT_UNLOCK(pi);
}
static void
cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
{
struct ifnet *vlan;
if (arg != ifp || ifp->if_type != IFT_ETHER)
return;
vlan = VLAN_DEVAT(ifp, vid);
VLAN_SETCOOKIE(vlan, ifp);
}
static int
cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
#ifdef INVARIANTS
panic("%s: opcode 0x%02x on iq %p with payload %p",
__func__, rss->opcode, iq, m);
#else
log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
__func__, rss->opcode, iq, m);
m_freem(m);
#endif
return (EDOOFUS);
}
int
t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
{
uintptr_t *loc, new;
if (opcode >= nitems(sc->cpl_handler))
return (EINVAL);
new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
loc = (uintptr_t *) &sc->cpl_handler[opcode];
atomic_store_rel_ptr(loc, new);
return (0);
}
static int
an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
{
#ifdef INVARIANTS
panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
#else
log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
__func__, iq, ctrl);
#endif
return (EDOOFUS);
}
int
t4_register_an_handler(struct adapter *sc, an_handler_t h)
{
uintptr_t *loc, new;
new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
loc = (uintptr_t *) &sc->an_handler;
atomic_store_rel_ptr(loc, new);
return (0);
}
static int
fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
{
const struct cpl_fw6_msg *cpl =
__containerof(rpl, struct cpl_fw6_msg, data[0]);
#ifdef INVARIANTS
panic("%s: fw_msg type %d", __func__, cpl->type);
#else
log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
#endif
return (EDOOFUS);
}
int
t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
{
uintptr_t *loc, new;
if (type >= nitems(sc->fw_msg_handler))
return (EINVAL);
/*
* These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
* handler dispatch table. Reject any attempt to install a handler for
* this subtype.
*/
if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
return (EINVAL);
new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
loc = (uintptr_t *) &sc->fw_msg_handler[type];
atomic_store_rel_ptr(loc, new);
return (0);
}
static int
t4_sysctls(struct adapter *sc)
{
struct sysctl_ctx_list *ctx;
struct sysctl_oid *oid;
struct sysctl_oid_list *children, *c0;
static char *caps[] = {
"\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */
"\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */
"\6HASHFILTER\7ETHOFLD",
"\20\1TOE", /* caps[2] toecaps */
"\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */
"\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */
"\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
"\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
"\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */
"\4PO_INITIAOR\5PO_TARGET"
};
static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
ctx = device_get_sysctl_ctx(sc->dev);
/*
* dev.t4nex.X.
*/
oid = device_get_sysctl_tree(sc->dev);
c0 = children = SYSCTL_CHILDREN(oid);
sc->sc_do_rxcopy = 1;
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
&sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
sc->params.nports, "# of ports");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
NULL, chip_rev(sc), "chip hardware revision");
SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
CTLFLAG_RD, sc->fw_version, 0, "firmware version");
SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
sc->cfcsum, "config file checksum");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
sysctl_bitfield, "A", "available doorbells");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
sysctl_bitfield, "A", "available link capabilities");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
sysctl_bitfield, "A", "available NIC capabilities");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
sysctl_bitfield, "A", "available TCP offload capabilities");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
sysctl_bitfield, "A", "available RDMA capabilities");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
sysctl_bitfield, "A", "available iSCSI capabilities");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
sysctl_bitfield, "A", "available FCoE capabilities");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
sc->params.vpd.cclk, "core clock frequency (in KHz)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
sizeof(sc->sge.timer_val), sysctl_int_array, "A",
"interrupt holdoff timer values (us)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
sizeof(sc->sge.counter_val), sysctl_int_array, "A",
"interrupt holdoff packet counter values");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
NULL, sc->tids.nftids, "number of filters");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
"chip temperature (in Celsius)");
t4_sge_sysctls(sc, ctx, children);
sc->lro_timeout = 100;
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
&sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
#ifdef SBUF_DRAIN
/*
* dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload.
*/
oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
CTLFLAG_RD | CTLFLAG_SKIP, NULL,
"logs and miscellaneous information");
children = SYSCTL_CHILDREN(oid);
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_cctrl, "A", "congestion control");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_cim_la, "A", "CIM logic analyzer");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
if (is_t5(sc)) {
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
}
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_cim_qcfg, "A", "CIM queue configuration");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_cpl_stats, "A", "CPL statistics");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_ddp_stats, "A", "non-TCP DDP statistics");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_devlog, "A", "firmware's device log");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_fcoe_stats, "A", "FCoE statistics");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_hw_sched, "A", "hardware scheduler ");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_l2t, "A", "hardware L2 table");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_lb_stats, "A", "loopback statistics");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_meminfo, "A", "memory regions");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_mps_tcam, "A", "MPS TCAM entries");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_path_mtus, "A", "path MTUs");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_pm_stats, "A", "PM statistics");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_rdma_stats, "A", "RDMA statistics");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_tcp_stats, "A", "TCP statistics");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_tids, "A", "TID information");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_tp_err_stats, "A", "TP error statistics");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_tp_la, "A", "TP logic analyzer");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_tx_rate, "A", "Tx rate");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_ulprx_la, "A", "ULPRX logic analyzer");
if (is_t5(sc)) {
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
sysctl_wcwr_stats, "A", "write combined work requests");
}
#endif
#ifdef TCP_OFFLOAD
if (is_offload(sc)) {
/*
* dev.t4nex.X.toe.
*/
oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
NULL, "TOE parameters");
children = SYSCTL_CHILDREN(oid);
sc->tt.sndbuf = 256 * 1024;
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
&sc->tt.sndbuf, 0, "max hardware send buffer size");
sc->tt.ddp = 0;
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
&sc->tt.ddp, 0, "DDP allowed");
sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
&sc->tt.indsz, 0, "DDP max indicate size allowed");
sc->tt.ddp_thres =
G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
&sc->tt.ddp_thres, 0, "DDP threshold");
sc->tt.rx_coalesce = 1;
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
sc->tt.tx_align = 1;
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
}
#endif
return (0);
}
static int
cxgbe_sysctls(struct port_info *pi)
{
struct sysctl_ctx_list *ctx;
struct sysctl_oid *oid;
struct sysctl_oid_list *children;
struct adapter *sc = pi->adapter;
ctx = device_get_sysctl_ctx(pi->dev);
/*
* dev.cxgbe.X.
*/
oid = device_get_sysctl_tree(pi->dev);
children = SYSCTL_CHILDREN(oid);
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
"PHY temperature (in Celsius)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
"PHY firmware version");
}
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
&pi->nrxq, 0, "# of rx queues");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
&pi->ntxq, 0, "# of tx queues");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
&pi->first_rxq, 0, "index of first rx queue");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
&pi->first_txq, 0, "index of first tx queue");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
"Reserve queue 0 for non-flowid packets");
#ifdef TCP_OFFLOAD
if (is_offload(sc)) {
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
&pi->nofldrxq, 0,
"# of rx queues for offloaded TCP connections");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
&pi->nofldtxq, 0,
"# of tx queues for offloaded TCP connections");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
CTLFLAG_RD, &pi->first_ofld_rxq, 0,
"index of first TOE rx queue");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
CTLFLAG_RD, &pi->first_ofld_txq, 0,
"index of first TOE tx queue");
}
#endif
#ifdef DEV_NETMAP
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
&pi->nnmrxq, 0, "# of rx queues for netmap");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
&pi->nnmtxq, 0, "# of tx queues for netmap");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
CTLFLAG_RD, &pi->first_nm_rxq, 0,
"index of first netmap rx queue");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
CTLFLAG_RD, &pi->first_nm_txq, 0,
"index of first netmap tx queue");
#endif
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
"holdoff timer index");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
"holdoff packet counter index");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
"rx queue size");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
"tx queue size");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
"A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
/*
* dev.cxgbe.X.stats.
*/
oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
NULL, "port statistics");
children = SYSCTL_CHILDREN(oid);
SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
&pi->tx_parse_error, 0,
"# of tx packets with invalid length or # of segments");
#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
sysctl_handle_t4_reg64, "QU", desc)
SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
"# of tx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
"# of tx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
"# of tx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
"# of tx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
"# of tx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
"# of tx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
"# of tx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
"# of frames received with bad FCS",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
"# of frames received with length error",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
"# of rx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
"# of rx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
"# of rx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
"# of rx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
"# of rx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
"# of rx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
"# of rx frames in this range",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
#undef SYSCTL_ADD_T4_REG64
#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
&pi->stats.name, desc)
/* We get these from port_stats and they may be stale by upto 1s */
SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
"# drops due to buffer-group 0 overflows");
SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
"# drops due to buffer-group 1 overflows");
SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
"# drops due to buffer-group 2 overflows");
SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
"# drops due to buffer-group 3 overflows");
SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
"# of buffer-group 0 truncated packets");
SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
"# of buffer-group 1 truncated packets");
SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
"# of buffer-group 2 truncated packets");
SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
"# of buffer-group 3 truncated packets");
#undef SYSCTL_ADD_T4_PORTSTAT
return (0);
}
static int
sysctl_int_array(SYSCTL_HANDLER_ARGS)
{
int rc, *i, space = 0;
struct sbuf sb;
sbuf_new_for_sysctl(&sb, NULL, 64, req);
for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
if (space)
sbuf_printf(&sb, " ");
sbuf_printf(&sb, "%d", *i);
space = 1;
}
rc = sbuf_finish(&sb);
sbuf_delete(&sb);
return (rc);
}
static int
sysctl_bitfield(SYSCTL_HANDLER_ARGS)
{
int rc;
struct sbuf *sb;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return(rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
if (sb == NULL)
return (ENOMEM);
sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_btphy(SYSCTL_HANDLER_ARGS)
{
struct port_info *pi = arg1;
int op = arg2;
struct adapter *sc = pi->adapter;
u_int v;
int rc;
rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
if (rc)
return (rc);
/* XXX: magic numbers */
rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
&v);
end_synchronized_op(sc, 0);
if (rc)
return (rc);
if (op == 0)
v /= 256;
rc = sysctl_handle_int(oidp, &v, 0, req);
return (rc);
}
static int
sysctl_noflowq(SYSCTL_HANDLER_ARGS)
{
struct port_info *pi = arg1;
int rc, val;
val = pi->rsrv_noflowq;
rc = sysctl_handle_int(oidp, &val, 0, req);
if (rc != 0 || req->newptr == NULL)
return (rc);
if ((val >= 1) && (pi->ntxq > 1))
pi->rsrv_noflowq = 1;
else
pi->rsrv_noflowq = 0;
return (rc);
}
static int
sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
{
struct port_info *pi = arg1;
struct adapter *sc = pi->adapter;
int idx, rc, i;
struct sge_rxq *rxq;
#ifdef TCP_OFFLOAD
struct sge_ofld_rxq *ofld_rxq;
#endif
uint8_t v;
idx = pi->tmr_idx;
rc = sysctl_handle_int(oidp, &idx, 0, req);
if (rc != 0 || req->newptr == NULL)
return (rc);
if (idx < 0 || idx >= SGE_NTIMERS)
return (EINVAL);
rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
"t4tmr");
if (rc)
return (rc);
v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
for_each_rxq(pi, i, rxq) {
#ifdef atomic_store_rel_8
atomic_store_rel_8(&rxq->iq.intr_params, v);
#else
rxq->iq.intr_params = v;
#endif
}
#ifdef TCP_OFFLOAD
for_each_ofld_rxq(pi, i, ofld_rxq) {
#ifdef atomic_store_rel_8
atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
#else
ofld_rxq->iq.intr_params = v;
#endif
}
#endif
pi->tmr_idx = idx;
end_synchronized_op(sc, LOCK_HELD);
return (0);
}
static int
sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
{
struct port_info *pi = arg1;
struct adapter *sc = pi->adapter;
int idx, rc;
idx = pi->pktc_idx;
rc = sysctl_handle_int(oidp, &idx, 0, req);
if (rc != 0 || req->newptr == NULL)
return (rc);
if (idx < -1 || idx >= SGE_NCOUNTERS)
return (EINVAL);
rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
"t4pktc");
if (rc)
return (rc);
if (pi->flags & PORT_INIT_DONE)
rc = EBUSY; /* cannot be changed once the queues are created */
else
pi->pktc_idx = idx;
end_synchronized_op(sc, LOCK_HELD);
return (rc);
}
static int
sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
{
struct port_info *pi = arg1;
struct adapter *sc = pi->adapter;
int qsize, rc;
qsize = pi->qsize_rxq;
rc = sysctl_handle_int(oidp, &qsize, 0, req);
if (rc != 0 || req->newptr == NULL)
return (rc);
if (qsize < 128 || (qsize & 7))
return (EINVAL);
rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
"t4rxqs");
if (rc)
return (rc);
if (pi->flags & PORT_INIT_DONE)
rc = EBUSY; /* cannot be changed once the queues are created */
else
pi->qsize_rxq = qsize;
end_synchronized_op(sc, LOCK_HELD);
return (rc);
}
static int
sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
{
struct port_info *pi = arg1;
struct adapter *sc = pi->adapter;
int qsize, rc;
qsize = pi->qsize_txq;
rc = sysctl_handle_int(oidp, &qsize, 0, req);
if (rc != 0 || req->newptr == NULL)
return (rc);
if (qsize < 128 || qsize > 65536)
return (EINVAL);
rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
"t4txqs");
if (rc)
return (rc);
if (pi->flags & PORT_INIT_DONE)
rc = EBUSY; /* cannot be changed once the queues are created */
else
pi->qsize_txq = qsize;
end_synchronized_op(sc, LOCK_HELD);
return (rc);
}
static int
sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
{
struct port_info *pi = arg1;
struct adapter *sc = pi->adapter;
struct link_config *lc = &pi->link_cfg;
int rc;
if (req->newptr == NULL) {
struct sbuf *sb;
static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return(rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
if (sb == NULL)
return (ENOMEM);
sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
rc = sbuf_finish(sb);
sbuf_delete(sb);
} else {
char s[2];
int n;
s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
s[1] = 0;
rc = sysctl_handle_string(oidp, s, sizeof(s), req);
if (rc != 0)
return(rc);
if (s[1] != 0)
return (EINVAL);
if (s[0] < '0' || s[0] > '9')
return (EINVAL); /* not a number */
n = s[0] - '0';
if (n & ~(PAUSE_TX | PAUSE_RX))
return (EINVAL); /* some other bit is set too */
rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
if (rc)
return (rc);
if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
int link_ok = lc->link_ok;
lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
lc->requested_fc |= n;
rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
lc->link_ok = link_ok; /* restore */
}
end_synchronized_op(sc, 0);
}
return (rc);
}
static int
sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
int reg = arg2;
uint64_t val;
val = t4_read_reg64(sc, reg);
return (sysctl_handle_64(oidp, &val, 0, req));
}
static int
sysctl_temperature(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
int rc, t;
uint32_t param, val;
rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
if (rc)
return (rc);
param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
end_synchronized_op(sc, 0);
if (rc)
return (rc);
/* unknown is returned as 0 but we display -1 in that case */
t = val == 0 ? -1 : val;
rc = sysctl_handle_int(oidp, &t, 0, req);
return (rc);
}
#ifdef SBUF_DRAIN
static int
sysctl_cctrl(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc, i;
uint16_t incr[NMTUS][NCCTRL_WIN];
static const char *dec_fac[] = {
"0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
"0.9375"
};
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
t4_read_cong_tbl(sc, incr);
for (i = 0; i < NCCTRL_WIN; ++i) {
sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
incr[5][i], incr[6][i], incr[7][i]);
sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
incr[8][i], incr[9][i], incr[10][i], incr[11][i],
incr[12][i], incr[13][i], incr[14][i], incr[15][i],
sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
}
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
"TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */
"ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
"SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */
};
static int
sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc, i, n, qid = arg2;
uint32_t *buf, *p;
char *qtype;
u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
("%s: bad qid %d\n", __func__, qid));
if (qid < CIM_NUM_IBQ) {
/* inbound queue */
qtype = "IBQ";
n = 4 * CIM_IBQ_SIZE;
buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
rc = t4_read_cim_ibq(sc, qid, buf, n);
} else {
/* outbound queue */
qtype = "OBQ";
qid -= CIM_NUM_IBQ;
n = 4 * cim_num_obq * CIM_OBQ_SIZE;
buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
rc = t4_read_cim_obq(sc, qid, buf, n);
}
if (rc < 0) {
rc = -rc;
goto done;
}
n = rc * sizeof(uint32_t); /* rc has # of words actually read */
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
goto done;
sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
if (sb == NULL) {
rc = ENOMEM;
goto done;
}
sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
for (i = 0, p = buf; i < n; i += 16, p += 4)
sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
p[2], p[3]);
rc = sbuf_finish(sb);
sbuf_delete(sb);
done:
free(buf, M_CXGBE);
return (rc);
}
static int
sysctl_cim_la(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
u_int cfg;
struct sbuf *sb;
uint32_t *buf, *p;
int rc;
rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
if (rc != 0)
return (rc);
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
M_ZERO | M_WAITOK);
rc = -t4_cim_read_la(sc, buf, NULL);
if (rc != 0)
goto done;
sbuf_printf(sb, "Status Data PC%s",
cfg & F_UPDBGLACAPTPCONLY ? "" :
" LS0Stat LS0Addr LS0Data");
KASSERT((sc->params.cim_la_size & 7) == 0,
("%s: p will walk off the end of buf", __func__));
for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
if (cfg & F_UPDBGLACAPTPCONLY) {
sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff,
p[6], p[7]);
sbuf_printf(sb, "\n %02x %02x%06x %02x%06x",
(p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
p[4] & 0xff, p[5] >> 8);
sbuf_printf(sb, "\n %02x %x%07x %x%07x",
(p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
p[1] & 0xf, p[2] >> 4);
} else {
sbuf_printf(sb,
"\n %02x %x%07x %x%07x %08x %08x "
"%08x%08x%08x%08x",
(p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
p[6], p[7]);
}
}
rc = sbuf_finish(sb);
sbuf_delete(sb);
done:
free(buf, M_CXGBE);
return (rc);
}
static int
sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
u_int i;
struct sbuf *sb;
uint32_t *buf, *p;
int rc;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
M_ZERO | M_WAITOK);
t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
p = buf;
for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
p[1], p[0]);
}
sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD");
for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u",
(p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
(p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
(p[1] >> 2) | ((p[2] & 3) << 30),
(p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
p[0] & 1);
}
rc = sbuf_finish(sb);
sbuf_delete(sb);
free(buf, M_CXGBE);
return (rc);
}
static int
sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
u_int i;
struct sbuf *sb;
uint32_t *buf, *p;
int rc;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
M_ZERO | M_WAITOK);
t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
p = buf;
sbuf_printf(sb, "Cntl ID DataBE Addr Data");
for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x",
(p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
p[4], p[3], p[2], p[1], p[0]);
}
sbuf_printf(sb, "\n\nCntl ID Data");
for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x",
(p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
}
rc = sbuf_finish(sb);
sbuf_delete(sb);
free(buf, M_CXGBE);
return (rc);
}
static int
sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc, i;
uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
uint16_t thres[CIM_NUM_IBQ];
uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
if (is_t4(sc)) {
cim_num_obq = CIM_NUM_OBQ;
ibq_rdaddr = A_UP_IBQ_0_RDADDR;
obq_rdaddr = A_UP_OBQ_0_REALADDR;
} else {
cim_num_obq = CIM_NUM_OBQ_T5;
ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
}
nq = CIM_NUM_IBQ + cim_num_obq;
rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
if (rc == 0)
rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
if (rc != 0)
return (rc);
t4_read_cimq_cfg(sc, base, size, thres);
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
if (sb == NULL)
return (ENOMEM);
sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail");
for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u",
qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
G_QUEREMFLITS(p[2]) * 16);
for ( ; i < nq; i++, p += 4, wr += 2)
sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i],
base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
G_QUEREMFLITS(p[2]) * 16);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc;
struct tp_cpl_stats stats;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
t4_tp_get_cpl_stats(sc, &stats);
sbuf_printf(sb, " channel 0 channel 1 channel 2 "
"channel 3\n");
sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n",
stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u",
stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc;
struct tp_usm_stats stats;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return(rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
t4_get_usm_stats(sc, &stats);
sbuf_printf(sb, "Frames: %u\n", stats.frames);
sbuf_printf(sb, "Octets: %ju\n", stats.octets);
sbuf_printf(sb, "Drops: %u", stats.drops);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
const char *devlog_level_strings[] = {
[FW_DEVLOG_LEVEL_EMERG] = "EMERG",
[FW_DEVLOG_LEVEL_CRIT] = "CRIT",
[FW_DEVLOG_LEVEL_ERR] = "ERR",
[FW_DEVLOG_LEVEL_NOTICE] = "NOTICE",
[FW_DEVLOG_LEVEL_INFO] = "INFO",
[FW_DEVLOG_LEVEL_DEBUG] = "DEBUG"
};
const char *devlog_facility_strings[] = {
[FW_DEVLOG_FACILITY_CORE] = "CORE",
[FW_DEVLOG_FACILITY_CF] = "CF",
[FW_DEVLOG_FACILITY_SCHED] = "SCHED",
[FW_DEVLOG_FACILITY_TIMER] = "TIMER",
[FW_DEVLOG_FACILITY_RES] = "RES",
[FW_DEVLOG_FACILITY_HW] = "HW",
[FW_DEVLOG_FACILITY_FLR] = "FLR",
[FW_DEVLOG_FACILITY_DMAQ] = "DMAQ",
[FW_DEVLOG_FACILITY_PHY] = "PHY",
[FW_DEVLOG_FACILITY_MAC] = "MAC",
[FW_DEVLOG_FACILITY_PORT] = "PORT",
[FW_DEVLOG_FACILITY_VI] = "VI",
[FW_DEVLOG_FACILITY_FILTER] = "FILTER",
[FW_DEVLOG_FACILITY_ACL] = "ACL",
[FW_DEVLOG_FACILITY_TM] = "TM",
[FW_DEVLOG_FACILITY_QFC] = "QFC",
[FW_DEVLOG_FACILITY_DCB] = "DCB",
[FW_DEVLOG_FACILITY_ETH] = "ETH",
[FW_DEVLOG_FACILITY_OFLD] = "OFLD",
[FW_DEVLOG_FACILITY_RI] = "RI",
[FW_DEVLOG_FACILITY_ISCSI] = "ISCSI",
[FW_DEVLOG_FACILITY_FCOE] = "FCOE",
[FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI",
[FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE"
};
static int
sysctl_devlog(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct devlog_params *dparams = &sc->params.devlog;
struct fw_devlog_e *buf, *e;
int i, j, rc, nentries, first = 0, m;
struct sbuf *sb;
uint64_t ftstamp = UINT64_MAX;
if (dparams->start == 0) {
dparams->memtype = FW_MEMTYPE_EDC0;
dparams->start = 0x84000;
dparams->size = 32768;
}
nentries = dparams->size / sizeof(struct fw_devlog_e);
buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
if (buf == NULL)
return (ENOMEM);
m = fwmtype_to_hwmtype(dparams->memtype);
rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
if (rc != 0)
goto done;
for (i = 0; i < nentries; i++) {
e = &buf[i];
if (e->timestamp == 0)
break; /* end */
e->timestamp = be64toh(e->timestamp);
e->seqno = be32toh(e->seqno);
for (j = 0; j < 8; j++)
e->params[j] = be32toh(e->params[j]);
if (e->timestamp < ftstamp) {
ftstamp = e->timestamp;
first = i;
}
}
if (buf[first].timestamp == 0)
goto done; /* nothing in the log */
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
goto done;
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL) {
rc = ENOMEM;
goto done;
}
sbuf_printf(sb, "%10s %15s %8s %8s %s\n",
"Seq#", "Tstamp", "Level", "Facility", "Message");
i = first;
do {
e = &buf[i];
if (e->timestamp == 0)
break; /* end */
sbuf_printf(sb, "%10d %15ju %8s %8s ",
e->seqno, e->timestamp,
(e->level < nitems(devlog_level_strings) ?
devlog_level_strings[e->level] : "UNKNOWN"),
(e->facility < nitems(devlog_facility_strings) ?
devlog_facility_strings[e->facility] : "UNKNOWN"));
sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
e->params[2], e->params[3], e->params[4],
e->params[5], e->params[6], e->params[7]);
if (++i == nentries)
i = 0;
} while (i != first);
rc = sbuf_finish(sb);
sbuf_delete(sb);
done:
free(buf, M_CXGBE);
return (rc);
}
static int
sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc;
struct tp_fcoe_stats stats[4];
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
t4_get_fcoe_stats(sc, 0, &stats[0]);
t4_get_fcoe_stats(sc, 1, &stats[1]);
t4_get_fcoe_stats(sc, 2, &stats[2]);
t4_get_fcoe_stats(sc, 3, &stats[3]);
sbuf_printf(sb, " channel 0 channel 1 "
"channel 2 channel 3\n");
sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n",
stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
stats[3].octetsDDP);
sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP,
stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
stats[3].framesDrop);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc, i;
unsigned int map, kbps, ipg, mode;
unsigned int pace_tab[NTX_SCHED];
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
t4_read_pace_tbl(sc, pace_tab);
sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) "
"Class IPG (0.1 ns) Flow IPG (us)");
for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
t4_get_tx_sched(sc, i, &kbps, &ipg);
sbuf_printf(sb, "\n %u %-5s %u ", i,
(mode & (1 << i)) ? "flow" : "class", map & 3);
if (kbps)
sbuf_printf(sb, "%9u ", kbps);
else
sbuf_printf(sb, " disabled ");
if (ipg)
sbuf_printf(sb, "%13u ", ipg);
else
sbuf_printf(sb, " disabled ");
if (pace_tab[i])
sbuf_printf(sb, "%10u", pace_tab[i]);
else
sbuf_printf(sb, " disabled");
}
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc, i, j;
uint64_t *p0, *p1;
struct lb_port_stats s[2];
static const char *stat_name[] = {
"OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
"UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
"Frames128To255:", "Frames256To511:", "Frames512To1023:",
"Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
"BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
"BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
"BG2FramesTrunc:", "BG3FramesTrunc:"
};
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
memset(s, 0, sizeof(s));
for (i = 0; i < 4; i += 2) {
t4_get_lb_stats(sc, i, &s[0]);
t4_get_lb_stats(sc, i + 1, &s[1]);
p0 = &s[0].octets;
p1 = &s[1].octets;
sbuf_printf(sb, "%s Loopback %u"
" Loopback %u", i == 0 ? "" : "\n", i, i + 1);
for (j = 0; j < nitems(stat_name); j++)
sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
*p0++, *p1++);
}
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
{
int rc = 0;
struct port_info *pi = arg1;
struct sbuf *sb;
static const char *linkdnreasons[] = {
"non-specific", "remote fault", "autoneg failed", "reserved3",
"PHY overheated", "unknown", "rx los", "reserved7"
};
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return(rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
if (sb == NULL)
return (ENOMEM);
if (pi->linkdnrc < 0)
sbuf_printf(sb, "n/a");
else if (pi->linkdnrc < nitems(linkdnreasons))
sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
else
sbuf_printf(sb, "%d", pi->linkdnrc);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
struct mem_desc {
unsigned int base;
unsigned int limit;
unsigned int idx;
};
static int
mem_desc_cmp(const void *a, const void *b)
{
return ((const struct mem_desc *)a)->base -
((const struct mem_desc *)b)->base;
}
static void
mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
unsigned int to)
{
unsigned int size;
size = to - from + 1;
if (size == 0)
return;
/* XXX: need humanize_number(3) in libkern for a more readable 'size' */
sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
}
static int
sysctl_meminfo(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc, i, n;
uint32_t lo, hi, used, alloc;
static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
static const char *region[] = {
"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
"RQUDP region:", "PBL region:", "TXPBL region:",
"DBVFIFO region:", "ULPRX state:", "ULPTX state:",
"On-chip queues:"
};
struct mem_desc avail[4];
struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */
struct mem_desc *md = mem;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
for (i = 0; i < nitems(mem); i++) {
mem[i].limit = 0;
mem[i].idx = i;
}
/* Find and sort the populated memory ranges */
i = 0;
lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
if (lo & F_EDRAM0_ENABLE) {
hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
avail[i].base = G_EDRAM0_BASE(hi) << 20;
avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
avail[i].idx = 0;
i++;
}
if (lo & F_EDRAM1_ENABLE) {
hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
avail[i].base = G_EDRAM1_BASE(hi) << 20;
avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
avail[i].idx = 1;
i++;
}
if (lo & F_EXT_MEM_ENABLE) {
hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
avail[i].base = G_EXT_MEM_BASE(hi) << 20;
avail[i].limit = avail[i].base +
(G_EXT_MEM_SIZE(hi) << 20);
avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */
i++;
}
if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
avail[i].limit = avail[i].base +
(G_EXT_MEM1_SIZE(hi) << 20);
avail[i].idx = 4;
i++;
}
if (!i) /* no memory available */
return 0;
qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
(md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
(md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
(md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
(md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
(md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
/* the next few have explicit upper bounds */
md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
md->limit = md->base - 1 +
t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
md++;
md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
md->limit = md->base - 1 +
t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
md++;
if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
} else {
md->base = 0;
md->idx = nitems(region); /* hide it */
}
md++;
#define ulp_region(reg) \
md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
(md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
ulp_region(RX_ISCSI);
ulp_region(RX_TDDP);
ulp_region(TX_TPT);
ulp_region(RX_STAG);
ulp_region(RX_RQ);
ulp_region(RX_RQUDP);
ulp_region(RX_PBL);
ulp_region(TX_PBL);
#undef ulp_region
md->base = 0;
md->idx = nitems(region);
if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
A_SGE_DBVFIFO_SIZE))) << 2) - 1;
}
md++;
md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
md->limit = md->base + sc->tids.ntids - 1;
md++;
md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
md->limit = md->base + sc->tids.ntids - 1;
md++;
md->base = sc->vres.ocq.start;
if (sc->vres.ocq.size)
md->limit = md->base + sc->vres.ocq.size - 1;
else
md->idx = nitems(region); /* hide it */
md++;
/* add any address-space holes, there can be up to 3 */
for (n = 0; n < i - 1; n++)
if (avail[n].limit < avail[n + 1].base)
(md++)->base = avail[n].limit;
if (avail[n].limit)
(md++)->base = avail[n].limit;
n = md - mem;
qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
for (lo = 0; lo < i; lo++)
mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
avail[lo].limit - 1);
sbuf_printf(sb, "\n");
for (i = 0; i < n; i++) {
if (mem[i].idx >= nitems(region))
continue; /* skip holes */
if (!mem[i].limit)
mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
mem_region_show(sb, region[mem[i].idx], mem[i].base,
mem[i].limit);
}
sbuf_printf(sb, "\n");
lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
mem_region_show(sb, "uP RAM:", lo, hi);
lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
mem_region_show(sb, "uP Extmem2:", lo, hi);
lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
G_PMRXMAXPAGE(lo),
t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
(lo & F_PMRXNUMCHN) ? 2 : 1);
lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
G_PMTXMAXPAGE(lo),
hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
sbuf_printf(sb, "%u p-structs\n",
t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
for (i = 0; i < 4; i++) {
lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
if (is_t4(sc)) {
used = G_USED(lo);
alloc = G_ALLOC(lo);
} else {
used = G_T5_USED(lo);
alloc = G_T5_ALLOC(lo);
}
sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
i, used, alloc);
}
for (i = 0; i < 4; i++) {
lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
if (is_t4(sc)) {
used = G_USED(lo);
alloc = G_ALLOC(lo);
} else {
used = G_T5_USED(lo);
alloc = G_T5_ALLOC(lo);
}
sbuf_printf(sb,
"\nLoopback %d using %u pages out of %u allocated",
i, used, alloc);
}
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static inline void
tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
{
*mask = x | y;
y = htobe64(y);
memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
}
static int
sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc, i, n;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
sbuf_printf(sb,
"Idx Ethernet address Mask Vld Ports PF"
" VF Replication P0 P1 P2 P3 ML");
n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
for (i = 0; i < n; i++) {
uint64_t tcamx, tcamy, mask;
uint32_t cls_lo, cls_hi;
uint8_t addr[ETHER_ADDR_LEN];
tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
if (tcamx & tcamy)
continue;
tcamxy2valmask(tcamx, tcamy, addr, &mask);
sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
" %c %#x%4u%4d", i, addr[0], addr[1], addr[2],
addr[3], addr[4], addr[5], (uintmax_t)mask,
(cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
G_PORTMAP(cls_hi), G_PF(cls_lo),
(cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
if (cls_lo & F_REPLICATE) {
struct fw_ldst_cmd ldst_cmd;
memset(&ldst_cmd, 0, sizeof(ldst_cmd));
ldst_cmd.op_to_addrspace =
htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
F_FW_CMD_REQUEST | F_FW_CMD_READ |
V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
ldst_cmd.u.mps.fid_ctl =
htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
V_FW_LDST_CMD_CTL(i));
rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
"t4mps");
if (rc)
break;
rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
sizeof(ldst_cmd), &ldst_cmd);
end_synchronized_op(sc, 0);
if (rc != 0) {
sbuf_printf(sb,
" ------------ error %3u ------------", rc);
rc = 0;
} else {
sbuf_printf(sb, " %08x %08x %08x %08x",
be32toh(ldst_cmd.u.mps.rplc127_96),
be32toh(ldst_cmd.u.mps.rplc95_64),
be32toh(ldst_cmd.u.mps.rplc63_32),
be32toh(ldst_cmd.u.mps.rplc31_0));
}
} else
sbuf_printf(sb, "%36s", "");
sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
}
if (rc)
(void) sbuf_finish(sb);
else
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc;
uint16_t mtus[NMTUS];
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
t4_read_mtu_tbl(sc, mtus, NULL);
sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
mtus[14], mtus[15]);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc, i;
uint32_t cnt[PM_NSTATS];
uint64_t cyc[PM_NSTATS];
static const char *rx_stats[] = {
"Read:", "Write bypass:", "Write mem:", "Flush:"
};
static const char *tx_stats[] = {
"Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
};
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
t4_pmtx_get_stats(sc, cnt, cyc);
sbuf_printf(sb, " Tx pcmds Tx bytes");
for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
cyc[i]);
t4_pmrx_get_stats(sc, cnt, cyc);
sbuf_printf(sb, "\n Rx pcmds Rx bytes");
for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
cyc[i]);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc;
struct tp_rdma_stats stats;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
t4_tp_get_rdma_stats(sc, &stats);
sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc;
struct tp_tcp_stats v4, v6;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
t4_tp_get_tcp_stats(sc, &v4, &v6);
sbuf_printf(sb,
" IP IPv6\n");
sbuf_printf(sb, "OutRsts: %20u %20u\n",
v4.tcpOutRsts, v6.tcpOutRsts);
sbuf_printf(sb, "InSegs: %20ju %20ju\n",
v4.tcpInSegs, v6.tcpInSegs);
sbuf_printf(sb, "OutSegs: %20ju %20ju\n",
v4.tcpOutSegs, v6.tcpOutSegs);
sbuf_printf(sb, "RetransSegs: %20ju %20ju",
v4.tcpRetransSegs, v6.tcpRetransSegs);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_tids(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc;
struct tid_info *t = &sc->tids;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
if (t->natids) {
sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
t->atids_in_use);
}
if (t->ntids) {
if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
if (b) {
sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
t->ntids - 1);
} else {
sbuf_printf(sb, "TID range: %u-%u",
t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
t->ntids - 1);
}
} else
sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
sbuf_printf(sb, ", in use: %u\n",
atomic_load_acq_int(&t->tids_in_use));
}
if (t->nstids) {
sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
t->stid_base + t->nstids - 1, t->stids_in_use);
}
if (t->nftids) {
sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
t->ftid_base + t->nftids - 1);
}
if (t->netids) {
sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
t->etid_base + t->netids - 1);
}
sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc;
struct tp_err_stats stats;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
t4_tp_get_err_stats(sc, &stats);
sbuf_printf(sb, " channel 0 channel 1 channel 2 "
"channel 3\n");
sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n",
stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
stats.macInErrs[3]);
sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n",
stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
stats.hdrInErrs[3]);
sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n",
stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
stats.tcpInErrs[3]);
sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n",
stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
stats.tcp6InErrs[3]);
sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n",
stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
stats.tnlCongDrops[3]);
sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n",
stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
stats.tnlTxDrops[3]);
sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n",
stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n",
stats.ofldChanDrops[0], stats.ofldChanDrops[1],
stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u",
stats.ofldNoNeigh, stats.ofldCongDefer);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
struct field_desc {
const char *name;
u_int start;
u_int width;
};
static void
field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
{
char buf[32];
int line_size = 0;
while (f->name) {
uint64_t mask = (1ULL << f->width) - 1;
int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
((uintmax_t)v >> f->start) & mask);
if (line_size + len >= 79) {
line_size = 8;
sbuf_printf(sb, "\n ");
}
sbuf_printf(sb, "%s ", buf);
line_size += len + 1;
f++;
}
sbuf_printf(sb, "\n");
}
static struct field_desc tp_la0[] = {
{ "RcfOpCodeOut", 60, 4 },
{ "State", 56, 4 },
{ "WcfState", 52, 4 },
{ "RcfOpcSrcOut", 50, 2 },
{ "CRxError", 49, 1 },
{ "ERxError", 48, 1 },
{ "SanityFailed", 47, 1 },
{ "SpuriousMsg", 46, 1 },
{ "FlushInputMsg", 45, 1 },
{ "FlushInputCpl", 44, 1 },
{ "RssUpBit", 43, 1 },
{ "RssFilterHit", 42, 1 },
{ "Tid", 32, 10 },
{ "InitTcb", 31, 1 },
{ "LineNumber", 24, 7 },
{ "Emsg", 23, 1 },
{ "EdataOut", 22, 1 },
{ "Cmsg", 21, 1 },
{ "CdataOut", 20, 1 },
{ "EreadPdu", 19, 1 },
{ "CreadPdu", 18, 1 },
{ "TunnelPkt", 17, 1 },
{ "RcfPeerFin", 16, 1 },
{ "RcfReasonOut", 12, 4 },
{ "TxCchannel", 10, 2 },
{ "RcfTxChannel", 8, 2 },
{ "RxEchannel", 6, 2 },
{ "RcfRxChannel", 5, 1 },
{ "RcfDataOutSrdy", 4, 1 },
{ "RxDvld", 3, 1 },
{ "RxOoDvld", 2, 1 },
{ "RxCongestion", 1, 1 },
{ "TxCongestion", 0, 1 },
{ NULL }
};
static struct field_desc tp_la1[] = {
{ "CplCmdIn", 56, 8 },
{ "CplCmdOut", 48, 8 },
{ "ESynOut", 47, 1 },
{ "EAckOut", 46, 1 },
{ "EFinOut", 45, 1 },
{ "ERstOut", 44, 1 },
{ "SynIn", 43, 1 },
{ "AckIn", 42, 1 },
{ "FinIn", 41, 1 },
{ "RstIn", 40, 1 },
{ "DataIn", 39, 1 },
{ "DataInVld", 38, 1 },
{ "PadIn", 37, 1 },
{ "RxBufEmpty", 36, 1 },
{ "RxDdp", 35, 1 },
{ "RxFbCongestion", 34, 1 },
{ "TxFbCongestion", 33, 1 },
{ "TxPktSumSrdy", 32, 1 },
{ "RcfUlpType", 28, 4 },
{ "Eread", 27, 1 },
{ "Ebypass", 26, 1 },
{ "Esave", 25, 1 },
{ "Static0", 24, 1 },
{ "Cread", 23, 1 },
{ "Cbypass", 22, 1 },
{ "Csave", 21, 1 },
{ "CPktOut", 20, 1 },
{ "RxPagePoolFull", 18, 2 },
{ "RxLpbkPkt", 17, 1 },
{ "TxLpbkPkt", 16, 1 },
{ "RxVfValid", 15, 1 },
{ "SynLearned", 14, 1 },
{ "SetDelEntry", 13, 1 },
{ "SetInvEntry", 12, 1 },
{ "CpcmdDvld", 11, 1 },
{ "CpcmdSave", 10, 1 },
{ "RxPstructsFull", 8, 2 },
{ "EpcmdDvld", 7, 1 },
{ "EpcmdFlush", 6, 1 },
{ "EpcmdTrimPrefix", 5, 1 },
{ "EpcmdTrimPostfix", 4, 1 },
{ "ERssIp4Pkt", 3, 1 },
{ "ERssIp6Pkt", 2, 1 },
{ "ERssTcpUdpPkt", 1, 1 },
{ "ERssFceFipPkt", 0, 1 },
{ NULL }
};
static struct field_desc tp_la2[] = {
{ "CplCmdIn", 56, 8 },
{ "MpsVfVld", 55, 1 },
{ "MpsPf", 52, 3 },
{ "MpsVf", 44, 8 },
{ "SynIn", 43, 1 },
{ "AckIn", 42, 1 },
{ "FinIn", 41, 1 },
{ "RstIn", 40, 1 },
{ "DataIn", 39, 1 },
{ "DataInVld", 38, 1 },
{ "PadIn", 37, 1 },
{ "RxBufEmpty", 36, 1 },
{ "RxDdp", 35, 1 },
{ "RxFbCongestion", 34, 1 },
{ "TxFbCongestion", 33, 1 },
{ "TxPktSumSrdy", 32, 1 },
{ "RcfUlpType", 28, 4 },
{ "Eread", 27, 1 },
{ "Ebypass", 26, 1 },
{ "Esave", 25, 1 },
{ "Static0", 24, 1 },
{ "Cread", 23, 1 },
{ "Cbypass", 22, 1 },
{ "Csave", 21, 1 },
{ "CPktOut", 20, 1 },
{ "RxPagePoolFull", 18, 2 },
{ "RxLpbkPkt", 17, 1 },
{ "TxLpbkPkt", 16, 1 },
{ "RxVfValid", 15, 1 },
{ "SynLearned", 14, 1 },
{ "SetDelEntry", 13, 1 },
{ "SetInvEntry", 12, 1 },
{ "CpcmdDvld", 11, 1 },
{ "CpcmdSave", 10, 1 },
{ "RxPstructsFull", 8, 2 },
{ "EpcmdDvld", 7, 1 },
{ "EpcmdFlush", 6, 1 },
{ "EpcmdTrimPrefix", 5, 1 },
{ "EpcmdTrimPostfix", 4, 1 },
{ "ERssIp4Pkt", 3, 1 },
{ "ERssIp6Pkt", 2, 1 },
{ "ERssTcpUdpPkt", 1, 1 },
{ "ERssFceFipPkt", 0, 1 },
{ NULL }
};
static void
tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
{
field_desc_show(sb, *p, tp_la0);
}
static void
tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
{
if (idx)
sbuf_printf(sb, "\n");
field_desc_show(sb, p[0], tp_la0);
if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
field_desc_show(sb, p[1], tp_la0);
}
static void
tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
{
if (idx)
sbuf_printf(sb, "\n");
field_desc_show(sb, p[0], tp_la0);
if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
}
static int
sysctl_tp_la(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
uint64_t *buf, *p;
int rc;
u_int i, inc;
void (*show_func)(struct sbuf *, uint64_t *, int);
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
t4_tp_read_la(sc, buf, NULL);
p = buf;
switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
case 2:
inc = 2;
show_func = tp_la_show2;
break;
case 3:
inc = 2;
show_func = tp_la_show3;
break;
default:
inc = 1;
show_func = tp_la_show;
}
for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
(*show_func)(sb, p, i);
rc = sbuf_finish(sb);
sbuf_delete(sb);
free(buf, M_CXGBE);
return (rc);
}
static int
sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc;
u64 nrate[NCHAN], orate[NCHAN];
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
if (sb == NULL)
return (ENOMEM);
t4_get_chan_txrate(sc, nrate, orate);
sbuf_printf(sb, " channel 0 channel 1 channel 2 "
"channel 3\n");
sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n",
nrate[0], nrate[1], nrate[2], nrate[3]);
sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju",
orate[0], orate[1], orate[2], orate[3]);
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
static int
sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
uint32_t *buf, *p;
int rc, i;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
M_ZERO | M_WAITOK);
t4_ulprx_read_la(sc, buf);
p = buf;
sbuf_printf(sb, " Pcmd Type Message"
" Data");
for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x",
p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
}
rc = sbuf_finish(sb);
sbuf_delete(sb);
free(buf, M_CXGBE);
return (rc);
}
static int
sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
{
struct adapter *sc = arg1;
struct sbuf *sb;
int rc, v;
rc = sysctl_wire_old_buffer(req, 0);
if (rc != 0)
return (rc);
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
if (sb == NULL)
return (ENOMEM);
v = t4_read_reg(sc, A_SGE_STAT_CFG);
if (G_STATSOURCE_T5(v) == 7) {
if (G_STATMODE(v) == 0) {
sbuf_printf(sb, "total %d, incomplete %d",
t4_read_reg(sc, A_SGE_STAT_TOTAL),
t4_read_reg(sc, A_SGE_STAT_MATCH));
} else if (G_STATMODE(v) == 1) {
sbuf_printf(sb, "total %d, data overflow %d",
t4_read_reg(sc, A_SGE_STAT_TOTAL),
t4_read_reg(sc, A_SGE_STAT_MATCH));
}
}
rc = sbuf_finish(sb);
sbuf_delete(sb);
return (rc);
}
#endif
static uint32_t
fconf_to_mode(uint32_t fconf)
{
uint32_t mode;
mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
if (fconf & F_FRAGMENTATION)
mode |= T4_FILTER_IP_FRAGMENT;
if (fconf & F_MPSHITTYPE)
mode |= T4_FILTER_MPS_HIT_TYPE;
if (fconf & F_MACMATCH)
mode |= T4_FILTER_MAC_IDX;
if (fconf & F_ETHERTYPE)
mode |= T4_FILTER_ETH_TYPE;
if (fconf & F_PROTOCOL)
mode |= T4_FILTER_IP_PROTO;
if (fconf & F_TOS)
mode |= T4_FILTER_IP_TOS;
if (fconf & F_VLAN)
mode |= T4_FILTER_VLAN;
if (fconf & F_VNIC_ID)
mode |= T4_FILTER_VNIC;
if (fconf & F_PORT)
mode |= T4_FILTER_PORT;
if (fconf & F_FCOE)
mode |= T4_FILTER_FCoE;
return (mode);
}
static uint32_t
mode_to_fconf(uint32_t mode)
{
uint32_t fconf = 0;
if (mode & T4_FILTER_IP_FRAGMENT)
fconf |= F_FRAGMENTATION;
if (mode & T4_FILTER_MPS_HIT_TYPE)
fconf |= F_MPSHITTYPE;
if (mode & T4_FILTER_MAC_IDX)
fconf |= F_MACMATCH;
if (mode & T4_FILTER_ETH_TYPE)
fconf |= F_ETHERTYPE;
if (mode & T4_FILTER_IP_PROTO)
fconf |= F_PROTOCOL;
if (mode & T4_FILTER_IP_TOS)
fconf |= F_TOS;
if (mode & T4_FILTER_VLAN)
fconf |= F_VLAN;
if (mode & T4_FILTER_VNIC)
fconf |= F_VNIC_ID;
if (mode & T4_FILTER_PORT)
fconf |= F_PORT;
if (mode & T4_FILTER_FCoE)
fconf |= F_FCOE;
return (fconf);
}
static uint32_t
fspec_to_fconf(struct t4_filter_specification *fs)
{
uint32_t fconf = 0;
if (fs->val.frag || fs->mask.frag)
fconf |= F_FRAGMENTATION;
if (fs->val.matchtype || fs->mask.matchtype)
fconf |= F_MPSHITTYPE;
if (fs->val.macidx || fs->mask.macidx)
fconf |= F_MACMATCH;
if (fs->val.ethtype || fs->mask.ethtype)
fconf |= F_ETHERTYPE;
if (fs->val.proto || fs->mask.proto)
fconf |= F_PROTOCOL;
if (fs->val.tos || fs->mask.tos)
fconf |= F_TOS;
if (fs->val.vlan_vld || fs->mask.vlan_vld)
fconf |= F_VLAN;
if (fs->val.vnic_vld || fs->mask.vnic_vld)
fconf |= F_VNIC_ID;
if (fs->val.iport || fs->mask.iport)
fconf |= F_PORT;
if (fs->val.fcoe || fs->mask.fcoe)
fconf |= F_FCOE;
return (fconf);
}
static int
get_filter_mode(struct adapter *sc, uint32_t *mode)
{
int rc;
uint32_t fconf;
rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
"t4getfm");
if (rc)
return (rc);
t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
A_TP_VLAN_PRI_MAP);
if (sc->params.tp.vlan_pri_map != fconf) {
log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
fconf);
}
*mode = fconf_to_mode(fconf);
end_synchronized_op(sc, LOCK_HELD);
return (0);
}
static int
set_filter_mode(struct adapter *sc, uint32_t mode)
{
uint32_t fconf;
int rc;
fconf = mode_to_fconf(mode);
rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
"t4setfm");
if (rc)
return (rc);
if (sc->tids.ftids_in_use > 0) {
rc = EBUSY;
goto done;
}
#ifdef TCP_OFFLOAD
if (uld_active(sc, ULD_TOM)) {
rc = EBUSY;
goto done;
}
#endif
rc = -t4_set_filter_mode(sc, fconf);
done:
end_synchronized_op(sc, LOCK_HELD);
return (rc);
}
static inline uint64_t
get_filter_hits(struct adapter *sc, uint32_t fid)
{
uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
uint64_t hits;
memwin_info(sc, 0, &mw_base, NULL);
off = position_memwin(sc, 0,
tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
if (is_t4(sc)) {
hits = t4_read_reg64(sc, mw_base + off + 16);
hits = be64toh(hits);
} else {
hits = t4_read_reg(sc, mw_base + off + 24);
hits = be32toh(hits);
}
return (hits);
}
static int
get_filter(struct adapter *sc, struct t4_filter *t)
{
int i, rc, nfilters = sc->tids.nftids;
struct filter_entry *f;
rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
"t4getf");
if (rc)
return (rc);
if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
t->idx >= nfilters) {
t->idx = 0xffffffff;
goto done;
}
f = &sc->tids.ftid_tab[t->idx];
for (i = t->idx; i < nfilters; i++, f++) {
if (f->valid) {
t->idx = i;
t->l2tidx = f->l2t ? f->l2t->idx : 0;
t->smtidx = f->smtidx;
if (f->fs.hitcnts)
t->hits = get_filter_hits(sc, t->idx);
else
t->hits = UINT64_MAX;
t->fs = f->fs;
goto done;
}
}
t->idx = 0xffffffff;
done:
end_synchronized_op(sc, LOCK_HELD);
return (0);
}
static int
set_filter(struct adapter *sc, struct t4_filter *t)
{
unsigned int nfilters, nports;
struct filter_entry *f;
int i, rc;
rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
if (rc)
return (rc);
nfilters = sc->tids.nftids;
nports = sc->params.nports;
if (nfilters == 0) {
rc = ENOTSUP;
goto done;
}
if (!(sc->flags & FULL_INIT_DONE)) {
rc = EAGAIN;
goto done;
}
if (t->idx >= nfilters) {
rc = EINVAL;
goto done;
}
/* Validate against the global filter mode */
if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
sc->params.tp.vlan_pri_map) {
rc = E2BIG;
goto done;
}
if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
rc = EINVAL;
goto done;
}
if (t->fs.val.iport >= nports) {
rc = EINVAL;
goto done;
}
/* Can't specify an iq if not steering to it */
if (!t->fs.dirsteer && t->fs.iq) {
rc = EINVAL;
goto done;
}
/* IPv6 filter idx must be 4 aligned */
if (t->fs.type == 1 &&
((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
rc = EINVAL;
goto done;
}
if (sc->tids.ftid_tab == NULL) {
KASSERT(sc->tids.ftids_in_use == 0,
("%s: no memory allocated but filters_in_use > 0",
__func__));
sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
if (sc->tids.ftid_tab == NULL) {
rc = ENOMEM;
goto done;
}
mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
}
for (i = 0; i < 4; i++) {
f = &sc->tids.ftid_tab[t->idx + i];
if (f->pending || f->valid) {
rc = EBUSY;
goto done;
}
if (f->locked) {
rc = EPERM;
goto done;
}
if (t->fs.type == 0)
break;
}
f = &sc->tids.ftid_tab[t->idx];
f->fs = t->fs;
rc = set_filter_wr(sc, t->idx);
done:
end_synchronized_op(sc, 0);
if (rc == 0) {
mtx_lock(&sc->tids.ftid_lock);
for (;;) {
if (f->pending == 0) {
rc = f->valid ? 0 : EIO;
break;
}
if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
PCATCH, "t4setfw", 0)) {
rc = EINPROGRESS;
break;
}
}
mtx_unlock(&sc->tids.ftid_lock);
}
return (rc);
}
static int
del_filter(struct adapter *sc, struct t4_filter *t)
{
unsigned int nfilters;
struct filter_entry *f;
int rc;
rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
if (rc)
return (rc);
nfilters = sc->tids.nftids;
if (nfilters == 0) {
rc = ENOTSUP;
goto done;
}
if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
t->idx >= nfilters) {
rc = EINVAL;
goto done;
}
if (!(sc->flags & FULL_INIT_DONE)) {
rc = EAGAIN;
goto done;
}
f = &sc->tids.ftid_tab[t->idx];
if (f->pending) {
rc = EBUSY;
goto done;
}
if (f->locked) {
rc = EPERM;
goto done;
}
if (f->valid) {
t->fs = f->fs; /* extra info for the caller */
rc = del_filter_wr(sc, t->idx);
}
done:
end_synchronized_op(sc, 0);
if (rc == 0) {
mtx_lock(&sc->tids.ftid_lock);
for (;;) {
if (f->pending == 0) {
rc = f->valid ? EIO : 0;
break;
}
if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
PCATCH, "t4delfw", 0)) {
rc = EINPROGRESS;
break;
}
}
mtx_unlock(&sc->tids.ftid_lock);
}
return (rc);
}
static void
clear_filter(struct filter_entry *f)
{
if (f->l2t)
t4_l2t_release(f->l2t);
bzero(f, sizeof (*f));
}
static int
set_filter_wr(struct adapter *sc, int fidx)
{
struct filter_entry *f = &sc->tids.ftid_tab[fidx];
struct fw_filter_wr *fwr;
unsigned int ftid;
struct wrq_cookie cookie;
ASSERT_SYNCHRONIZED_OP(sc);
if (f->fs.newdmac || f->fs.newvlan) {
/* This filter needs an L2T entry; allocate one. */
f->l2t = t4_l2t_alloc_switching(sc->l2t);
if (f->l2t == NULL)
return (EAGAIN);
if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
f->fs.dmac)) {
t4_l2t_release(f->l2t);
f->l2t = NULL;
return (ENOMEM);
}
}
ftid = sc->tids.ftid_base + fidx;
fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
if (fwr == NULL)
return (ENOMEM);
bzero(fwr, sizeof(*fwr));
fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
fwr->tid_to_iq =
htobe32(V_FW_FILTER_WR_TID(ftid) |
V_FW_FILTER_WR_RQTYPE(f->fs.type) |
V_FW_FILTER_WR_NOREPLY(0) |
V_FW_FILTER_WR_IQ(f->fs.iq));
fwr->del_filter_to_l2tix =
htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
f->fs.newvlan == VLAN_REWRITE) |
V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
f->fs.newvlan == VLAN_REWRITE) |
V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
V_FW_FILTER_WR_PRIO(f->fs.prio) |
V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
fwr->ethtype = htobe16(f->fs.val.ethtype);
fwr->ethtypem = htobe16(f->fs.mask.ethtype);
fwr->frag_to_ovlan_vldm =
(V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
fwr->smac_sel = 0;
fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
fwr->maci_to_matchtypem =
htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
V_FW_FILTER_WR_PORT(f->fs.val.iport) |
V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
fwr->ptcl = f->fs.val.proto;
fwr->ptclm = f->fs.mask.proto;
fwr->ttyp = f->fs.val.tos;
fwr->ttypm = f->fs.mask.tos;
fwr->ivlan = htobe16(f->fs.val.vlan);
fwr->ivlanm = htobe16(f->fs.mask.vlan);
fwr->ovlan = htobe16(f->fs.val.vnic);
fwr->ovlanm = htobe16(f->fs.mask.vnic);
bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
fwr->lp = htobe16(f->fs.val.dport);
fwr->lpm = htobe16(f->fs.mask.dport);
fwr->fp = htobe16(f->fs.val.sport);
fwr->fpm = htobe16(f->fs.mask.sport);
if (f->fs.newsmac)
bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
f->pending = 1;
sc->tids.ftids_in_use++;
commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
return (0);
}
static int
del_filter_wr(struct adapter *sc, int fidx)
{
struct filter_entry *f = &sc->tids.ftid_tab[fidx];
struct fw_filter_wr *fwr;
unsigned int ftid;
struct wrq_cookie cookie;
ftid = sc->tids.ftid_base + fidx;
fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
if (fwr == NULL)
return (ENOMEM);
bzero(fwr, sizeof (*fwr));
t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
f->pending = 1;
commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
return (0);
}
int
t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
unsigned int idx = GET_TID(rpl);
unsigned int rc;
struct filter_entry *f;
KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
rss->opcode));
if (is_ftid(sc, idx)) {
idx -= sc->tids.ftid_base;
f = &sc->tids.ftid_tab[idx];
rc = G_COOKIE(rpl->cookie);
mtx_lock(&sc->tids.ftid_lock);
if (rc == FW_FILTER_WR_FLT_ADDED) {
KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
__func__, idx));
f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
f->pending = 0; /* asynchronous setup completed */
f->valid = 1;
} else {
if (rc != FW_FILTER_WR_FLT_DELETED) {
/* Add or delete failed, display an error */
log(LOG_ERR,
"filter %u setup failed with error %u\n",
idx, rc);
}
clear_filter(f);
sc->tids.ftids_in_use--;
}
wakeup(&sc->tids.ftid_tab);
mtx_unlock(&sc->tids.ftid_lock);
}
return (0);
}
static int
get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
{
int rc;
if (cntxt->cid > M_CTXTQID)
return (EINVAL);
if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
return (EINVAL);
rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
if (rc)
return (rc);
if (sc->flags & FW_OK) {
rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
&cntxt->data[0]);
if (rc == 0)
goto done;
}
/*
* Read via firmware failed or wasn't even attempted. Read directly via
* the backdoor.
*/
rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
done:
end_synchronized_op(sc, 0);
return (rc);
}
static int
load_fw(struct adapter *sc, struct t4_data *fw)
{
int rc;
uint8_t *fw_data;
rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
if (rc)
return (rc);
if (sc->flags & FULL_INIT_DONE) {
rc = EBUSY;
goto done;
}
fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
if (fw_data == NULL) {
rc = ENOMEM;
goto done;
}
rc = copyin(fw->data, fw_data, fw->len);
if (rc == 0)
rc = -t4_load_fw(sc, fw_data, fw->len);
free(fw_data, M_CXGBE);
done:
end_synchronized_op(sc, 0);
return (rc);
}
static int
read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
{
uint32_t addr, off, remaining, i, n;
uint32_t *buf, *b;
uint32_t mw_base, mw_aperture;
int rc;
uint8_t *dst;
rc = validate_mem_range(sc, mr->addr, mr->len);
if (rc != 0)
return (rc);
memwin_info(sc, win, &mw_base, &mw_aperture);
buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
addr = mr->addr;
remaining = mr->len;
dst = (void *)mr->data;
while (remaining) {
off = position_memwin(sc, win, addr);
/* number of bytes that we'll copy in the inner loop */
n = min(remaining, mw_aperture - off);
for (i = 0; i < n; i += 4)
*b++ = t4_read_reg(sc, mw_base + off + i);
rc = copyout(buf, dst, n);
if (rc != 0)
break;
b = buf;
dst += n;
remaining -= n;
addr += n;
}
free(buf, M_CXGBE);
return (rc);
}
static int
read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
{
int rc;
if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
return (EINVAL);
if (i2cd->len > sizeof(i2cd->data))
return (EFBIG);
rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
if (rc)
return (rc);
rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
i2cd->offset, i2cd->len, &i2cd->data[0]);
end_synchronized_op(sc, 0);
return (rc);
}
static int
in_range(int val, int lo, int hi)
{
return (val < 0 || (val <= hi && val >= lo));
}
static int
set_sched_class(struct adapter *sc, struct t4_sched_params *p)
{
int fw_subcmd, fw_type, rc;
rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
if (rc)
return (rc);
if (!(sc->flags & FULL_INIT_DONE)) {
rc = EAGAIN;
goto done;
}
/*
* Translate the cxgbetool parameters into T4 firmware parameters. (The
* sub-command and type are in common locations.)
*/
if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
fw_subcmd = FW_SCHED_SC_CONFIG;
else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
fw_subcmd = FW_SCHED_SC_PARAMS;
else {
rc = EINVAL;
goto done;
}
if (p->type == SCHED_CLASS_TYPE_PACKET)
fw_type = FW_SCHED_TYPE_PKTSCHED;
else {
rc = EINVAL;
goto done;
}
if (fw_subcmd == FW_SCHED_SC_CONFIG) {
/* Vet our parameters ..*/
if (p->u.config.minmax < 0) {
rc = EINVAL;
goto done;
}
/* And pass the request to the firmware ...*/
rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
goto done;
}
if (fw_subcmd == FW_SCHED_SC_PARAMS) {
int fw_level;
int fw_mode;
int fw_rateunit;
int fw_ratemode;
if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
else {
rc = EINVAL;
goto done;
}
if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
else {
rc = EINVAL;
goto done;
}
if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
else {
rc = EINVAL;
goto done;
}
if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
else {
rc = EINVAL;
goto done;
}
/* Vet our parameters ... */
if (!in_range(p->u.params.channel, 0, 3) ||
!in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
!in_range(p->u.params.minrate, 0, 10000000) ||
!in_range(p->u.params.maxrate, 0, 10000000) ||
!in_range(p->u.params.weight, 0, 100)) {
rc = ERANGE;
goto done;
}
/*
* Translate any unset parameters into the firmware's
* nomenclature and/or fail the call if the parameters
* are required ...
*/
if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
p->u.params.channel < 0 || p->u.params.cl < 0) {
rc = EINVAL;
goto done;
}
if (p->u.params.minrate < 0)
p->u.params.minrate = 0;
if (p->u.params.maxrate < 0) {
if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
rc = EINVAL;
goto done;
} else
p->u.params.maxrate = 0;
}
if (p->u.params.weight < 0) {
if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
rc = EINVAL;
goto done;
} else
p->u.params.weight = 0;
}
if (p->u.params.pktsize < 0) {
if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
rc = EINVAL;
goto done;
} else
p->u.params.pktsize = 0;
}
/* See what the firmware thinks of the request ... */
rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
fw_rateunit, fw_ratemode, p->u.params.channel,
p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
p->u.params.weight, p->u.params.pktsize, 1);
goto done;
}
rc = EINVAL;
done:
end_synchronized_op(sc, 0);
return (rc);
}
static int
set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
{
struct port_info *pi = NULL;
struct sge_txq *txq;
uint32_t fw_mnem, fw_queue, fw_class;
int i, rc;
rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
if (rc)
return (rc);
if (!(sc->flags & FULL_INIT_DONE)) {
rc = EAGAIN;
goto done;
}
if (p->port >= sc->params.nports) {
rc = EINVAL;
goto done;
}
pi = sc->port[p->port];
if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
rc = EINVAL;
goto done;
}
/*
* Create a template for the FW_PARAMS_CMD mnemonic and value (TX
* Scheduling Class in this case).
*/
fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
fw_class = p->cl < 0 ? 0xffffffff : p->cl;
/*
* If op.queue is non-negative, then we're only changing the scheduling
* on a single specified TX queue.
*/
if (p->queue >= 0) {
txq = &sc->sge.txq[pi->first_txq + p->queue];
fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
&fw_class);
goto done;
}
/*
* Change the scheduling on all the TX queues for the
* interface.
*/
for_each_txq(pi, i, txq) {
fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
&fw_class);
if (rc)
goto done;
}
rc = 0;
done:
end_synchronized_op(sc, 0);
return (rc);
}
int
t4_os_find_pci_capability(struct adapter *sc, int cap)
{
int i;
return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
}
int
t4_os_pci_save_state(struct adapter *sc)
{
device_t dev;
struct pci_devinfo *dinfo;
dev = sc->dev;
dinfo = device_get_ivars(dev);
pci_cfg_save(dev, dinfo, 0);
return (0);
}
int
t4_os_pci_restore_state(struct adapter *sc)
{
device_t dev;
struct pci_devinfo *dinfo;
dev = sc->dev;
dinfo = device_get_ivars(dev);
pci_cfg_restore(dev, dinfo);
return (0);
}
void
t4_os_portmod_changed(const struct adapter *sc, int idx)
{
struct port_info *pi = sc->port[idx];
static const char *mod_str[] = {
NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
};
build_medialist(pi, &pi->media);
#ifdef DEV_NETMAP
build_medialist(pi, &pi->nm_media);
#endif
if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
if_printf(pi->ifp, "transceiver unplugged.\n");
else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
if_printf(pi->ifp, "unknown transceiver inserted.\n");
else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
if_printf(pi->ifp, "unsupported transceiver inserted.\n");
else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
if_printf(pi->ifp, "%s transceiver inserted.\n",
mod_str[pi->mod_type]);
} else {
if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
pi->mod_type);
}
}
void
t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
{
struct port_info *pi = sc->port[idx];
struct ifnet *ifp = pi->ifp;
if (link_stat) {
pi->linkdnrc = -1;
ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
if_link_state_change(ifp, LINK_STATE_UP);
} else {
if (reason >= 0)
pi->linkdnrc = reason;
if_link_state_change(ifp, LINK_STATE_DOWN);
}
}
void
t4_iterate(void (*func)(struct adapter *, void *), void *arg)
{
struct adapter *sc;
sx_slock(&t4_list_lock);
SLIST_FOREACH(sc, &t4_list, link) {
/*
* func should not make any assumptions about what state sc is
* in - the only guarantee is that sc->sc_lock is a valid lock.
*/
func(sc, arg);
}
sx_sunlock(&t4_list_lock);
}
static int
t4_open(struct cdev *dev, int flags, int type, struct thread *td)
{
return (0);
}
static int
t4_close(struct cdev *dev, int flags, int type, struct thread *td)
{
return (0);
}
static int
t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
struct thread *td)
{
int rc;
struct adapter *sc = dev->si_drv1;
rc = priv_check(td, PRIV_DRIVER);
if (rc != 0)
return (rc);
switch (cmd) {
case CHELSIO_T4_GETREG: {
struct t4_reg *edata = (struct t4_reg *)data;
if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
return (EFAULT);
if (edata->size == 4)
edata->val = t4_read_reg(sc, edata->addr);
else if (edata->size == 8)
edata->val = t4_read_reg64(sc, edata->addr);
else
return (EINVAL);
break;
}
case CHELSIO_T4_SETREG: {
struct t4_reg *edata = (struct t4_reg *)data;
if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
return (EFAULT);
if (edata->size == 4) {
if (edata->val & 0xffffffff00000000)
return (EINVAL);
t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
} else if (edata->size == 8)
t4_write_reg64(sc, edata->addr, edata->val);
else
return (EINVAL);
break;
}
case CHELSIO_T4_REGDUMP: {
struct t4_regdump *regs = (struct t4_regdump *)data;
int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
uint8_t *buf;
if (regs->len < reglen) {
regs->len = reglen; /* hint to the caller */
return (ENOBUFS);
}
regs->len = reglen;
buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
t4_get_regs(sc, regs, buf);
rc = copyout(buf, regs->data, reglen);
free(buf, M_CXGBE);
break;
}
case CHELSIO_T4_GET_FILTER_MODE:
rc = get_filter_mode(sc, (uint32_t *)data);
break;
case CHELSIO_T4_SET_FILTER_MODE:
rc = set_filter_mode(sc, *(uint32_t *)data);
break;
case CHELSIO_T4_GET_FILTER:
rc = get_filter(sc, (struct t4_filter *)data);
break;
case CHELSIO_T4_SET_FILTER:
rc = set_filter(sc, (struct t4_filter *)data);
break;
case CHELSIO_T4_DEL_FILTER:
rc = del_filter(sc, (struct t4_filter *)data);
break;
case CHELSIO_T4_GET_SGE_CONTEXT:
rc = get_sge_context(sc, (struct t4_sge_context *)data);
break;
case CHELSIO_T4_LOAD_FW:
rc = load_fw(sc, (struct t4_data *)data);
break;
case CHELSIO_T4_GET_MEM:
rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
break;
case CHELSIO_T4_GET_I2C:
rc = read_i2c(sc, (struct t4_i2c_data *)data);
break;
case CHELSIO_T4_CLEAR_STATS: {
int i;
u_int port_id = *(uint32_t *)data;
struct port_info *pi;
if (port_id >= sc->params.nports)
return (EINVAL);
pi = sc->port[port_id];
/* MAC stats */
t4_clr_port_stats(sc, pi->tx_chan);
pi->tx_parse_error = 0;
if (pi->flags & PORT_INIT_DONE) {
struct sge_rxq *rxq;
struct sge_txq *txq;
struct sge_wrq *wrq;
for_each_rxq(pi, i, rxq) {
#if defined(INET) || defined(INET6)
rxq->lro.lro_queued = 0;
rxq->lro.lro_flushed = 0;
#endif
rxq->rxcsum = 0;
rxq->vlan_extraction = 0;
}
for_each_txq(pi, i, txq) {
txq->txcsum = 0;
txq->tso_wrs = 0;
txq->vlan_insertion = 0;
txq->imm_wrs = 0;
txq->sgl_wrs = 0;
txq->txpkt_wrs = 0;
txq->txpkts0_wrs = 0;
txq->txpkts1_wrs = 0;
txq->txpkts0_pkts = 0;
txq->txpkts1_pkts = 0;
mp_ring_reset_stats(txq->r);
}
#ifdef TCP_OFFLOAD
/* nothing to clear for each ofld_rxq */
for_each_ofld_txq(pi, i, wrq) {
wrq->tx_wrs_direct = 0;
wrq->tx_wrs_copied = 0;
}
#endif
wrq = &sc->sge.ctrlq[pi->port_id];
wrq->tx_wrs_direct = 0;
wrq->tx_wrs_copied = 0;
}
break;
}
case CHELSIO_T4_SCHED_CLASS:
rc = set_sched_class(sc, (struct t4_sched_params *)data);
break;
case CHELSIO_T4_SCHED_QUEUE:
rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
break;
case CHELSIO_T4_GET_TRACER:
rc = t4_get_tracer(sc, (struct t4_tracer *)data);
break;
case CHELSIO_T4_SET_TRACER:
rc = t4_set_tracer(sc, (struct t4_tracer *)data);
break;
default:
rc = EINVAL;
}
return (rc);
}
#ifdef TCP_OFFLOAD
void
t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
const unsigned int *pgsz_order)
{
struct port_info *pi = ifp->if_softc;
struct adapter *sc = pi->adapter;
t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
V_HPZ3(pgsz_order[3]));
}
static int
toe_capability(struct port_info *pi, int enable)
{
int rc;
struct adapter *sc = pi->adapter;
ASSERT_SYNCHRONIZED_OP(sc);
if (!is_offload(sc))
return (ENODEV);
if (enable) {
/*
* We need the port's queues around so that we're able to send
* and receive CPLs to/from the TOE even if the ifnet for this
* port has never been UP'd administratively.
*/
if (!(pi->flags & PORT_INIT_DONE)) {
rc = cxgbe_init_synchronized(pi);
if (rc)
return (rc);
}
if (isset(&sc->offload_map, pi->port_id))
return (0);
if (!uld_active(sc, ULD_TOM)) {
rc = t4_activate_uld(sc, ULD_TOM);
if (rc == EAGAIN) {
log(LOG_WARNING,
"You must kldload t4_tom.ko before trying "
"to enable TOE on a cxgbe interface.\n");
}
if (rc != 0)
return (rc);
KASSERT(sc->tom_softc != NULL,
("%s: TOM activated but softc NULL", __func__));
KASSERT(uld_active(sc, ULD_TOM),
("%s: TOM activated but flag not set", __func__));
}
/* Activate iWARP and iSCSI too, if the modules are loaded. */
if (!uld_active(sc, ULD_IWARP))
(void) t4_activate_uld(sc, ULD_IWARP);
if (!uld_active(sc, ULD_ISCSI))
(void) t4_activate_uld(sc, ULD_ISCSI);
setbit(&sc->offload_map, pi->port_id);
} else {
if (!isset(&sc->offload_map, pi->port_id))
return (0);
KASSERT(uld_active(sc, ULD_TOM),
("%s: TOM never initialized?", __func__));
clrbit(&sc->offload_map, pi->port_id);
}
return (0);
}
/*
* Add an upper layer driver to the global list.
*/
int
t4_register_uld(struct uld_info *ui)
{
int rc = 0;
struct uld_info *u;
sx_xlock(&t4_uld_list_lock);
SLIST_FOREACH(u, &t4_uld_list, link) {
if (u->uld_id == ui->uld_id) {
rc = EEXIST;
goto done;
}
}
SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
ui->refcount = 0;
done:
sx_xunlock(&t4_uld_list_lock);
return (rc);
}
int
t4_unregister_uld(struct uld_info *ui)
{
int rc = EINVAL;
struct uld_info *u;
sx_xlock(&t4_uld_list_lock);
SLIST_FOREACH(u, &t4_uld_list, link) {
if (u == ui) {
if (ui->refcount > 0) {
rc = EBUSY;
goto done;
}
SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
rc = 0;
goto done;
}
}
done:
sx_xunlock(&t4_uld_list_lock);
return (rc);
}
int
t4_activate_uld(struct adapter *sc, int id)
{
int rc;
struct uld_info *ui;
ASSERT_SYNCHRONIZED_OP(sc);
if (id < 0 || id > ULD_MAX)
return (EINVAL);
rc = EAGAIN; /* kldoad the module with this ULD and try again. */
sx_slock(&t4_uld_list_lock);
SLIST_FOREACH(ui, &t4_uld_list, link) {
if (ui->uld_id == id) {
if (!(sc->flags & FULL_INIT_DONE)) {
rc = adapter_full_init(sc);
if (rc != 0)
break;
}
rc = ui->activate(sc);
if (rc == 0) {
setbit(&sc->active_ulds, id);
ui->refcount++;
}
break;
}
}
sx_sunlock(&t4_uld_list_lock);
return (rc);
}
int
t4_deactivate_uld(struct adapter *sc, int id)
{
int rc;
struct uld_info *ui;
ASSERT_SYNCHRONIZED_OP(sc);
if (id < 0 || id > ULD_MAX)
return (EINVAL);
rc = ENXIO;
sx_slock(&t4_uld_list_lock);
SLIST_FOREACH(ui, &t4_uld_list, link) {
if (ui->uld_id == id) {
rc = ui->deactivate(sc);
if (rc == 0) {
clrbit(&sc->active_ulds, id);
ui->refcount--;
}
break;
}
}
sx_sunlock(&t4_uld_list_lock);
return (rc);
}
int
uld_active(struct adapter *sc, int uld_id)
{
MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
return (isset(&sc->active_ulds, uld_id));
}
#endif
/*
* Come up with reasonable defaults for some of the tunables, provided they're
* not set by the user (in which case we'll use the values as is).
*/
static void
tweak_tunables(void)
{
int nc = mp_ncpus; /* our snapshot of the number of CPUs */
if (t4_ntxq10g < 1)
t4_ntxq10g = min(nc, NTXQ_10G);
if (t4_ntxq1g < 1)
t4_ntxq1g = min(nc, NTXQ_1G);
if (t4_nrxq10g < 1)
t4_nrxq10g = min(nc, NRXQ_10G);
if (t4_nrxq1g < 1)
t4_nrxq1g = min(nc, NRXQ_1G);
#ifdef TCP_OFFLOAD
if (t4_nofldtxq10g < 1)
t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
if (t4_nofldtxq1g < 1)
t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
if (t4_nofldrxq10g < 1)
t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
if (t4_nofldrxq1g < 1)
t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
if (t4_toecaps_allowed == -1)
t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
#else
if (t4_toecaps_allowed == -1)
t4_toecaps_allowed = 0;
#endif
#ifdef DEV_NETMAP
if (t4_nnmtxq10g < 1)
t4_nnmtxq10g = min(nc, NNMTXQ_10G);
if (t4_nnmtxq1g < 1)
t4_nnmtxq1g = min(nc, NNMTXQ_1G);
if (t4_nnmrxq10g < 1)
t4_nnmrxq10g = min(nc, NNMRXQ_10G);
if (t4_nnmrxq1g < 1)
t4_nnmrxq1g = min(nc, NNMRXQ_1G);
#endif
if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
t4_tmr_idx_10g = TMR_IDX_10G;
if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
t4_pktc_idx_10g = PKTC_IDX_10G;
if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
t4_tmr_idx_1g = TMR_IDX_1G;
if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
t4_pktc_idx_1g = PKTC_IDX_1G;
if (t4_qsize_txq < 128)
t4_qsize_txq = 128;
if (t4_qsize_rxq < 128)
t4_qsize_rxq = 128;
while (t4_qsize_rxq & 7)
t4_qsize_rxq++;
t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
}
static struct sx mlu; /* mod load unload */
SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
static int
mod_event(module_t mod, int cmd, void *arg)
{
int rc = 0;
static int loaded = 0;
switch (cmd) {
case MOD_LOAD:
sx_xlock(&mlu);
if (loaded++ == 0) {
t4_sge_modload();
sx_init(&t4_list_lock, "T4/T5 adapters");
SLIST_INIT(&t4_list);
#ifdef TCP_OFFLOAD
sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
SLIST_INIT(&t4_uld_list);
#endif
t4_tracer_modload();
tweak_tunables();
}
sx_xunlock(&mlu);
break;
case MOD_UNLOAD:
sx_xlock(&mlu);
if (--loaded == 0) {
int tries;
sx_slock(&t4_list_lock);
if (!SLIST_EMPTY(&t4_list)) {
rc = EBUSY;
sx_sunlock(&t4_list_lock);
goto done_unload;
}
#ifdef TCP_OFFLOAD
sx_slock(&t4_uld_list_lock);
if (!SLIST_EMPTY(&t4_uld_list)) {
rc = EBUSY;
sx_sunlock(&t4_uld_list_lock);
sx_sunlock(&t4_list_lock);
goto done_unload;
}
#endif
tries = 0;
while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
uprintf("%ju clusters with custom free routine "
"still is use.\n", t4_sge_extfree_refs());
pause("t4unload", 2 * hz);
}
#ifdef TCP_OFFLOAD
sx_sunlock(&t4_uld_list_lock);
#endif
sx_sunlock(&t4_list_lock);
if (t4_sge_extfree_refs() == 0) {
t4_tracer_modunload();
#ifdef TCP_OFFLOAD
sx_destroy(&t4_uld_list_lock);
#endif
sx_destroy(&t4_list_lock);
t4_sge_modunload();
loaded = 0;
} else {
rc = EBUSY;
loaded++; /* undo earlier decrement */
}
}
done_unload:
sx_xunlock(&mlu);
break;
}
return (rc);
}
static devclass_t t4_devclass, t5_devclass;
static devclass_t cxgbe_devclass, cxl_devclass;
DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
MODULE_VERSION(t4nex, 1);
MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
MODULE_VERSION(t5nex, 1);
MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
MODULE_VERSION(cxgbe, 1);
DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
MODULE_VERSION(cxl, 1);
Index: head/sys/dev/dcons/dcons_os.c
===================================================================
--- head/sys/dev/dcons/dcons_os.c (revision 283290)
+++ head/sys/dev/dcons/dcons_os.c (revision 283291)
@@ -1,488 +1,488 @@
/*-
* Copyright (C) 2003,2004
* Hidetoshi Shimokawa. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
*
* This product includes software developed by Hidetoshi Shimokawa.
*
* 4. Neither the name of the author nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include <sys/param.h>
#include <sys/kdb.h>
#include <gdb/gdb.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/cons.h>
#include <sys/consio.h>
#include <sys/tty.h>
#include <sys/malloc.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/ucred.h>
#include <machine/bus.h>
#include <dev/dcons/dcons.h>
#include <dev/dcons/dcons_os.h>
#include <ddb/ddb.h>
#include <sys/reboot.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
#include "opt_dcons.h"
#include "opt_kdb.h"
#include "opt_gdb.h"
#include "opt_ddb.h"
#ifndef DCONS_POLL_HZ
#define DCONS_POLL_HZ 25
#endif
#ifndef DCONS_POLL_IDLE
#define DCONS_POLL_IDLE 256
#endif
#ifndef DCONS_BUF_SIZE
#define DCONS_BUF_SIZE (16*1024)
#endif
#ifndef DCONS_FORCE_CONSOLE
#define DCONS_FORCE_CONSOLE 0 /* Mostly for FreeBSD-4/DragonFly */
#endif
#ifndef KLD_MODULE
static char bssbuf[DCONS_BUF_SIZE]; /* buf in bss */
#endif
/* global data */
static struct dcons_global dg;
struct dcons_global *dcons_conf;
static int poll_hz = DCONS_POLL_HZ;
static u_int poll_idle = DCONS_POLL_HZ * DCONS_POLL_IDLE;
static struct dcons_softc sc[DCONS_NPORT];
static SYSCTL_NODE(_kern, OID_AUTO, dcons, CTLFLAG_RD, 0, "Dumb Console");
SYSCTL_INT(_kern_dcons, OID_AUTO, poll_hz, CTLFLAG_RW, &poll_hz, 0,
"dcons polling rate");
static int drv_init = 0;
static struct callout dcons_callout;
struct dcons_buf *dcons_buf; /* for local dconschat */
static void dcons_timeout(void *);
static int dcons_drv_init(int);
static cn_probe_t dcons_cnprobe;
static cn_init_t dcons_cninit;
static cn_term_t dcons_cnterm;
static cn_getc_t dcons_cngetc;
static cn_putc_t dcons_cnputc;
static cn_grab_t dcons_cngrab;
static cn_ungrab_t dcons_cnungrab;
CONSOLE_DRIVER(dcons);
#if defined(GDB)
static gdb_probe_f dcons_dbg_probe;
static gdb_init_f dcons_dbg_init;
static gdb_term_f dcons_dbg_term;
static gdb_getc_f dcons_dbg_getc;
static gdb_putc_f dcons_dbg_putc;
GDB_DBGPORT(dcons, dcons_dbg_probe, dcons_dbg_init, dcons_dbg_term,
dcons_dbg_getc, dcons_dbg_putc);
extern struct gdb_dbgport *gdb_cur;
#endif
static tsw_outwakeup_t dcons_outwakeup;
static struct ttydevsw dcons_ttydevsw = {
.tsw_flags = TF_NOPREFIX,
.tsw_outwakeup = dcons_outwakeup,
};
#if (defined(GDB) || defined(DDB))
static int
dcons_check_break(struct dcons_softc *dc, int c)
{
if (c < 0)
return (c);
#ifdef GDB
if ((dc->flags & DC_GDB) != 0 && gdb_cur == &dcons_gdb_dbgport)
kdb_alt_break_gdb(c, &dc->brk_state);
else
#endif
kdb_alt_break(c, &dc->brk_state);
return (c);
}
#else
#define dcons_check_break(dc, c) (c)
#endif
static int
dcons_os_checkc_nopoll(struct dcons_softc *dc)
{
int c;
if (dg.dma_tag != NULL)
bus_dmamap_sync(dg.dma_tag, dg.dma_map, BUS_DMASYNC_POSTREAD);
c = dcons_check_break(dc, dcons_checkc(dc));
if (dg.dma_tag != NULL)
bus_dmamap_sync(dg.dma_tag, dg.dma_map, BUS_DMASYNC_PREREAD);
return (c);
}
static int
dcons_os_checkc(struct dcons_softc *dc)
{
EVENTHANDLER_INVOKE(dcons_poll, 0);
return (dcons_os_checkc_nopoll(dc));
}
static void
dcons_os_putc(struct dcons_softc *dc, int c)
{
if (dg.dma_tag != NULL)
bus_dmamap_sync(dg.dma_tag, dg.dma_map, BUS_DMASYNC_POSTWRITE);
dcons_putc(dc, c);
if (dg.dma_tag != NULL)
bus_dmamap_sync(dg.dma_tag, dg.dma_map, BUS_DMASYNC_PREWRITE);
}
static void
dcons_outwakeup(struct tty *tp)
{
struct dcons_softc *dc;
char ch;
dc = tty_softc(tp);
while (ttydisc_getc(tp, &ch, sizeof ch) != 0)
dcons_os_putc(dc, ch);
}
static void
dcons_timeout(void *v)
{
struct tty *tp;
struct dcons_softc *dc;
int i, c, polltime;
for (i = 0; i < DCONS_NPORT; i ++) {
dc = &sc[i];
tp = dc->tty;
tty_lock(tp);
while ((c = dcons_os_checkc_nopoll(dc)) != -1) {
ttydisc_rint(tp, c, 0);
poll_idle = 0;
}
ttydisc_rint_done(tp);
tty_unlock(tp);
}
poll_idle++;
polltime = hz;
if (poll_idle <= (poll_hz * DCONS_POLL_IDLE))
polltime /= poll_hz;
callout_reset(&dcons_callout, polltime, dcons_timeout, tp);
}
static void
dcons_cnprobe(struct consdev *cp)
{
sprintf(cp->cn_name, "dcons");
#if DCONS_FORCE_CONSOLE
cp->cn_pri = CN_REMOTE;
#else
cp->cn_pri = CN_NORMAL;
#endif
}
static void
dcons_cninit(struct consdev *cp)
{
dcons_drv_init(0);
cp->cn_arg = (void *)&sc[DCONS_CON]; /* share port0 with unit0 */
}
static void
dcons_cnterm(struct consdev *cp)
{
}
static void
dcons_cngrab(struct consdev *cp)
{
}
static void
dcons_cnungrab(struct consdev *cp)
{
}
static int
dcons_cngetc(struct consdev *cp)
{
struct dcons_softc *dc = (struct dcons_softc *)cp->cn_arg;
return (dcons_os_checkc(dc));
}
static void
dcons_cnputc(struct consdev *cp, int c)
{
struct dcons_softc *dc = (struct dcons_softc *)cp->cn_arg;
dcons_os_putc(dc, c);
}
static int
dcons_drv_init(int stage)
{
#if defined(__i386__) || defined(__amd64__)
quad_t addr, size;
#endif
if (drv_init)
return(drv_init);
drv_init = -1;
bzero(&dg, sizeof(dg));
dcons_conf = &dg;
dg.cdev = &dcons_consdev;
dg.buf = NULL;
dg.size = DCONS_BUF_SIZE;
#if defined(__i386__) || defined(__amd64__)
if (getenv_quad("dcons.addr", &addr) > 0 &&
getenv_quad("dcons.size", &size) > 0) {
#ifdef __i386__
vm_paddr_t pa;
/*
* Allow read/write access to dcons buffer.
*/
for (pa = trunc_page(addr); pa < addr + size; pa += PAGE_SIZE)
*vtopte(KERNBASE + pa) |= PG_RW;
invltlb();
#endif
/* XXX P to V */
dg.buf = (struct dcons_buf *)(vm_offset_t)(KERNBASE + addr);
dg.size = size;
if (dcons_load_buffer(dg.buf, dg.size, sc) < 0)
dg.buf = NULL;
}
#endif
if (dg.buf != NULL)
goto ok;
#ifndef KLD_MODULE
if (stage == 0) { /* XXX or cold */
/*
* DCONS_FORCE_CONSOLE == 1 and statically linked.
* called from cninit(). can't use contigmalloc yet .
*/
dg.buf = (struct dcons_buf *) bssbuf;
dcons_init(dg.buf, dg.size, sc);
} else
#endif
{
/*
* DCONS_FORCE_CONSOLE == 0 or kernel module case.
* if the module is loaded after boot,
* bssbuf could be non-continuous.
*/
dg.buf = (struct dcons_buf *) contigmalloc(dg.size,
M_DEVBUF, 0, 0x10000, 0xffffffff, PAGE_SIZE, 0ul);
if (dg.buf == NULL)
return (-1);
dcons_init(dg.buf, dg.size, sc);
}
ok:
dcons_buf = dg.buf;
drv_init = 1;
return 0;
}
static int
dcons_attach_port(int port, char *name, int flags)
{
struct dcons_softc *dc;
struct tty *tp;
dc = &sc[port];
tp = tty_alloc(&dcons_ttydevsw, dc);
dc->flags = flags;
dc->tty = tp;
tty_init_console(tp, 0);
tty_makedev(tp, NULL, "%s", name);
return(0);
}
static int
dcons_attach(void)
{
int polltime;
dcons_attach_port(DCONS_CON, "dcons", 0);
dcons_attach_port(DCONS_GDB, "dgdb", DC_GDB);
- callout_init(&dcons_callout, CALLOUT_MPSAFE);
+ callout_init(&dcons_callout, 1);
polltime = hz / poll_hz;
callout_reset(&dcons_callout, polltime, dcons_timeout, NULL);
return(0);
}
static int
dcons_detach(int port)
{
struct tty *tp;
struct dcons_softc *dc;
dc = &sc[port];
tp = dc->tty;
tty_lock(tp);
tty_rel_gone(tp);
return(0);
}
static int
dcons_modevent(module_t mode, int type, void *data)
{
int err = 0, ret;
switch (type) {
case MOD_LOAD:
ret = dcons_drv_init(1);
if (ret != -1)
dcons_attach();
if (ret == 0) {
dcons_cnprobe(&dcons_consdev);
dcons_cninit(&dcons_consdev);
cnadd(&dcons_consdev);
}
break;
case MOD_UNLOAD:
printf("dcons: unload\n");
if (drv_init == 1) {
callout_stop(&dcons_callout);
cnremove(&dcons_consdev);
dcons_detach(DCONS_CON);
dcons_detach(DCONS_GDB);
dg.buf->magic = 0;
contigfree(dg.buf, DCONS_BUF_SIZE, M_DEVBUF);
}
break;
case MOD_SHUTDOWN:
#if 0 /* Keep connection after halt */
dg.buf->magic = 0;
#endif
break;
default:
err = EOPNOTSUPP;
break;
}
return(err);
}
#if defined(GDB)
/* Debugger interface */
static int
dcons_os_getc(struct dcons_softc *dc)
{
int c;
while ((c = dcons_os_checkc(dc)) == -1);
return (c & 0xff);
}
static int
dcons_dbg_probe(void)
{
int dcons_gdb;
if (getenv_int("dcons_gdb", &dcons_gdb) == 0)
return (-1);
return (dcons_gdb);
}
static void
dcons_dbg_init(void)
{
}
static void
dcons_dbg_term(void)
{
}
static void
dcons_dbg_putc(int c)
{
struct dcons_softc *dc = &sc[DCONS_GDB];
dcons_os_putc(dc, c);
}
static int
dcons_dbg_getc(void)
{
struct dcons_softc *dc = &sc[DCONS_GDB];
return (dcons_os_getc(dc));
}
#endif
DEV_MODULE(dcons, dcons_modevent, NULL);
MODULE_VERSION(dcons, DCONS_VERSION);
Index: head/sys/dev/drm2/drm_irq.c
===================================================================
--- head/sys/dev/drm2/drm_irq.c (revision 283290)
+++ head/sys/dev/drm2/drm_irq.c (revision 283291)
@@ -1,1404 +1,1404 @@
/**
* \file drm_irq.c
* IRQ support
*
* \author Rickard E. (Rik) Faith <faith@valinux.com>
* \author Gareth Hughes <gareth@valinux.com>
*/
/*
* Created: Fri Mar 19 14:30:16 1999 by faith@valinux.com
*
* Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
* Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <dev/drm2/drmP.h>
/* Access macro for slots in vblank timestamp ringbuffer. */
#define vblanktimestamp(dev, crtc, count) ( \
(dev)->_vblank_time[(crtc) * DRM_VBLANKTIME_RBSIZE + \
((count) % DRM_VBLANKTIME_RBSIZE)])
/* Retry timestamp calculation up to 3 times to satisfy
* drm_timestamp_precision before giving up.
*/
#define DRM_TIMESTAMP_MAXRETRIES 3
/* Threshold in nanoseconds for detection of redundant
* vblank irq in drm_handle_vblank(). 1 msec should be ok.
*/
#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
/**
* Get interrupt from bus id.
*
* \param inode device inode.
* \param file_priv DRM file private.
* \param cmd command.
* \param arg user argument, pointing to a drm_irq_busid structure.
* \return zero on success or a negative number on failure.
*
* Finds the PCI device with the specified bus id and gets its IRQ number.
* This IOCTL is deprecated, and will now return EINVAL for any busid not equal
* to that of the device that this DRM instance attached to.
*/
int drm_irq_by_busid(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_irq_busid *p = data;
if (!dev->driver->bus->irq_by_busid)
return -EINVAL;
if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
return -EINVAL;
return dev->driver->bus->irq_by_busid(dev, p);
}
/*
* Clear vblank timestamp buffer for a crtc.
*/
static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
{
memset(&dev->_vblank_time[crtc * DRM_VBLANKTIME_RBSIZE], 0,
DRM_VBLANKTIME_RBSIZE * sizeof(struct timeval));
}
/*
* Disable vblank irq's on crtc, make sure that last vblank count
* of hardware and corresponding consistent software vblank counter
* are preserved, even if there are any spurious vblank irq's after
* disable.
*/
static void vblank_disable_and_save(struct drm_device *dev, int crtc)
{
u32 vblcount;
s64 diff_ns;
int vblrc;
struct timeval tvblank;
int count = DRM_TIMESTAMP_MAXRETRIES;
/* Prevent vblank irq processing while disabling vblank irqs,
* so no updates of timestamps or count can happen after we've
* disabled. Needed to prevent races in case of delayed irq's.
*/
mtx_lock(&dev->vblank_time_lock);
dev->driver->disable_vblank(dev, crtc);
dev->vblank_enabled[crtc] = 0;
/* No further vblank irq's will be processed after
* this point. Get current hardware vblank count and
* vblank timestamp, repeat until they are consistent.
*
* FIXME: There is still a race condition here and in
* drm_update_vblank_count() which can cause off-by-one
* reinitialization of software vblank counter. If gpu
* vblank counter doesn't increment exactly at the leading
* edge of a vblank interval, then we can lose 1 count if
* we happen to execute between start of vblank and the
* delayed gpu counter increment.
*/
do {
dev->last_vblank[crtc] = dev->driver->get_vblank_counter(dev, crtc);
vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0);
} while (dev->last_vblank[crtc] != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblrc);
if (!count)
vblrc = 0;
/* Compute time difference to stored timestamp of last vblank
* as updated by last invocation of drm_handle_vblank() in vblank irq.
*/
vblcount = atomic_read(&dev->_vblank_count[crtc]);
diff_ns = timeval_to_ns(&tvblank) -
timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
/* If there is at least 1 msec difference between the last stored
* timestamp and tvblank, then we are currently executing our
* disable inside a new vblank interval, the tvblank timestamp
* corresponds to this new vblank interval and the irq handler
* for this vblank didn't run yet and won't run due to our disable.
* Therefore we need to do the job of drm_handle_vblank() and
* increment the vblank counter by one to account for this vblank.
*
* Skip this step if there isn't any high precision timestamp
* available. In that case we can't account for this and just
* hope for the best.
*/
if ((vblrc > 0) && (abs64(diff_ns) > 1000000)) {
atomic_inc(&dev->_vblank_count[crtc]);
smp_mb__after_atomic_inc();
}
/* Invalidate all timestamps while vblank irq's are off. */
clear_vblank_timestamps(dev, crtc);
mtx_unlock(&dev->vblank_time_lock);
}
static void vblank_disable_fn(void *arg)
{
struct drm_device *dev = (struct drm_device *)arg;
int i;
if (!dev->vblank_disable_allowed)
return;
for (i = 0; i < dev->num_crtcs; i++) {
mtx_lock(&dev->vbl_lock);
if (atomic_read(&dev->vblank_refcount[i]) == 0 &&
dev->vblank_enabled[i]) {
DRM_DEBUG("disabling vblank on crtc %d\n", i);
vblank_disable_and_save(dev, i);
}
mtx_unlock(&dev->vbl_lock);
}
}
void drm_vblank_cleanup(struct drm_device *dev)
{
/* Bail if the driver didn't call drm_vblank_init() */
if (dev->num_crtcs == 0)
return;
callout_stop(&dev->vblank_disable_callout);
vblank_disable_fn(dev);
free(dev->_vblank_count, DRM_MEM_VBLANK);
free(dev->vblank_refcount, DRM_MEM_VBLANK);
free(dev->vblank_enabled, DRM_MEM_VBLANK);
free(dev->last_vblank, DRM_MEM_VBLANK);
free(dev->last_vblank_wait, DRM_MEM_VBLANK);
free(dev->vblank_inmodeset, DRM_MEM_VBLANK);
free(dev->_vblank_time, DRM_MEM_VBLANK);
mtx_destroy(&dev->vbl_lock);
mtx_destroy(&dev->vblank_time_lock);
dev->num_crtcs = 0;
}
EXPORT_SYMBOL(drm_vblank_cleanup);
int drm_vblank_init(struct drm_device *dev, int num_crtcs)
{
int i, ret = -ENOMEM;
- callout_init(&dev->vblank_disable_callout, CALLOUT_MPSAFE);
+ callout_init(&dev->vblank_disable_callout, 1);
mtx_init(&dev->vbl_lock, "drmvbl", NULL, MTX_DEF);
mtx_init(&dev->vblank_time_lock, "drmvtl", NULL, MTX_DEF);
dev->num_crtcs = num_crtcs;
dev->_vblank_count = malloc(sizeof(atomic_t) * num_crtcs,
DRM_MEM_VBLANK, M_NOWAIT);
if (!dev->_vblank_count)
goto err;
dev->vblank_refcount = malloc(sizeof(atomic_t) * num_crtcs,
DRM_MEM_VBLANK, M_NOWAIT);
if (!dev->vblank_refcount)
goto err;
dev->vblank_enabled = malloc(num_crtcs * sizeof(int),
DRM_MEM_VBLANK, M_NOWAIT | M_ZERO);
if (!dev->vblank_enabled)
goto err;
dev->last_vblank = malloc(num_crtcs * sizeof(u32),
DRM_MEM_VBLANK, M_NOWAIT | M_ZERO);
if (!dev->last_vblank)
goto err;
dev->last_vblank_wait = malloc(num_crtcs * sizeof(u32),
DRM_MEM_VBLANK, M_NOWAIT | M_ZERO);
if (!dev->last_vblank_wait)
goto err;
dev->vblank_inmodeset = malloc(num_crtcs * sizeof(int),
DRM_MEM_VBLANK, M_NOWAIT | M_ZERO);
if (!dev->vblank_inmodeset)
goto err;
dev->_vblank_time = malloc(num_crtcs * DRM_VBLANKTIME_RBSIZE *
sizeof(struct timeval), DRM_MEM_VBLANK, M_NOWAIT | M_ZERO);
if (!dev->_vblank_time)
goto err;
DRM_INFO("Supports vblank timestamp caching Rev 1 (10.10.2010).\n");
/* Driver specific high-precision vblank timestamping supported? */
if (dev->driver->get_vblank_timestamp)
DRM_INFO("Driver supports precise vblank timestamp query.\n");
else
DRM_INFO("No driver support for vblank timestamp query.\n");
/* Zero per-crtc vblank stuff */
for (i = 0; i < num_crtcs; i++) {
atomic_set(&dev->_vblank_count[i], 0);
atomic_set(&dev->vblank_refcount[i], 0);
}
dev->vblank_disable_allowed = 0;
return 0;
err:
drm_vblank_cleanup(dev);
return ret;
}
EXPORT_SYMBOL(drm_vblank_init);
/**
* Install IRQ handler.
*
* \param dev DRM device.
*
* Initializes the IRQ related data. Installs the handler, calling the driver
* \c irq_preinstall() and \c irq_postinstall() functions
* before and after the installation.
*/
int drm_irq_install(struct drm_device *dev)
{
int ret;
unsigned long sh_flags = 0;
if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
return -EINVAL;
if (drm_dev_to_irq(dev) == 0)
return -EINVAL;
DRM_LOCK(dev);
/* Driver must have been initialized */
if (!dev->dev_private) {
DRM_UNLOCK(dev);
return -EINVAL;
}
if (dev->irq_enabled) {
DRM_UNLOCK(dev);
return -EBUSY;
}
dev->irq_enabled = 1;
DRM_UNLOCK(dev);
DRM_DEBUG("irq=%d\n", drm_dev_to_irq(dev));
/* Before installing handler */
if (dev->driver->irq_preinstall)
dev->driver->irq_preinstall(dev);
/* Install handler */
sh_flags = INTR_TYPE_TTY | INTR_MPSAFE;
if (!drm_core_check_feature(dev, DRIVER_IRQ_SHARED))
/*
* FIXME Linux<->FreeBSD: This seems to make
* bus_setup_intr() unhappy: it was reported to return
* EINVAL on an i915 board (8086:2592 in a Thinkpad
* X41).
*
* For now, no driver we have use that.
*/
sh_flags |= INTR_EXCL;
ret = -bus_setup_intr(dev->dev, dev->irqr, sh_flags, NULL,
dev->driver->irq_handler, dev, &dev->irqh);
if (ret < 0) {
device_printf(dev->dev, "Error setting interrupt: %d\n", -ret);
DRM_LOCK(dev);
dev->irq_enabled = 0;
DRM_UNLOCK(dev);
return ret;
}
/* After installing handler */
if (dev->driver->irq_postinstall)
ret = dev->driver->irq_postinstall(dev);
if (ret < 0) {
DRM_LOCK(dev);
dev->irq_enabled = 0;
DRM_UNLOCK(dev);
bus_teardown_intr(dev->dev, dev->irqr, dev->irqh);
dev->driver->bus->free_irq(dev);
}
return ret;
}
EXPORT_SYMBOL(drm_irq_install);
/**
* Uninstall the IRQ handler.
*
* \param dev DRM device.
*
* Calls the driver's \c irq_uninstall() function, and stops the irq.
*/
int drm_irq_uninstall(struct drm_device *dev)
{
int irq_enabled, i;
if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
return -EINVAL;
DRM_LOCK(dev);
irq_enabled = dev->irq_enabled;
dev->irq_enabled = 0;
DRM_UNLOCK(dev);
/*
* Wake up any waiters so they don't hang.
*/
if (dev->num_crtcs) {
mtx_lock(&dev->vbl_lock);
for (i = 0; i < dev->num_crtcs; i++) {
DRM_WAKEUP(&dev->_vblank_count[i]);
dev->vblank_enabled[i] = 0;
dev->last_vblank[i] =
dev->driver->get_vblank_counter(dev, i);
}
mtx_unlock(&dev->vbl_lock);
}
if (!irq_enabled)
return -EINVAL;
DRM_DEBUG("irq=%d\n", drm_dev_to_irq(dev));
if (dev->driver->irq_uninstall)
dev->driver->irq_uninstall(dev);
bus_teardown_intr(dev->dev, dev->irqr, dev->irqh);
dev->driver->bus->free_irq(dev);
return 0;
}
EXPORT_SYMBOL(drm_irq_uninstall);
/**
* IRQ control ioctl.
*
* \param inode device inode.
* \param file_priv DRM file private.
* \param cmd command.
* \param arg user argument, pointing to a drm_control structure.
* \return zero on success or a negative number on failure.
*
* Calls irq_install() or irq_uninstall() according to \p arg.
*/
int drm_control(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_control *ctl = data;
/* if we haven't irq we fallback for compatibility reasons -
* this used to be a separate function in drm_dma.h
*/
switch (ctl->func) {
case DRM_INST_HANDLER:
if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
return 0;
if (drm_core_check_feature(dev, DRIVER_MODESET))
return 0;
if (dev->if_version < DRM_IF_VERSION(1, 2) &&
ctl->irq != drm_dev_to_irq(dev))
return -EINVAL;
return drm_irq_install(dev);
case DRM_UNINST_HANDLER:
if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
return 0;
if (drm_core_check_feature(dev, DRIVER_MODESET))
return 0;
return drm_irq_uninstall(dev);
default:
return -EINVAL;
}
}
/**
* drm_calc_timestamping_constants - Calculate and
* store various constants which are later needed by
* vblank and swap-completion timestamping, e.g, by
* drm_calc_vbltimestamp_from_scanoutpos().
* They are derived from crtc's true scanout timing,
* so they take things like panel scaling or other
* adjustments into account.
*
* @crtc drm_crtc whose timestamp constants should be updated.
*
*/
void drm_calc_timestamping_constants(struct drm_crtc *crtc)
{
s64 linedur_ns = 0, pixeldur_ns = 0, framedur_ns = 0;
u64 dotclock;
/* Dot clock in Hz: */
dotclock = (u64) crtc->hwmode.clock * 1000;
/* Fields of interlaced scanout modes are only halve a frame duration.
* Double the dotclock to get halve the frame-/line-/pixelduration.
*/
if (crtc->hwmode.flags & DRM_MODE_FLAG_INTERLACE)
dotclock *= 2;
/* Valid dotclock? */
if (dotclock > 0) {
/* Convert scanline length in pixels and video dot clock to
* line duration, frame duration and pixel duration in
* nanoseconds:
*/
pixeldur_ns = (s64) div64_u64(1000000000, dotclock);
linedur_ns = (s64) div64_u64(((u64) crtc->hwmode.crtc_htotal *
1000000000), dotclock);
framedur_ns = (s64) crtc->hwmode.crtc_vtotal * linedur_ns;
} else
DRM_ERROR("crtc %d: Can't calculate constants, dotclock = 0!\n",
crtc->base.id);
crtc->pixeldur_ns = pixeldur_ns;
crtc->linedur_ns = linedur_ns;
crtc->framedur_ns = framedur_ns;
DRM_DEBUG("crtc %d: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
crtc->base.id, crtc->hwmode.crtc_htotal,
crtc->hwmode.crtc_vtotal, crtc->hwmode.crtc_vdisplay);
DRM_DEBUG("crtc %d: clock %d kHz framedur %d linedur %d, pixeldur %d\n",
crtc->base.id, (int) dotclock/1000, (int) framedur_ns,
(int) linedur_ns, (int) pixeldur_ns);
}
EXPORT_SYMBOL(drm_calc_timestamping_constants);
/**
* drm_calc_vbltimestamp_from_scanoutpos - helper routine for kms
* drivers. Implements calculation of exact vblank timestamps from
* given drm_display_mode timings and current video scanout position
* of a crtc. This can be called from within get_vblank_timestamp()
* implementation of a kms driver to implement the actual timestamping.
*
* Should return timestamps conforming to the OML_sync_control OpenML
* extension specification. The timestamp corresponds to the end of
* the vblank interval, aka start of scanout of topmost-leftmost display
* pixel in the following video frame.
*
* Requires support for optional dev->driver->get_scanout_position()
* in kms driver, plus a bit of setup code to provide a drm_display_mode
* that corresponds to the true scanout timing.
*
* The current implementation only handles standard video modes. It
* returns as no operation if a doublescan or interlaced video mode is
* active. Higher level code is expected to handle this.
*
* @dev: DRM device.
* @crtc: Which crtc's vblank timestamp to retrieve.
* @max_error: Desired maximum allowable error in timestamps (nanosecs).
* On return contains true maximum error of timestamp.
* @vblank_time: Pointer to struct timeval which should receive the timestamp.
* @flags: Flags to pass to driver:
* 0 = Default.
* DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler.
* @refcrtc: drm_crtc* of crtc which defines scanout timing.
*
* Returns negative value on error, failure or if not supported in current
* video mode:
*
* -EINVAL - Invalid crtc.
* -EAGAIN - Temporary unavailable, e.g., called before initial modeset.
* -ENOTSUPP - Function not supported in current display mode.
* -EIO - Failed, e.g., due to failed scanout position query.
*
* Returns or'ed positive status flags on success:
*
* DRM_VBLANKTIME_SCANOUTPOS_METHOD - Signal this method used for timestamping.
* DRM_VBLANKTIME_INVBL - Timestamp taken while scanout was in vblank interval.
*
*/
int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
int *max_error,
struct timeval *vblank_time,
unsigned flags,
struct drm_crtc *refcrtc)
{
struct timeval stime, raw_time;
struct drm_display_mode *mode;
int vbl_status, vtotal, vdisplay;
int vpos, hpos, i;
s64 framedur_ns, linedur_ns, pixeldur_ns, delta_ns, duration_ns;
bool invbl;
if (crtc < 0 || crtc >= dev->num_crtcs) {
DRM_ERROR("Invalid crtc %d\n", crtc);
return -EINVAL;
}
/* Scanout position query not supported? Should not happen. */
if (!dev->driver->get_scanout_position) {
DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
return -EIO;
}
mode = &refcrtc->hwmode;
vtotal = mode->crtc_vtotal;
vdisplay = mode->crtc_vdisplay;
/* Durations of frames, lines, pixels in nanoseconds. */
framedur_ns = refcrtc->framedur_ns;
linedur_ns = refcrtc->linedur_ns;
pixeldur_ns = refcrtc->pixeldur_ns;
/* If mode timing undefined, just return as no-op:
* Happens during initial modesetting of a crtc.
*/
if (vtotal <= 0 || vdisplay <= 0 || framedur_ns == 0) {
DRM_DEBUG("crtc %d: Noop due to uninitialized mode.\n", crtc);
return -EAGAIN;
}
/* Get current scanout position with system timestamp.
* Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
* if single query takes longer than max_error nanoseconds.
*
* This guarantees a tight bound on maximum error if
* code gets preempted or delayed for some reason.
*/
for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
/* Disable preemption to make it very likely to
* succeed in the first iteration even on PREEMPT_RT kernel.
*/
critical_enter();
/* Get system timestamp before query. */
getmicrouptime(&stime);
/* Get vertical and horizontal scanout pos. vpos, hpos. */
vbl_status = dev->driver->get_scanout_position(dev, crtc, &vpos, &hpos);
/* Get system timestamp after query. */
getmicrouptime(&raw_time);
#ifdef FREEBSD_NOTYET
if (!drm_timestamp_monotonic)
mono_time_offset = ktime_get_monotonic_offset();
#endif /* FREEBSD_NOTYET */
critical_exit();
/* Return as no-op if scanout query unsupported or failed. */
if (!(vbl_status & DRM_SCANOUTPOS_VALID)) {
DRM_DEBUG("crtc %d : scanoutpos query failed [%d].\n",
crtc, vbl_status);
return -EIO;
}
duration_ns = timeval_to_ns(&raw_time) - timeval_to_ns(&stime);
/* Accept result with < max_error nsecs timing uncertainty. */
if (duration_ns <= (s64) *max_error)
break;
}
/* Noisy system timing? */
if (i == DRM_TIMESTAMP_MAXRETRIES) {
DRM_DEBUG("crtc %d: Noisy timestamp %d us > %d us [%d reps].\n",
crtc, (int) duration_ns/1000, *max_error/1000, i);
}
/* Return upper bound of timestamp precision error. */
*max_error = (int) duration_ns;
/* Check if in vblank area:
* vpos is >=0 in video scanout area, but negative
* within vblank area, counting down the number of lines until
* start of scanout.
*/
invbl = vbl_status & DRM_SCANOUTPOS_INVBL;
/* Convert scanout position into elapsed time at raw_time query
* since start of scanout at first display scanline. delta_ns
* can be negative if start of scanout hasn't happened yet.
*/
delta_ns = (s64) vpos * linedur_ns + (s64) hpos * pixeldur_ns;
/* Is vpos outside nominal vblank area, but less than
* 1/100 of a frame height away from start of vblank?
* If so, assume this isn't a massively delayed vblank
* interrupt, but a vblank interrupt that fired a few
* microseconds before true start of vblank. Compensate
* by adding a full frame duration to the final timestamp.
* Happens, e.g., on ATI R500, R600.
*
* We only do this if DRM_CALLED_FROM_VBLIRQ.
*/
if ((flags & DRM_CALLED_FROM_VBLIRQ) && !invbl &&
((vdisplay - vpos) < vtotal / 100)) {
delta_ns = delta_ns - framedur_ns;
/* Signal this correction as "applied". */
vbl_status |= 0x8;
}
#ifdef FREEBSD_NOTYET
if (!drm_timestamp_monotonic)
etime = ktime_sub(etime, mono_time_offset);
/* save this only for debugging purposes */
tv_etime = ktime_to_timeval(etime);
#endif /* FREEBSD_NOTYET */
/* Subtract time delta from raw timestamp to get final
* vblank_time timestamp for end of vblank.
*/
*vblank_time = ns_to_timeval(timeval_to_ns(&raw_time) - delta_ns);
DRM_DEBUG("crtc %d : v %d p(%d,%d)@ %jd.%jd -> %jd.%jd [e %d us, %d rep]\n",
crtc, (int)vbl_status, hpos, vpos, (uintmax_t)raw_time.tv_sec,
(uintmax_t)raw_time.tv_usec, (uintmax_t)vblank_time->tv_sec,
(uintmax_t)vblank_time->tv_usec, (int)duration_ns/1000, i);
vbl_status = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
if (invbl)
vbl_status |= DRM_VBLANKTIME_INVBL;
return vbl_status;
}
EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
static struct timeval get_drm_timestamp(void)
{
struct timeval now;
microtime(&now);
#ifdef FREEBSD_NOTYET
if (!drm_timestamp_monotonic)
now = ktime_sub(now, ktime_get_monotonic_offset());
#endif /* defined(FREEBSD_NOTYET) */
return now;
}
/**
* drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
* vblank interval.
*
* @dev: DRM device
* @crtc: which crtc's vblank timestamp to retrieve
* @tvblank: Pointer to target struct timeval which should receive the timestamp
* @flags: Flags to pass to driver:
* 0 = Default.
* DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler.
*
* Fetches the system timestamp corresponding to the time of the most recent
* vblank interval on specified crtc. May call into kms-driver to
* compute the timestamp with a high-precision GPU specific method.
*
* Returns zero if timestamp originates from uncorrected do_gettimeofday()
* call, i.e., it isn't very precisely locked to the true vblank.
*
* Returns non-zero if timestamp is considered to be very precise.
*/
u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
struct timeval *tvblank, unsigned flags)
{
int ret;
/* Define requested maximum error on timestamps (nanoseconds). */
int max_error = (int) drm_timestamp_precision * 1000;
/* Query driver if possible and precision timestamping enabled. */
if (dev->driver->get_vblank_timestamp && (max_error > 0)) {
ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error,
tvblank, flags);
if (ret > 0)
return (u32) ret;
}
/* GPU high precision timestamp query unsupported or failed.
* Return current monotonic/gettimeofday timestamp as best estimate.
*/
*tvblank = get_drm_timestamp();
return 0;
}
EXPORT_SYMBOL(drm_get_last_vbltimestamp);
/**
* drm_vblank_count - retrieve "cooked" vblank counter value
* @dev: DRM device
* @crtc: which counter to retrieve
*
* Fetches the "cooked" vblank count value that represents the number of
* vblank events since the system was booted, including lost events due to
* modesetting activity.
*/
u32 drm_vblank_count(struct drm_device *dev, int crtc)
{
return atomic_read(&dev->_vblank_count[crtc]);
}
EXPORT_SYMBOL(drm_vblank_count);
/**
* drm_vblank_count_and_time - retrieve "cooked" vblank counter value
* and the system timestamp corresponding to that vblank counter value.
*
* @dev: DRM device
* @crtc: which counter to retrieve
* @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
*
* Fetches the "cooked" vblank count value that represents the number of
* vblank events since the system was booted, including lost events due to
* modesetting activity. Returns corresponding system timestamp of the time
* of the vblank interval that corresponds to the current value vblank counter
* value.
*/
u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
struct timeval *vblanktime)
{
u32 cur_vblank;
/* Read timestamp from slot of _vblank_time ringbuffer
* that corresponds to current vblank count. Retry if
* count has incremented during readout. This works like
* a seqlock.
*/
do {
cur_vblank = atomic_read(&dev->_vblank_count[crtc]);
*vblanktime = vblanktimestamp(dev, crtc, cur_vblank);
smp_rmb();
} while (cur_vblank != atomic_read(&dev->_vblank_count[crtc]));
return cur_vblank;
}
EXPORT_SYMBOL(drm_vblank_count_and_time);
static void send_vblank_event(struct drm_device *dev,
struct drm_pending_vblank_event *e,
unsigned long seq, struct timeval *now)
{
WARN_ON_SMP(!mtx_owned(&dev->event_lock));
e->event.sequence = seq;
e->event.tv_sec = now->tv_sec;
e->event.tv_usec = now->tv_usec;
list_add_tail(&e->base.link,
&e->base.file_priv->event_list);
drm_event_wakeup(&e->base);
CTR3(KTR_DRM, "vblank_event_delivered %d %d %d",
e->base.pid, e->pipe, e->event.sequence);
}
/**
* drm_send_vblank_event - helper to send vblank event after pageflip
* @dev: DRM device
* @crtc: CRTC in question
* @e: the event to send
*
* Updates sequence # and timestamp on event, and sends it to userspace.
* Caller must hold event lock.
*/
void drm_send_vblank_event(struct drm_device *dev, int crtc,
struct drm_pending_vblank_event *e)
{
struct timeval now;
unsigned int seq;
if (crtc >= 0) {
seq = drm_vblank_count_and_time(dev, crtc, &now);
} else {
seq = 0;
now = get_drm_timestamp();
}
send_vblank_event(dev, e, seq, &now);
}
EXPORT_SYMBOL(drm_send_vblank_event);
/**
* drm_update_vblank_count - update the master vblank counter
* @dev: DRM device
* @crtc: counter to update
*
* Call back into the driver to update the appropriate vblank counter
* (specified by @crtc). Deal with wraparound, if it occurred, and
* update the last read value so we can deal with wraparound on the next
* call if necessary.
*
* Only necessary when going from off->on, to account for frames we
* didn't get an interrupt for.
*
* Note: caller must hold dev->vbl_lock since this reads & writes
* device vblank fields.
*/
static void drm_update_vblank_count(struct drm_device *dev, int crtc)
{
u32 cur_vblank, diff, tslot, rc;
struct timeval t_vblank;
/*
* Interrupts were disabled prior to this call, so deal with counter
* wrap if needed.
* NOTE! It's possible we lost a full dev->max_vblank_count events
* here if the register is small or we had vblank interrupts off for
* a long time.
*
* We repeat the hardware vblank counter & timestamp query until
* we get consistent results. This to prevent races between gpu
* updating its hardware counter while we are retrieving the
* corresponding vblank timestamp.
*/
do {
cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
} while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
/* Deal with counter wrap */
diff = cur_vblank - dev->last_vblank[crtc];
if (cur_vblank < dev->last_vblank[crtc]) {
diff += dev->max_vblank_count;
DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
crtc, dev->last_vblank[crtc], cur_vblank, diff);
}
DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n",
crtc, diff);
/* Reinitialize corresponding vblank timestamp if high-precision query
* available. Skip this step if query unsupported or failed. Will
* reinitialize delayed at next vblank interrupt in that case.
*/
if (rc) {
tslot = atomic_read(&dev->_vblank_count[crtc]) + diff;
vblanktimestamp(dev, crtc, tslot) = t_vblank;
}
smp_mb__before_atomic_inc();
atomic_add(diff, &dev->_vblank_count[crtc]);
smp_mb__after_atomic_inc();
}
/**
* drm_vblank_get - get a reference count on vblank events
* @dev: DRM device
* @crtc: which CRTC to own
*
* Acquire a reference count on vblank events to avoid having them disabled
* while in use.
*
* RETURNS
* Zero on success, nonzero on failure.
*/
int drm_vblank_get(struct drm_device *dev, int crtc)
{
int ret = 0;
mtx_lock(&dev->vbl_lock);
/* Going from 0->1 means we have to enable interrupts again */
if (atomic_add_return(1, &dev->vblank_refcount[crtc]) == 1) {
mtx_lock(&dev->vblank_time_lock);
if (!dev->vblank_enabled[crtc]) {
/* Enable vblank irqs under vblank_time_lock protection.
* All vblank count & timestamp updates are held off
* until we are done reinitializing master counter and
* timestamps. Filtercode in drm_handle_vblank() will
* prevent double-accounting of same vblank interval.
*/
ret = dev->driver->enable_vblank(dev, crtc);
DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n",
crtc, ret);
if (ret)
atomic_dec(&dev->vblank_refcount[crtc]);
else {
dev->vblank_enabled[crtc] = 1;
drm_update_vblank_count(dev, crtc);
}
}
mtx_unlock(&dev->vblank_time_lock);
} else {
if (!dev->vblank_enabled[crtc]) {
atomic_dec(&dev->vblank_refcount[crtc]);
ret = -EINVAL;
}
}
mtx_unlock(&dev->vbl_lock);
return ret;
}
EXPORT_SYMBOL(drm_vblank_get);
/**
* drm_vblank_put - give up ownership of vblank events
* @dev: DRM device
* @crtc: which counter to give up
*
* Release ownership of a given vblank counter, turning off interrupts
* if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
*/
void drm_vblank_put(struct drm_device *dev, int crtc)
{
BUG_ON(atomic_read(&dev->vblank_refcount[crtc]) == 0);
/* Last user schedules interrupt disable */
if (atomic_dec_and_test(&dev->vblank_refcount[crtc]) &&
(drm_vblank_offdelay > 0))
callout_reset(&dev->vblank_disable_callout,
(drm_vblank_offdelay * DRM_HZ) / 1000,
vblank_disable_fn, dev);
}
EXPORT_SYMBOL(drm_vblank_put);
/**
* drm_vblank_off - disable vblank events on a CRTC
* @dev: DRM device
* @crtc: CRTC in question
*
* Caller must hold event lock.
*/
void drm_vblank_off(struct drm_device *dev, int crtc)
{
struct drm_pending_vblank_event *e, *t;
struct timeval now;
unsigned int seq;
mtx_lock(&dev->vbl_lock);
vblank_disable_and_save(dev, crtc);
DRM_WAKEUP(&dev->_vblank_count[crtc]);
/* Send any queued vblank events, lest the natives grow disquiet */
seq = drm_vblank_count_and_time(dev, crtc, &now);
mtx_lock(&dev->event_lock);
list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
if (e->pipe != crtc)
continue;
DRM_DEBUG("Sending premature vblank event on disable: \
wanted %d, current %d\n",
e->event.sequence, seq);
list_del(&e->base.link);
drm_vblank_put(dev, e->pipe);
send_vblank_event(dev, e, seq, &now);
}
mtx_unlock(&dev->event_lock);
mtx_unlock(&dev->vbl_lock);
}
EXPORT_SYMBOL(drm_vblank_off);
/**
* drm_vblank_pre_modeset - account for vblanks across mode sets
* @dev: DRM device
* @crtc: CRTC in question
*
* Account for vblank events across mode setting events, which will likely
* reset the hardware frame counter.
*/
void drm_vblank_pre_modeset(struct drm_device *dev, int crtc)
{
/* vblank is not initialized (IRQ not installed ?), or has been freed */
if (!dev->num_crtcs)
return;
/*
* To avoid all the problems that might happen if interrupts
* were enabled/disabled around or between these calls, we just
* have the kernel take a reference on the CRTC (just once though
* to avoid corrupting the count if multiple, mismatch calls occur),
* so that interrupts remain enabled in the interim.
*/
if (!dev->vblank_inmodeset[crtc]) {
dev->vblank_inmodeset[crtc] = 0x1;
if (drm_vblank_get(dev, crtc) == 0)
dev->vblank_inmodeset[crtc] |= 0x2;
}
}
EXPORT_SYMBOL(drm_vblank_pre_modeset);
void drm_vblank_post_modeset(struct drm_device *dev, int crtc)
{
/* vblank is not initialized (IRQ not installed ?), or has been freed */
if (!dev->num_crtcs)
return;
if (dev->vblank_inmodeset[crtc]) {
mtx_lock(&dev->vbl_lock);
dev->vblank_disable_allowed = 1;
mtx_unlock(&dev->vbl_lock);
if (dev->vblank_inmodeset[crtc] & 0x2)
drm_vblank_put(dev, crtc);
dev->vblank_inmodeset[crtc] = 0;
}
}
EXPORT_SYMBOL(drm_vblank_post_modeset);
/**
* drm_modeset_ctl - handle vblank event counter changes across mode switch
* @DRM_IOCTL_ARGS: standard ioctl arguments
*
* Applications should call the %_DRM_PRE_MODESET and %_DRM_POST_MODESET
* ioctls around modesetting so that any lost vblank events are accounted for.
*
* Generally the counter will reset across mode sets. If interrupts are
* enabled around this call, we don't have to do anything since the counter
* will have already been incremented.
*/
int drm_modeset_ctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_modeset_ctl *modeset = data;
unsigned int crtc;
/* If drm_vblank_init() hasn't been called yet, just no-op */
if (!dev->num_crtcs)
return 0;
/* KMS drivers handle this internally */
if (drm_core_check_feature(dev, DRIVER_MODESET))
return 0;
crtc = modeset->crtc;
if (crtc >= dev->num_crtcs)
return -EINVAL;
switch (modeset->cmd) {
case _DRM_PRE_MODESET:
drm_vblank_pre_modeset(dev, crtc);
break;
case _DRM_POST_MODESET:
drm_vblank_post_modeset(dev, crtc);
break;
default:
return -EINVAL;
}
return 0;
}
static void
drm_vblank_event_destroy(struct drm_pending_event *e)
{
free(e, DRM_MEM_VBLANK);
}
static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
union drm_wait_vblank *vblwait,
struct drm_file *file_priv)
{
struct drm_pending_vblank_event *e;
struct timeval now;
unsigned int seq;
int ret;
e = malloc(sizeof *e, DRM_MEM_VBLANK, M_NOWAIT | M_ZERO);
if (e == NULL) {
ret = -ENOMEM;
goto err_put;
}
e->pipe = pipe;
e->base.pid = curproc->p_pid;
e->event.base.type = DRM_EVENT_VBLANK;
e->event.base.length = sizeof e->event;
e->event.user_data = vblwait->request.signal;
e->base.event = &e->event.base;
e->base.file_priv = file_priv;
e->base.destroy = drm_vblank_event_destroy;
mtx_lock(&dev->event_lock);
if (file_priv->event_space < sizeof e->event) {
ret = -EBUSY;
goto err_unlock;
}
file_priv->event_space -= sizeof e->event;
seq = drm_vblank_count_and_time(dev, pipe, &now);
if ((vblwait->request.type & _DRM_VBLANK_NEXTONMISS) &&
(seq - vblwait->request.sequence) <= (1 << 23)) {
vblwait->request.sequence = seq + 1;
vblwait->reply.sequence = vblwait->request.sequence;
}
DRM_DEBUG("event on vblank count %d, current %d, crtc %d\n",
vblwait->request.sequence, seq, pipe);
CTR4(KTR_DRM, "vblank_event_queued %d %d rt %x %d", curproc->p_pid, pipe,
vblwait->request.type, vblwait->request.sequence);
e->event.sequence = vblwait->request.sequence;
if ((seq - vblwait->request.sequence) <= (1 << 23)) {
drm_vblank_put(dev, pipe);
send_vblank_event(dev, e, seq, &now);
vblwait->reply.sequence = seq;
} else {
/* drm_handle_vblank_events will call drm_vblank_put */
list_add_tail(&e->base.link, &dev->vblank_event_list);
vblwait->reply.sequence = vblwait->request.sequence;
}
mtx_unlock(&dev->event_lock);
return 0;
err_unlock:
mtx_unlock(&dev->event_lock);
free(e, DRM_MEM_VBLANK);
err_put:
drm_vblank_put(dev, pipe);
return ret;
}
/**
* Wait for VBLANK.
*
* \param inode device inode.
* \param file_priv DRM file private.
* \param cmd command.
* \param data user argument, pointing to a drm_wait_vblank structure.
* \return zero on success or a negative number on failure.
*
* This function enables the vblank interrupt on the pipe requested, then
* sleeps waiting for the requested sequence number to occur, and drops
* the vblank interrupt refcount afterwards. (vblank irq disable follows that
* after a timeout with no further vblank waits scheduled).
*/
int drm_wait_vblank(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
union drm_wait_vblank *vblwait = data;
int ret;
unsigned int flags, seq, crtc, high_crtc;
if (/*(!drm_dev_to_irq(dev)) || */(!dev->irq_enabled))
return -EINVAL;
if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
return -EINVAL;
if (vblwait->request.type &
~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
_DRM_VBLANK_HIGH_CRTC_MASK)) {
DRM_ERROR("Unsupported type value 0x%x, supported mask 0x%x\n",
vblwait->request.type,
(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
_DRM_VBLANK_HIGH_CRTC_MASK));
return -EINVAL;
}
flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
high_crtc = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
if (high_crtc)
crtc = high_crtc >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
else
crtc = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
if (crtc >= dev->num_crtcs)
return -EINVAL;
ret = drm_vblank_get(dev, crtc);
if (ret) {
DRM_DEBUG("failed to acquire vblank counter, %d\n", ret);
return ret;
}
seq = drm_vblank_count(dev, crtc);
switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
case _DRM_VBLANK_RELATIVE:
vblwait->request.sequence += seq;
vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
case _DRM_VBLANK_ABSOLUTE:
break;
default:
ret = -EINVAL;
goto done;
}
if (flags & _DRM_VBLANK_EVENT) {
/* must hold on to the vblank ref until the event fires
* drm_vblank_put will be called asynchronously
*/
return drm_queue_vblank_event(dev, crtc, vblwait, file_priv);
}
if ((flags & _DRM_VBLANK_NEXTONMISS) &&
(seq - vblwait->request.sequence) <= (1<<23)) {
vblwait->request.sequence = seq + 1;
}
DRM_DEBUG("waiting on vblank count %d, crtc %d\n",
vblwait->request.sequence, crtc);
dev->last_vblank_wait[crtc] = vblwait->request.sequence;
mtx_lock(&dev->vblank_time_lock);
while (((drm_vblank_count(dev, crtc) - vblwait->request.sequence) >
(1 << 23)) && dev->irq_enabled) {
/*
* The wakeups from the drm_irq_uninstall() and
* drm_vblank_off() may be lost there since vbl_lock
* is not held. Then, the timeout will wake us; the 3
* seconds delay should not be a problem for
* application when crtc is disabled or irq
* uninstalled anyway.
*/
ret = -msleep(&dev->_vblank_count[crtc], &dev->vblank_time_lock,
PCATCH, "drmvbl", 3 * hz);
if (ret == -ERESTART)
ret = -ERESTARTSYS;
if (ret != 0)
break;
}
mtx_unlock(&dev->vblank_time_lock);
if (ret != -EINTR) {
struct timeval now;
long reply_seq;
reply_seq = drm_vblank_count_and_time(dev, crtc, &now);
CTR5(KTR_DRM, "wait_vblank %d %d rt %x success %d %d",
curproc->p_pid, crtc, vblwait->request.type,
vblwait->request.sequence, reply_seq);
vblwait->reply.sequence = reply_seq;
vblwait->reply.tval_sec = now.tv_sec;
vblwait->reply.tval_usec = now.tv_usec;
DRM_DEBUG("returning %d to client\n",
vblwait->reply.sequence);
} else {
CTR5(KTR_DRM, "wait_vblank %d %d rt %x error %d %d",
curproc->p_pid, crtc, vblwait->request.type, ret,
vblwait->request.sequence);
DRM_DEBUG("vblank wait interrupted by signal\n");
}
done:
drm_vblank_put(dev, crtc);
return ret;
}
static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
{
struct drm_pending_vblank_event *e, *t;
struct timeval now;
unsigned int seq;
seq = drm_vblank_count_and_time(dev, crtc, &now);
mtx_lock(&dev->event_lock);
list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
if (e->pipe != crtc)
continue;
if ((seq - e->event.sequence) > (1<<23))
continue;
DRM_DEBUG("vblank event on %d, current %d\n",
e->event.sequence, seq);
list_del(&e->base.link);
drm_vblank_put(dev, e->pipe);
send_vblank_event(dev, e, seq, &now);
}
mtx_unlock(&dev->event_lock);
CTR2(KTR_DRM, "drm_handle_vblank_events %d %d", seq, crtc);
}
/**
* drm_handle_vblank - handle a vblank event
* @dev: DRM device
* @crtc: where this event occurred
*
* Drivers should call this routine in their vblank interrupt handlers to
* update the vblank counter and send any signals that may be pending.
*/
bool drm_handle_vblank(struct drm_device *dev, int crtc)
{
u32 vblcount;
s64 diff_ns;
struct timeval tvblank;
if (!dev->num_crtcs)
return false;
/* Need timestamp lock to prevent concurrent execution with
* vblank enable/disable, as this would cause inconsistent
* or corrupted timestamps and vblank counts.
*/
mtx_lock(&dev->vblank_time_lock);
/* Vblank irq handling disabled. Nothing to do. */
if (!dev->vblank_enabled[crtc]) {
mtx_unlock(&dev->vblank_time_lock);
return false;
}
/* Fetch corresponding timestamp for this vblank interval from
* driver and store it in proper slot of timestamp ringbuffer.
*/
/* Get current timestamp and count. */
vblcount = atomic_read(&dev->_vblank_count[crtc]);
drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ);
/* Compute time difference to timestamp of last vblank */
diff_ns = timeval_to_ns(&tvblank) -
timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
/* Update vblank timestamp and count if at least
* DRM_REDUNDANT_VBLIRQ_THRESH_NS nanoseconds
* difference between last stored timestamp and current
* timestamp. A smaller difference means basically
* identical timestamps. Happens if this vblank has
* been already processed and this is a redundant call,
* e.g., due to spurious vblank interrupts. We need to
* ignore those for accounting.
*/
if (abs64(diff_ns) > DRM_REDUNDANT_VBLIRQ_THRESH_NS) {
/* Store new timestamp in ringbuffer. */
vblanktimestamp(dev, crtc, vblcount + 1) = tvblank;
/* Increment cooked vblank count. This also atomically commits
* the timestamp computed above.
*/
smp_mb__before_atomic_inc();
atomic_inc(&dev->_vblank_count[crtc]);
smp_mb__after_atomic_inc();
} else {
DRM_DEBUG("crtc %d: Redundant vblirq ignored. diff_ns = %d\n",
crtc, (int) diff_ns);
}
DRM_WAKEUP(&dev->_vblank_count[crtc]);
drm_handle_vblank_events(dev, crtc);
mtx_unlock(&dev->vblank_time_lock);
return true;
}
EXPORT_SYMBOL(drm_handle_vblank);
Index: head/sys/dev/drm2/i915/intel_display.c
===================================================================
--- head/sys/dev/drm2/i915/intel_display.c (revision 283290)
+++ head/sys/dev/drm2/i915/intel_display.c (revision 283291)
@@ -1,7249 +1,7249 @@
/*
* Copyright © 2006-2007 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Authors:
* Eric Anholt <eric@anholt.net>
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <dev/drm2/drmP.h>
#include <dev/drm2/drm.h>
#include <dev/drm2/i915/i915_drm.h>
#include <dev/drm2/i915/i915_drv.h>
#include <dev/drm2/i915/intel_drv.h>
#include <dev/drm2/drm_edid.h>
#include <dev/drm2/drm_dp_helper.h>
#include <dev/drm2/drm_crtc_helper.h>
#include <sys/limits.h>
#define HAS_eDP (intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
bool intel_pipe_has_type(struct drm_crtc *crtc, int type);
static void intel_increase_pllclock(struct drm_crtc *crtc);
static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
typedef struct {
/* given values */
int n;
int m1, m2;
int p1, p2;
/* derived values */
int dot;
int vco;
int m;
int p;
} intel_clock_t;
typedef struct {
int min, max;
} intel_range_t;
typedef struct {
int dot_limit;
int p2_slow, p2_fast;
} intel_p2_t;
#define INTEL_P2_NUM 2
typedef struct intel_limit intel_limit_t;
struct intel_limit {
intel_range_t dot, vco, n, m, m1, m2, p, p1;
intel_p2_t p2;
bool (* find_pll)(const intel_limit_t *, struct drm_crtc *,
int, int, intel_clock_t *, intel_clock_t *);
};
/* FDI */
#define IRONLAKE_FDI_FREQ 2700000 /* in kHz for mode->clock */
static bool
intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
int target, int refclk, intel_clock_t *match_clock,
intel_clock_t *best_clock);
static bool
intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
int target, int refclk, intel_clock_t *match_clock,
intel_clock_t *best_clock);
static bool
intel_find_pll_g4x_dp(const intel_limit_t *, struct drm_crtc *crtc,
int target, int refclk, intel_clock_t *match_clock,
intel_clock_t *best_clock);
static bool
intel_find_pll_ironlake_dp(const intel_limit_t *, struct drm_crtc *crtc,
int target, int refclk, intel_clock_t *match_clock,
intel_clock_t *best_clock);
static inline u32 /* units of 100MHz */
intel_fdi_link_freq(struct drm_device *dev)
{
if (IS_GEN5(dev)) {
struct drm_i915_private *dev_priv = dev->dev_private;
return (I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2;
} else
return 27;
}
static const intel_limit_t intel_limits_i8xx_dvo = {
.dot = { .min = 25000, .max = 350000 },
.vco = { .min = 930000, .max = 1400000 },
.n = { .min = 3, .max = 16 },
.m = { .min = 96, .max = 140 },
.m1 = { .min = 18, .max = 26 },
.m2 = { .min = 6, .max = 16 },
.p = { .min = 4, .max = 128 },
.p1 = { .min = 2, .max = 33 },
.p2 = { .dot_limit = 165000,
.p2_slow = 4, .p2_fast = 2 },
.find_pll = intel_find_best_PLL,
};
static const intel_limit_t intel_limits_i8xx_lvds = {
.dot = { .min = 25000, .max = 350000 },
.vco = { .min = 930000, .max = 1400000 },
.n = { .min = 3, .max = 16 },
.m = { .min = 96, .max = 140 },
.m1 = { .min = 18, .max = 26 },
.m2 = { .min = 6, .max = 16 },
.p = { .min = 4, .max = 128 },
.p1 = { .min = 1, .max = 6 },
.p2 = { .dot_limit = 165000,
.p2_slow = 14, .p2_fast = 7 },
.find_pll = intel_find_best_PLL,
};
static const intel_limit_t intel_limits_i9xx_sdvo = {
.dot = { .min = 20000, .max = 400000 },
.vco = { .min = 1400000, .max = 2800000 },
.n = { .min = 1, .max = 6 },
.m = { .min = 70, .max = 120 },
.m1 = { .min = 10, .max = 22 },
.m2 = { .min = 5, .max = 9 },
.p = { .min = 5, .max = 80 },
.p1 = { .min = 1, .max = 8 },
.p2 = { .dot_limit = 200000,
.p2_slow = 10, .p2_fast = 5 },
.find_pll = intel_find_best_PLL,
};
static const intel_limit_t intel_limits_i9xx_lvds = {
.dot = { .min = 20000, .max = 400000 },
.vco = { .min = 1400000, .max = 2800000 },
.n = { .min = 1, .max = 6 },
.m = { .min = 70, .max = 120 },
.m1 = { .min = 10, .max = 22 },
.m2 = { .min = 5, .max = 9 },
.p = { .min = 7, .max = 98 },
.p1 = { .min = 1, .max = 8 },
.p2 = { .dot_limit = 112000,
.p2_slow = 14, .p2_fast = 7 },
.find_pll = intel_find_best_PLL,
};
static const intel_limit_t intel_limits_g4x_sdvo = {
.dot = { .min = 25000, .max = 270000 },
.vco = { .min = 1750000, .max = 3500000},
.n = { .min = 1, .max = 4 },
.m = { .min = 104, .max = 138 },
.m1 = { .min = 17, .max = 23 },
.m2 = { .min = 5, .max = 11 },
.p = { .min = 10, .max = 30 },
.p1 = { .min = 1, .max = 3},
.p2 = { .dot_limit = 270000,
.p2_slow = 10,
.p2_fast = 10
},
.find_pll = intel_g4x_find_best_PLL,
};
static const intel_limit_t intel_limits_g4x_hdmi = {
.dot = { .min = 22000, .max = 400000 },
.vco = { .min = 1750000, .max = 3500000},
.n = { .min = 1, .max = 4 },
.m = { .min = 104, .max = 138 },
.m1 = { .min = 16, .max = 23 },
.m2 = { .min = 5, .max = 11 },
.p = { .min = 5, .max = 80 },
.p1 = { .min = 1, .max = 8},
.p2 = { .dot_limit = 165000,
.p2_slow = 10, .p2_fast = 5 },
.find_pll = intel_g4x_find_best_PLL,
};
static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
.dot = { .min = 20000, .max = 115000 },
.vco = { .min = 1750000, .max = 3500000 },
.n = { .min = 1, .max = 3 },
.m = { .min = 104, .max = 138 },
.m1 = { .min = 17, .max = 23 },
.m2 = { .min = 5, .max = 11 },
.p = { .min = 28, .max = 112 },
.p1 = { .min = 2, .max = 8 },
.p2 = { .dot_limit = 0,
.p2_slow = 14, .p2_fast = 14
},
.find_pll = intel_g4x_find_best_PLL,
};
static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
.dot = { .min = 80000, .max = 224000 },
.vco = { .min = 1750000, .max = 3500000 },
.n = { .min = 1, .max = 3 },
.m = { .min = 104, .max = 138 },
.m1 = { .min = 17, .max = 23 },
.m2 = { .min = 5, .max = 11 },
.p = { .min = 14, .max = 42 },
.p1 = { .min = 2, .max = 6 },
.p2 = { .dot_limit = 0,
.p2_slow = 7, .p2_fast = 7
},
.find_pll = intel_g4x_find_best_PLL,
};
static const intel_limit_t intel_limits_g4x_display_port = {
.dot = { .min = 161670, .max = 227000 },
.vco = { .min = 1750000, .max = 3500000},
.n = { .min = 1, .max = 2 },
.m = { .min = 97, .max = 108 },
.m1 = { .min = 0x10, .max = 0x12 },
.m2 = { .min = 0x05, .max = 0x06 },
.p = { .min = 10, .max = 20 },
.p1 = { .min = 1, .max = 2},
.p2 = { .dot_limit = 0,
.p2_slow = 10, .p2_fast = 10 },
.find_pll = intel_find_pll_g4x_dp,
};
static const intel_limit_t intel_limits_pineview_sdvo = {
.dot = { .min = 20000, .max = 400000},
.vco = { .min = 1700000, .max = 3500000 },
/* Pineview's Ncounter is a ring counter */
.n = { .min = 3, .max = 6 },
.m = { .min = 2, .max = 256 },
/* Pineview only has one combined m divider, which we treat as m2. */
.m1 = { .min = 0, .max = 0 },
.m2 = { .min = 0, .max = 254 },
.p = { .min = 5, .max = 80 },
.p1 = { .min = 1, .max = 8 },
.p2 = { .dot_limit = 200000,
.p2_slow = 10, .p2_fast = 5 },
.find_pll = intel_find_best_PLL,
};
static const intel_limit_t intel_limits_pineview_lvds = {
.dot = { .min = 20000, .max = 400000 },
.vco = { .min = 1700000, .max = 3500000 },
.n = { .min = 3, .max = 6 },
.m = { .min = 2, .max = 256 },
.m1 = { .min = 0, .max = 0 },
.m2 = { .min = 0, .max = 254 },
.p = { .min = 7, .max = 112 },
.p1 = { .min = 1, .max = 8 },
.p2 = { .dot_limit = 112000,
.p2_slow = 14, .p2_fast = 14 },
.find_pll = intel_find_best_PLL,
};
/* Ironlake / Sandybridge
*
* We calculate clock using (register_value + 2) for N/M1/M2, so here
* the range value for them is (actual_value - 2).
*/
static const intel_limit_t intel_limits_ironlake_dac = {
.dot = { .min = 25000, .max = 350000 },
.vco = { .min = 1760000, .max = 3510000 },
.n = { .min = 1, .max = 5 },
.m = { .min = 79, .max = 127 },
.m1 = { .min = 12, .max = 22 },
.m2 = { .min = 5, .max = 9 },
.p = { .min = 5, .max = 80 },
.p1 = { .min = 1, .max = 8 },
.p2 = { .dot_limit = 225000,
.p2_slow = 10, .p2_fast = 5 },
.find_pll = intel_g4x_find_best_PLL,
};
static const intel_limit_t intel_limits_ironlake_single_lvds = {
.dot = { .min = 25000, .max = 350000 },
.vco = { .min = 1760000, .max = 3510000 },
.n = { .min = 1, .max = 3 },
.m = { .min = 79, .max = 118 },
.m1 = { .min = 12, .max = 22 },
.m2 = { .min = 5, .max = 9 },
.p = { .min = 28, .max = 112 },
.p1 = { .min = 2, .max = 8 },
.p2 = { .dot_limit = 225000,
.p2_slow = 14, .p2_fast = 14 },
.find_pll = intel_g4x_find_best_PLL,
};
static const intel_limit_t intel_limits_ironlake_dual_lvds = {
.dot = { .min = 25000, .max = 350000 },
.vco = { .min = 1760000, .max = 3510000 },
.n = { .min = 1, .max = 3 },
.m = { .min = 79, .max = 127 },
.m1 = { .min = 12, .max = 22 },
.m2 = { .min = 5, .max = 9 },
.p = { .min = 14, .max = 56 },
.p1 = { .min = 2, .max = 8 },
.p2 = { .dot_limit = 225000,
.p2_slow = 7, .p2_fast = 7 },
.find_pll = intel_g4x_find_best_PLL,
};
/* LVDS 100mhz refclk limits. */
static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
.dot = { .min = 25000, .max = 350000 },
.vco = { .min = 1760000, .max = 3510000 },
.n = { .min = 1, .max = 2 },
.m = { .min = 79, .max = 126 },
.m1 = { .min = 12, .max = 22 },
.m2 = { .min = 5, .max = 9 },
.p = { .min = 28, .max = 112 },
.p1 = { .min = 2, .max = 8 },
.p2 = { .dot_limit = 225000,
.p2_slow = 14, .p2_fast = 14 },
.find_pll = intel_g4x_find_best_PLL,
};
static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
.dot = { .min = 25000, .max = 350000 },
.vco = { .min = 1760000, .max = 3510000 },
.n = { .min = 1, .max = 3 },
.m = { .min = 79, .max = 126 },
.m1 = { .min = 12, .max = 22 },
.m2 = { .min = 5, .max = 9 },
.p = { .min = 14, .max = 42 },
.p1 = { .min = 2, .max = 6 },
.p2 = { .dot_limit = 225000,
.p2_slow = 7, .p2_fast = 7 },
.find_pll = intel_g4x_find_best_PLL,
};
static const intel_limit_t intel_limits_ironlake_display_port = {
.dot = { .min = 25000, .max = 350000 },
.vco = { .min = 1760000, .max = 3510000},
.n = { .min = 1, .max = 2 },
.m = { .min = 81, .max = 90 },
.m1 = { .min = 12, .max = 22 },
.m2 = { .min = 5, .max = 9 },
.p = { .min = 10, .max = 20 },
.p1 = { .min = 1, .max = 2},
.p2 = { .dot_limit = 0,
.p2_slow = 10, .p2_fast = 10 },
.find_pll = intel_find_pll_ironlake_dp,
};
u32 intel_dpio_read(struct drm_i915_private *dev_priv, int reg)
{
u32 val = 0;
mtx_lock(&dev_priv->dpio_lock);
if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
DRM_ERROR("DPIO idle wait timed out\n");
goto out_unlock;
}
I915_WRITE(DPIO_REG, reg);
I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_READ | DPIO_PORTID |
DPIO_BYTE);
if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
DRM_ERROR("DPIO read wait timed out\n");
goto out_unlock;
}
val = I915_READ(DPIO_DATA);
out_unlock:
mtx_unlock(&dev_priv->dpio_lock);
return val;
}
#if 0
static void intel_dpio_write(struct drm_i915_private *dev_priv, int reg,
u32 val)
{
mtx_lock(&dev_priv->dpio_lock);
if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
DRM_ERROR("DPIO idle wait timed out\n");
goto out_unlock;
}
I915_WRITE(DPIO_DATA, val);
I915_WRITE(DPIO_REG, reg);
I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_WRITE | DPIO_PORTID |
DPIO_BYTE);
if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100))
DRM_ERROR("DPIO write wait timed out\n");
out_unlock:
mtx_unlock(&dev_priv->dpio_lock);
}
#endif
static void vlv_init_dpio(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
/* Reset the DPIO config */
I915_WRITE(DPIO_CTL, 0);
POSTING_READ(DPIO_CTL);
I915_WRITE(DPIO_CTL, 1);
POSTING_READ(DPIO_CTL);
}
static int intel_dual_link_lvds_callback(const struct dmi_system_id *id)
{
DRM_INFO("Forcing lvds to dual link mode on %s\n", id->ident);
return 1;
}
static const struct dmi_system_id intel_dual_link_lvds[] = {
{
.callback = intel_dual_link_lvds_callback,
.ident = "Apple MacBook Pro (Core i5/i7 Series)",
.matches = {
DMI_MATCH(DMI_SYS_VENDOR, "Apple Inc."),
DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro8,2"),
},
},
{ } /* terminating entry */
};
static bool is_dual_link_lvds(struct drm_i915_private *dev_priv,
unsigned int reg)
{
unsigned int val;
/* use the module option value if specified */
if (i915_lvds_channel_mode > 0)
return i915_lvds_channel_mode == 2;
if (dmi_check_system(intel_dual_link_lvds))
return true;
if (dev_priv->lvds_val)
val = dev_priv->lvds_val;
else {
/* BIOS should set the proper LVDS register value at boot, but
* in reality, it doesn't set the value when the lid is closed;
* we need to check "the value to be set" in VBT when LVDS
* register is uninitialized.
*/
val = I915_READ(reg);
if (!(val & ~LVDS_DETECTED))
val = dev_priv->bios_lvds_val;
dev_priv->lvds_val = val;
}
return (val & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP;
}
static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
int refclk)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
const intel_limit_t *limit;
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
if (is_dual_link_lvds(dev_priv, PCH_LVDS)) {
/* LVDS dual channel */
if (refclk == 100000)
limit = &intel_limits_ironlake_dual_lvds_100m;
else
limit = &intel_limits_ironlake_dual_lvds;
} else {
if (refclk == 100000)
limit = &intel_limits_ironlake_single_lvds_100m;
else
limit = &intel_limits_ironlake_single_lvds;
}
} else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
HAS_eDP)
limit = &intel_limits_ironlake_display_port;
else
limit = &intel_limits_ironlake_dac;
return limit;
}
static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
const intel_limit_t *limit;
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
if (is_dual_link_lvds(dev_priv, LVDS))
/* LVDS with dual channel */
limit = &intel_limits_g4x_dual_channel_lvds;
else
/* LVDS with dual channel */
limit = &intel_limits_g4x_single_channel_lvds;
} else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI) ||
intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG)) {
limit = &intel_limits_g4x_hdmi;
} else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO)) {
limit = &intel_limits_g4x_sdvo;
} else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
limit = &intel_limits_g4x_display_port;
} else /* The option is for other outputs */
limit = &intel_limits_i9xx_sdvo;
return limit;
}
static const intel_limit_t *intel_limit(struct drm_crtc *crtc, int refclk)
{
struct drm_device *dev = crtc->dev;
const intel_limit_t *limit;
if (HAS_PCH_SPLIT(dev))
limit = intel_ironlake_limit(crtc, refclk);
else if (IS_G4X(dev)) {
limit = intel_g4x_limit(crtc);
} else if (IS_PINEVIEW(dev)) {
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
limit = &intel_limits_pineview_lvds;
else
limit = &intel_limits_pineview_sdvo;
} else if (!IS_GEN2(dev)) {
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
limit = &intel_limits_i9xx_lvds;
else
limit = &intel_limits_i9xx_sdvo;
} else {
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
limit = &intel_limits_i8xx_lvds;
else
limit = &intel_limits_i8xx_dvo;
}
return limit;
}
/* m1 is reserved as 0 in Pineview, n is a ring counter */
static void pineview_clock(int refclk, intel_clock_t *clock)
{
clock->m = clock->m2 + 2;
clock->p = clock->p1 * clock->p2;
clock->vco = refclk * clock->m / clock->n;
clock->dot = clock->vco / clock->p;
}
static void intel_clock(struct drm_device *dev, int refclk, intel_clock_t *clock)
{
if (IS_PINEVIEW(dev)) {
pineview_clock(refclk, clock);
return;
}
clock->m = 5 * (clock->m1 + 2) + (clock->m2 + 2);
clock->p = clock->p1 * clock->p2;
clock->vco = refclk * clock->m / (clock->n + 2);
clock->dot = clock->vco / clock->p;
}
/**
* Returns whether any output on the specified pipe is of the specified type
*/
bool intel_pipe_has_type(struct drm_crtc *crtc, int type)
{
struct drm_device *dev = crtc->dev;
struct drm_mode_config *mode_config = &dev->mode_config;
struct intel_encoder *encoder;
list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
if (encoder->base.crtc == crtc && encoder->type == type)
return true;
return false;
}
#define INTELPllInvalid(s) do { /* DRM_DEBUG(s); */ return false; } while (0)
/**
* Returns whether the given set of divisors are valid for a given refclk with
* the given connectors.
*/
static bool intel_PLL_is_valid(struct drm_device *dev,
const intel_limit_t *limit,
const intel_clock_t *clock)
{
if (clock->p1 < limit->p1.min || limit->p1.max < clock->p1)
INTELPllInvalid("p1 out of range\n");
if (clock->p < limit->p.min || limit->p.max < clock->p)
INTELPllInvalid("p out of range\n");
if (clock->m2 < limit->m2.min || limit->m2.max < clock->m2)
INTELPllInvalid("m2 out of range\n");
if (clock->m1 < limit->m1.min || limit->m1.max < clock->m1)
INTELPllInvalid("m1 out of range\n");
if (clock->m1 <= clock->m2 && !IS_PINEVIEW(dev))
INTELPllInvalid("m1 <= m2\n");
if (clock->m < limit->m.min || limit->m.max < clock->m)
INTELPllInvalid("m out of range\n");
if (clock->n < limit->n.min || limit->n.max < clock->n)
INTELPllInvalid("n out of range\n");
if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
INTELPllInvalid("vco out of range\n");
/* XXX: We may need to be checking "Dot clock" depending on the multiplier,
* connector, etc., rather than just a single range.
*/
if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
INTELPllInvalid("dot out of range\n");
return true;
}
static bool
intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
int target, int refclk, intel_clock_t *match_clock,
intel_clock_t *best_clock)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
intel_clock_t clock;
int err = target;
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
(I915_READ(LVDS)) != 0) {
/*
* For LVDS, if the panel is on, just rely on its current
* settings for dual-channel. We haven't figured out how to
* reliably set up different single/dual channel state, if we
* even can.
*/
if (is_dual_link_lvds(dev_priv, LVDS))
clock.p2 = limit->p2.p2_fast;
else
clock.p2 = limit->p2.p2_slow;
} else {
if (target < limit->p2.dot_limit)
clock.p2 = limit->p2.p2_slow;
else
clock.p2 = limit->p2.p2_fast;
}
memset(best_clock, 0, sizeof(*best_clock));
for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
clock.m1++) {
for (clock.m2 = limit->m2.min;
clock.m2 <= limit->m2.max; clock.m2++) {
/* m1 is always 0 in Pineview */
if (clock.m2 >= clock.m1 && !IS_PINEVIEW(dev))
break;
for (clock.n = limit->n.min;
clock.n <= limit->n.max; clock.n++) {
for (clock.p1 = limit->p1.min;
clock.p1 <= limit->p1.max; clock.p1++) {
int this_err;
intel_clock(dev, refclk, &clock);
if (!intel_PLL_is_valid(dev, limit,
&clock))
continue;
if (match_clock &&
clock.p != match_clock->p)
continue;
this_err = abs(clock.dot - target);
if (this_err < err) {
*best_clock = clock;
err = this_err;
}
}
}
}
}
return (err != target);
}
static bool
intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
int target, int refclk, intel_clock_t *match_clock,
intel_clock_t *best_clock)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
intel_clock_t clock;
int max_n;
bool found;
/* approximately equals target * 0.00585 */
int err_most = (target >> 8) + (target >> 9);
found = false;
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
int lvds_reg;
if (HAS_PCH_SPLIT(dev))
lvds_reg = PCH_LVDS;
else
lvds_reg = LVDS;
if ((I915_READ(lvds_reg) & LVDS_CLKB_POWER_MASK) ==
LVDS_CLKB_POWER_UP)
clock.p2 = limit->p2.p2_fast;
else
clock.p2 = limit->p2.p2_slow;
} else {
if (target < limit->p2.dot_limit)
clock.p2 = limit->p2.p2_slow;
else
clock.p2 = limit->p2.p2_fast;
}
memset(best_clock, 0, sizeof(*best_clock));
max_n = limit->n.max;
/* based on hardware requirement, prefer smaller n to precision */
for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
/* based on hardware requirement, prefere larger m1,m2 */
for (clock.m1 = limit->m1.max;
clock.m1 >= limit->m1.min; clock.m1--) {
for (clock.m2 = limit->m2.max;
clock.m2 >= limit->m2.min; clock.m2--) {
for (clock.p1 = limit->p1.max;
clock.p1 >= limit->p1.min; clock.p1--) {
int this_err;
intel_clock(dev, refclk, &clock);
if (!intel_PLL_is_valid(dev, limit,
&clock))
continue;
if (match_clock &&
clock.p != match_clock->p)
continue;
this_err = abs(clock.dot - target);
if (this_err < err_most) {
*best_clock = clock;
err_most = this_err;
max_n = clock.n;
found = true;
}
}
}
}
}
return found;
}
static bool
intel_find_pll_ironlake_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
int target, int refclk, intel_clock_t *match_clock,
intel_clock_t *best_clock)
{
struct drm_device *dev = crtc->dev;
intel_clock_t clock;
if (target < 200000) {
clock.n = 1;
clock.p1 = 2;
clock.p2 = 10;
clock.m1 = 12;
clock.m2 = 9;
} else {
clock.n = 2;
clock.p1 = 1;
clock.p2 = 10;
clock.m1 = 14;
clock.m2 = 8;
}
intel_clock(dev, refclk, &clock);
memcpy(best_clock, &clock, sizeof(intel_clock_t));
return true;
}
/* DisplayPort has only two frequencies, 162MHz and 270MHz */
static bool
intel_find_pll_g4x_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
int target, int refclk, intel_clock_t *match_clock,
intel_clock_t *best_clock)
{
intel_clock_t clock;
if (target < 200000) {
clock.p1 = 2;
clock.p2 = 10;
clock.n = 2;
clock.m1 = 23;
clock.m2 = 8;
} else {
clock.p1 = 1;
clock.p2 = 10;
clock.n = 1;
clock.m1 = 14;
clock.m2 = 2;
}
clock.m = 5 * (clock.m1 + 2) + (clock.m2 + 2);
clock.p = (clock.p1 * clock.p2);
clock.dot = 96000 * clock.m / (clock.n + 2) / clock.p;
clock.vco = 0;
memcpy(best_clock, &clock, sizeof(intel_clock_t));
return true;
}
static void ironlake_wait_for_vblank(struct drm_device *dev, int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
u32 frame, frame_reg = PIPEFRAME(pipe);
frame = I915_READ(frame_reg);
if (wait_for(I915_READ_NOTRACE(frame_reg) != frame, 50))
DRM_DEBUG_KMS("vblank wait timed out\n");
}
/**
* intel_wait_for_vblank - wait for vblank on a given pipe
* @dev: drm device
* @pipe: pipe to wait for
*
* Wait for vblank to occur on a given pipe. Needed for various bits of
* mode setting code.
*/
void intel_wait_for_vblank(struct drm_device *dev, int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
int pipestat_reg = PIPESTAT(pipe);
if (INTEL_INFO(dev)->gen >= 5) {
ironlake_wait_for_vblank(dev, pipe);
return;
}
/* Clear existing vblank status. Note this will clear any other
* sticky status fields as well.
*
* This races with i915_driver_irq_handler() with the result
* that either function could miss a vblank event. Here it is not
* fatal, as we will either wait upon the next vblank interrupt or
* timeout. Generally speaking intel_wait_for_vblank() is only
* called during modeset at which time the GPU should be idle and
* should *not* be performing page flips and thus not waiting on
* vblanks...
* Currently, the result of us stealing a vblank from the irq
* handler is that a single frame will be skipped during swapbuffers.
*/
I915_WRITE(pipestat_reg,
I915_READ(pipestat_reg) | PIPE_VBLANK_INTERRUPT_STATUS);
/* Wait for vblank interrupt bit to set */
if (_intel_wait_for(dev,
I915_READ(pipestat_reg) & PIPE_VBLANK_INTERRUPT_STATUS,
50, 1, "915vbl"))
DRM_DEBUG_KMS("vblank wait timed out\n");
}
/*
* intel_wait_for_pipe_off - wait for pipe to turn off
* @dev: drm device
* @pipe: pipe to wait for
*
* After disabling a pipe, we can't wait for vblank in the usual way,
* spinning on the vblank interrupt status bit, since we won't actually
* see an interrupt when the pipe is disabled.
*
* On Gen4 and above:
* wait for the pipe register state bit to turn off
*
* Otherwise:
* wait for the display line value to settle (it usually
* ends up stopping at the start of the next frame).
*
*/
void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
if (INTEL_INFO(dev)->gen >= 4) {
int reg = PIPECONF(pipe);
/* Wait for the Pipe State to go off */
if (_intel_wait_for(dev,
(I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0, 100,
1, "915pip"))
DRM_DEBUG_KMS("pipe_off wait timed out\n");
} else {
u32 last_line, line_mask;
int reg = PIPEDSL(pipe);
unsigned long timeout = jiffies + msecs_to_jiffies(100);
if (IS_GEN2(dev))
line_mask = DSL_LINEMASK_GEN2;
else
line_mask = DSL_LINEMASK_GEN3;
/* Wait for the display line to settle */
do {
last_line = I915_READ(reg) & line_mask;
DELAY(5000);
} while (((I915_READ(reg) & line_mask) != last_line) &&
time_after(timeout, jiffies));
if (time_after(jiffies, timeout))
DRM_DEBUG_KMS("pipe_off wait timed out\n");
}
}
static const char *state_string(bool enabled)
{
return enabled ? "on" : "off";
}
/* Only for pre-ILK configs */
static void assert_pll(struct drm_i915_private *dev_priv,
enum pipe pipe, bool state)
{
int reg;
u32 val;
bool cur_state;
reg = DPLL(pipe);
val = I915_READ(reg);
cur_state = !!(val & DPLL_VCO_ENABLE);
if (cur_state != state)
printf("PLL state assertion failure (expected %s, current %s)\n",
state_string(state), state_string(cur_state));
}
#define assert_pll_enabled(d, p) assert_pll(d, p, true)
#define assert_pll_disabled(d, p) assert_pll(d, p, false)
/* For ILK+ */
static void assert_pch_pll(struct drm_i915_private *dev_priv,
struct intel_crtc *intel_crtc, bool state)
{
int reg;
u32 val;
bool cur_state;
if (HAS_PCH_LPT(dev_priv->dev)) {
DRM_DEBUG_DRIVER("LPT detected: skipping PCH PLL test\n");
return;
}
if (!intel_crtc->pch_pll) {
printf("asserting PCH PLL enabled with no PLL\n");
return;
}
if (HAS_PCH_CPT(dev_priv->dev)) {
u32 pch_dpll;
pch_dpll = I915_READ(PCH_DPLL_SEL);
/* Make sure the selected PLL is enabled to the transcoder */
KASSERT(((pch_dpll >> (4 * intel_crtc->pipe)) & 8) != 0,
("transcoder %d PLL not enabled\n", intel_crtc->pipe));
}
reg = intel_crtc->pch_pll->pll_reg;
val = I915_READ(reg);
cur_state = !!(val & DPLL_VCO_ENABLE);
if (cur_state != state)
printf("PCH PLL state assertion failure (expected %s, current %s)\n",
state_string(state), state_string(cur_state));
}
#define assert_pch_pll_enabled(d, p) assert_pch_pll(d, p, true)
#define assert_pch_pll_disabled(d, p) assert_pch_pll(d, p, false)
static void assert_fdi_tx(struct drm_i915_private *dev_priv,
enum pipe pipe, bool state)
{
int reg;
u32 val;
bool cur_state;
if (IS_HASWELL(dev_priv->dev)) {
/* On Haswell, DDI is used instead of FDI_TX_CTL */
reg = DDI_FUNC_CTL(pipe);
val = I915_READ(reg);
cur_state = !!(val & PIPE_DDI_FUNC_ENABLE);
} else {
reg = FDI_TX_CTL(pipe);
val = I915_READ(reg);
cur_state = !!(val & FDI_TX_ENABLE);
}
if (cur_state != state)
printf("FDI TX state assertion failure (expected %s, current %s)\n",
state_string(state), state_string(cur_state));
}
#define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
#define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
static void assert_fdi_rx(struct drm_i915_private *dev_priv,
enum pipe pipe, bool state)
{
int reg;
u32 val;
bool cur_state;
if (IS_HASWELL(dev_priv->dev) && pipe > 0) {
DRM_ERROR("Attempting to enable FDI_RX on Haswell pipe > 0\n");
return;
} else {
reg = FDI_RX_CTL(pipe);
val = I915_READ(reg);
cur_state = !!(val & FDI_RX_ENABLE);
}
if (cur_state != state)
printf("FDI RX state assertion failure (expected %s, current %s)\n",
state_string(state), state_string(cur_state));
}
#define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
#define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
int reg;
u32 val;
/* ILK FDI PLL is always enabled */
if (dev_priv->info->gen == 5)
return;
/* On Haswell, DDI ports are responsible for the FDI PLL setup */
if (IS_HASWELL(dev_priv->dev))
return;
reg = FDI_TX_CTL(pipe);
val = I915_READ(reg);
if (!(val & FDI_TX_PLL_ENABLE))
printf("FDI TX PLL assertion failure, should be active but is disabled\n");
}
static void assert_fdi_rx_pll_enabled(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
int reg;
u32 val;
if (IS_HASWELL(dev_priv->dev) && pipe > 0) {
DRM_ERROR("Attempting to enable FDI on Haswell with pipe > 0\n");
return;
}
reg = FDI_RX_CTL(pipe);
val = I915_READ(reg);
if (!(val & FDI_RX_PLL_ENABLE))
printf("FDI RX PLL assertion failure, should be active but is disabled\n");
}
static void assert_panel_unlocked(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
int pp_reg, lvds_reg;
u32 val;
enum pipe panel_pipe = PIPE_A;
bool locked = true;
if (HAS_PCH_SPLIT(dev_priv->dev)) {
pp_reg = PCH_PP_CONTROL;
lvds_reg = PCH_LVDS;
} else {
pp_reg = PP_CONTROL;
lvds_reg = LVDS;
}
val = I915_READ(pp_reg);
if (!(val & PANEL_POWER_ON) ||
((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS))
locked = false;
if (I915_READ(lvds_reg) & LVDS_PIPEB_SELECT)
panel_pipe = PIPE_B;
if (panel_pipe == pipe && locked)
printf("panel assertion failure, pipe %c regs locked\n",
pipe_name(pipe));
}
void assert_pipe(struct drm_i915_private *dev_priv,
enum pipe pipe, bool state)
{
int reg;
u32 val;
bool cur_state;
/* if we need the pipe A quirk it must be always on */
if (pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE)
state = true;
reg = PIPECONF(pipe);
val = I915_READ(reg);
cur_state = !!(val & PIPECONF_ENABLE);
if (cur_state != state)
printf("pipe %c assertion failure (expected %s, current %s)\n",
pipe_name(pipe), state_string(state), state_string(cur_state));
}
static void assert_plane(struct drm_i915_private *dev_priv,
enum plane plane, bool state)
{
int reg;
u32 val;
bool cur_state;
reg = DSPCNTR(plane);
val = I915_READ(reg);
cur_state = !!(val & DISPLAY_PLANE_ENABLE);
if (cur_state != state)
printf("plane %c assertion failure, (expected %s, current %s)\n",
plane_name(plane), state_string(state), state_string(cur_state));
}
#define assert_plane_enabled(d, p) assert_plane(d, p, true)
#define assert_plane_disabled(d, p) assert_plane(d, p, false)
static void assert_planes_disabled(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
int reg, i;
u32 val;
int cur_pipe;
/* Planes are fixed to pipes on ILK+ */
if (HAS_PCH_SPLIT(dev_priv->dev)) {
reg = DSPCNTR(pipe);
val = I915_READ(reg);
if ((val & DISPLAY_PLANE_ENABLE) != 0)
printf("plane %c assertion failure, should be disabled but not\n",
plane_name(pipe));
return;
}
/* Need to check both planes against the pipe */
for (i = 0; i < 2; i++) {
reg = DSPCNTR(i);
val = I915_READ(reg);
cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
DISPPLANE_SEL_PIPE_SHIFT;
if ((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe)
printf("plane %c assertion failure, should be off on pipe %c but is still active\n",
plane_name(i), pipe_name(pipe));
}
}
static void assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
{
u32 val;
bool enabled;
if (HAS_PCH_LPT(dev_priv->dev)) {
DRM_DEBUG_DRIVER("LPT does not has PCH refclk, skipping check\n");
return;
}
val = I915_READ(PCH_DREF_CONTROL);
enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK |
DREF_SUPERSPREAD_SOURCE_MASK));
if (!enabled)
printf("PCH refclk assertion failure, should be active but is disabled\n");
}
static void assert_transcoder_disabled(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
int reg;
u32 val;
bool enabled;
reg = TRANSCONF(pipe);
val = I915_READ(reg);
enabled = !!(val & TRANS_ENABLE);
if (enabled)
printf("transcoder assertion failed, should be off on pipe %c but is still active\n",
pipe_name(pipe));
}
static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv,
enum pipe pipe, u32 val)
{
if ((val & PORT_ENABLE) == 0)
return false;
if (HAS_PCH_CPT(dev_priv->dev)) {
if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
return false;
} else {
if ((val & TRANSCODER_MASK) != TRANSCODER(pipe))
return false;
}
return true;
}
static bool lvds_pipe_enabled(struct drm_i915_private *dev_priv,
enum pipe pipe, u32 val)
{
if ((val & LVDS_PORT_EN) == 0)
return false;
if (HAS_PCH_CPT(dev_priv->dev)) {
if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
return false;
} else {
if ((val & LVDS_PIPE_MASK) != LVDS_PIPE(pipe))
return false;
}
return true;
}
static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv,
enum pipe pipe, u32 val)
{
if ((val & ADPA_DAC_ENABLE) == 0)
return false;
if (HAS_PCH_CPT(dev_priv->dev)) {
if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
return false;
} else {
if ((val & ADPA_PIPE_SELECT_MASK) != ADPA_PIPE_SELECT(pipe))
return false;
}
return true;
}
static bool dp_pipe_enabled(struct drm_i915_private *dev_priv,
enum pipe pipe, u32 port_sel, u32 val)
{
if ((val & DP_PORT_EN) == 0)
return false;
if (HAS_PCH_CPT(dev_priv->dev)) {
u32 trans_dp_ctl_reg = TRANS_DP_CTL(pipe);
u32 trans_dp_ctl = I915_READ(trans_dp_ctl_reg);
if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel)
return false;
} else {
if ((val & DP_PIPE_MASK) != (pipe << 30))
return false;
}
return true;
}
static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
enum pipe pipe, int reg, u32 port_sel)
{
u32 val = I915_READ(reg);
if (dp_pipe_enabled(dev_priv, pipe, port_sel, val))
printf("PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
reg, pipe_name(pipe));
}
static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
enum pipe pipe, int reg)
{
u32 val = I915_READ(reg);
if (hdmi_pipe_enabled(dev_priv, val, pipe))
printf("PCH HDMI (0x%08x) enabled on transcoder %c, should be disabled\n",
reg, pipe_name(pipe));
}
static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
int reg;
u32 val;
assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
reg = PCH_ADPA;
val = I915_READ(reg);
if (adpa_pipe_enabled(dev_priv, val, pipe))
printf("PCH VGA enabled on transcoder %c, should be disabled\n",
pipe_name(pipe));
reg = PCH_LVDS;
val = I915_READ(reg);
if (lvds_pipe_enabled(dev_priv, val, pipe))
printf("PCH LVDS enabled on transcoder %c, should be disabled\n",
pipe_name(pipe));
assert_pch_hdmi_disabled(dev_priv, pipe, HDMIB);
assert_pch_hdmi_disabled(dev_priv, pipe, HDMIC);
assert_pch_hdmi_disabled(dev_priv, pipe, HDMID);
}
/**
* intel_enable_pll - enable a PLL
* @dev_priv: i915 private structure
* @pipe: pipe PLL to enable
*
* Enable @pipe's PLL so we can start pumping pixels from a plane. Check to
* make sure the PLL reg is writable first though, since the panel write
* protect mechanism may be enabled.
*
* Note! This is for pre-ILK only.
*/
static void intel_enable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
{
int reg;
u32 val;
/* No really, not for ILK+ */
KASSERT(dev_priv->info->gen < 5, ("Wrong device gen"));
/* PLL is protected by panel, make sure we can write it */
if (IS_MOBILE(dev_priv->dev) && !IS_I830(dev_priv->dev))
assert_panel_unlocked(dev_priv, pipe);
reg = DPLL(pipe);
val = I915_READ(reg);
val |= DPLL_VCO_ENABLE;
/* We do this three times for luck */
I915_WRITE(reg, val);
POSTING_READ(reg);
DELAY(150); /* wait for warmup */
I915_WRITE(reg, val);
POSTING_READ(reg);
DELAY(150); /* wait for warmup */
I915_WRITE(reg, val);
POSTING_READ(reg);
DELAY(150); /* wait for warmup */
}
/**
* intel_disable_pll - disable a PLL
* @dev_priv: i915 private structure
* @pipe: pipe PLL to disable
*
* Disable the PLL for @pipe, making sure the pipe is off first.
*
* Note! This is for pre-ILK only.
*/
static void intel_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
{
int reg;
u32 val;
/* Don't disable pipe A or pipe A PLLs if needed */
if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
return;
/* Make sure the pipe isn't still relying on us */
assert_pipe_disabled(dev_priv, pipe);
reg = DPLL(pipe);
val = I915_READ(reg);
val &= ~DPLL_VCO_ENABLE;
I915_WRITE(reg, val);
POSTING_READ(reg);
}
/* SBI access */
static void
intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value)
{
mtx_lock(&dev_priv->dpio_lock);
if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_READY) == 0,
100)) {
DRM_ERROR("timeout waiting for SBI to become ready\n");
goto out_unlock;
}
I915_WRITE(SBI_ADDR,
(reg << 16));
I915_WRITE(SBI_DATA,
value);
I915_WRITE(SBI_CTL_STAT,
SBI_BUSY |
SBI_CTL_OP_CRWR);
if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_READY | SBI_RESPONSE_SUCCESS)) == 0,
100)) {
DRM_ERROR("timeout waiting for SBI to complete write transaction\n");
goto out_unlock;
}
out_unlock:
mtx_unlock(&dev_priv->dpio_lock);
}
static u32
intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg)
{
u32 value;
value = 0;
mtx_lock(&dev_priv->dpio_lock);
if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_READY) == 0,
100)) {
DRM_ERROR("timeout waiting for SBI to become ready\n");
goto out_unlock;
}
I915_WRITE(SBI_ADDR,
(reg << 16));
I915_WRITE(SBI_CTL_STAT,
SBI_BUSY |
SBI_CTL_OP_CRRD);
if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_READY | SBI_RESPONSE_SUCCESS)) == 0,
100)) {
DRM_ERROR("timeout waiting for SBI to complete read transaction\n");
goto out_unlock;
}
value = I915_READ(SBI_DATA);
out_unlock:
mtx_unlock(&dev_priv->dpio_lock);
return value;
}
/**
* intel_enable_pch_pll - enable PCH PLL
* @dev_priv: i915 private structure
* @pipe: pipe PLL to enable
*
* The PCH PLL needs to be enabled before the PCH transcoder, since it
* drives the transcoder clock.
*/
static void intel_enable_pch_pll(struct intel_crtc *intel_crtc)
{
struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
struct intel_pch_pll *pll;
int reg;
u32 val;
/* PCH PLLs only available on ILK, SNB and IVB */
KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen"));
pll = intel_crtc->pch_pll;
if (pll == NULL)
return;
if (pll->refcount == 0) {
DRM_DEBUG_KMS("pll->refcount == 0\n");
return;
}
DRM_DEBUG_KMS("enable PCH PLL %x (active %d, on? %d)for crtc %d\n",
pll->pll_reg, pll->active, pll->on,
intel_crtc->base.base.id);
/* PCH refclock must be enabled first */
assert_pch_refclk_enabled(dev_priv);
if (pll->active++ && pll->on) {
assert_pch_pll_enabled(dev_priv, intel_crtc);
return;
}
DRM_DEBUG_KMS("enabling PCH PLL %x\n", pll->pll_reg);
reg = pll->pll_reg;
val = I915_READ(reg);
val |= DPLL_VCO_ENABLE;
I915_WRITE(reg, val);
POSTING_READ(reg);
DELAY(200);
pll->on = true;
}
static void intel_disable_pch_pll(struct intel_crtc *intel_crtc)
{
struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
struct intel_pch_pll *pll = intel_crtc->pch_pll;
int reg;
u32 val;
/* PCH only available on ILK+ */
KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen"));
if (pll == NULL)
return;
if (pll->refcount == 0) {
DRM_DEBUG_KMS("pll->refcount == 0\n");
return;
}
DRM_DEBUG_KMS("disable PCH PLL %x (active %d, on? %d) for crtc %d\n",
pll->pll_reg, pll->active, pll->on,
intel_crtc->base.base.id);
if (pll->active == 0) {
DRM_DEBUG_KMS("pll->active == 0\n");
assert_pch_pll_disabled(dev_priv, intel_crtc);
return;
}
if (--pll->active) {
assert_pch_pll_enabled(dev_priv, intel_crtc);
return;
}
DRM_DEBUG_KMS("disabling PCH PLL %x\n", pll->pll_reg);
/* Make sure transcoder isn't still depending on us */
assert_transcoder_disabled(dev_priv, intel_crtc->pipe);
reg = pll->pll_reg;
val = I915_READ(reg);
val &= ~DPLL_VCO_ENABLE;
I915_WRITE(reg, val);
POSTING_READ(reg);
DELAY(200);
pll->on = false;
}
static void intel_enable_transcoder(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
int reg;
u32 val, pipeconf_val;
struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
/* PCH only available on ILK+ */
KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen"));
/* Make sure PCH DPLL is enabled */
assert_pch_pll_enabled(dev_priv, to_intel_crtc(crtc));
/* FDI must be feeding us bits for PCH ports */
assert_fdi_tx_enabled(dev_priv, pipe);
assert_fdi_rx_enabled(dev_priv, pipe);
if (IS_HASWELL(dev_priv->dev) && pipe > 0) {
DRM_ERROR("Attempting to enable transcoder on Haswell with pipe > 0\n");
return;
}
reg = TRANSCONF(pipe);
val = I915_READ(reg);
pipeconf_val = I915_READ(PIPECONF(pipe));
if (HAS_PCH_IBX(dev_priv->dev)) {
/*
* make the BPC in transcoder be consistent with
* that in pipeconf reg.
*/
val &= ~PIPE_BPC_MASK;
val |= pipeconf_val & PIPE_BPC_MASK;
}
val &= ~TRANS_INTERLACE_MASK;
if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
if (HAS_PCH_IBX(dev_priv->dev) &&
intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO))
val |= TRANS_LEGACY_INTERLACED_ILK;
else
val |= TRANS_INTERLACED;
else
val |= TRANS_PROGRESSIVE;
I915_WRITE(reg, val | TRANS_ENABLE);
if (_intel_wait_for(dev_priv->dev, I915_READ(reg) & TRANS_STATE_ENABLE,
100, 1, "915trc"))
DRM_ERROR("failed to enable transcoder %d\n", pipe);
}
static void intel_disable_transcoder(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
int reg;
u32 val;
/* FDI relies on the transcoder */
assert_fdi_tx_disabled(dev_priv, pipe);
assert_fdi_rx_disabled(dev_priv, pipe);
/* Ports must be off as well */
assert_pch_ports_disabled(dev_priv, pipe);
reg = TRANSCONF(pipe);
val = I915_READ(reg);
val &= ~TRANS_ENABLE;
I915_WRITE(reg, val);
/* wait for PCH transcoder off, transcoder state */
if (_intel_wait_for(dev_priv->dev,
(I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50,
1, "915trd"))
DRM_ERROR("failed to disable transcoder %d\n", pipe);
}
/**
* intel_enable_pipe - enable a pipe, asserting requirements
* @dev_priv: i915 private structure
* @pipe: pipe to enable
* @pch_port: on ILK+, is this pipe driving a PCH port or not
*
* Enable @pipe, making sure that various hardware specific requirements
* are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
*
* @pipe should be %PIPE_A or %PIPE_B.
*
* Will wait until the pipe is actually running (i.e. first vblank) before
* returning.
*/
static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe,
bool pch_port)
{
int reg;
u32 val;
/*
* A pipe without a PLL won't actually be able to drive bits from
* a plane. On ILK+ the pipe PLLs are integrated, so we don't
* need the check.
*/
if (!HAS_PCH_SPLIT(dev_priv->dev))
assert_pll_enabled(dev_priv, pipe);
else {
if (pch_port) {
/* if driving the PCH, we need FDI enabled */
assert_fdi_rx_pll_enabled(dev_priv, pipe);
assert_fdi_tx_pll_enabled(dev_priv, pipe);
}
/* FIXME: assert CPU port conditions for SNB+ */
}
reg = PIPECONF(pipe);
val = I915_READ(reg);
if (val & PIPECONF_ENABLE)
return;
I915_WRITE(reg, val | PIPECONF_ENABLE);
intel_wait_for_vblank(dev_priv->dev, pipe);
}
/**
* intel_disable_pipe - disable a pipe, asserting requirements
* @dev_priv: i915 private structure
* @pipe: pipe to disable
*
* Disable @pipe, making sure that various hardware specific requirements
* are met, if applicable, e.g. plane disabled, panel fitter off, etc.
*
* @pipe should be %PIPE_A or %PIPE_B.
*
* Will wait until the pipe has shut down before returning.
*/
static void intel_disable_pipe(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
int reg;
u32 val;
/*
* Make sure planes won't keep trying to pump pixels to us,
* or we might hang the display.
*/
assert_planes_disabled(dev_priv, pipe);
/* Don't disable pipe A or pipe A PLLs if needed */
if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
return;
reg = PIPECONF(pipe);
val = I915_READ(reg);
if ((val & PIPECONF_ENABLE) == 0)
return;
I915_WRITE(reg, val & ~PIPECONF_ENABLE);
intel_wait_for_pipe_off(dev_priv->dev, pipe);
}
/*
* Plane regs are double buffered, going from enabled->disabled needs a
* trigger in order to latch. The display address reg provides this.
*/
void intel_flush_display_plane(struct drm_i915_private *dev_priv,
enum plane plane)
{
I915_WRITE(DSPADDR(plane), I915_READ(DSPADDR(plane)));
I915_WRITE(DSPSURF(plane), I915_READ(DSPSURF(plane)));
}
/**
* intel_enable_plane - enable a display plane on a given pipe
* @dev_priv: i915 private structure
* @plane: plane to enable
* @pipe: pipe being fed
*
* Enable @plane on @pipe, making sure that @pipe is running first.
*/
static void intel_enable_plane(struct drm_i915_private *dev_priv,
enum plane plane, enum pipe pipe)
{
int reg;
u32 val;
/* If the pipe isn't enabled, we can't pump pixels and may hang */
assert_pipe_enabled(dev_priv, pipe);
reg = DSPCNTR(plane);
val = I915_READ(reg);
if (val & DISPLAY_PLANE_ENABLE)
return;
I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE);
intel_flush_display_plane(dev_priv, plane);
intel_wait_for_vblank(dev_priv->dev, pipe);
}
/**
* intel_disable_plane - disable a display plane
* @dev_priv: i915 private structure
* @plane: plane to disable
* @pipe: pipe consuming the data
*
* Disable @plane; should be an independent operation.
*/
static void intel_disable_plane(struct drm_i915_private *dev_priv,
enum plane plane, enum pipe pipe)
{
int reg;
u32 val;
reg = DSPCNTR(plane);
val = I915_READ(reg);
if ((val & DISPLAY_PLANE_ENABLE) == 0)
return;
I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE);
intel_flush_display_plane(dev_priv, plane);
intel_wait_for_vblank(dev_priv->dev, pipe);
}
static void disable_pch_dp(struct drm_i915_private *dev_priv,
enum pipe pipe, int reg, u32 port_sel)
{
u32 val = I915_READ(reg);
if (dp_pipe_enabled(dev_priv, pipe, port_sel, val)) {
DRM_DEBUG_KMS("Disabling pch dp %x on pipe %d\n", reg, pipe);
I915_WRITE(reg, val & ~DP_PORT_EN);
}
}
static void disable_pch_hdmi(struct drm_i915_private *dev_priv,
enum pipe pipe, int reg)
{
u32 val = I915_READ(reg);
if (hdmi_pipe_enabled(dev_priv, val, pipe)) {
DRM_DEBUG_KMS("Disabling pch HDMI %x on pipe %d\n",
reg, pipe);
I915_WRITE(reg, val & ~PORT_ENABLE);
}
}
/* Disable any ports connected to this transcoder */
static void intel_disable_pch_ports(struct drm_i915_private *dev_priv,
enum pipe pipe)
{
u32 reg, val;
val = I915_READ(PCH_PP_CONTROL);
I915_WRITE(PCH_PP_CONTROL, val | PANEL_UNLOCK_REGS);
disable_pch_dp(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
disable_pch_dp(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
disable_pch_dp(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
reg = PCH_ADPA;
val = I915_READ(reg);
if (adpa_pipe_enabled(dev_priv, val, pipe))
I915_WRITE(reg, val & ~ADPA_DAC_ENABLE);
reg = PCH_LVDS;
val = I915_READ(reg);
if (lvds_pipe_enabled(dev_priv, val, pipe)) {
DRM_DEBUG_KMS("disable lvds on pipe %d val 0x%08x\n", pipe, val);
I915_WRITE(reg, val & ~LVDS_PORT_EN);
POSTING_READ(reg);
DELAY(100);
}
disable_pch_hdmi(dev_priv, pipe, HDMIB);
disable_pch_hdmi(dev_priv, pipe, HDMIC);
disable_pch_hdmi(dev_priv, pipe, HDMID);
}
int
intel_pin_and_fence_fb_obj(struct drm_device *dev,
struct drm_i915_gem_object *obj,
struct intel_ring_buffer *pipelined)
{
struct drm_i915_private *dev_priv = dev->dev_private;
u32 alignment;
int ret;
alignment = 0; /* shut gcc */
switch (obj->tiling_mode) {
case I915_TILING_NONE:
if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
alignment = 128 * 1024;
else if (INTEL_INFO(dev)->gen >= 4)
alignment = 4 * 1024;
else
alignment = 64 * 1024;
break;
case I915_TILING_X:
/* pin() will align the object as required by fence */
alignment = 0;
break;
case I915_TILING_Y:
/* FIXME: Is this true? */
DRM_ERROR("Y tiled not allowed for scan out buffers\n");
return -EINVAL;
default:
KASSERT(0, ("Wrong tiling for fb obj"));
}
dev_priv->mm.interruptible = false;
ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined);
if (ret)
goto err_interruptible;
/* Install a fence for tiled scan-out. Pre-i965 always needs a
* fence, whereas 965+ only requires a fence if using
* framebuffer compression. For simplicity, we always install
* a fence as the cost is not that onerous.
*/
ret = i915_gem_object_get_fence(obj);
if (ret)
goto err_unpin;
i915_gem_object_pin_fence(obj);
dev_priv->mm.interruptible = true;
return 0;
err_unpin:
i915_gem_object_unpin_from_display_plane(obj);
err_interruptible:
dev_priv->mm.interruptible = true;
return ret;
}
void intel_unpin_fb_obj(struct drm_i915_gem_object *obj)
{
i915_gem_object_unpin_fence(obj);
i915_gem_object_unpin_from_display_plane(obj);
}
static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
int x, int y)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct intel_framebuffer *intel_fb;
struct drm_i915_gem_object *obj;
int plane = intel_crtc->plane;
unsigned long Start, Offset;
u32 dspcntr;
u32 reg;
switch (plane) {
case 0:
case 1:
break;
default:
DRM_ERROR("Can't update plane %d in SAREA\n", plane);
return -EINVAL;
}
intel_fb = to_intel_framebuffer(fb);
obj = intel_fb->obj;
reg = DSPCNTR(plane);
dspcntr = I915_READ(reg);
/* Mask out pixel format bits in case we change it */
dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
switch (fb->bits_per_pixel) {
case 8:
dspcntr |= DISPPLANE_8BPP;
break;
case 16:
if (fb->depth == 15)
dspcntr |= DISPPLANE_15_16BPP;
else
dspcntr |= DISPPLANE_16BPP;
break;
case 24:
case 32:
dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
break;
default:
DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
return -EINVAL;
}
if (INTEL_INFO(dev)->gen >= 4) {
if (obj->tiling_mode != I915_TILING_NONE)
dspcntr |= DISPPLANE_TILED;
else
dspcntr &= ~DISPPLANE_TILED;
}
I915_WRITE(reg, dspcntr);
Start = obj->gtt_offset;
Offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8);
DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
Start, Offset, x, y, fb->pitches[0]);
I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
if (INTEL_INFO(dev)->gen >= 4) {
I915_MODIFY_DISPBASE(DSPSURF(plane), Start);
I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
I915_WRITE(DSPADDR(plane), Offset);
} else
I915_WRITE(DSPADDR(plane), Start + Offset);
POSTING_READ(reg);
return (0);
}
static int ironlake_update_plane(struct drm_crtc *crtc,
struct drm_framebuffer *fb, int x, int y)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct intel_framebuffer *intel_fb;
struct drm_i915_gem_object *obj;
int plane = intel_crtc->plane;
unsigned long Start, Offset;
u32 dspcntr;
u32 reg;
switch (plane) {
case 0:
case 1:
case 2:
break;
default:
DRM_ERROR("Can't update plane %d in SAREA\n", plane);
return -EINVAL;
}
intel_fb = to_intel_framebuffer(fb);
obj = intel_fb->obj;
reg = DSPCNTR(plane);
dspcntr = I915_READ(reg);
/* Mask out pixel format bits in case we change it */
dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
switch (fb->bits_per_pixel) {
case 8:
dspcntr |= DISPPLANE_8BPP;
break;
case 16:
if (fb->depth != 16) {
DRM_ERROR("bpp 16, depth %d\n", fb->depth);
return -EINVAL;
}
dspcntr |= DISPPLANE_16BPP;
break;
case 24:
case 32:
if (fb->depth == 24)
dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
else if (fb->depth == 30)
dspcntr |= DISPPLANE_32BPP_30BIT_NO_ALPHA;
else {
DRM_ERROR("bpp %d depth %d\n", fb->bits_per_pixel,
fb->depth);
return -EINVAL;
}
break;
default:
DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
return -EINVAL;
}
if (obj->tiling_mode != I915_TILING_NONE)
dspcntr |= DISPPLANE_TILED;
else
dspcntr &= ~DISPPLANE_TILED;
/* must disable */
dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
I915_WRITE(reg, dspcntr);
Start = obj->gtt_offset;
Offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8);
DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
Start, Offset, x, y, fb->pitches[0]);
I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
I915_MODIFY_DISPBASE(DSPSURF(plane), Start);
I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
I915_WRITE(DSPADDR(plane), Offset);
POSTING_READ(reg);
return 0;
}
/* Assume fb object is pinned & idle & fenced and just update base pointers */
static int
intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
int x, int y, enum mode_set_atomic state)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
if (dev_priv->display.disable_fbc)
dev_priv->display.disable_fbc(dev);
intel_increase_pllclock(crtc);
return dev_priv->display.update_plane(crtc, fb, x, y);
}
static int
intel_finish_fb(struct drm_framebuffer *old_fb)
{
struct drm_i915_gem_object *obj = to_intel_framebuffer(old_fb)->obj;
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
bool was_interruptible = dev_priv->mm.interruptible;
int ret;
mtx_lock(&dev->event_lock);
while (!atomic_load_acq_int(&dev_priv->mm.wedged) &&
atomic_load_acq_int(&obj->pending_flip) != 0) {
msleep(&obj->pending_flip, &dev->event_lock,
0, "915flp", 0);
}
mtx_unlock(&dev->event_lock);
/* Big Hammer, we also need to ensure that any pending
* MI_WAIT_FOR_EVENT inside a user batch buffer on the
* current scanout is retired before unpinning the old
* framebuffer.
*
* This should only fail upon a hung GPU, in which case we
* can safely continue.
*/
dev_priv->mm.interruptible = false;
ret = i915_gem_object_finish_gpu(obj);
dev_priv->mm.interruptible = was_interruptible;
return ret;
}
static int
intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
struct drm_framebuffer *old_fb)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_master_private *master_priv;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int ret;
/* no fb bound */
if (!crtc->fb) {
DRM_ERROR("No FB bound\n");
return 0;
}
if(intel_crtc->plane > dev_priv->num_pipe) {
DRM_ERROR("no plane for crtc: plane %d, num_pipes %d\n",
intel_crtc->plane,
dev_priv->num_pipe);
return -EINVAL;
}
DRM_LOCK(dev);
ret = intel_pin_and_fence_fb_obj(dev,
to_intel_framebuffer(crtc->fb)->obj,
NULL);
if (ret != 0) {
DRM_UNLOCK(dev);
DRM_ERROR("pin & fence failed\n");
return ret;
}
if (old_fb)
intel_finish_fb(old_fb);
ret = dev_priv->display.update_plane(crtc, crtc->fb, x, y);
if (ret) {
intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj);
DRM_UNLOCK(dev);
DRM_ERROR("failed to update base address\n");
return ret;
}
if (old_fb) {
intel_wait_for_vblank(dev, intel_crtc->pipe);
intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj);
}
intel_update_fbc(dev);
DRM_UNLOCK(dev);
if (!dev->primary->master)
return 0;
master_priv = dev->primary->master->driver_priv;
if (!master_priv->sarea_priv)
return 0;
if (intel_crtc->pipe) {
master_priv->sarea_priv->pipeB_x = x;
master_priv->sarea_priv->pipeB_y = y;
} else {
master_priv->sarea_priv->pipeA_x = x;
master_priv->sarea_priv->pipeA_y = y;
}
return 0;
}
static void ironlake_set_pll_edp(struct drm_crtc *crtc, int clock)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
u32 dpa_ctl;
DRM_DEBUG_KMS("eDP PLL enable for clock %d\n", clock);
dpa_ctl = I915_READ(DP_A);
dpa_ctl &= ~DP_PLL_FREQ_MASK;
if (clock < 200000) {
u32 temp;
dpa_ctl |= DP_PLL_FREQ_160MHZ;
/* workaround for 160Mhz:
1) program 0x4600c bits 15:0 = 0x8124
2) program 0x46010 bit 0 = 1
3) program 0x46034 bit 24 = 1
4) program 0x64000 bit 14 = 1
*/
temp = I915_READ(0x4600c);
temp &= 0xffff0000;
I915_WRITE(0x4600c, temp | 0x8124);
temp = I915_READ(0x46010);
I915_WRITE(0x46010, temp | 1);
temp = I915_READ(0x46034);
I915_WRITE(0x46034, temp | (1 << 24));
} else {
dpa_ctl |= DP_PLL_FREQ_270MHZ;
}
I915_WRITE(DP_A, dpa_ctl);
POSTING_READ(DP_A);
DELAY(500);
}
static void intel_fdi_normal_train(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 reg, temp;
/* enable normal train */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
if (IS_IVYBRIDGE(dev)) {
temp &= ~FDI_LINK_TRAIN_NONE_IVB;
temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
} else {
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
}
I915_WRITE(reg, temp);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
if (HAS_PCH_CPT(dev)) {
temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
temp |= FDI_LINK_TRAIN_NORMAL_CPT;
} else {
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_NONE;
}
I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
/* wait one idle pattern time */
POSTING_READ(reg);
DELAY(1000);
/* IVB wants error correction enabled */
if (IS_IVYBRIDGE(dev))
I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
FDI_FE_ERRC_ENABLE);
}
static void cpt_phase_pointer_enable(struct drm_device *dev, int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
u32 flags = I915_READ(SOUTH_CHICKEN1);
flags |= FDI_PHASE_SYNC_OVR(pipe);
I915_WRITE(SOUTH_CHICKEN1, flags); /* once to unlock... */
flags |= FDI_PHASE_SYNC_EN(pipe);
I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to enable */
POSTING_READ(SOUTH_CHICKEN1);
}
/* The FDI link training functions for ILK/Ibexpeak. */
static void ironlake_fdi_link_train(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int plane = intel_crtc->plane;
u32 reg, temp, tries;
/* FDI needs bits from pipe & plane first */
assert_pipe_enabled(dev_priv, pipe);
assert_plane_enabled(dev_priv, plane);
/* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
for train result */
reg = FDI_RX_IMR(pipe);
temp = I915_READ(reg);
temp &= ~FDI_RX_SYMBOL_LOCK;
temp &= ~FDI_RX_BIT_LOCK;
I915_WRITE(reg, temp);
I915_READ(reg);
DELAY(150);
/* enable CPU FDI TX and PCH FDI RX */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~(7 << 19);
temp |= (intel_crtc->fdi_lanes - 1) << 19;
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_1;
I915_WRITE(reg, temp | FDI_TX_ENABLE);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_1;
I915_WRITE(reg, temp | FDI_RX_ENABLE);
POSTING_READ(reg);
DELAY(150);
/* Ironlake workaround, enable clock pointer after FDI enable*/
if (HAS_PCH_IBX(dev)) {
I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
FDI_RX_PHASE_SYNC_POINTER_EN);
}
reg = FDI_RX_IIR(pipe);
for (tries = 0; tries < 5; tries++) {
temp = I915_READ(reg);
DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
if ((temp & FDI_RX_BIT_LOCK)) {
DRM_DEBUG_KMS("FDI train 1 done.\n");
I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
break;
}
}
if (tries == 5)
DRM_ERROR("FDI train 1 fail!\n");
/* Train 2 */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_2;
I915_WRITE(reg, temp);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_2;
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(150);
reg = FDI_RX_IIR(pipe);
for (tries = 0; tries < 5; tries++) {
temp = I915_READ(reg);
DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
if (temp & FDI_RX_SYMBOL_LOCK) {
I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
DRM_DEBUG_KMS("FDI train 2 done.\n");
break;
}
}
if (tries == 5)
DRM_ERROR("FDI train 2 fail!\n");
DRM_DEBUG_KMS("FDI train done\n");
}
static const int snb_b_fdi_train_param[] = {
FDI_LINK_TRAIN_400MV_0DB_SNB_B,
FDI_LINK_TRAIN_400MV_6DB_SNB_B,
FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
FDI_LINK_TRAIN_800MV_0DB_SNB_B,
};
/* The FDI link training functions for SNB/Cougarpoint. */
static void gen6_fdi_link_train(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 reg, temp, i, retry;
/* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
for train result */
reg = FDI_RX_IMR(pipe);
temp = I915_READ(reg);
temp &= ~FDI_RX_SYMBOL_LOCK;
temp &= ~FDI_RX_BIT_LOCK;
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(150);
/* enable CPU FDI TX and PCH FDI RX */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~(7 << 19);
temp |= (intel_crtc->fdi_lanes - 1) << 19;
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_1;
temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
/* SNB-B */
temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
I915_WRITE(reg, temp | FDI_TX_ENABLE);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
if (HAS_PCH_CPT(dev)) {
temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
} else {
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_1;
}
I915_WRITE(reg, temp | FDI_RX_ENABLE);
POSTING_READ(reg);
DELAY(150);
if (HAS_PCH_CPT(dev))
cpt_phase_pointer_enable(dev, pipe);
for (i = 0; i < 4; i++) {
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
temp |= snb_b_fdi_train_param[i];
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(500);
for (retry = 0; retry < 5; retry++) {
reg = FDI_RX_IIR(pipe);
temp = I915_READ(reg);
DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
if (temp & FDI_RX_BIT_LOCK) {
I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
DRM_DEBUG_KMS("FDI train 1 done.\n");
break;
}
DELAY(50);
}
if (retry < 5)
break;
}
if (i == 4)
DRM_ERROR("FDI train 1 fail!\n");
/* Train 2 */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_2;
if (IS_GEN6(dev)) {
temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
/* SNB-B */
temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
}
I915_WRITE(reg, temp);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
if (HAS_PCH_CPT(dev)) {
temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
} else {
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_2;
}
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(150);
for (i = 0; i < 4; i++) {
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
temp |= snb_b_fdi_train_param[i];
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(500);
for (retry = 0; retry < 5; retry++) {
reg = FDI_RX_IIR(pipe);
temp = I915_READ(reg);
DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
if (temp & FDI_RX_SYMBOL_LOCK) {
I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
DRM_DEBUG_KMS("FDI train 2 done.\n");
break;
}
DELAY(50);
}
if (retry < 5)
break;
}
if (i == 4)
DRM_ERROR("FDI train 2 fail!\n");
DRM_DEBUG_KMS("FDI train done.\n");
}
/* Manual link training for Ivy Bridge A0 parts */
static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 reg, temp, i;
/* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
for train result */
reg = FDI_RX_IMR(pipe);
temp = I915_READ(reg);
temp &= ~FDI_RX_SYMBOL_LOCK;
temp &= ~FDI_RX_BIT_LOCK;
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(150);
/* enable CPU FDI TX and PCH FDI RX */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~(7 << 19);
temp |= (intel_crtc->fdi_lanes - 1) << 19;
temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
temp |= FDI_COMPOSITE_SYNC;
I915_WRITE(reg, temp | FDI_TX_ENABLE);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_AUTO;
temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
temp |= FDI_COMPOSITE_SYNC;
I915_WRITE(reg, temp | FDI_RX_ENABLE);
POSTING_READ(reg);
DELAY(150);
for (i = 0; i < 4; i++) {
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
temp |= snb_b_fdi_train_param[i];
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(500);
reg = FDI_RX_IIR(pipe);
temp = I915_READ(reg);
DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
if (temp & FDI_RX_BIT_LOCK ||
(I915_READ(reg) & FDI_RX_BIT_LOCK)) {
I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
DRM_DEBUG_KMS("FDI train 1 done.\n");
break;
}
}
if (i == 4)
DRM_ERROR("FDI train 1 fail!\n");
/* Train 2 */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_NONE_IVB;
temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
I915_WRITE(reg, temp);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(150);
for (i = 0; i < 4; i++ ) {
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
temp |= snb_b_fdi_train_param[i];
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(500);
reg = FDI_RX_IIR(pipe);
temp = I915_READ(reg);
DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
if (temp & FDI_RX_SYMBOL_LOCK) {
I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
DRM_DEBUG_KMS("FDI train 2 done.\n");
break;
}
}
if (i == 4)
DRM_ERROR("FDI train 2 fail!\n");
DRM_DEBUG_KMS("FDI train done.\n");
}
static void ironlake_fdi_pll_enable(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 reg, temp;
/* Write the TU size bits so error detection works */
I915_WRITE(FDI_RX_TUSIZE1(pipe),
I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
/* enable PCH FDI RX PLL, wait warmup plus DMI latency */
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~((0x7 << 19) | (0x7 << 16));
temp |= (intel_crtc->fdi_lanes - 1) << 19;
temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
POSTING_READ(reg);
DELAY(200);
/* Switch from Rawclk to PCDclk */
temp = I915_READ(reg);
I915_WRITE(reg, temp | FDI_PCDCLK);
POSTING_READ(reg);
DELAY(200);
/* On Haswell, the PLL configuration for ports and pipes is handled
* separately, as part of DDI setup */
if (!IS_HASWELL(dev)) {
/* Enable CPU FDI TX PLL, always on for Ironlake */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
if ((temp & FDI_TX_PLL_ENABLE) == 0) {
I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
POSTING_READ(reg);
DELAY(100);
}
}
}
static void cpt_phase_pointer_disable(struct drm_device *dev, int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
u32 flags = I915_READ(SOUTH_CHICKEN1);
flags &= ~(FDI_PHASE_SYNC_EN(pipe));
I915_WRITE(SOUTH_CHICKEN1, flags); /* once to disable... */
flags &= ~(FDI_PHASE_SYNC_OVR(pipe));
I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to lock */
POSTING_READ(SOUTH_CHICKEN1);
}
static void ironlake_fdi_disable(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 reg, temp;
/* disable CPU FDI tx and PCH FDI rx */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
POSTING_READ(reg);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~(0x7 << 16);
temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
POSTING_READ(reg);
DELAY(100);
/* Ironlake workaround, disable clock pointer after downing FDI */
if (HAS_PCH_IBX(dev)) {
I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
I915_WRITE(FDI_RX_CHICKEN(pipe),
I915_READ(FDI_RX_CHICKEN(pipe) &
~FDI_RX_PHASE_SYNC_POINTER_EN));
} else if (HAS_PCH_CPT(dev)) {
cpt_phase_pointer_disable(dev, pipe);
}
/* still set train pattern 1 */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_1;
I915_WRITE(reg, temp);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
if (HAS_PCH_CPT(dev)) {
temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
} else {
temp &= ~FDI_LINK_TRAIN_NONE;
temp |= FDI_LINK_TRAIN_PATTERN_1;
}
/* BPC in FDI rx is consistent with that in PIPECONF */
temp &= ~(0x07 << 16);
temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
I915_WRITE(reg, temp);
POSTING_READ(reg);
DELAY(100);
}
static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
if (crtc->fb == NULL)
return;
DRM_LOCK(dev);
intel_finish_fb(crtc->fb);
DRM_UNLOCK(dev);
}
static bool intel_crtc_driving_pch(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_mode_config *mode_config = &dev->mode_config;
struct intel_encoder *encoder;
/*
* If there's a non-PCH eDP on this crtc, it must be DP_A, and that
* must be driven by its own crtc; no sharing is possible.
*/
list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
if (encoder->base.crtc != crtc)
continue;
/* On Haswell, LPT PCH handles the VGA connection via FDI, and Haswell
* CPU handles all others */
if (IS_HASWELL(dev)) {
/* It is still unclear how this will work on PPT, so throw up a warning */
if (!HAS_PCH_LPT(dev))
DRM_DEBUG_KMS("Haswell: PPT\n");
if (encoder->type == DRM_MODE_ENCODER_DAC) {
DRM_DEBUG_KMS("Haswell detected DAC encoder, assuming is PCH\n");
return true;
} else {
DRM_DEBUG_KMS("Haswell detected encoder %d, assuming is CPU\n",
encoder->type);
return false;
}
}
switch (encoder->type) {
case INTEL_OUTPUT_EDP:
if (!intel_encoder_is_pch_edp(&encoder->base))
return false;
continue;
}
}
return true;
}
/* Program iCLKIP clock to the desired frequency */
static void lpt_program_iclkip(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
u32 divsel, phaseinc, auxdiv, phasedir = 0;
u32 temp;
/* It is necessary to ungate the pixclk gate prior to programming
* the divisors, and gate it back when it is done.
*/
I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
/* Disable SSCCTL */
intel_sbi_write(dev_priv, SBI_SSCCTL6,
intel_sbi_read(dev_priv, SBI_SSCCTL6) |
SBI_SSCCTL_DISABLE);
/* 20MHz is a corner case which is out of range for the 7-bit divisor */
if (crtc->mode.clock == 20000) {
auxdiv = 1;
divsel = 0x41;
phaseinc = 0x20;
} else {
/* The iCLK virtual clock root frequency is in MHz,
* but the crtc->mode.clock in in KHz. To get the divisors,
* it is necessary to divide one by another, so we
* convert the virtual clock precision to KHz here for higher
* precision.
*/
u32 iclk_virtual_root_freq = 172800 * 1000;
u32 iclk_pi_range = 64;
u32 desired_divisor, msb_divisor_value, pi_value;
desired_divisor = (iclk_virtual_root_freq / crtc->mode.clock);
msb_divisor_value = desired_divisor / iclk_pi_range;
pi_value = desired_divisor % iclk_pi_range;
auxdiv = 0;
divsel = msb_divisor_value - 2;
phaseinc = pi_value;
}
/* This should not happen with any sane values */
if ((SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
~SBI_SSCDIVINTPHASE_DIVSEL_MASK))
DRM_DEBUG_KMS("DIVSEL_MASK");
if ((SBI_SSCDIVINTPHASE_DIR(phasedir) &
~SBI_SSCDIVINTPHASE_INCVAL_MASK))
DRM_DEBUG_KMS("INCVAL_MASK");
DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
crtc->mode.clock,
auxdiv,
divsel,
phasedir,
phaseinc);
/* Program SSCDIVINTPHASE6 */
temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6);
temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
intel_sbi_write(dev_priv,
SBI_SSCDIVINTPHASE6,
temp);
/* Program SSCAUXDIV */
temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6);
temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
intel_sbi_write(dev_priv,
SBI_SSCAUXDIV6,
temp);
/* Enable modulator and associated divider */
temp = intel_sbi_read(dev_priv, SBI_SSCCTL6);
temp &= ~SBI_SSCCTL_DISABLE;
intel_sbi_write(dev_priv,
SBI_SSCCTL6,
temp);
/* Wait for initialization time */
DELAY(24);
I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
}
/*
* Enable PCH resources required for PCH ports:
* - PCH PLLs
* - FDI training & RX/TX
* - update transcoder timings
* - DP transcoding bits
* - transcoder
*/
static void ironlake_pch_enable(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 reg, temp;
assert_transcoder_disabled(dev_priv, pipe);
/* For PCH output, training FDI link */
dev_priv->display.fdi_link_train(crtc);
intel_enable_pch_pll(intel_crtc);
if (HAS_PCH_LPT(dev)) {
DRM_DEBUG_KMS("LPT detected: programming iCLKIP\n");
lpt_program_iclkip(crtc);
} else if (HAS_PCH_CPT(dev)) {
u32 sel;
temp = I915_READ(PCH_DPLL_SEL);
switch (pipe) {
default:
case 0:
temp |= TRANSA_DPLL_ENABLE;
sel = TRANSA_DPLLB_SEL;
break;
case 1:
temp |= TRANSB_DPLL_ENABLE;
sel = TRANSB_DPLLB_SEL;
break;
case 2:
temp |= TRANSC_DPLL_ENABLE;
sel = TRANSC_DPLLB_SEL;
break;
}
if (intel_crtc->pch_pll->pll_reg == _PCH_DPLL_B)
temp |= sel;
else
temp &= ~sel;
I915_WRITE(PCH_DPLL_SEL, temp);
}
/* set transcoder timing, panel must allow it */
assert_panel_unlocked(dev_priv, pipe);
I915_WRITE(TRANS_HTOTAL(pipe), I915_READ(HTOTAL(pipe)));
I915_WRITE(TRANS_HBLANK(pipe), I915_READ(HBLANK(pipe)));
I915_WRITE(TRANS_HSYNC(pipe), I915_READ(HSYNC(pipe)));
I915_WRITE(TRANS_VTOTAL(pipe), I915_READ(VTOTAL(pipe)));
I915_WRITE(TRANS_VBLANK(pipe), I915_READ(VBLANK(pipe)));
I915_WRITE(TRANS_VSYNC(pipe), I915_READ(VSYNC(pipe)));
I915_WRITE(TRANS_VSYNCSHIFT(pipe), I915_READ(VSYNCSHIFT(pipe)));
if (!IS_HASWELL(dev))
intel_fdi_normal_train(crtc);
/* For PCH DP, enable TRANS_DP_CTL */
if (HAS_PCH_CPT(dev) &&
(intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))) {
u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) >> 5;
reg = TRANS_DP_CTL(pipe);
temp = I915_READ(reg);
temp &= ~(TRANS_DP_PORT_SEL_MASK |
TRANS_DP_SYNC_MASK |
TRANS_DP_BPC_MASK);
temp |= (TRANS_DP_OUTPUT_ENABLE |
TRANS_DP_ENH_FRAMING);
temp |= bpc << 9; /* same format but at 11:9 */
if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC)
temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
switch (intel_trans_dp_port_sel(crtc)) {
case PCH_DP_B:
temp |= TRANS_DP_PORT_SEL_B;
break;
case PCH_DP_C:
temp |= TRANS_DP_PORT_SEL_C;
break;
case PCH_DP_D:
temp |= TRANS_DP_PORT_SEL_D;
break;
default:
DRM_DEBUG_KMS("Wrong PCH DP port return. Guess port B\n");
temp |= TRANS_DP_PORT_SEL_B;
break;
}
I915_WRITE(reg, temp);
}
intel_enable_transcoder(dev_priv, pipe);
}
static void intel_put_pch_pll(struct intel_crtc *intel_crtc)
{
struct intel_pch_pll *pll = intel_crtc->pch_pll;
if (pll == NULL)
return;
if (pll->refcount == 0) {
printf("bad PCH PLL refcount\n");
return;
}
--pll->refcount;
intel_crtc->pch_pll = NULL;
}
static struct intel_pch_pll *intel_get_pch_pll(struct intel_crtc *intel_crtc, u32 dpll, u32 fp)
{
struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
struct intel_pch_pll *pll;
int i;
pll = intel_crtc->pch_pll;
if (pll) {
DRM_DEBUG_KMS("CRTC:%d reusing existing PCH PLL %x\n",
intel_crtc->base.base.id, pll->pll_reg);
goto prepare;
}
if (HAS_PCH_IBX(dev_priv->dev)) {
/* Ironlake PCH has a fixed PLL->PCH pipe mapping. */
i = intel_crtc->pipe;
pll = &dev_priv->pch_plls[i];
DRM_DEBUG_KMS("CRTC:%d using pre-allocated PCH PLL %x\n",
intel_crtc->base.base.id, pll->pll_reg);
goto found;
}
for (i = 0; i < dev_priv->num_pch_pll; i++) {
pll = &dev_priv->pch_plls[i];
/* Only want to check enabled timings first */
if (pll->refcount == 0)
continue;
if (dpll == (I915_READ(pll->pll_reg) & 0x7fffffff) &&
fp == I915_READ(pll->fp0_reg)) {
DRM_DEBUG_KMS("CRTC:%d sharing existing PCH PLL %x (refcount %d, ative %d)\n",
intel_crtc->base.base.id,
pll->pll_reg, pll->refcount, pll->active);
goto found;
}
}
/* Ok no matching timings, maybe there's a free one? */
for (i = 0; i < dev_priv->num_pch_pll; i++) { /* XXXKIB: HACK */
pll = &dev_priv->pch_plls[i];
if (pll->refcount == 0) {
DRM_DEBUG_KMS("CRTC:%d allocated PCH PLL %x\n",
intel_crtc->base.base.id, pll->pll_reg);
goto found;
}
}
return NULL;
found:
intel_crtc->pch_pll = pll;
pll->refcount++;
DRM_DEBUG_DRIVER("using pll %d for pipe %d\n", i, intel_crtc->pipe);
prepare: /* separate function? */
DRM_DEBUG_DRIVER("switching PLL %x off\n", pll->pll_reg);
/* Wait for the clocks to stabilize before rewriting the regs */
I915_WRITE(pll->pll_reg, dpll & ~DPLL_VCO_ENABLE);
POSTING_READ(pll->pll_reg);
DELAY(150);
I915_WRITE(pll->fp0_reg, fp);
I915_WRITE(pll->pll_reg, dpll & ~DPLL_VCO_ENABLE);
pll->on = false;
return pll;
}
void intel_cpt_verify_modeset(struct drm_device *dev, int pipe)
{
struct drm_i915_private *dev_priv = dev->dev_private;
int dslreg = PIPEDSL(pipe), tc2reg = TRANS_CHICKEN2(pipe);
u32 temp;
temp = I915_READ(dslreg);
DELAY(500);
if (_intel_wait_for(dev, I915_READ(dslreg) != temp, 5, 1, "915cp1")) {
/* Without this, mode sets may fail silently on FDI */
I915_WRITE(tc2reg, TRANS_AUTOTRAIN_GEN_STALL_DIS);
DELAY(250);
I915_WRITE(tc2reg, 0);
if (_intel_wait_for(dev, I915_READ(dslreg) != temp, 5, 1,
"915cp2"))
DRM_ERROR("mode set failed: pipe %d stuck\n", pipe);
}
}
static void ironlake_crtc_enable(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int plane = intel_crtc->plane;
u32 temp;
bool is_pch_port;
if (intel_crtc->active)
return;
intel_crtc->active = true;
intel_update_watermarks(dev);
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
temp = I915_READ(PCH_LVDS);
if ((temp & LVDS_PORT_EN) == 0)
I915_WRITE(PCH_LVDS, temp | LVDS_PORT_EN);
}
is_pch_port = intel_crtc_driving_pch(crtc);
if (is_pch_port)
ironlake_fdi_pll_enable(crtc);
else
ironlake_fdi_disable(crtc);
/* Enable panel fitting for LVDS */
if (dev_priv->pch_pf_size &&
(intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) || HAS_eDP)) {
/* Force use of hard-coded filter coefficients
* as some pre-programmed values are broken,
* e.g. x201.
*/
I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
I915_WRITE(PF_WIN_POS(pipe), dev_priv->pch_pf_pos);
I915_WRITE(PF_WIN_SZ(pipe), dev_priv->pch_pf_size);
}
intel_enable_pipe(dev_priv, pipe, is_pch_port);
intel_enable_plane(dev_priv, plane, pipe);
if (is_pch_port)
ironlake_pch_enable(crtc);
intel_crtc_load_lut(crtc);
DRM_LOCK(dev);
intel_update_fbc(dev);
DRM_UNLOCK(dev);
intel_crtc_update_cursor(crtc, true);
}
static void ironlake_crtc_disable(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int plane = intel_crtc->plane;
u32 reg, temp;
if (!intel_crtc->active)
return;
intel_crtc_wait_for_pending_flips(crtc);
drm_vblank_off(dev, pipe);
intel_crtc_update_cursor(crtc, false);
intel_disable_plane(dev_priv, plane, pipe);
if (dev_priv->cfb_plane == plane)
intel_disable_fbc(dev);
intel_disable_pipe(dev_priv, pipe);
/* Disable PF */
I915_WRITE(PF_CTL(pipe), 0);
I915_WRITE(PF_WIN_SZ(pipe), 0);
ironlake_fdi_disable(crtc);
/* This is a horrible layering violation; we should be doing this in
* the connector/encoder ->prepare instead, but we don't always have
* enough information there about the config to know whether it will
* actually be necessary or just cause undesired flicker.
*/
intel_disable_pch_ports(dev_priv, pipe);
intel_disable_transcoder(dev_priv, pipe);
if (HAS_PCH_CPT(dev)) {
/* disable TRANS_DP_CTL */
reg = TRANS_DP_CTL(pipe);
temp = I915_READ(reg);
temp &= ~(TRANS_DP_OUTPUT_ENABLE | TRANS_DP_PORT_SEL_MASK);
temp |= TRANS_DP_PORT_SEL_NONE;
I915_WRITE(reg, temp);
/* disable DPLL_SEL */
temp = I915_READ(PCH_DPLL_SEL);
switch (pipe) {
case 0:
temp &= ~(TRANSA_DPLL_ENABLE | TRANSA_DPLLB_SEL);
break;
case 1:
temp &= ~(TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
break;
case 2:
/* C shares PLL A or B */
temp &= ~(TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL);
break;
default:
KASSERT(1, ("Wrong pipe %d", pipe)); /* wtf */
}
I915_WRITE(PCH_DPLL_SEL, temp);
}
/* disable PCH DPLL */
intel_disable_pch_pll(intel_crtc);
/* Switch from PCDclk to Rawclk */
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
I915_WRITE(reg, temp & ~FDI_PCDCLK);
/* Disable CPU FDI TX PLL */
reg = FDI_TX_CTL(pipe);
temp = I915_READ(reg);
I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
POSTING_READ(reg);
DELAY(100);
reg = FDI_RX_CTL(pipe);
temp = I915_READ(reg);
I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
/* Wait for the clocks to turn off. */
POSTING_READ(reg);
DELAY(100);
intel_crtc->active = false;
intel_update_watermarks(dev);
DRM_LOCK(dev);
intel_update_fbc(dev);
DRM_UNLOCK(dev);
}
static void ironlake_crtc_dpms(struct drm_crtc *crtc, int mode)
{
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int plane = intel_crtc->plane;
/* XXX: When our outputs are all unaware of DPMS modes other than off
* and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
*/
switch (mode) {
case DRM_MODE_DPMS_ON:
case DRM_MODE_DPMS_STANDBY:
case DRM_MODE_DPMS_SUSPEND:
DRM_DEBUG_KMS("crtc %d/%d dpms on\n", pipe, plane);
ironlake_crtc_enable(crtc);
break;
case DRM_MODE_DPMS_OFF:
DRM_DEBUG_KMS("crtc %d/%d dpms off\n", pipe, plane);
ironlake_crtc_disable(crtc);
break;
}
}
static void ironlake_crtc_off(struct drm_crtc *crtc)
{
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
intel_put_pch_pll(intel_crtc);
}
static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable)
{
if (!enable && intel_crtc->overlay) {
struct drm_device *dev = intel_crtc->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
DRM_LOCK(dev);
dev_priv->mm.interruptible = false;
(void) intel_overlay_switch_off(intel_crtc->overlay);
dev_priv->mm.interruptible = true;
DRM_UNLOCK(dev);
}
/* Let userspace switch the overlay on again. In most cases userspace
* has to recompute where to put it anyway.
*/
}
static void i9xx_crtc_enable(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int plane = intel_crtc->plane;
if (intel_crtc->active)
return;
intel_crtc->active = true;
intel_update_watermarks(dev);
intel_enable_pll(dev_priv, pipe);
intel_enable_pipe(dev_priv, pipe, false);
intel_enable_plane(dev_priv, plane, pipe);
intel_crtc_load_lut(crtc);
intel_update_fbc(dev);
/* Give the overlay scaler a chance to enable if it's on this pipe */
intel_crtc_dpms_overlay(intel_crtc, true);
intel_crtc_update_cursor(crtc, true);
}
static void i9xx_crtc_disable(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int plane = intel_crtc->plane;
if (!intel_crtc->active)
return;
/* Give the overlay scaler a chance to disable if it's on this pipe */
intel_crtc_wait_for_pending_flips(crtc);
drm_vblank_off(dev, pipe);
intel_crtc_dpms_overlay(intel_crtc, false);
intel_crtc_update_cursor(crtc, false);
if (dev_priv->cfb_plane == plane)
intel_disable_fbc(dev);
intel_disable_plane(dev_priv, plane, pipe);
intel_disable_pipe(dev_priv, pipe);
intel_disable_pll(dev_priv, pipe);
intel_crtc->active = false;
intel_update_fbc(dev);
intel_update_watermarks(dev);
}
static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
{
/* XXX: When our outputs are all unaware of DPMS modes other than off
* and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
*/
switch (mode) {
case DRM_MODE_DPMS_ON:
case DRM_MODE_DPMS_STANDBY:
case DRM_MODE_DPMS_SUSPEND:
i9xx_crtc_enable(crtc);
break;
case DRM_MODE_DPMS_OFF:
i9xx_crtc_disable(crtc);
break;
}
}
static void i9xx_crtc_off(struct drm_crtc *crtc)
{
}
/**
* Sets the power management mode of the pipe and plane.
*/
static void intel_crtc_dpms(struct drm_crtc *crtc, int mode)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_master_private *master_priv;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
bool enabled;
if (intel_crtc->dpms_mode == mode)
return;
intel_crtc->dpms_mode = mode;
dev_priv->display.dpms(crtc, mode);
if (!dev->primary->master)
return;
master_priv = dev->primary->master->driver_priv;
if (!master_priv->sarea_priv)
return;
enabled = crtc->enabled && mode != DRM_MODE_DPMS_OFF;
switch (pipe) {
case 0:
master_priv->sarea_priv->pipeA_w = enabled ? crtc->mode.hdisplay : 0;
master_priv->sarea_priv->pipeA_h = enabled ? crtc->mode.vdisplay : 0;
break;
case 1:
master_priv->sarea_priv->pipeB_w = enabled ? crtc->mode.hdisplay : 0;
master_priv->sarea_priv->pipeB_h = enabled ? crtc->mode.vdisplay : 0;
break;
default:
DRM_ERROR("Can't update pipe %c in SAREA\n", pipe_name(pipe));
break;
}
}
static void intel_crtc_disable(struct drm_crtc *crtc)
{
struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
dev_priv->display.off(crtc);
assert_plane_disabled(dev->dev_private, to_intel_crtc(crtc)->plane);
assert_pipe_disabled(dev->dev_private, to_intel_crtc(crtc)->pipe);
if (crtc->fb) {
DRM_LOCK(dev);
intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj);
DRM_UNLOCK(dev);
}
}
/* Prepare for a mode set.
*
* Note we could be a lot smarter here. We need to figure out which outputs
* will be enabled, which disabled (in short, how the config will changes)
* and perform the minimum necessary steps to accomplish that, e.g. updating
* watermarks, FBC configuration, making sure PLLs are programmed correctly,
* panel fitting is in the proper state, etc.
*/
static void i9xx_crtc_prepare(struct drm_crtc *crtc)
{
i9xx_crtc_disable(crtc);
}
static void i9xx_crtc_commit(struct drm_crtc *crtc)
{
i9xx_crtc_enable(crtc);
}
static void ironlake_crtc_prepare(struct drm_crtc *crtc)
{
ironlake_crtc_disable(crtc);
}
static void ironlake_crtc_commit(struct drm_crtc *crtc)
{
ironlake_crtc_enable(crtc);
}
void intel_encoder_prepare(struct drm_encoder *encoder)
{
struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
/* lvds has its own version of prepare see intel_lvds_prepare */
encoder_funcs->dpms(encoder, DRM_MODE_DPMS_OFF);
}
void intel_encoder_commit(struct drm_encoder *encoder)
{
struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
struct drm_device *dev = encoder->dev;
struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
/* lvds has its own version of commit see intel_lvds_commit */
encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
if (HAS_PCH_CPT(dev))
intel_cpt_verify_modeset(dev, intel_crtc->pipe);
}
void intel_encoder_destroy(struct drm_encoder *encoder)
{
struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
drm_encoder_cleanup(encoder);
free(intel_encoder, DRM_MEM_KMS);
}
static bool intel_crtc_mode_fixup(struct drm_crtc *crtc,
const struct drm_display_mode *mode,
struct drm_display_mode *adjusted_mode)
{
struct drm_device *dev = crtc->dev;
if (HAS_PCH_SPLIT(dev)) {
/* FDI link clock is fixed at 2.7G */
if (mode->clock * 3 > IRONLAKE_FDI_FREQ * 4)
return false;
}
/* All interlaced capable intel hw wants timings in frames. Note though
* that intel_lvds_mode_fixup does some funny tricks with the crtc
* timings, so we need to be careful not to clobber these.*/
if (!(adjusted_mode->private_flags & INTEL_MODE_CRTC_TIMINGS_SET))
drm_mode_set_crtcinfo(adjusted_mode, 0);
return true;
}
static int valleyview_get_display_clock_speed(struct drm_device *dev)
{
return 400000; /* FIXME */
}
static int i945_get_display_clock_speed(struct drm_device *dev)
{
return 400000;
}
static int i915_get_display_clock_speed(struct drm_device *dev)
{
return 333000;
}
static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
{
return 200000;
}
static int i915gm_get_display_clock_speed(struct drm_device *dev)
{
u16 gcfgc = 0;
gcfgc = pci_read_config(dev->dev, GCFGC, 2);
if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
return 133000;
else {
switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
case GC_DISPLAY_CLOCK_333_MHZ:
return 333000;
default:
case GC_DISPLAY_CLOCK_190_200_MHZ:
return 190000;
}
}
}
static int i865_get_display_clock_speed(struct drm_device *dev)
{
return 266000;
}
static int i855_get_display_clock_speed(struct drm_device *dev)
{
u16 hpllcc = 0;
/* Assume that the hardware is in the high speed state. This
* should be the default.
*/
switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
case GC_CLOCK_133_200:
case GC_CLOCK_100_200:
return 200000;
case GC_CLOCK_166_250:
return 250000;
case GC_CLOCK_100_133:
return 133000;
}
/* Shouldn't happen */
return 0;
}
static int i830_get_display_clock_speed(struct drm_device *dev)
{
return 133000;
}
struct fdi_m_n {
u32 tu;
u32 gmch_m;
u32 gmch_n;
u32 link_m;
u32 link_n;
};
static void
fdi_reduce_ratio(u32 *num, u32 *den)
{
while (*num > 0xffffff || *den > 0xffffff) {
*num >>= 1;
*den >>= 1;
}
}
static void
ironlake_compute_m_n(int bits_per_pixel, int nlanes, int pixel_clock,
int link_clock, struct fdi_m_n *m_n)
{
m_n->tu = 64; /* default size */
/* BUG_ON(pixel_clock > INT_MAX / 36); */
m_n->gmch_m = bits_per_pixel * pixel_clock;
m_n->gmch_n = link_clock * nlanes * 8;
fdi_reduce_ratio(&m_n->gmch_m, &m_n->gmch_n);
m_n->link_m = pixel_clock;
m_n->link_n = link_clock;
fdi_reduce_ratio(&m_n->link_m, &m_n->link_n);
}
static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
{
if (i915_panel_use_ssc >= 0)
return i915_panel_use_ssc != 0;
return dev_priv->lvds_use_ssc
&& !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
}
/**
* intel_choose_pipe_bpp_dither - figure out what color depth the pipe should send
* @crtc: CRTC structure
* @mode: requested mode
*
* A pipe may be connected to one or more outputs. Based on the depth of the
* attached framebuffer, choose a good color depth to use on the pipe.
*
* If possible, match the pipe depth to the fb depth. In some cases, this
* isn't ideal, because the connected output supports a lesser or restricted
* set of depths. Resolve that here:
* LVDS typically supports only 6bpc, so clamp down in that case
* HDMI supports only 8bpc or 12bpc, so clamp to 8bpc with dither for 10bpc
* Displays may support a restricted set as well, check EDID and clamp as
* appropriate.
* DP may want to dither down to 6bpc to fit larger modes
*
* RETURNS:
* Dithering requirement (i.e. false if display bpc and pipe bpc match,
* true if they don't match).
*/
static bool intel_choose_pipe_bpp_dither(struct drm_crtc *crtc,
unsigned int *pipe_bpp,
struct drm_display_mode *mode)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_encoder *encoder;
struct drm_connector *connector;
unsigned int display_bpc = UINT_MAX, bpc;
/* Walk the encoders & connectors on this crtc, get min bpc */
list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
if (encoder->crtc != crtc)
continue;
if (intel_encoder->type == INTEL_OUTPUT_LVDS) {
unsigned int lvds_bpc;
if ((I915_READ(PCH_LVDS) & LVDS_A3_POWER_MASK) ==
LVDS_A3_POWER_UP)
lvds_bpc = 8;
else
lvds_bpc = 6;
if (lvds_bpc < display_bpc) {
DRM_DEBUG_KMS("clamping display bpc (was %d) to LVDS (%d)\n", display_bpc, lvds_bpc);
display_bpc = lvds_bpc;
}
continue;
}
if (intel_encoder->type == INTEL_OUTPUT_EDP) {
/* Use VBT settings if we have an eDP panel */
unsigned int edp_bpc = dev_priv->edp.bpp / 3;
if (edp_bpc < display_bpc) {
DRM_DEBUG_KMS("clamping display bpc (was %d) to eDP (%d)\n", display_bpc, edp_bpc);
display_bpc = edp_bpc;
}
continue;
}
/* Not one of the known troublemakers, check the EDID */
list_for_each_entry(connector, &dev->mode_config.connector_list,
head) {
if (connector->encoder != encoder)
continue;
/* Don't use an invalid EDID bpc value */
if (connector->display_info.bpc &&
connector->display_info.bpc < display_bpc) {
DRM_DEBUG_KMS("clamping display bpc (was %d) to EDID reported max of %d\n", display_bpc, connector->display_info.bpc);
display_bpc = connector->display_info.bpc;
}
}
/*
* HDMI is either 12 or 8, so if the display lets 10bpc sneak
* through, clamp it down. (Note: >12bpc will be caught below.)
*/
if (intel_encoder->type == INTEL_OUTPUT_HDMI) {
if (display_bpc > 8 && display_bpc < 12) {
DRM_DEBUG_KMS("forcing bpc to 12 for HDMI\n");
display_bpc = 12;
} else {
DRM_DEBUG_KMS("forcing bpc to 8 for HDMI\n");
display_bpc = 8;
}
}
}
if (mode->private_flags & INTEL_MODE_DP_FORCE_6BPC) {
DRM_DEBUG_KMS("Dithering DP to 6bpc\n");
display_bpc = 6;
}
/*
* We could just drive the pipe at the highest bpc all the time and
* enable dithering as needed, but that costs bandwidth. So choose
* the minimum value that expresses the full color range of the fb but
* also stays within the max display bpc discovered above.
*/
switch (crtc->fb->depth) {
case 8:
bpc = 8; /* since we go through a colormap */
break;
case 15:
case 16:
bpc = 6; /* min is 18bpp */
break;
case 24:
bpc = 8;
break;
case 30:
bpc = 10;
break;
case 48:
bpc = 12;
break;
default:
DRM_DEBUG("unsupported depth, assuming 24 bits\n");
bpc = min((unsigned int)8, display_bpc);
break;
}
display_bpc = min(display_bpc, bpc);
DRM_DEBUG_KMS("setting pipe bpc to %d (max display bpc %d)\n",
bpc, display_bpc);
*pipe_bpp = display_bpc * 3;
return display_bpc != bpc;
}
static int i9xx_get_refclk(struct drm_crtc *crtc, int num_connectors)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
int refclk;
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
refclk = dev_priv->lvds_ssc_freq * 1000;
DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
refclk / 1000);
} else if (!IS_GEN2(dev)) {
refclk = 96000;
} else {
refclk = 48000;
}
return refclk;
}
static void i9xx_adjust_sdvo_tv_clock(struct drm_display_mode *adjusted_mode,
intel_clock_t *clock)
{
/* SDVO TV has fixed PLL values depend on its clock range,
this mirrors vbios setting. */
if (adjusted_mode->clock >= 100000
&& adjusted_mode->clock < 140500) {
clock->p1 = 2;
clock->p2 = 10;
clock->n = 3;
clock->m1 = 16;
clock->m2 = 8;
} else if (adjusted_mode->clock >= 140500
&& adjusted_mode->clock <= 200000) {
clock->p1 = 1;
clock->p2 = 10;
clock->n = 6;
clock->m1 = 12;
clock->m2 = 8;
}
}
static void i9xx_update_pll_dividers(struct drm_crtc *crtc,
intel_clock_t *clock,
intel_clock_t *reduced_clock)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 fp, fp2 = 0;
if (IS_PINEVIEW(dev)) {
fp = (1 << clock->n) << 16 | clock->m1 << 8 | clock->m2;
if (reduced_clock)
fp2 = (1 << reduced_clock->n) << 16 |
reduced_clock->m1 << 8 | reduced_clock->m2;
} else {
fp = clock->n << 16 | clock->m1 << 8 | clock->m2;
if (reduced_clock)
fp2 = reduced_clock->n << 16 | reduced_clock->m1 << 8 |
reduced_clock->m2;
}
I915_WRITE(FP0(pipe), fp);
intel_crtc->lowfreq_avail = false;
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
reduced_clock && i915_powersave) {
I915_WRITE(FP1(pipe), fp2);
intel_crtc->lowfreq_avail = true;
} else {
I915_WRITE(FP1(pipe), fp);
}
}
static void intel_update_lvds(struct drm_crtc *crtc, intel_clock_t *clock,
struct drm_display_mode *adjusted_mode)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 temp;
temp = I915_READ(LVDS);
temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
if (pipe == 1) {
temp |= LVDS_PIPEB_SELECT;
} else {
temp &= ~LVDS_PIPEB_SELECT;
}
/* set the corresponsding LVDS_BORDER bit */
temp |= dev_priv->lvds_border_bits;
/* Set the B0-B3 data pairs corresponding to whether we're going to
* set the DPLLs for dual-channel mode or not.
*/
if (clock->p2 == 7)
temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
else
temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
/* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
* appropriately here, but we need to look more thoroughly into how
* panels behave in the two modes.
*/
/* set the dithering flag on LVDS as needed */
if (INTEL_INFO(dev)->gen >= 4) {
if (dev_priv->lvds_dither)
temp |= LVDS_ENABLE_DITHER;
else
temp &= ~LVDS_ENABLE_DITHER;
}
temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
temp |= LVDS_HSYNC_POLARITY;
if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
temp |= LVDS_VSYNC_POLARITY;
I915_WRITE(LVDS, temp);
}
static void i9xx_update_pll(struct drm_crtc *crtc,
struct drm_display_mode *mode,
struct drm_display_mode *adjusted_mode,
intel_clock_t *clock, intel_clock_t *reduced_clock,
int num_connectors)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 dpll;
bool is_sdvo;
is_sdvo = intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO) ||
intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI);
dpll = DPLL_VGA_MODE_DIS;
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
dpll |= DPLLB_MODE_LVDS;
else
dpll |= DPLLB_MODE_DAC_SERIAL;
if (is_sdvo) {
int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
if (pixel_multiplier > 1) {
if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
dpll |= (pixel_multiplier - 1) << SDVO_MULTIPLIER_SHIFT_HIRES;
}
dpll |= DPLL_DVO_HIGH_SPEED;
}
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT))
dpll |= DPLL_DVO_HIGH_SPEED;
/* compute bitmask from p1 value */
if (IS_PINEVIEW(dev))
dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
else {
dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
if (IS_G4X(dev) && reduced_clock)
dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
}
switch (clock->p2) {
case 5:
dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
break;
case 7:
dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
break;
case 10:
dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
break;
case 14:
dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
break;
}
if (INTEL_INFO(dev)->gen >= 4)
dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
if (is_sdvo && intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT))
dpll |= PLL_REF_INPUT_TVCLKINBC;
else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT))
/* XXX: just matching BIOS for now */
/* dpll |= PLL_REF_INPUT_TVCLKINBC; */
dpll |= 3;
else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
intel_panel_use_ssc(dev_priv) && num_connectors < 2)
dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
else
dpll |= PLL_REF_INPUT_DREFCLK;
dpll |= DPLL_VCO_ENABLE;
I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
POSTING_READ(DPLL(pipe));
DELAY(150);
/* The LVDS pin pair needs to be on before the DPLLs are enabled.
* This is an exception to the general rule that mode_set doesn't turn
* things on.
*/
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
intel_update_lvds(crtc, clock, adjusted_mode);
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT))
intel_dp_set_m_n(crtc, mode, adjusted_mode);
I915_WRITE(DPLL(pipe), dpll);
/* Wait for the clocks to stabilize. */
POSTING_READ(DPLL(pipe));
DELAY(150);
if (INTEL_INFO(dev)->gen >= 4) {
u32 temp = 0;
if (is_sdvo) {
temp = intel_mode_get_pixel_multiplier(adjusted_mode);
if (temp > 1)
temp = (temp - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
else
temp = 0;
}
I915_WRITE(DPLL_MD(pipe), temp);
} else {
/* The pixel multiplier can only be updated once the
* DPLL is enabled and the clocks are stable.
*
* So write it again.
*/
I915_WRITE(DPLL(pipe), dpll);
}
}
static void i8xx_update_pll(struct drm_crtc *crtc,
struct drm_display_mode *adjusted_mode,
intel_clock_t *clock,
int num_connectors)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 dpll;
dpll = DPLL_VGA_MODE_DIS;
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
} else {
if (clock->p1 == 2)
dpll |= PLL_P1_DIVIDE_BY_TWO;
else
dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
if (clock->p2 == 4)
dpll |= PLL_P2_DIVIDE_BY_4;
}
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT))
/* XXX: just matching BIOS for now */
/* dpll |= PLL_REF_INPUT_TVCLKINBC; */
dpll |= 3;
else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
intel_panel_use_ssc(dev_priv) && num_connectors < 2)
dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
else
dpll |= PLL_REF_INPUT_DREFCLK;
dpll |= DPLL_VCO_ENABLE;
I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
POSTING_READ(DPLL(pipe));
DELAY(150);
I915_WRITE(DPLL(pipe), dpll);
/* Wait for the clocks to stabilize. */
POSTING_READ(DPLL(pipe));
DELAY(150);
/* The LVDS pin pair needs to be on before the DPLLs are enabled.
* This is an exception to the general rule that mode_set doesn't turn
* things on.
*/
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
intel_update_lvds(crtc, clock, adjusted_mode);
/* The pixel multiplier can only be updated once the
* DPLL is enabled and the clocks are stable.
*
* So write it again.
*/
I915_WRITE(DPLL(pipe), dpll);
}
static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
struct drm_display_mode *mode,
struct drm_display_mode *adjusted_mode,
int x, int y,
struct drm_framebuffer *old_fb)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int plane = intel_crtc->plane;
int refclk, num_connectors = 0;
intel_clock_t clock, reduced_clock;
u32 dspcntr, pipeconf, vsyncshift;
bool ok, has_reduced_clock = false, is_sdvo = false;
bool is_lvds = false, is_tv = false, is_dp = false;
struct drm_mode_config *mode_config = &dev->mode_config;
struct intel_encoder *encoder;
const intel_limit_t *limit;
int ret;
list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
if (encoder->base.crtc != crtc)
continue;
switch (encoder->type) {
case INTEL_OUTPUT_LVDS:
is_lvds = true;
break;
case INTEL_OUTPUT_SDVO:
case INTEL_OUTPUT_HDMI:
is_sdvo = true;
if (encoder->needs_tv_clock)
is_tv = true;
break;
case INTEL_OUTPUT_TVOUT:
is_tv = true;
break;
case INTEL_OUTPUT_DISPLAYPORT:
is_dp = true;
break;
}
num_connectors++;
}
refclk = i9xx_get_refclk(crtc, num_connectors);
/*
* Returns a set of divisors for the desired target clock with the given
* refclk, or false. The returned values represent the clock equation:
* reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
*/
limit = intel_limit(crtc, refclk);
ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL,
&clock);
if (!ok) {
DRM_ERROR("Couldn't find PLL settings for mode!\n");
return -EINVAL;
}
/* Ensure that the cursor is valid for the new mode before changing... */
intel_crtc_update_cursor(crtc, true);
if (is_lvds && dev_priv->lvds_downclock_avail) {
/*
* Ensure we match the reduced clock's P to the target clock.
* If the clocks don't match, we can't switch the display clock
* by using the FP0/FP1. In such case we will disable the LVDS
* downclock feature.
*/
has_reduced_clock = limit->find_pll(limit, crtc,
dev_priv->lvds_downclock,
refclk,
&clock,
&reduced_clock);
}
if (is_sdvo && is_tv)
i9xx_adjust_sdvo_tv_clock(adjusted_mode, &clock);
i9xx_update_pll_dividers(crtc, &clock, has_reduced_clock ?
&reduced_clock : NULL);
if (IS_GEN2(dev))
i8xx_update_pll(crtc, adjusted_mode, &clock, num_connectors);
else
i9xx_update_pll(crtc, mode, adjusted_mode, &clock,
has_reduced_clock ? &reduced_clock : NULL,
num_connectors);
/* setup pipeconf */
pipeconf = I915_READ(PIPECONF(pipe));
/* Set up the display plane register */
dspcntr = DISPPLANE_GAMMA_ENABLE;
if (pipe == 0)
dspcntr &= ~DISPPLANE_SEL_PIPE_MASK;
else
dspcntr |= DISPPLANE_SEL_PIPE_B;
if (pipe == 0 && INTEL_INFO(dev)->gen < 4) {
/* Enable pixel doubling when the dot clock is > 90% of the (display)
* core speed.
*
* XXX: No double-wide on 915GM pipe B. Is that the only reason for the
* pipe == 0 check?
*/
if (mode->clock >
dev_priv->display.get_display_clock_speed(dev) * 9 / 10)
pipeconf |= PIPECONF_DOUBLE_WIDE;
else
pipeconf &= ~PIPECONF_DOUBLE_WIDE;
}
/* default to 8bpc */
pipeconf &= ~(PIPECONF_BPP_MASK | PIPECONF_DITHER_EN);
if (is_dp) {
if (mode->private_flags & INTEL_MODE_DP_FORCE_6BPC) {
pipeconf |= PIPECONF_BPP_6 |
PIPECONF_DITHER_EN |
PIPECONF_DITHER_TYPE_SP;
}
}
DRM_DEBUG_KMS("Mode for pipe %c:\n", pipe == 0 ? 'A' : 'B');
drm_mode_debug_printmodeline(mode);
if (HAS_PIPE_CXSR(dev)) {
if (intel_crtc->lowfreq_avail) {
DRM_DEBUG_KMS("enabling CxSR downclocking\n");
pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
} else {
DRM_DEBUG_KMS("disabling CxSR downclocking\n");
pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
}
}
pipeconf &= ~PIPECONF_INTERLACE_MASK;
if (!IS_GEN2(dev) &&
adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
/* the chip adds 2 halflines automatically */
adjusted_mode->crtc_vtotal -= 1;
adjusted_mode->crtc_vblank_end -= 1;
vsyncshift = adjusted_mode->crtc_hsync_start
- adjusted_mode->crtc_htotal/2;
} else {
pipeconf |= PIPECONF_PROGRESSIVE;
vsyncshift = 0;
}
if (!IS_GEN3(dev))
I915_WRITE(VSYNCSHIFT(pipe), vsyncshift);
I915_WRITE(HTOTAL(pipe),
(adjusted_mode->crtc_hdisplay - 1) |
((adjusted_mode->crtc_htotal - 1) << 16));
I915_WRITE(HBLANK(pipe),
(adjusted_mode->crtc_hblank_start - 1) |
((adjusted_mode->crtc_hblank_end - 1) << 16));
I915_WRITE(HSYNC(pipe),
(adjusted_mode->crtc_hsync_start - 1) |
((adjusted_mode->crtc_hsync_end - 1) << 16));
I915_WRITE(VTOTAL(pipe),
(adjusted_mode->crtc_vdisplay - 1) |
((adjusted_mode->crtc_vtotal - 1) << 16));
I915_WRITE(VBLANK(pipe),
(adjusted_mode->crtc_vblank_start - 1) |
((adjusted_mode->crtc_vblank_end - 1) << 16));
I915_WRITE(VSYNC(pipe),
(adjusted_mode->crtc_vsync_start - 1) |
((adjusted_mode->crtc_vsync_end - 1) << 16));
/* pipesrc and dspsize control the size that is scaled from,
* which should always be the user's requested size.
*/
I915_WRITE(DSPSIZE(plane),
((mode->vdisplay - 1) << 16) |
(mode->hdisplay - 1));
I915_WRITE(DSPPOS(plane), 0);
I915_WRITE(PIPESRC(pipe),
((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
I915_WRITE(PIPECONF(pipe), pipeconf);
POSTING_READ(PIPECONF(pipe));
intel_enable_pipe(dev_priv, pipe, false);
intel_wait_for_vblank(dev, pipe);
I915_WRITE(DSPCNTR(plane), dspcntr);
POSTING_READ(DSPCNTR(plane));
ret = intel_pipe_set_base(crtc, x, y, old_fb);
intel_update_watermarks(dev);
return ret;
}
/*
* Initialize reference clocks when the driver loads
*/
void ironlake_init_pch_refclk(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_mode_config *mode_config = &dev->mode_config;
struct intel_encoder *encoder;
u32 temp;
bool has_lvds = false;
bool has_cpu_edp = false;
bool has_pch_edp = false;
bool has_panel = false;
bool has_ck505 = false;
bool can_ssc = false;
/* We need to take the global config into account */
list_for_each_entry(encoder, &mode_config->encoder_list,
base.head) {
switch (encoder->type) {
case INTEL_OUTPUT_LVDS:
has_panel = true;
has_lvds = true;
break;
case INTEL_OUTPUT_EDP:
has_panel = true;
if (intel_encoder_is_pch_edp(&encoder->base))
has_pch_edp = true;
else
has_cpu_edp = true;
break;
}
}
if (HAS_PCH_IBX(dev)) {
has_ck505 = dev_priv->display_clock_mode;
can_ssc = has_ck505;
} else {
has_ck505 = false;
can_ssc = true;
}
DRM_DEBUG_KMS("has_panel %d has_lvds %d has_pch_edp %d has_cpu_edp %d has_ck505 %d\n",
has_panel, has_lvds, has_pch_edp, has_cpu_edp,
has_ck505);
/* Ironlake: try to setup display ref clock before DPLL
* enabling. This is only under driver's control after
* PCH B stepping, previous chipset stepping should be
* ignoring this setting.
*/
temp = I915_READ(PCH_DREF_CONTROL);
/* Always enable nonspread source */
temp &= ~DREF_NONSPREAD_SOURCE_MASK;
if (has_ck505)
temp |= DREF_NONSPREAD_CK505_ENABLE;
else
temp |= DREF_NONSPREAD_SOURCE_ENABLE;
if (has_panel) {
temp &= ~DREF_SSC_SOURCE_MASK;
temp |= DREF_SSC_SOURCE_ENABLE;
/* SSC must be turned on before enabling the CPU output */
if (intel_panel_use_ssc(dev_priv) && can_ssc) {
DRM_DEBUG_KMS("Using SSC on panel\n");
temp |= DREF_SSC1_ENABLE;
} else
temp &= ~DREF_SSC1_ENABLE;
/* Get SSC going before enabling the outputs */
I915_WRITE(PCH_DREF_CONTROL, temp);
POSTING_READ(PCH_DREF_CONTROL);
DELAY(200);
temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
/* Enable CPU source on CPU attached eDP */
if (has_cpu_edp) {
if (intel_panel_use_ssc(dev_priv) && can_ssc) {
DRM_DEBUG_KMS("Using SSC on eDP\n");
temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
}
else
temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
} else
temp |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
I915_WRITE(PCH_DREF_CONTROL, temp);
POSTING_READ(PCH_DREF_CONTROL);
DELAY(200);
} else {
DRM_DEBUG_KMS("Disabling SSC entirely\n");
temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
/* Turn off CPU output */
temp |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
I915_WRITE(PCH_DREF_CONTROL, temp);
POSTING_READ(PCH_DREF_CONTROL);
DELAY(200);
/* Turn off the SSC source */
temp &= ~DREF_SSC_SOURCE_MASK;
temp |= DREF_SSC_SOURCE_DISABLE;
/* Turn off SSC1 */
temp &= ~ DREF_SSC1_ENABLE;
I915_WRITE(PCH_DREF_CONTROL, temp);
POSTING_READ(PCH_DREF_CONTROL);
DELAY(200);
}
}
static int ironlake_get_refclk(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_encoder *encoder;
struct drm_mode_config *mode_config = &dev->mode_config;
struct intel_encoder *edp_encoder = NULL;
int num_connectors = 0;
bool is_lvds = false;
list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
if (encoder->base.crtc != crtc)
continue;
switch (encoder->type) {
case INTEL_OUTPUT_LVDS:
is_lvds = true;
break;
case INTEL_OUTPUT_EDP:
edp_encoder = encoder;
break;
}
num_connectors++;
}
if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
dev_priv->lvds_ssc_freq);
return dev_priv->lvds_ssc_freq * 1000;
}
return 120000;
}
static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
struct drm_display_mode *mode,
struct drm_display_mode *adjusted_mode,
int x, int y,
struct drm_framebuffer *old_fb)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int plane = intel_crtc->plane;
int refclk, num_connectors = 0;
intel_clock_t clock, reduced_clock;
u32 dpll, fp = 0, fp2 = 0, dspcntr, pipeconf;
bool ok, has_reduced_clock = false, is_sdvo = false;
bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false;
struct drm_mode_config *mode_config = &dev->mode_config;
struct intel_encoder *encoder, *edp_encoder = NULL;
const intel_limit_t *limit;
int ret;
struct fdi_m_n m_n = {0};
u32 temp;
int target_clock, pixel_multiplier, lane, link_bw, factor;
unsigned int pipe_bpp;
bool dither;
bool is_cpu_edp = false, is_pch_edp = false;
list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
if (encoder->base.crtc != crtc)
continue;
switch (encoder->type) {
case INTEL_OUTPUT_LVDS:
is_lvds = true;
break;
case INTEL_OUTPUT_SDVO:
case INTEL_OUTPUT_HDMI:
is_sdvo = true;
if (encoder->needs_tv_clock)
is_tv = true;
break;
case INTEL_OUTPUT_TVOUT:
is_tv = true;
break;
case INTEL_OUTPUT_ANALOG:
is_crt = true;
break;
case INTEL_OUTPUT_DISPLAYPORT:
is_dp = true;
break;
case INTEL_OUTPUT_EDP:
is_dp = true;
if (intel_encoder_is_pch_edp(&encoder->base))
is_pch_edp = true;
else
is_cpu_edp = true;
edp_encoder = encoder;
break;
}
num_connectors++;
}
refclk = ironlake_get_refclk(crtc);
/*
* Returns a set of divisors for the desired target clock with the given
* refclk, or false. The returned values represent the clock equation:
* reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
*/
limit = intel_limit(crtc, refclk);
ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL,
&clock);
if (!ok) {
DRM_ERROR("Couldn't find PLL settings for mode!\n");
return -EINVAL;
}
/* Ensure that the cursor is valid for the new mode before changing... */
intel_crtc_update_cursor(crtc, true);
if (is_lvds && dev_priv->lvds_downclock_avail) {
/*
* Ensure we match the reduced clock's P to the target clock.
* If the clocks don't match, we can't switch the display clock
* by using the FP0/FP1. In such case we will disable the LVDS
* downclock feature.
*/
has_reduced_clock = limit->find_pll(limit, crtc,
dev_priv->lvds_downclock,
refclk,
&clock,
&reduced_clock);
}
/* SDVO TV has fixed PLL values depend on its clock range,
this mirrors vbios setting. */
if (is_sdvo && is_tv) {
if (adjusted_mode->clock >= 100000
&& adjusted_mode->clock < 140500) {
clock.p1 = 2;
clock.p2 = 10;
clock.n = 3;
clock.m1 = 16;
clock.m2 = 8;
} else if (adjusted_mode->clock >= 140500
&& adjusted_mode->clock <= 200000) {
clock.p1 = 1;
clock.p2 = 10;
clock.n = 6;
clock.m1 = 12;
clock.m2 = 8;
}
}
/* FDI link */
pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
lane = 0;
/* CPU eDP doesn't require FDI link, so just set DP M/N
according to current link config */
if (is_cpu_edp) {
target_clock = mode->clock;
intel_edp_link_config(edp_encoder, &lane, &link_bw);
} else {
/* [e]DP over FDI requires target mode clock
instead of link clock */
if (is_dp)
target_clock = mode->clock;
else
target_clock = adjusted_mode->clock;
/* FDI is a binary signal running at ~2.7GHz, encoding
* each output octet as 10 bits. The actual frequency
* is stored as a divider into a 100MHz clock, and the
* mode pixel clock is stored in units of 1KHz.
* Hence the bw of each lane in terms of the mode signal
* is:
*/
link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10;
}
/* determine panel color depth */
temp = I915_READ(PIPECONF(pipe));
temp &= ~PIPE_BPC_MASK;
dither = intel_choose_pipe_bpp_dither(crtc, &pipe_bpp, mode);
switch (pipe_bpp) {
case 18:
temp |= PIPE_6BPC;
break;
case 24:
temp |= PIPE_8BPC;
break;
case 30:
temp |= PIPE_10BPC;
break;
case 36:
temp |= PIPE_12BPC;
break;
default:
printf("intel_choose_pipe_bpp returned invalid value %d\n",
pipe_bpp);
temp |= PIPE_8BPC;
pipe_bpp = 24;
break;
}
intel_crtc->bpp = pipe_bpp;
I915_WRITE(PIPECONF(pipe), temp);
if (!lane) {
/*
* Account for spread spectrum to avoid
* oversubscribing the link. Max center spread
* is 2.5%; use 5% for safety's sake.
*/
u32 bps = target_clock * intel_crtc->bpp * 21 / 20;
lane = bps / (link_bw * 8) + 1;
}
intel_crtc->fdi_lanes = lane;
if (pixel_multiplier > 1)
link_bw *= pixel_multiplier;
ironlake_compute_m_n(intel_crtc->bpp, lane, target_clock, link_bw,
&m_n);
fp = clock.n << 16 | clock.m1 << 8 | clock.m2;
if (has_reduced_clock)
fp2 = reduced_clock.n << 16 | reduced_clock.m1 << 8 |
reduced_clock.m2;
/* Enable autotuning of the PLL clock (if permissible) */
factor = 21;
if (is_lvds) {
if ((intel_panel_use_ssc(dev_priv) &&
dev_priv->lvds_ssc_freq == 100) ||
(I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP)
factor = 25;
} else if (is_sdvo && is_tv)
factor = 20;
if (clock.m < factor * clock.n)
fp |= FP_CB_TUNE;
dpll = 0;
if (is_lvds)
dpll |= DPLLB_MODE_LVDS;
else
dpll |= DPLLB_MODE_DAC_SERIAL;
if (is_sdvo) {
int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
if (pixel_multiplier > 1) {
dpll |= (pixel_multiplier - 1) << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
}
dpll |= DPLL_DVO_HIGH_SPEED;
}
if (is_dp && !is_cpu_edp)
dpll |= DPLL_DVO_HIGH_SPEED;
/* compute bitmask from p1 value */
dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
/* also FPA1 */
dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
switch (clock.p2) {
case 5:
dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
break;
case 7:
dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
break;
case 10:
dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
break;
case 14:
dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
break;
}
if (is_sdvo && is_tv)
dpll |= PLL_REF_INPUT_TVCLKINBC;
else if (is_tv)
/* XXX: just matching BIOS for now */
/* dpll |= PLL_REF_INPUT_TVCLKINBC; */
dpll |= 3;
else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
else
dpll |= PLL_REF_INPUT_DREFCLK;
/* setup pipeconf */
pipeconf = I915_READ(PIPECONF(pipe));
/* Set up the display plane register */
dspcntr = DISPPLANE_GAMMA_ENABLE;
DRM_DEBUG_KMS("Mode for pipe %d:\n", pipe);
drm_mode_debug_printmodeline(mode);
/* CPU eDP is the only output that doesn't need a PCH PLL of its own on
* pre-Haswell/LPT generation */
if (HAS_PCH_LPT(dev)) {
DRM_DEBUG_KMS("LPT detected: no PLL for pipe %d necessary\n",
pipe);
} else if (!is_cpu_edp) {
struct intel_pch_pll *pll;
pll = intel_get_pch_pll(intel_crtc, dpll, fp);
if (pll == NULL) {
DRM_DEBUG_DRIVER("failed to find PLL for pipe %d\n",
pipe);
return -EINVAL;
}
} else
intel_put_pch_pll(intel_crtc);
/* The LVDS pin pair needs to be on before the DPLLs are enabled.
* This is an exception to the general rule that mode_set doesn't turn
* things on.
*/
if (is_lvds) {
temp = I915_READ(PCH_LVDS);
temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
if (HAS_PCH_CPT(dev)) {
temp &= ~PORT_TRANS_SEL_MASK;
temp |= PORT_TRANS_SEL_CPT(pipe);
} else {
if (pipe == 1)
temp |= LVDS_PIPEB_SELECT;
else
temp &= ~LVDS_PIPEB_SELECT;
}
/* set the corresponsding LVDS_BORDER bit */
temp |= dev_priv->lvds_border_bits;
/* Set the B0-B3 data pairs corresponding to whether we're going to
* set the DPLLs for dual-channel mode or not.
*/
if (clock.p2 == 7)
temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
else
temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
/* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
* appropriately here, but we need to look more thoroughly into how
* panels behave in the two modes.
*/
temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
temp |= LVDS_HSYNC_POLARITY;
if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
temp |= LVDS_VSYNC_POLARITY;
I915_WRITE(PCH_LVDS, temp);
}
pipeconf &= ~PIPECONF_DITHER_EN;
pipeconf &= ~PIPECONF_DITHER_TYPE_MASK;
if ((is_lvds && dev_priv->lvds_dither) || dither) {
pipeconf |= PIPECONF_DITHER_EN;
pipeconf |= PIPECONF_DITHER_TYPE_SP;
}
if (is_dp && !is_cpu_edp) {
intel_dp_set_m_n(crtc, mode, adjusted_mode);
} else {
/* For non-DP output, clear any trans DP clock recovery setting.*/
I915_WRITE(TRANSDATA_M1(pipe), 0);
I915_WRITE(TRANSDATA_N1(pipe), 0);
I915_WRITE(TRANSDPLINK_M1(pipe), 0);
I915_WRITE(TRANSDPLINK_N1(pipe), 0);
}
if (intel_crtc->pch_pll) {
I915_WRITE(intel_crtc->pch_pll->pll_reg, dpll);
/* Wait for the clocks to stabilize. */
POSTING_READ(intel_crtc->pch_pll->pll_reg);
DELAY(150);
/* The pixel multiplier can only be updated once the
* DPLL is enabled and the clocks are stable.
*
* So write it again.
*/
I915_WRITE(intel_crtc->pch_pll->pll_reg, dpll);
}
intel_crtc->lowfreq_avail = false;
if (intel_crtc->pch_pll) {
if (is_lvds && has_reduced_clock && i915_powersave) {
I915_WRITE(intel_crtc->pch_pll->fp1_reg, fp2);
intel_crtc->lowfreq_avail = true;
if (HAS_PIPE_CXSR(dev)) {
DRM_DEBUG_KMS("enabling CxSR downclocking\n");
pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
}
} else {
I915_WRITE(intel_crtc->pch_pll->fp1_reg, fp);
if (HAS_PIPE_CXSR(dev)) {
DRM_DEBUG_KMS("disabling CxSR downclocking\n");
pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
}
}
}
pipeconf &= ~PIPECONF_INTERLACE_MASK;
if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
pipeconf |= PIPECONF_INTERLACED_ILK;
/* the chip adds 2 halflines automatically */
adjusted_mode->crtc_vtotal -= 1;
adjusted_mode->crtc_vblank_end -= 1;
I915_WRITE(VSYNCSHIFT(pipe),
adjusted_mode->crtc_hsync_start
- adjusted_mode->crtc_htotal/2);
} else {
pipeconf |= PIPECONF_PROGRESSIVE;
I915_WRITE(VSYNCSHIFT(pipe), 0);
}
I915_WRITE(HTOTAL(pipe),
(adjusted_mode->crtc_hdisplay - 1) |
((adjusted_mode->crtc_htotal - 1) << 16));
I915_WRITE(HBLANK(pipe),
(adjusted_mode->crtc_hblank_start - 1) |
((adjusted_mode->crtc_hblank_end - 1) << 16));
I915_WRITE(HSYNC(pipe),
(adjusted_mode->crtc_hsync_start - 1) |
((adjusted_mode->crtc_hsync_end - 1) << 16));
I915_WRITE(VTOTAL(pipe),
(adjusted_mode->crtc_vdisplay - 1) |
((adjusted_mode->crtc_vtotal - 1) << 16));
I915_WRITE(VBLANK(pipe),
(adjusted_mode->crtc_vblank_start - 1) |
((adjusted_mode->crtc_vblank_end - 1) << 16));
I915_WRITE(VSYNC(pipe),
(adjusted_mode->crtc_vsync_start - 1) |
((adjusted_mode->crtc_vsync_end - 1) << 16));
/* pipesrc controls the size that is scaled from, which should
* always be the user's requested size.
*/
I915_WRITE(PIPESRC(pipe),
((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
I915_WRITE(PIPE_DATA_M1(pipe), TU_SIZE(m_n.tu) | m_n.gmch_m);
I915_WRITE(PIPE_DATA_N1(pipe), m_n.gmch_n);
I915_WRITE(PIPE_LINK_M1(pipe), m_n.link_m);
I915_WRITE(PIPE_LINK_N1(pipe), m_n.link_n);
if (is_cpu_edp)
ironlake_set_pll_edp(crtc, adjusted_mode->clock);
I915_WRITE(PIPECONF(pipe), pipeconf);
POSTING_READ(PIPECONF(pipe));
intel_wait_for_vblank(dev, pipe);
I915_WRITE(DSPCNTR(plane), dspcntr);
POSTING_READ(DSPCNTR(plane));
ret = intel_pipe_set_base(crtc, x, y, old_fb);
intel_update_watermarks(dev);
intel_update_linetime_watermarks(dev, pipe, adjusted_mode);
return ret;
}
static int intel_crtc_mode_set(struct drm_crtc *crtc,
struct drm_display_mode *mode,
struct drm_display_mode *adjusted_mode,
int x, int y,
struct drm_framebuffer *old_fb)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int ret;
drm_vblank_pre_modeset(dev, pipe);
ret = dev_priv->display.crtc_mode_set(crtc, mode, adjusted_mode,
x, y, old_fb);
drm_vblank_post_modeset(dev, pipe);
if (ret)
intel_crtc->dpms_mode = DRM_MODE_DPMS_OFF;
else
intel_crtc->dpms_mode = DRM_MODE_DPMS_ON;
return ret;
}
static bool intel_eld_uptodate(struct drm_connector *connector,
int reg_eldv, uint32_t bits_eldv,
int reg_elda, uint32_t bits_elda,
int reg_edid)
{
struct drm_i915_private *dev_priv = connector->dev->dev_private;
uint8_t *eld = connector->eld;
uint32_t i;
i = I915_READ(reg_eldv);
i &= bits_eldv;
if (!eld[0])
return !i;
if (!i)
return false;
i = I915_READ(reg_elda);
i &= ~bits_elda;
I915_WRITE(reg_elda, i);
for (i = 0; i < eld[2]; i++)
if (I915_READ(reg_edid) != *((uint32_t *)eld + i))
return false;
return true;
}
static void g4x_write_eld(struct drm_connector *connector,
struct drm_crtc *crtc)
{
struct drm_i915_private *dev_priv = connector->dev->dev_private;
uint8_t *eld = connector->eld;
uint32_t eldv;
uint32_t len;
uint32_t i;
i = I915_READ(G4X_AUD_VID_DID);
if (i == INTEL_AUDIO_DEVBLC || i == INTEL_AUDIO_DEVCL)
eldv = G4X_ELDV_DEVCL_DEVBLC;
else
eldv = G4X_ELDV_DEVCTG;
if (intel_eld_uptodate(connector,
G4X_AUD_CNTL_ST, eldv,
G4X_AUD_CNTL_ST, G4X_ELD_ADDR,
G4X_HDMIW_HDMIEDID))
return;
i = I915_READ(G4X_AUD_CNTL_ST);
i &= ~(eldv | G4X_ELD_ADDR);
len = (i >> 9) & 0x1f; /* ELD buffer size */
I915_WRITE(G4X_AUD_CNTL_ST, i);
if (!eld[0])
return;
if (eld[2] < (uint8_t)len)
len = eld[2];
DRM_DEBUG_KMS("ELD size %d\n", len);
for (i = 0; i < len; i++)
I915_WRITE(G4X_HDMIW_HDMIEDID, *((uint32_t *)eld + i));
i = I915_READ(G4X_AUD_CNTL_ST);
i |= eldv;
I915_WRITE(G4X_AUD_CNTL_ST, i);
}
static void ironlake_write_eld(struct drm_connector *connector,
struct drm_crtc *crtc)
{
struct drm_i915_private *dev_priv = connector->dev->dev_private;
uint8_t *eld = connector->eld;
uint32_t eldv;
uint32_t i;
int len;
int hdmiw_hdmiedid;
int aud_config;
int aud_cntl_st;
int aud_cntrl_st2;
if (HAS_PCH_IBX(connector->dev)) {
hdmiw_hdmiedid = IBX_HDMIW_HDMIEDID_A;
aud_config = IBX_AUD_CONFIG_A;
aud_cntl_st = IBX_AUD_CNTL_ST_A;
aud_cntrl_st2 = IBX_AUD_CNTL_ST2;
} else {
hdmiw_hdmiedid = CPT_HDMIW_HDMIEDID_A;
aud_config = CPT_AUD_CONFIG_A;
aud_cntl_st = CPT_AUD_CNTL_ST_A;
aud_cntrl_st2 = CPT_AUD_CNTRL_ST2;
}
i = to_intel_crtc(crtc)->pipe;
hdmiw_hdmiedid += i * 0x100;
aud_cntl_st += i * 0x100;
aud_config += i * 0x100;
DRM_DEBUG_KMS("ELD on pipe %c\n", pipe_name(i));
i = I915_READ(aud_cntl_st);
i = (i >> 29) & 0x3; /* DIP_Port_Select, 0x1 = PortB */
if (!i) {
DRM_DEBUG_KMS("Audio directed to unknown port\n");
/* operate blindly on all ports */
eldv = IBX_ELD_VALIDB;
eldv |= IBX_ELD_VALIDB << 4;
eldv |= IBX_ELD_VALIDB << 8;
} else {
DRM_DEBUG_KMS("ELD on port %c\n", 'A' + i);
eldv = IBX_ELD_VALIDB << ((i - 1) * 4);
}
if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
DRM_DEBUG_DRIVER("ELD: DisplayPort detected\n");
eld[5] |= (1 << 2); /* Conn_Type, 0x1 = DisplayPort */
I915_WRITE(aud_config, AUD_CONFIG_N_VALUE_INDEX); /* 0x1 = DP */
} else
I915_WRITE(aud_config, 0);
if (intel_eld_uptodate(connector,
aud_cntrl_st2, eldv,
aud_cntl_st, IBX_ELD_ADDRESS,
hdmiw_hdmiedid))
return;
i = I915_READ(aud_cntrl_st2);
i &= ~eldv;
I915_WRITE(aud_cntrl_st2, i);
if (!eld[0])
return;
i = I915_READ(aud_cntl_st);
i &= ~IBX_ELD_ADDRESS;
I915_WRITE(aud_cntl_st, i);
/* 84 bytes of hw ELD buffer */
len = 21;
if (eld[2] < (uint8_t)len)
len = eld[2];
DRM_DEBUG_KMS("ELD size %d\n", len);
for (i = 0; i < len; i++)
I915_WRITE(hdmiw_hdmiedid, *((uint32_t *)eld + i));
i = I915_READ(aud_cntrl_st2);
i |= eldv;
I915_WRITE(aud_cntrl_st2, i);
}
void intel_write_eld(struct drm_encoder *encoder,
struct drm_display_mode *mode)
{
struct drm_crtc *crtc = encoder->crtc;
struct drm_connector *connector;
struct drm_device *dev = encoder->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
connector = drm_select_eld(encoder, mode);
if (!connector)
return;
DRM_DEBUG_KMS("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
connector->base.id,
drm_get_connector_name(connector),
connector->encoder->base.id,
drm_get_encoder_name(connector->encoder));
connector->eld[6] = drm_av_sync_delay(connector, mode) / 2;
if (dev_priv->display.write_eld)
dev_priv->display.write_eld(connector, crtc);
}
/** Loads the palette/gamma unit for the CRTC with the prepared values */
void intel_crtc_load_lut(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int palreg = PALETTE(intel_crtc->pipe);
int i;
/* The clocks have to be on to load the palette. */
if (!crtc->enabled || !intel_crtc->active)
return;
/* use legacy palette for Ironlake */
if (HAS_PCH_SPLIT(dev))
palreg = LGC_PALETTE(intel_crtc->pipe);
for (i = 0; i < 256; i++) {
I915_WRITE(palreg + 4 * i,
(intel_crtc->lut_r[i] << 16) |
(intel_crtc->lut_g[i] << 8) |
intel_crtc->lut_b[i]);
}
}
static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
bool visible = base != 0;
u32 cntl;
if (intel_crtc->cursor_visible == visible)
return;
cntl = I915_READ(_CURACNTR);
if (visible) {
/* On these chipsets we can only modify the base whilst
* the cursor is disabled.
*/
I915_WRITE(_CURABASE, base);
cntl &= ~(CURSOR_FORMAT_MASK);
/* XXX width must be 64, stride 256 => 0x00 << 28 */
cntl |= CURSOR_ENABLE |
CURSOR_GAMMA_ENABLE |
CURSOR_FORMAT_ARGB;
} else
cntl &= ~(CURSOR_ENABLE | CURSOR_GAMMA_ENABLE);
I915_WRITE(_CURACNTR, cntl);
intel_crtc->cursor_visible = visible;
}
static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
bool visible = base != 0;
if (intel_crtc->cursor_visible != visible) {
uint32_t cntl = I915_READ(CURCNTR(pipe));
if (base) {
cntl &= ~(CURSOR_MODE | MCURSOR_PIPE_SELECT);
cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE;
cntl |= pipe << 28; /* Connect to correct pipe */
} else {
cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
cntl |= CURSOR_MODE_DISABLE;
}
I915_WRITE(CURCNTR(pipe), cntl);
intel_crtc->cursor_visible = visible;
}
/* and commit changes on next vblank */
I915_WRITE(CURBASE(pipe), base);
}
static void ivb_update_cursor(struct drm_crtc *crtc, u32 base)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
bool visible = base != 0;
if (intel_crtc->cursor_visible != visible) {
uint32_t cntl = I915_READ(CURCNTR_IVB(pipe));
if (base) {
cntl &= ~CURSOR_MODE;
cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE;
} else {
cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
cntl |= CURSOR_MODE_DISABLE;
}
I915_WRITE(CURCNTR_IVB(pipe), cntl);
intel_crtc->cursor_visible = visible;
}
/* and commit changes on next vblank */
I915_WRITE(CURBASE_IVB(pipe), base);
}
/* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
static void intel_crtc_update_cursor(struct drm_crtc *crtc,
bool on)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int x = intel_crtc->cursor_x;
int y = intel_crtc->cursor_y;
u32 base, pos;
bool visible;
pos = 0;
if (on && crtc->enabled && crtc->fb) {
base = intel_crtc->cursor_addr;
if (x > (int) crtc->fb->width)
base = 0;
if (y > (int) crtc->fb->height)
base = 0;
} else
base = 0;
if (x < 0) {
if (x + intel_crtc->cursor_width < 0)
base = 0;
pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
x = -x;
}
pos |= x << CURSOR_X_SHIFT;
if (y < 0) {
if (y + intel_crtc->cursor_height < 0)
base = 0;
pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
y = -y;
}
pos |= y << CURSOR_Y_SHIFT;
visible = base != 0;
if (!visible && !intel_crtc->cursor_visible)
return;
if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
I915_WRITE(CURPOS_IVB(pipe), pos);
ivb_update_cursor(crtc, base);
} else {
I915_WRITE(CURPOS(pipe), pos);
if (IS_845G(dev) || IS_I865G(dev))
i845_update_cursor(crtc, base);
else
i9xx_update_cursor(crtc, base);
}
}
static int intel_crtc_cursor_set(struct drm_crtc *crtc,
struct drm_file *file,
uint32_t handle,
uint32_t width, uint32_t height)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct drm_i915_gem_object *obj;
uint32_t addr;
int ret;
DRM_DEBUG_KMS("\n");
/* if we want to turn off the cursor ignore width and height */
if (!handle) {
DRM_DEBUG_KMS("cursor off\n");
addr = 0;
obj = NULL;
DRM_LOCK(dev);
goto finish;
}
/* Currently we only support 64x64 cursors */
if (width != 64 || height != 64) {
DRM_ERROR("we currently only support 64x64 cursors\n");
return -EINVAL;
}
obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
if (&obj->base == NULL)
return -ENOENT;
if (obj->base.size < width * height * 4) {
DRM_ERROR("buffer is to small\n");
ret = -ENOMEM;
goto fail;
}
/* we only need to pin inside GTT if cursor is non-phy */
DRM_LOCK(dev);
if (!dev_priv->info->cursor_needs_physical) {
if (obj->tiling_mode) {
DRM_ERROR("cursor cannot be tiled\n");
ret = -EINVAL;
goto fail_locked;
}
ret = i915_gem_object_pin_to_display_plane(obj, 0, NULL);
if (ret) {
DRM_ERROR("failed to move cursor bo into the GTT\n");
goto fail_locked;
}
ret = i915_gem_object_put_fence(obj);
if (ret) {
DRM_ERROR("failed to release fence for cursor\n");
goto fail_unpin;
}
addr = obj->gtt_offset;
} else {
int align = IS_I830(dev) ? 16 * 1024 : 256;
ret = i915_gem_attach_phys_object(dev, obj,
(intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1,
align);
if (ret) {
DRM_ERROR("failed to attach phys object\n");
goto fail_locked;
}
addr = obj->phys_obj->handle->busaddr;
}
if (IS_GEN2(dev))
I915_WRITE(CURSIZE, (height << 12) | width);
finish:
if (intel_crtc->cursor_bo) {
if (dev_priv->info->cursor_needs_physical) {
if (intel_crtc->cursor_bo != obj)
i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo);
} else
i915_gem_object_unpin_from_display_plane(intel_crtc->cursor_bo);
drm_gem_object_unreference(&intel_crtc->cursor_bo->base);
}
DRM_UNLOCK(dev);
intel_crtc->cursor_addr = addr;
intel_crtc->cursor_bo = obj;
intel_crtc->cursor_width = width;
intel_crtc->cursor_height = height;
intel_crtc_update_cursor(crtc, true);
return 0;
fail_unpin:
i915_gem_object_unpin_from_display_plane(obj);
fail_locked:
DRM_UNLOCK(dev);
fail:
drm_gem_object_unreference_unlocked(&obj->base);
return ret;
}
static int intel_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
{
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
intel_crtc->cursor_x = x;
intel_crtc->cursor_y = y;
intel_crtc_update_cursor(crtc, true);
return 0;
}
/** Sets the color ramps on behalf of RandR */
void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
u16 blue, int regno)
{
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
intel_crtc->lut_r[regno] = red >> 8;
intel_crtc->lut_g[regno] = green >> 8;
intel_crtc->lut_b[regno] = blue >> 8;
}
void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
u16 *blue, int regno)
{
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
*red = intel_crtc->lut_r[regno] << 8;
*green = intel_crtc->lut_g[regno] << 8;
*blue = intel_crtc->lut_b[regno] << 8;
}
static void intel_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
u16 *blue, uint32_t start, uint32_t size)
{
int end = (start + size > 256) ? 256 : start + size, i;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
for (i = start; i < end; i++) {
intel_crtc->lut_r[i] = red[i] >> 8;
intel_crtc->lut_g[i] = green[i] >> 8;
intel_crtc->lut_b[i] = blue[i] >> 8;
}
intel_crtc_load_lut(crtc);
}
/**
* Get a pipe with a simple mode set on it for doing load-based monitor
* detection.
*
* It will be up to the load-detect code to adjust the pipe as appropriate for
* its requirements. The pipe will be connected to no other encoders.
*
* Currently this code will only succeed if there is a pipe with no encoders
* configured for it. In the future, it could choose to temporarily disable
* some outputs to free up a pipe for its use.
*
* \return crtc, or NULL if no pipes are available.
*/
/* VESA 640x480x72Hz mode to set on the pipe */
static struct drm_display_mode load_detect_mode = {
DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
};
static int
intel_framebuffer_create(struct drm_device *dev,
struct drm_mode_fb_cmd2 *mode_cmd, struct drm_i915_gem_object *obj,
struct drm_framebuffer **res)
{
struct intel_framebuffer *intel_fb;
int ret;
intel_fb = malloc(sizeof(*intel_fb), DRM_MEM_KMS, M_WAITOK | M_ZERO);
ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
if (ret) {
drm_gem_object_unreference_unlocked(&obj->base);
free(intel_fb, DRM_MEM_KMS);
return (ret);
}
*res = &intel_fb->base;
return (0);
}
static u32
intel_framebuffer_pitch_for_width(int width, int bpp)
{
u32 pitch = howmany(width * bpp, 8);
return roundup2(pitch, 64);
}
static u32
intel_framebuffer_size_for_mode(struct drm_display_mode *mode, int bpp)
{
u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp);
return roundup2(pitch * mode->vdisplay, PAGE_SIZE);
}
static int
intel_framebuffer_create_for_mode(struct drm_device *dev,
struct drm_display_mode *mode, int depth, int bpp,
struct drm_framebuffer **res)
{
struct drm_i915_gem_object *obj;
struct drm_mode_fb_cmd2 mode_cmd;
obj = i915_gem_alloc_object(dev,
intel_framebuffer_size_for_mode(mode, bpp));
if (obj == NULL)
return (-ENOMEM);
mode_cmd.width = mode->hdisplay;
mode_cmd.height = mode->vdisplay;
mode_cmd.pitches[0] = intel_framebuffer_pitch_for_width(mode_cmd.width,
bpp);
mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
return (intel_framebuffer_create(dev, &mode_cmd, obj, res));
}
static int
mode_fits_in_fbdev(struct drm_device *dev,
struct drm_display_mode *mode, struct drm_framebuffer **res)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_object *obj;
struct drm_framebuffer *fb;
if (dev_priv->fbdev == NULL) {
*res = NULL;
return (0);
}
obj = dev_priv->fbdev->ifb.obj;
if (obj == NULL) {
*res = NULL;
return (0);
}
fb = &dev_priv->fbdev->ifb.base;
if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay,
fb->bits_per_pixel)) {
*res = NULL;
return (0);
}
if (obj->base.size < mode->vdisplay * fb->pitches[0]) {
*res = NULL;
return (0);
}
*res = fb;
return (0);
}
bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
struct drm_connector *connector,
struct drm_display_mode *mode,
struct intel_load_detect_pipe *old)
{
struct intel_crtc *intel_crtc;
struct drm_crtc *possible_crtc;
struct drm_encoder *encoder = &intel_encoder->base;
struct drm_crtc *crtc = NULL;
struct drm_device *dev = encoder->dev;
struct drm_framebuffer *old_fb;
int i = -1, r;
DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
connector->base.id, drm_get_connector_name(connector),
encoder->base.id, drm_get_encoder_name(encoder));
/*
* Algorithm gets a little messy:
*
* - if the connector already has an assigned crtc, use it (but make
* sure it's on first)
*
* - try to find the first unused crtc that can drive this connector,
* and use that if we find one
*/
/* See if we already have a CRTC for this connector */
if (encoder->crtc) {
crtc = encoder->crtc;
intel_crtc = to_intel_crtc(crtc);
old->dpms_mode = intel_crtc->dpms_mode;
old->load_detect_temp = false;
/* Make sure the crtc and connector are running */
if (intel_crtc->dpms_mode != DRM_MODE_DPMS_ON) {
struct drm_encoder_helper_funcs *encoder_funcs;
struct drm_crtc_helper_funcs *crtc_funcs;
crtc_funcs = crtc->helper_private;
crtc_funcs->dpms(crtc, DRM_MODE_DPMS_ON);
encoder_funcs = encoder->helper_private;
encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
}
return true;
}
/* Find an unused one (if possible) */
list_for_each_entry(possible_crtc, &dev->mode_config.crtc_list, head) {
i++;
if (!(encoder->possible_crtcs & (1 << i)))
continue;
if (!possible_crtc->enabled) {
crtc = possible_crtc;
break;
}
}
/*
* If we didn't find an unused CRTC, don't use any.
*/
if (!crtc) {
DRM_DEBUG_KMS("no pipe available for load-detect\n");
return false;
}
encoder->crtc = crtc;
connector->encoder = encoder;
intel_crtc = to_intel_crtc(crtc);
old->dpms_mode = intel_crtc->dpms_mode;
old->load_detect_temp = true;
old->release_fb = NULL;
if (!mode)
mode = &load_detect_mode;
old_fb = crtc->fb;
/* We need a framebuffer large enough to accommodate all accesses
* that the plane may generate whilst we perform load detection.
* We can not rely on the fbcon either being present (we get called
* during its initialisation to detect all boot displays, or it may
* not even exist) or that it is large enough to satisfy the
* requested mode.
*/
r = mode_fits_in_fbdev(dev, mode, &crtc->fb);
if (crtc->fb == NULL) {
DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
r = intel_framebuffer_create_for_mode(dev, mode, 24, 32,
&crtc->fb);
old->release_fb = crtc->fb;
} else
DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
if (r != 0) {
DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
crtc->fb = old_fb;
return false;
}
if (!drm_crtc_helper_set_mode(crtc, mode, 0, 0, old_fb)) {
DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
if (old->release_fb)
old->release_fb->funcs->destroy(old->release_fb);
crtc->fb = old_fb;
return false;
}
/* let the connector get through one full cycle before testing */
intel_wait_for_vblank(dev, intel_crtc->pipe);
return true;
}
void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder,
struct drm_connector *connector,
struct intel_load_detect_pipe *old)
{
struct drm_encoder *encoder = &intel_encoder->base;
struct drm_device *dev = encoder->dev;
struct drm_crtc *crtc = encoder->crtc;
struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
connector->base.id, drm_get_connector_name(connector),
encoder->base.id, drm_get_encoder_name(encoder));
if (old->load_detect_temp) {
connector->encoder = NULL;
drm_helper_disable_unused_functions(dev);
if (old->release_fb)
old->release_fb->funcs->destroy(old->release_fb);
return;
}
/* Switch crtc and encoder back off if necessary */
if (old->dpms_mode != DRM_MODE_DPMS_ON) {
encoder_funcs->dpms(encoder, old->dpms_mode);
crtc_funcs->dpms(crtc, old->dpms_mode);
}
}
/* Returns the clock of the currently programmed mode of the given pipe. */
static int intel_crtc_clock_get(struct drm_device *dev, struct drm_crtc *crtc)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
u32 dpll = I915_READ(DPLL(pipe));
u32 fp;
intel_clock_t clock;
if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
fp = I915_READ(FP0(pipe));
else
fp = I915_READ(FP1(pipe));
clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
if (IS_PINEVIEW(dev)) {
clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
} else {
clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
}
if (!IS_GEN2(dev)) {
if (IS_PINEVIEW(dev))
clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
else
clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
DPLL_FPA01_P1_POST_DIV_SHIFT);
switch (dpll & DPLL_MODE_MASK) {
case DPLLB_MODE_DAC_SERIAL:
clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
5 : 10;
break;
case DPLLB_MODE_LVDS:
clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
7 : 14;
break;
default:
DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
"mode\n", (int)(dpll & DPLL_MODE_MASK));
return 0;
}
/* XXX: Handle the 100Mhz refclk */
intel_clock(dev, 96000, &clock);
} else {
bool is_lvds = (pipe == 1) && (I915_READ(LVDS) & LVDS_PORT_EN);
if (is_lvds) {
clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
DPLL_FPA01_P1_POST_DIV_SHIFT);
clock.p2 = 14;
if ((dpll & PLL_REF_INPUT_MASK) ==
PLLB_REF_INPUT_SPREADSPECTRUMIN) {
/* XXX: might not be 66MHz */
intel_clock(dev, 66000, &clock);
} else
intel_clock(dev, 48000, &clock);
} else {
if (dpll & PLL_P1_DIVIDE_BY_TWO)
clock.p1 = 2;
else {
clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
}
if (dpll & PLL_P2_DIVIDE_BY_4)
clock.p2 = 4;
else
clock.p2 = 2;
intel_clock(dev, 48000, &clock);
}
}
/* XXX: It would be nice to validate the clocks, but we can't reuse
* i830PllIsValid() because it relies on the xf86_config connector
* configuration being accurate, which it isn't necessarily.
*/
return clock.dot;
}
/** Returns the currently programmed mode of the given pipe. */
struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
struct drm_crtc *crtc)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
struct drm_display_mode *mode;
int htot = I915_READ(HTOTAL(pipe));
int hsync = I915_READ(HSYNC(pipe));
int vtot = I915_READ(VTOTAL(pipe));
int vsync = I915_READ(VSYNC(pipe));
mode = malloc(sizeof(*mode), DRM_MEM_KMS, M_WAITOK | M_ZERO);
mode->clock = intel_crtc_clock_get(dev, crtc);
mode->hdisplay = (htot & 0xffff) + 1;
mode->htotal = ((htot & 0xffff0000) >> 16) + 1;
mode->hsync_start = (hsync & 0xffff) + 1;
mode->hsync_end = ((hsync & 0xffff0000) >> 16) + 1;
mode->vdisplay = (vtot & 0xffff) + 1;
mode->vtotal = ((vtot & 0xffff0000) >> 16) + 1;
mode->vsync_start = (vsync & 0xffff) + 1;
mode->vsync_end = ((vsync & 0xffff0000) >> 16) + 1;
drm_mode_set_name(mode);
return mode;
}
#define GPU_IDLE_TIMEOUT (500 /* ms */ * 1000 / hz)
/* When this timer fires, we've been idle for awhile */
static void intel_gpu_idle_timer(void *arg)
{
struct drm_device *dev = arg;
drm_i915_private_t *dev_priv = dev->dev_private;
if (!list_empty(&dev_priv->mm.active_list)) {
/* Still processing requests, so just re-arm the timer. */
callout_schedule(&dev_priv->idle_callout, GPU_IDLE_TIMEOUT);
return;
}
dev_priv->busy = false;
taskqueue_enqueue(dev_priv->tq, &dev_priv->idle_task);
}
#define CRTC_IDLE_TIMEOUT (1000 /* ms */ * 1000 / hz)
static void intel_crtc_idle_timer(void *arg)
{
struct intel_crtc *intel_crtc = arg;
struct drm_crtc *crtc = &intel_crtc->base;
drm_i915_private_t *dev_priv = crtc->dev->dev_private;
struct intel_framebuffer *intel_fb;
intel_fb = to_intel_framebuffer(crtc->fb);
if (intel_fb && intel_fb->obj->active) {
/* The framebuffer is still being accessed by the GPU. */
callout_schedule(&intel_crtc->idle_callout, CRTC_IDLE_TIMEOUT);
return;
}
intel_crtc->busy = false;
taskqueue_enqueue(dev_priv->tq, &dev_priv->idle_task);
}
static void intel_increase_pllclock(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
drm_i915_private_t *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
int pipe = intel_crtc->pipe;
int dpll_reg = DPLL(pipe);
int dpll;
if (HAS_PCH_SPLIT(dev))
return;
if (!dev_priv->lvds_downclock_avail)
return;
dpll = I915_READ(dpll_reg);
if (!HAS_PIPE_CXSR(dev) && (dpll & DISPLAY_RATE_SELECT_FPA1)) {
DRM_DEBUG_DRIVER("upclocking LVDS\n");
assert_panel_unlocked(dev_priv, pipe);
dpll &= ~DISPLAY_RATE_SELECT_FPA1;
I915_WRITE(dpll_reg, dpll);
intel_wait_for_vblank(dev, pipe);
dpll = I915_READ(dpll_reg);
if (dpll & DISPLAY_RATE_SELECT_FPA1)
DRM_DEBUG_DRIVER("failed to upclock LVDS!\n");
}
/* Schedule downclock */
callout_reset(&intel_crtc->idle_callout, CRTC_IDLE_TIMEOUT,
intel_crtc_idle_timer, intel_crtc);
}
static void intel_decrease_pllclock(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
drm_i915_private_t *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
if (HAS_PCH_SPLIT(dev))
return;
if (!dev_priv->lvds_downclock_avail)
return;
/*
* Since this is called by a timer, we should never get here in
* the manual case.
*/
if (!HAS_PIPE_CXSR(dev) && intel_crtc->lowfreq_avail) {
int pipe = intel_crtc->pipe;
int dpll_reg = DPLL(pipe);
u32 dpll;
DRM_DEBUG_DRIVER("downclocking LVDS\n");
assert_panel_unlocked(dev_priv, pipe);
dpll = I915_READ(dpll_reg);
dpll |= DISPLAY_RATE_SELECT_FPA1;
I915_WRITE(dpll_reg, dpll);
intel_wait_for_vblank(dev, pipe);
dpll = I915_READ(dpll_reg);
if (!(dpll & DISPLAY_RATE_SELECT_FPA1))
DRM_DEBUG_DRIVER("failed to downclock LVDS!\n");
}
}
/**
* intel_idle_update - adjust clocks for idleness
* @work: work struct
*
* Either the GPU or display (or both) went idle. Check the busy status
* here and adjust the CRTC and GPU clocks as necessary.
*/
static void intel_idle_update(void *arg, int pending)
{
drm_i915_private_t *dev_priv = arg;
struct drm_device *dev = dev_priv->dev;
struct drm_crtc *crtc;
struct intel_crtc *intel_crtc;
if (!i915_powersave)
return;
DRM_LOCK(dev);
i915_update_gfx_val(dev_priv);
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
/* Skip inactive CRTCs */
if (!crtc->fb)
continue;
intel_crtc = to_intel_crtc(crtc);
if (!intel_crtc->busy)
intel_decrease_pllclock(crtc);
}
DRM_UNLOCK(dev);
}
/**
* intel_mark_busy - mark the GPU and possibly the display busy
* @dev: drm device
* @obj: object we're operating on
*
* Callers can use this function to indicate that the GPU is busy processing
* commands. If @obj matches one of the CRTC objects (i.e. it's a scanout
* buffer), we'll also mark the display as busy, so we know to increase its
* clock frequency.
*/
void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj)
{
drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_crtc *crtc = NULL;
struct intel_framebuffer *intel_fb;
struct intel_crtc *intel_crtc;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return;
if (!dev_priv->busy) {
intel_sanitize_pm(dev);
dev_priv->busy = true;
} else
callout_reset(&dev_priv->idle_callout, GPU_IDLE_TIMEOUT,
intel_gpu_idle_timer, dev);
if (obj == NULL)
return;
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
if (!crtc->fb)
continue;
intel_crtc = to_intel_crtc(crtc);
intel_fb = to_intel_framebuffer(crtc->fb);
if (intel_fb->obj == obj) {
if (!intel_crtc->busy) {
/* Non-busy -> busy, upclock */
intel_increase_pllclock(crtc);
intel_crtc->busy = true;
} else {
/* Busy -> busy, put off timer */
callout_reset(&intel_crtc->idle_callout,
CRTC_IDLE_TIMEOUT, intel_crtc_idle_timer,
intel_crtc);
}
}
}
}
static void intel_crtc_destroy(struct drm_crtc *crtc)
{
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_unpin_work *work;
mtx_lock(&dev->event_lock);
work = intel_crtc->unpin_work;
intel_crtc->unpin_work = NULL;
mtx_unlock(&dev->event_lock);
if (work) {
taskqueue_cancel(dev_priv->tq, &work->task, NULL);
taskqueue_drain(dev_priv->tq, &work->task);
free(work, DRM_MEM_KMS);
}
drm_crtc_cleanup(crtc);
free(intel_crtc, DRM_MEM_KMS);
}
static void intel_unpin_work_fn(void *arg, int pending)
{
struct intel_unpin_work *work = arg;
struct drm_device *dev;
dev = work->dev;
DRM_LOCK(dev);
intel_unpin_fb_obj(work->old_fb_obj);
drm_gem_object_unreference(&work->pending_flip_obj->base);
drm_gem_object_unreference(&work->old_fb_obj->base);
intel_update_fbc(work->dev);
DRM_UNLOCK(dev);
free(work, DRM_MEM_KMS);
}
static void do_intel_finish_page_flip(struct drm_device *dev,
struct drm_crtc *crtc)
{
drm_i915_private_t *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct intel_unpin_work *work;
struct drm_i915_gem_object *obj;
struct drm_pending_vblank_event *e;
struct timeval tnow, tvbl;
/* Ignore early vblank irqs */
if (intel_crtc == NULL)
return;
microtime(&tnow);
mtx_lock(&dev->event_lock);
work = intel_crtc->unpin_work;
if (work == NULL || !work->pending) {
mtx_unlock(&dev->event_lock);
return;
}
intel_crtc->unpin_work = NULL;
if (work->event) {
e = work->event;
e->event.sequence = drm_vblank_count_and_time(dev, intel_crtc->pipe, &tvbl);
/* Called before vblank count and timestamps have
* been updated for the vblank interval of flip
* completion? Need to increment vblank count and
* add one videorefresh duration to returned timestamp
* to account for this. We assume this happened if we
* get called over 0.9 frame durations after the last
* timestamped vblank.
*
* This calculation can not be used with vrefresh rates
* below 5Hz (10Hz to be on the safe side) without
* promoting to 64 integers.
*/
if (10 * (timeval_to_ns(&tnow) - timeval_to_ns(&tvbl)) >
9 * crtc->framedur_ns) {
e->event.sequence++;
tvbl = ns_to_timeval(timeval_to_ns(&tvbl) +
crtc->framedur_ns);
}
e->event.tv_sec = tvbl.tv_sec;
e->event.tv_usec = tvbl.tv_usec;
list_add_tail(&e->base.link,
&e->base.file_priv->event_list);
drm_event_wakeup(&e->base);
}
drm_vblank_put(dev, intel_crtc->pipe);
obj = work->old_fb_obj;
atomic_clear_int(&obj->pending_flip, 1 << intel_crtc->plane);
if (atomic_load_acq_int(&obj->pending_flip) == 0)
wakeup(&obj->pending_flip);
mtx_unlock(&dev->event_lock);
taskqueue_enqueue(dev_priv->tq, &work->task);
CTR2(KTR_DRM, "i915_flip_complete %d %p", intel_crtc->plane,
work->pending_flip_obj);
}
void intel_finish_page_flip(struct drm_device *dev, int pipe)
{
drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
do_intel_finish_page_flip(dev, crtc);
}
void intel_finish_page_flip_plane(struct drm_device *dev, int plane)
{
drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane];
do_intel_finish_page_flip(dev, crtc);
}
void intel_prepare_page_flip(struct drm_device *dev, int plane)
{
drm_i915_private_t *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc =
to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]);
mtx_lock(&dev->event_lock);
if (intel_crtc->unpin_work) {
if ((++intel_crtc->unpin_work->pending) > 1)
DRM_ERROR("Prepared flip multiple times\n");
} else {
DRM_DEBUG("preparing flip with no unpin work?\n");
}
mtx_unlock(&dev->event_lock);
}
static int intel_gen2_queue_flip(struct drm_device *dev,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
struct drm_i915_gem_object *obj)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
unsigned long offset;
u32 flip_mask;
struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
int ret;
ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
if (ret)
goto err;
/* Offset into the new buffer for cases of shared fbs between CRTCs */
offset = crtc->y * fb->pitches[0] + crtc->x * fb->bits_per_pixel/8;
ret = intel_ring_begin(ring, 6);
if (ret)
goto err_unpin;
/* Can't queue multiple flips, so wait for the previous
* one to finish before executing the next.
*/
if (intel_crtc->plane)
flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
else
flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
intel_ring_emit(ring, MI_NOOP);
intel_ring_emit(ring, MI_DISPLAY_FLIP |
MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
intel_ring_emit(ring, fb->pitches[0]);
intel_ring_emit(ring, obj->gtt_offset + offset);
intel_ring_emit(ring, 0); /* aux display base address, unused */
intel_ring_advance(ring);
return 0;
err_unpin:
intel_unpin_fb_obj(obj);
err:
return ret;
}
static int intel_gen3_queue_flip(struct drm_device *dev,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
struct drm_i915_gem_object *obj)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
unsigned long offset;
u32 flip_mask;
struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
int ret;
ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
if (ret)
goto err;
/* Offset into the new buffer for cases of shared fbs between CRTCs */
offset = crtc->y * fb->pitches[0] + crtc->x * fb->bits_per_pixel/8;
ret = intel_ring_begin(ring, 6);
if (ret)
goto err_unpin;
if (intel_crtc->plane)
flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
else
flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
intel_ring_emit(ring, MI_NOOP);
intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 |
MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
intel_ring_emit(ring, fb->pitches[0]);
intel_ring_emit(ring, obj->gtt_offset + offset);
intel_ring_emit(ring, MI_NOOP);
intel_ring_advance(ring);
return 0;
err_unpin:
intel_unpin_fb_obj(obj);
err:
return ret;
}
static int intel_gen4_queue_flip(struct drm_device *dev,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
struct drm_i915_gem_object *obj)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
uint32_t pf, pipesrc;
struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
int ret;
ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
if (ret)
goto err;
ret = intel_ring_begin(ring, 4);
if (ret)
goto err_unpin;
/* i965+ uses the linear or tiled offsets from the
* Display Registers (which do not change across a page-flip)
* so we need only reprogram the base address.
*/
intel_ring_emit(ring, MI_DISPLAY_FLIP |
MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
intel_ring_emit(ring, fb->pitches[0]);
intel_ring_emit(ring, obj->gtt_offset | obj->tiling_mode);
/* XXX Enabling the panel-fitter across page-flip is so far
* untested on non-native modes, so ignore it for now.
* pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
*/
pf = 0;
pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
intel_ring_emit(ring, pf | pipesrc);
intel_ring_advance(ring);
return 0;
err_unpin:
intel_unpin_fb_obj(obj);
err:
return ret;
}
static int intel_gen6_queue_flip(struct drm_device *dev,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
struct drm_i915_gem_object *obj)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
uint32_t pf, pipesrc;
int ret;
ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
if (ret)
goto err;
ret = intel_ring_begin(ring, 4);
if (ret)
goto err_unpin;
intel_ring_emit(ring, MI_DISPLAY_FLIP |
MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode);
intel_ring_emit(ring, obj->gtt_offset);
/* Contrary to the suggestions in the documentation,
* "Enable Panel Fitter" does not seem to be required when page
* flipping with a non-native mode, and worse causes a normal
* modeset to fail.
* pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
*/
pf = 0;
pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
intel_ring_emit(ring, pf | pipesrc);
intel_ring_advance(ring);
return 0;
err_unpin:
intel_unpin_fb_obj(obj);
err:
return ret;
}
/*
* On gen7 we currently use the blit ring because (in early silicon at least)
* the render ring doesn't give us interrpts for page flip completion, which
* means clients will hang after the first flip is queued. Fortunately the
* blit ring generates interrupts properly, so use it instead.
*/
static int intel_gen7_queue_flip(struct drm_device *dev,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
struct drm_i915_gem_object *obj)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct intel_ring_buffer *ring = &dev_priv->rings[BCS];
int ret;
ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
if (ret)
goto err;
ret = intel_ring_begin(ring, 4);
if (ret)
goto err_unpin;
intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | (intel_crtc->plane << 19));
intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode));
intel_ring_emit(ring, (obj->gtt_offset));
intel_ring_emit(ring, (MI_NOOP));
intel_ring_advance(ring);
return 0;
err_unpin:
intel_unpin_fb_obj(obj);
err:
return ret;
}
static int intel_default_queue_flip(struct drm_device *dev,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
struct drm_i915_gem_object *obj)
{
return -ENODEV;
}
static int intel_crtc_page_flip(struct drm_crtc *crtc,
struct drm_framebuffer *fb,
struct drm_pending_vblank_event *event)
{
struct drm_device *dev = crtc->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_framebuffer *intel_fb;
struct drm_i915_gem_object *obj;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
struct intel_unpin_work *work;
int ret;
work = malloc(sizeof *work, DRM_MEM_KMS, M_WAITOK | M_ZERO);
work->event = event;
work->dev = crtc->dev;
intel_fb = to_intel_framebuffer(crtc->fb);
work->old_fb_obj = intel_fb->obj;
TASK_INIT(&work->task, 0, intel_unpin_work_fn, work);
ret = drm_vblank_get(dev, intel_crtc->pipe);
if (ret)
goto free_work;
/* We borrow the event spin lock for protecting unpin_work */
mtx_lock(&dev->event_lock);
if (intel_crtc->unpin_work) {
mtx_unlock(&dev->event_lock);
free(work, DRM_MEM_KMS);
drm_vblank_put(dev, intel_crtc->pipe);
DRM_DEBUG("flip queue: crtc already busy\n");
return -EBUSY;
}
intel_crtc->unpin_work = work;
mtx_unlock(&dev->event_lock);
intel_fb = to_intel_framebuffer(fb);
obj = intel_fb->obj;
DRM_LOCK(dev);
/* Reference the objects for the scheduled work. */
drm_gem_object_reference(&work->old_fb_obj->base);
drm_gem_object_reference(&obj->base);
crtc->fb = fb;
work->pending_flip_obj = obj;
work->enable_stall_check = true;
/* Block clients from rendering to the new back buffer until
* the flip occurs and the object is no longer visible.
*/
atomic_set_int(&work->old_fb_obj->pending_flip, 1 << intel_crtc->plane);
ret = dev_priv->display.queue_flip(dev, crtc, fb, obj);
if (ret)
goto cleanup_pending;
intel_disable_fbc(dev);
intel_mark_busy(dev, obj);
DRM_UNLOCK(dev);
CTR2(KTR_DRM, "i915_flip_request %d %p", intel_crtc->plane, obj);
return 0;
cleanup_pending:
atomic_clear_int(&work->old_fb_obj->pending_flip, 1 << intel_crtc->plane);
drm_gem_object_unreference(&work->old_fb_obj->base);
drm_gem_object_unreference(&obj->base);
DRM_UNLOCK(dev);
mtx_lock(&dev->event_lock);
intel_crtc->unpin_work = NULL;
mtx_unlock(&dev->event_lock);
drm_vblank_put(dev, intel_crtc->pipe);
free_work:
free(work, DRM_MEM_KMS);
return ret;
}
static void intel_sanitize_modesetting(struct drm_device *dev,
int pipe, int plane)
{
struct drm_i915_private *dev_priv = dev->dev_private;
u32 reg, val;
int i;
/* Clear any frame start delays used for debugging left by the BIOS */
for_each_pipe(i) {
reg = PIPECONF(i);
I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
}
if (HAS_PCH_SPLIT(dev))
return;
/* Who knows what state these registers were left in by the BIOS or
* grub?
*
* If we leave the registers in a conflicting state (e.g. with the
* display plane reading from the other pipe than the one we intend
* to use) then when we attempt to teardown the active mode, we will
* not disable the pipes and planes in the correct order -- leaving
* a plane reading from a disabled pipe and possibly leading to
* undefined behaviour.
*/
reg = DSPCNTR(plane);
val = I915_READ(reg);
if ((val & DISPLAY_PLANE_ENABLE) == 0)
return;
if (!!(val & DISPPLANE_SEL_PIPE_MASK) == pipe)
return;
/* This display plane is active and attached to the other CPU pipe. */
pipe = !pipe;
/* Disable the plane and wait for it to stop reading from the pipe. */
intel_disable_plane(dev_priv, plane, pipe);
intel_disable_pipe(dev_priv, pipe);
}
static void intel_crtc_reset(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
/* Reset flags back to the 'unknown' status so that they
* will be correctly set on the initial modeset.
*/
intel_crtc->dpms_mode = -1;
/* We need to fix up any BIOS configuration that conflicts with
* our expectations.
*/
intel_sanitize_modesetting(dev, intel_crtc->pipe, intel_crtc->plane);
}
static struct drm_crtc_helper_funcs intel_helper_funcs = {
.dpms = intel_crtc_dpms,
.mode_fixup = intel_crtc_mode_fixup,
.mode_set = intel_crtc_mode_set,
.mode_set_base = intel_pipe_set_base,
.mode_set_base_atomic = intel_pipe_set_base_atomic,
.load_lut = intel_crtc_load_lut,
.disable = intel_crtc_disable,
};
static const struct drm_crtc_funcs intel_crtc_funcs = {
.reset = intel_crtc_reset,
.cursor_set = intel_crtc_cursor_set,
.cursor_move = intel_crtc_cursor_move,
.gamma_set = intel_crtc_gamma_set,
.set_config = drm_crtc_helper_set_config,
.destroy = intel_crtc_destroy,
.page_flip = intel_crtc_page_flip,
};
static void intel_pch_pll_init(struct drm_device *dev)
{
drm_i915_private_t *dev_priv = dev->dev_private;
int i;
if (dev_priv->num_pch_pll == 0) {
DRM_DEBUG_KMS("No PCH PLLs on this hardware, skipping initialisation\n");
return;
}
for (i = 0; i < dev_priv->num_pch_pll; i++) {
dev_priv->pch_plls[i].pll_reg = _PCH_DPLL(i);
dev_priv->pch_plls[i].fp0_reg = _PCH_FP0(i);
dev_priv->pch_plls[i].fp1_reg = _PCH_FP1(i);
}
}
static void intel_crtc_init(struct drm_device *dev, int pipe)
{
drm_i915_private_t *dev_priv = dev->dev_private;
struct intel_crtc *intel_crtc;
int i;
intel_crtc = malloc(sizeof(struct intel_crtc) +
(INTELFB_CONN_LIMIT * sizeof(struct drm_connector *)),
DRM_MEM_KMS, M_WAITOK | M_ZERO);
drm_crtc_init(dev, &intel_crtc->base, &intel_crtc_funcs);
drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256);
for (i = 0; i < 256; i++) {
intel_crtc->lut_r[i] = i;
intel_crtc->lut_g[i] = i;
intel_crtc->lut_b[i] = i;
}
/* Swap pipes & planes for FBC on pre-965 */
intel_crtc->pipe = pipe;
intel_crtc->plane = pipe;
if (IS_MOBILE(dev) && IS_GEN3(dev)) {
DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
intel_crtc->plane = !pipe;
}
KASSERT(pipe < DRM_ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) &&
dev_priv->plane_to_crtc_mapping[intel_crtc->plane] == NULL,
("plane_to_crtc is already initialized"));
dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
intel_crtc_reset(&intel_crtc->base);
intel_crtc->active = true; /* force the pipe off on setup_init_config */
intel_crtc->bpp = 24; /* default for pre-Ironlake */
if (HAS_PCH_SPLIT(dev)) {
intel_helper_funcs.prepare = ironlake_crtc_prepare;
intel_helper_funcs.commit = ironlake_crtc_commit;
} else {
intel_helper_funcs.prepare = i9xx_crtc_prepare;
intel_helper_funcs.commit = i9xx_crtc_commit;
}
drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
intel_crtc->busy = false;
- callout_init(&intel_crtc->idle_callout, CALLOUT_MPSAFE);
+ callout_init(&intel_crtc->idle_callout, 1);
}
int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
struct drm_file *file)
{
struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
struct drm_mode_object *drmmode_obj;
struct intel_crtc *crtc;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -ENODEV;
drmmode_obj = drm_mode_object_find(dev, pipe_from_crtc_id->crtc_id,
DRM_MODE_OBJECT_CRTC);
if (!drmmode_obj) {
DRM_ERROR("no such CRTC id\n");
return -EINVAL;
}
crtc = to_intel_crtc(obj_to_crtc(drmmode_obj));
pipe_from_crtc_id->pipe = crtc->pipe;
return 0;
}
static int intel_encoder_clones(struct drm_device *dev, int type_mask)
{
struct intel_encoder *encoder;
int index_mask = 0;
int entry = 0;
list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
if (type_mask & encoder->clone_mask)
index_mask |= (1 << entry);
entry++;
}
return index_mask;
}
static bool has_edp_a(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
if (!IS_MOBILE(dev))
return false;
if ((I915_READ(DP_A) & DP_DETECTED) == 0)
return false;
if (IS_GEN5(dev) &&
(I915_READ(ILK_DISPLAY_CHICKEN_FUSES) & ILK_eDP_A_DISABLE))
return false;
return true;
}
static void intel_setup_outputs(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_encoder *encoder;
bool dpd_is_edp = false;
bool has_lvds;
has_lvds = intel_lvds_init(dev);
if (!has_lvds && !HAS_PCH_SPLIT(dev)) {
/* disable the panel fitter on everything but LVDS */
I915_WRITE(PFIT_CONTROL, 0);
}
if (HAS_PCH_SPLIT(dev)) {
dpd_is_edp = intel_dpd_is_edp(dev);
if (has_edp_a(dev))
intel_dp_init(dev, DP_A);
if (dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
intel_dp_init(dev, PCH_DP_D);
}
intel_crt_init(dev);
if (IS_HASWELL(dev)) {
int found;
/* Haswell uses DDI functions to detect digital outputs */
found = I915_READ(DDI_BUF_CTL_A) & DDI_INIT_DISPLAY_DETECTED;
/* DDI A only supports eDP */
if (found)
intel_ddi_init(dev, PORT_A);
/* DDI B, C and D detection is indicated by the SFUSE_STRAP
* register */
found = I915_READ(SFUSE_STRAP);
if (found & SFUSE_STRAP_DDIB_DETECTED)
intel_ddi_init(dev, PORT_B);
if (found & SFUSE_STRAP_DDIC_DETECTED)
intel_ddi_init(dev, PORT_C);
if (found & SFUSE_STRAP_DDID_DETECTED)
intel_ddi_init(dev, PORT_D);
} else if (HAS_PCH_SPLIT(dev)) {
int found;
DRM_DEBUG_KMS(
"HDMIB %d PCH_DP_B %d HDMIC %d HDMID %d PCH_DP_C %d PCH_DP_D %d LVDS %d\n",
(I915_READ(HDMIB) & PORT_DETECTED) != 0,
(I915_READ(PCH_DP_B) & DP_DETECTED) != 0,
(I915_READ(HDMIC) & PORT_DETECTED) != 0,
(I915_READ(HDMID) & PORT_DETECTED) != 0,
(I915_READ(PCH_DP_C) & DP_DETECTED) != 0,
(I915_READ(PCH_DP_D) & DP_DETECTED) != 0,
(I915_READ(PCH_LVDS) & LVDS_DETECTED) != 0);
if (I915_READ(HDMIB) & PORT_DETECTED) {
/* PCH SDVOB multiplex with HDMIB */
found = intel_sdvo_init(dev, PCH_SDVOB, true);
if (!found)
intel_hdmi_init(dev, HDMIB);
if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
intel_dp_init(dev, PCH_DP_B);
}
if (I915_READ(HDMIC) & PORT_DETECTED)
intel_hdmi_init(dev, HDMIC);
if (I915_READ(HDMID) & PORT_DETECTED)
intel_hdmi_init(dev, HDMID);
if (I915_READ(PCH_DP_C) & DP_DETECTED)
intel_dp_init(dev, PCH_DP_C);
if (!dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
intel_dp_init(dev, PCH_DP_D);
} else if (SUPPORTS_DIGITAL_OUTPUTS(dev)) {
bool found = false;
if (I915_READ(SDVOB) & SDVO_DETECTED) {
DRM_DEBUG_KMS("probing SDVOB\n");
found = intel_sdvo_init(dev, SDVOB, true);
if (!found && SUPPORTS_INTEGRATED_HDMI(dev)) {
DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
intel_hdmi_init(dev, SDVOB);
}
if (!found && SUPPORTS_INTEGRATED_DP(dev)) {
DRM_DEBUG_KMS("probing DP_B\n");
intel_dp_init(dev, DP_B);
}
}
/* Before G4X SDVOC doesn't have its own detect register */
if (I915_READ(SDVOB) & SDVO_DETECTED) {
DRM_DEBUG_KMS("probing SDVOC\n");
found = intel_sdvo_init(dev, SDVOC, false);
}
if (!found && (I915_READ(SDVOC) & SDVO_DETECTED)) {
if (SUPPORTS_INTEGRATED_HDMI(dev)) {
DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
intel_hdmi_init(dev, SDVOC);
}
if (SUPPORTS_INTEGRATED_DP(dev)) {
DRM_DEBUG_KMS("probing DP_C\n");
intel_dp_init(dev, DP_C);
}
}
if (SUPPORTS_INTEGRATED_DP(dev) &&
(I915_READ(DP_D) & DP_DETECTED)) {
DRM_DEBUG_KMS("probing DP_D\n");
intel_dp_init(dev, DP_D);
}
} else if (IS_GEN2(dev)) {
#if 1
KIB_NOTYET();
#else
intel_dvo_init(dev);
#endif
}
if (SUPPORTS_TV(dev))
intel_tv_init(dev);
list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
encoder->base.possible_crtcs = encoder->crtc_mask;
encoder->base.possible_clones =
intel_encoder_clones(dev, encoder->clone_mask);
}
/* disable all the possible outputs/crtcs before entering KMS mode */
drm_helper_disable_unused_functions(dev);
if (HAS_PCH_SPLIT(dev))
ironlake_init_pch_refclk(dev);
}
static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
{
struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
drm_framebuffer_cleanup(fb);
drm_gem_object_unreference_unlocked(&intel_fb->obj->base);
free(intel_fb, DRM_MEM_KMS);
}
static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
struct drm_file *file,
unsigned int *handle)
{
struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
struct drm_i915_gem_object *obj = intel_fb->obj;
return drm_gem_handle_create(file, &obj->base, handle);
}
static const struct drm_framebuffer_funcs intel_fb_funcs = {
.destroy = intel_user_framebuffer_destroy,
.create_handle = intel_user_framebuffer_create_handle,
};
int intel_framebuffer_init(struct drm_device *dev,
struct intel_framebuffer *intel_fb,
struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_i915_gem_object *obj)
{
int ret;
if (obj->tiling_mode == I915_TILING_Y)
return -EINVAL;
if (mode_cmd->pitches[0] & 63)
return -EINVAL;
switch (mode_cmd->pixel_format) {
case DRM_FORMAT_RGB332:
case DRM_FORMAT_RGB565:
case DRM_FORMAT_XRGB8888:
case DRM_FORMAT_XBGR8888:
case DRM_FORMAT_ARGB8888:
case DRM_FORMAT_XRGB2101010:
case DRM_FORMAT_ARGB2101010:
/* RGB formats are common across chipsets */
break;
case DRM_FORMAT_YUYV:
case DRM_FORMAT_UYVY:
case DRM_FORMAT_YVYU:
case DRM_FORMAT_VYUY:
break;
default:
DRM_DEBUG_KMS("unsupported pixel format %u\n",
mode_cmd->pixel_format);
return -EINVAL;
}
ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs);
if (ret) {
DRM_ERROR("framebuffer init failed %d\n", ret);
return ret;
}
drm_helper_mode_fill_fb_struct(&intel_fb->base, mode_cmd);
intel_fb->obj = obj;
return 0;
}
static int
intel_user_framebuffer_create(struct drm_device *dev,
struct drm_file *filp, struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_framebuffer **res)
{
struct drm_i915_gem_object *obj;
obj = to_intel_bo(drm_gem_object_lookup(dev, filp,
mode_cmd->handles[0]));
if (&obj->base == NULL)
return (-ENOENT);
return (intel_framebuffer_create(dev, mode_cmd, obj, res));
}
static const struct drm_mode_config_funcs intel_mode_funcs = {
.fb_create = intel_user_framebuffer_create,
.output_poll_changed = intel_fb_output_poll_changed,
};
/* Set up chip specific display functions */
static void intel_init_display(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
/* We always want a DPMS function */
if (HAS_PCH_SPLIT(dev)) {
dev_priv->display.dpms = ironlake_crtc_dpms;
dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set;
dev_priv->display.off = ironlake_crtc_off;
dev_priv->display.update_plane = ironlake_update_plane;
} else {
dev_priv->display.dpms = i9xx_crtc_dpms;
dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
dev_priv->display.off = i9xx_crtc_off;
dev_priv->display.update_plane = i9xx_update_plane;
}
/* Returns the core display clock speed */
if (IS_VALLEYVIEW(dev))
dev_priv->display.get_display_clock_speed =
valleyview_get_display_clock_speed;
else if (IS_I945G(dev) || (IS_G33(dev) && !IS_PINEVIEW_M(dev)))
dev_priv->display.get_display_clock_speed =
i945_get_display_clock_speed;
else if (IS_I915G(dev))
dev_priv->display.get_display_clock_speed =
i915_get_display_clock_speed;
else if (IS_I945GM(dev) || IS_845G(dev) || IS_PINEVIEW_M(dev))
dev_priv->display.get_display_clock_speed =
i9xx_misc_get_display_clock_speed;
else if (IS_I915GM(dev))
dev_priv->display.get_display_clock_speed =
i915gm_get_display_clock_speed;
else if (IS_I865G(dev))
dev_priv->display.get_display_clock_speed =
i865_get_display_clock_speed;
else if (IS_I85X(dev))
dev_priv->display.get_display_clock_speed =
i855_get_display_clock_speed;
else /* 852, 830 */
dev_priv->display.get_display_clock_speed =
i830_get_display_clock_speed;
if (HAS_PCH_SPLIT(dev)) {
if (IS_GEN5(dev)) {
dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
dev_priv->display.write_eld = ironlake_write_eld;
} else if (IS_GEN6(dev)) {
dev_priv->display.fdi_link_train = gen6_fdi_link_train;
dev_priv->display.write_eld = ironlake_write_eld;
} else if (IS_IVYBRIDGE(dev)) {
/* FIXME: detect B0+ stepping and use auto training */
dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
dev_priv->display.write_eld = ironlake_write_eld;
} else if (IS_HASWELL(dev)) {
dev_priv->display.fdi_link_train = hsw_fdi_link_train;
dev_priv->display.write_eld = ironlake_write_eld;
} else
dev_priv->display.update_wm = NULL;
} else if (IS_VALLEYVIEW(dev)) {
dev_priv->display.force_wake_get = vlv_force_wake_get;
dev_priv->display.force_wake_put = vlv_force_wake_put;
} else if (IS_G4X(dev)) {
dev_priv->display.write_eld = g4x_write_eld;
}
/* Default just returns -ENODEV to indicate unsupported */
dev_priv->display.queue_flip = intel_default_queue_flip;
switch (INTEL_INFO(dev)->gen) {
case 2:
dev_priv->display.queue_flip = intel_gen2_queue_flip;
break;
case 3:
dev_priv->display.queue_flip = intel_gen3_queue_flip;
break;
case 4:
case 5:
dev_priv->display.queue_flip = intel_gen4_queue_flip;
break;
case 6:
dev_priv->display.queue_flip = intel_gen6_queue_flip;
break;
case 7:
dev_priv->display.queue_flip = intel_gen7_queue_flip;
break;
}
}
/*
* Some BIOSes insist on assuming the GPU's pipe A is enabled at suspend,
* resume, or other times. This quirk makes sure that's the case for
* affected systems.
*/
static void quirk_pipea_force(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
dev_priv->quirks |= QUIRK_PIPEA_FORCE;
DRM_INFO("applying pipe a force quirk\n");
}
/*
* Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
*/
static void quirk_ssc_force_disable(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
DRM_INFO("applying lvds SSC disable quirk\n");
}
/*
* A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
* brightness value
*/
static void quirk_invert_brightness(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
DRM_INFO("applying inverted panel brightness quirk\n");
}
struct intel_quirk {
int device;
int subsystem_vendor;
int subsystem_device;
void (*hook)(struct drm_device *dev);
};
#define PCI_ANY_ID (~0u)
static struct intel_quirk intel_quirks[] = {
/* HP Mini needs pipe A force quirk (LP: #322104) */
{ 0x27ae, 0x103c, 0x361a, quirk_pipea_force },
/* Thinkpad R31 needs pipe A force quirk */
{ 0x3577, 0x1014, 0x0505, quirk_pipea_force },
/* Toshiba Protege R-205, S-209 needs pipe A force quirk */
{ 0x2592, 0x1179, 0x0001, quirk_pipea_force },
/* ThinkPad X30 needs pipe A force quirk (LP: #304614) */
{ 0x3577, 0x1014, 0x0513, quirk_pipea_force },
/* ThinkPad X40 needs pipe A force quirk */
/* ThinkPad T60 needs pipe A force quirk (bug #16494) */
{ 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
/* 855 & before need to leave pipe A & dpll A up */
{ 0x3582, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
{ 0x2562, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
/* Lenovo U160 cannot use SSC on LVDS */
{ 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
/* Sony Vaio Y cannot use SSC on LVDS */
{ 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
/* Acer Aspire 5734Z must invert backlight brightness */
{ 0x2a42, 0x1025, 0x0459, quirk_invert_brightness },
};
static void intel_init_quirks(struct drm_device *dev)
{
struct intel_quirk *q;
device_t d;
int i;
d = dev->dev;
for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
q = &intel_quirks[i];
if (pci_get_device(d) == q->device &&
(pci_get_subvendor(d) == q->subsystem_vendor ||
q->subsystem_vendor == PCI_ANY_ID) &&
(pci_get_subdevice(d) == q->subsystem_device ||
q->subsystem_device == PCI_ANY_ID))
q->hook(dev);
}
}
/* Disable the VGA plane that we never use */
static void i915_disable_vga(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
u8 sr1;
u32 vga_reg;
if (HAS_PCH_SPLIT(dev))
vga_reg = CPU_VGACNTRL;
else
vga_reg = VGACNTRL;
#if 0
vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
#endif
outb(VGA_SR_INDEX, SR01);
sr1 = inb(VGA_SR_DATA);
outb(VGA_SR_DATA, sr1 | 1 << 5);
#if 0
vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
#endif
DELAY(300);
I915_WRITE(vga_reg, VGA_DISP_DISABLE);
POSTING_READ(vga_reg);
}
static void ivb_pch_pwm_override(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
/*
* IVB has CPU eDP backlight regs too, set things up to let the
* PCH regs control the backlight
*/
I915_WRITE(BLC_PWM_CPU_CTL2, PWM_ENABLE);
I915_WRITE(BLC_PWM_CPU_CTL, 0);
I915_WRITE(BLC_PWM_PCH_CTL1, PWM_ENABLE);
}
void intel_modeset_init_hw(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
intel_init_clock_gating(dev);
if (IS_IRONLAKE_M(dev)) {
ironlake_enable_drps(dev);
ironlake_enable_rc6(dev);
intel_init_emon(dev);
}
if ((IS_GEN6(dev) || IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
gen6_enable_rps(dev_priv);
gen6_update_ring_freq(dev_priv);
}
if (IS_IVYBRIDGE(dev))
ivb_pch_pwm_override(dev);
}
void intel_modeset_init(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
int i, ret;
drm_mode_config_init(dev);
dev->mode_config.min_width = 0;
dev->mode_config.min_height = 0;
dev->mode_config.preferred_depth = 24;
dev->mode_config.prefer_shadow = 1;
dev->mode_config.funcs = &intel_mode_funcs;
intel_init_quirks(dev);
intel_init_pm(dev);
intel_prepare_ddi(dev);
intel_init_display(dev);
if (IS_GEN2(dev)) {
dev->mode_config.max_width = 2048;
dev->mode_config.max_height = 2048;
} else if (IS_GEN3(dev)) {
dev->mode_config.max_width = 4096;
dev->mode_config.max_height = 4096;
} else {
dev->mode_config.max_width = 8192;
dev->mode_config.max_height = 8192;
}
dev->mode_config.fb_base = dev->agp->base;
DRM_DEBUG_KMS("%d display pipe%s available.\n",
dev_priv->num_pipe, dev_priv->num_pipe > 1 ? "s" : "");
for (i = 0; i < dev_priv->num_pipe; i++) {
intel_crtc_init(dev, i);
ret = intel_plane_init(dev, i);
if (ret)
DRM_DEBUG_KMS("plane %d init failed: %d\n", i, ret);
}
intel_pch_pll_init(dev);
/* Just disable it once at startup */
i915_disable_vga(dev);
intel_setup_outputs(dev);
TASK_INIT(&dev_priv->idle_task, 0, intel_idle_update, dev_priv);
- callout_init(&dev_priv->idle_callout, CALLOUT_MPSAFE);
+ callout_init(&dev_priv->idle_callout, 1);
}
void intel_modeset_gem_init(struct drm_device *dev)
{
intel_modeset_init_hw(dev);
intel_setup_overlay(dev);
}
void intel_modeset_cleanup(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_crtc *crtc;
struct intel_crtc *intel_crtc;
drm_kms_helper_poll_fini(dev);
DRM_LOCK(dev);
#if 0
intel_unregister_dsm_handler();
#endif
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
/* Skip inactive CRTCs */
if (!crtc->fb)
continue;
intel_crtc = to_intel_crtc(crtc);
intel_increase_pllclock(crtc);
}
intel_disable_fbc(dev);
if (IS_IRONLAKE_M(dev))
ironlake_disable_drps(dev);
if ((IS_GEN6(dev) || IS_GEN7(dev)) && !IS_VALLEYVIEW(dev))
gen6_disable_rps(dev);
if (IS_IRONLAKE_M(dev))
ironlake_disable_rc6(dev);
if (IS_VALLEYVIEW(dev))
vlv_init_dpio(dev);
DRM_UNLOCK(dev);
/* Disable the irq before mode object teardown, for the irq might
* enqueue unpin/hotplug work. */
drm_irq_uninstall(dev);
if (taskqueue_cancel(dev_priv->tq, &dev_priv->hotplug_task, NULL))
taskqueue_drain(dev_priv->tq, &dev_priv->hotplug_task);
if (taskqueue_cancel(dev_priv->tq, &dev_priv->rps_task, NULL))
taskqueue_drain(dev_priv->tq, &dev_priv->rps_task);
/* Shut off idle work before the crtcs get freed. */
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
intel_crtc = to_intel_crtc(crtc);
callout_drain(&intel_crtc->idle_callout);
}
callout_drain(&dev_priv->idle_callout);
if (taskqueue_cancel(dev_priv->tq, &dev_priv->idle_task, NULL))
taskqueue_drain(dev_priv->tq, &dev_priv->idle_task);
drm_mode_config_cleanup(dev);
}
/*
* Return which encoder is currently attached for connector.
*/
struct drm_encoder *intel_best_encoder(struct drm_connector *connector)
{
return &intel_attached_encoder(connector)->base;
}
void intel_connector_attach_encoder(struct intel_connector *connector,
struct intel_encoder *encoder)
{
connector->encoder = encoder;
drm_mode_connector_attach_encoder(&connector->base,
&encoder->base);
}
/*
* set vga decode state - true == enable VGA decode
*/
int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
{
struct drm_i915_private *dev_priv;
device_t bridge_dev;
u16 gmch_ctrl;
dev_priv = dev->dev_private;
bridge_dev = intel_gtt_get_bridge_device();
gmch_ctrl = pci_read_config(bridge_dev, INTEL_GMCH_CTRL, 2);
if (state)
gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
else
gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
pci_write_config(bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl, 2);
return (0);
}
struct intel_display_error_state {
struct intel_cursor_error_state {
u32 control;
u32 position;
u32 base;
u32 size;
} cursor[2];
struct intel_pipe_error_state {
u32 conf;
u32 source;
u32 htotal;
u32 hblank;
u32 hsync;
u32 vtotal;
u32 vblank;
u32 vsync;
} pipe[2];
struct intel_plane_error_state {
u32 control;
u32 stride;
u32 size;
u32 pos;
u32 addr;
u32 surface;
u32 tile_offset;
} plane[2];
};
struct intel_display_error_state *
intel_display_capture_error_state(struct drm_device *dev)
{
drm_i915_private_t *dev_priv = dev->dev_private;
struct intel_display_error_state *error;
int i;
error = malloc(sizeof(*error), DRM_MEM_KMS, M_NOWAIT);
if (error == NULL)
return NULL;
for (i = 0; i < 2; i++) {
error->cursor[i].control = I915_READ(CURCNTR(i));
error->cursor[i].position = I915_READ(CURPOS(i));
error->cursor[i].base = I915_READ(CURBASE(i));
error->plane[i].control = I915_READ(DSPCNTR(i));
error->plane[i].stride = I915_READ(DSPSTRIDE(i));
error->plane[i].size = I915_READ(DSPSIZE(i));
error->plane[i].pos = I915_READ(DSPPOS(i));
error->plane[i].addr = I915_READ(DSPADDR(i));
if (INTEL_INFO(dev)->gen >= 4) {
error->plane[i].surface = I915_READ(DSPSURF(i));
error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
}
error->pipe[i].conf = I915_READ(PIPECONF(i));
error->pipe[i].source = I915_READ(PIPESRC(i));
error->pipe[i].htotal = I915_READ(HTOTAL(i));
error->pipe[i].hblank = I915_READ(HBLANK(i));
error->pipe[i].hsync = I915_READ(HSYNC(i));
error->pipe[i].vtotal = I915_READ(VTOTAL(i));
error->pipe[i].vblank = I915_READ(VBLANK(i));
error->pipe[i].vsync = I915_READ(VSYNC(i));
}
return error;
}
void
intel_display_print_error_state(struct sbuf *m,
struct drm_device *dev,
struct intel_display_error_state *error)
{
int i;
for (i = 0; i < 2; i++) {
sbuf_printf(m, "Pipe [%d]:\n", i);
sbuf_printf(m, " CONF: %08x\n", error->pipe[i].conf);
sbuf_printf(m, " SRC: %08x\n", error->pipe[i].source);
sbuf_printf(m, " HTOTAL: %08x\n", error->pipe[i].htotal);
sbuf_printf(m, " HBLANK: %08x\n", error->pipe[i].hblank);
sbuf_printf(m, " HSYNC: %08x\n", error->pipe[i].hsync);
sbuf_printf(m, " VTOTAL: %08x\n", error->pipe[i].vtotal);
sbuf_printf(m, " VBLANK: %08x\n", error->pipe[i].vblank);
sbuf_printf(m, " VSYNC: %08x\n", error->pipe[i].vsync);
sbuf_printf(m, "Plane [%d]:\n", i);
sbuf_printf(m, " CNTR: %08x\n", error->plane[i].control);
sbuf_printf(m, " STRIDE: %08x\n", error->plane[i].stride);
sbuf_printf(m, " SIZE: %08x\n", error->plane[i].size);
sbuf_printf(m, " POS: %08x\n", error->plane[i].pos);
sbuf_printf(m, " ADDR: %08x\n", error->plane[i].addr);
if (INTEL_INFO(dev)->gen >= 4) {
sbuf_printf(m, " SURF: %08x\n", error->plane[i].surface);
sbuf_printf(m, " TILEOFF: %08x\n", error->plane[i].tile_offset);
}
sbuf_printf(m, "Cursor [%d]:\n", i);
sbuf_printf(m, " CNTR: %08x\n", error->cursor[i].control);
sbuf_printf(m, " POS: %08x\n", error->cursor[i].position);
sbuf_printf(m, " BASE: %08x\n", error->cursor[i].base);
}
}
Index: head/sys/dev/glxsb/glxsb.c
===================================================================
--- head/sys/dev/glxsb/glxsb.c (revision 283290)
+++ head/sys/dev/glxsb/glxsb.c (revision 283291)
@@ -1,945 +1,945 @@
/* $OpenBSD: glxsb.c,v 1.7 2007/02/12 14:31:45 tom Exp $ */
/*
* Copyright (c) 2006 Tom Cosgrove <tom@openbsd.org>
* Copyright (c) 2003, 2004 Theo de Raadt
* Copyright (c) 2003 Jason Wright
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
* Driver for the security block on the AMD Geode LX processors
* http://www.amd.com/files/connectivitysolutions/geode/geode_lx/33234d_lx_ds.pdf
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/errno.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/random.h>
#include <sys/rman.h>
#include <sys/rwlock.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <machine/bus.h>
#include <machine/cpufunc.h>
#include <machine/resource.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/cryptosoft.h>
#include <opencrypto/xform.h>
#include "cryptodev_if.h"
#include "glxsb.h"
#define PCI_VENDOR_AMD 0x1022 /* AMD */
#define PCI_PRODUCT_AMD_GEODE_LX_CRYPTO 0x2082 /* Geode LX Crypto */
#define SB_GLD_MSR_CAP 0x58002000 /* RO - Capabilities */
#define SB_GLD_MSR_CONFIG 0x58002001 /* RW - Master Config */
#define SB_GLD_MSR_SMI 0x58002002 /* RW - SMI */
#define SB_GLD_MSR_ERROR 0x58002003 /* RW - Error */
#define SB_GLD_MSR_PM 0x58002004 /* RW - Power Mgmt */
#define SB_GLD_MSR_DIAG 0x58002005 /* RW - Diagnostic */
#define SB_GLD_MSR_CTRL 0x58002006 /* RW - Security Block Cntrl */
/* For GLD_MSR_CTRL: */
#define SB_GMC_DIV0 0x0000 /* AES update divisor values */
#define SB_GMC_DIV1 0x0001
#define SB_GMC_DIV2 0x0002
#define SB_GMC_DIV3 0x0003
#define SB_GMC_DIV_MASK 0x0003
#define SB_GMC_SBI 0x0004 /* AES swap bits */
#define SB_GMC_SBY 0x0008 /* AES swap bytes */
#define SB_GMC_TW 0x0010 /* Time write (EEPROM) */
#define SB_GMC_T_SEL0 0x0000 /* RNG post-proc: none */
#define SB_GMC_T_SEL1 0x0100 /* RNG post-proc: LFSR */
#define SB_GMC_T_SEL2 0x0200 /* RNG post-proc: whitener */
#define SB_GMC_T_SEL3 0x0300 /* RNG LFSR+whitener */
#define SB_GMC_T_SEL_MASK 0x0300
#define SB_GMC_T_NE 0x0400 /* Noise (generator) Enable */
#define SB_GMC_T_TM 0x0800 /* RNG test mode */
/* (deterministic) */
/* Security Block configuration/control registers (offsets from base) */
#define SB_CTL_A 0x0000 /* RW - SB Control A */
#define SB_CTL_B 0x0004 /* RW - SB Control B */
#define SB_AES_INT 0x0008 /* RW - SB AES Interrupt */
#define SB_SOURCE_A 0x0010 /* RW - Source A */
#define SB_DEST_A 0x0014 /* RW - Destination A */
#define SB_LENGTH_A 0x0018 /* RW - Length A */
#define SB_SOURCE_B 0x0020 /* RW - Source B */
#define SB_DEST_B 0x0024 /* RW - Destination B */
#define SB_LENGTH_B 0x0028 /* RW - Length B */
#define SB_WKEY 0x0030 /* WO - Writable Key 0-3 */
#define SB_WKEY_0 0x0030 /* WO - Writable Key 0 */
#define SB_WKEY_1 0x0034 /* WO - Writable Key 1 */
#define SB_WKEY_2 0x0038 /* WO - Writable Key 2 */
#define SB_WKEY_3 0x003C /* WO - Writable Key 3 */
#define SB_CBC_IV 0x0040 /* RW - CBC IV 0-3 */
#define SB_CBC_IV_0 0x0040 /* RW - CBC IV 0 */
#define SB_CBC_IV_1 0x0044 /* RW - CBC IV 1 */
#define SB_CBC_IV_2 0x0048 /* RW - CBC IV 2 */
#define SB_CBC_IV_3 0x004C /* RW - CBC IV 3 */
#define SB_RANDOM_NUM 0x0050 /* RW - Random Number */
#define SB_RANDOM_NUM_STATUS 0x0054 /* RW - Random Number Status */
#define SB_EEPROM_COMM 0x0800 /* RW - EEPROM Command */
#define SB_EEPROM_ADDR 0x0804 /* RW - EEPROM Address */
#define SB_EEPROM_DATA 0x0808 /* RW - EEPROM Data */
#define SB_EEPROM_SEC_STATE 0x080C /* RW - EEPROM Security State */
/* For SB_CTL_A and _B */
#define SB_CTL_ST 0x0001 /* Start operation (enc/dec) */
#define SB_CTL_ENC 0x0002 /* Encrypt (0 is decrypt) */
#define SB_CTL_DEC 0x0000 /* Decrypt */
#define SB_CTL_WK 0x0004 /* Use writable key (we set) */
#define SB_CTL_DC 0x0008 /* Destination coherent */
#define SB_CTL_SC 0x0010 /* Source coherent */
#define SB_CTL_CBC 0x0020 /* CBC (0 is ECB) */
/* For SB_AES_INT */
#define SB_AI_DISABLE_AES_A 0x0001 /* Disable AES A compl int */
#define SB_AI_ENABLE_AES_A 0x0000 /* Enable AES A compl int */
#define SB_AI_DISABLE_AES_B 0x0002 /* Disable AES B compl int */
#define SB_AI_ENABLE_AES_B 0x0000 /* Enable AES B compl int */
#define SB_AI_DISABLE_EEPROM 0x0004 /* Disable EEPROM op comp int */
#define SB_AI_ENABLE_EEPROM 0x0000 /* Enable EEPROM op compl int */
#define SB_AI_AES_A_COMPLETE 0x10000 /* AES A operation complete */
#define SB_AI_AES_B_COMPLETE 0x20000 /* AES B operation complete */
#define SB_AI_EEPROM_COMPLETE 0x40000 /* EEPROM operation complete */
#define SB_AI_CLEAR_INTR \
(SB_AI_DISABLE_AES_A | SB_AI_DISABLE_AES_B |\
SB_AI_DISABLE_EEPROM | SB_AI_AES_A_COMPLETE |\
SB_AI_AES_B_COMPLETE | SB_AI_EEPROM_COMPLETE)
#define SB_RNS_TRNG_VALID 0x0001 /* in SB_RANDOM_NUM_STATUS */
#define SB_MEM_SIZE 0x0810 /* Size of memory block */
#define SB_AES_ALIGN 0x0010 /* Source and dest buffers */
/* must be 16-byte aligned */
#define SB_AES_BLOCK_SIZE 0x0010
/*
* The Geode LX security block AES acceleration doesn't perform scatter-
* gather: it just takes source and destination addresses. Therefore the
* plain- and ciphertexts need to be contiguous. To this end, we allocate
* a buffer for both, and accept the overhead of copying in and out. If
* the number of bytes in one operation is bigger than allowed for by the
* buffer (buffer is twice the size of the max length, as it has both input
* and output) then we have to perform multiple encryptions/decryptions.
*/
#define GLXSB_MAX_AES_LEN 16384
MALLOC_DEFINE(M_GLXSB, "glxsb_data", "Glxsb Data");
struct glxsb_dma_map {
bus_dmamap_t dma_map; /* DMA map */
bus_dma_segment_t dma_seg; /* segments */
int dma_nsegs; /* #segments */
int dma_size; /* size */
caddr_t dma_vaddr; /* virtual address */
bus_addr_t dma_paddr; /* physical address */
};
struct glxsb_taskop {
struct glxsb_session *to_ses; /* crypto session */
struct cryptop *to_crp; /* cryptop to perfom */
struct cryptodesc *to_enccrd; /* enccrd to perform */
struct cryptodesc *to_maccrd; /* maccrd to perform */
};
struct glxsb_softc {
device_t sc_dev; /* device backpointer */
struct resource *sc_sr; /* resource */
int sc_rid; /* resource rid */
struct callout sc_rngco; /* RNG callout */
int sc_rnghz; /* RNG callout ticks */
bus_dma_tag_t sc_dmat; /* DMA tag */
struct glxsb_dma_map sc_dma; /* DMA map */
int32_t sc_cid; /* crypto tag */
uint32_t sc_sid; /* session id */
TAILQ_HEAD(ses_head, glxsb_session)
sc_sessions; /* crypto sessions */
struct rwlock sc_sessions_lock;/* sessions lock */
struct mtx sc_task_mtx; /* task mutex */
struct taskqueue *sc_tq; /* task queue */
struct task sc_cryptotask; /* task */
struct glxsb_taskop sc_to; /* task's crypto operation */
int sc_task_count; /* tasks count */
};
static int glxsb_probe(device_t);
static int glxsb_attach(device_t);
static int glxsb_detach(device_t);
static void glxsb_dmamap_cb(void *, bus_dma_segment_t *, int, int);
static int glxsb_dma_alloc(struct glxsb_softc *);
static void glxsb_dma_pre_op(struct glxsb_softc *, struct glxsb_dma_map *);
static void glxsb_dma_post_op(struct glxsb_softc *, struct glxsb_dma_map *);
static void glxsb_dma_free(struct glxsb_softc *, struct glxsb_dma_map *);
static void glxsb_rnd(void *);
static int glxsb_crypto_setup(struct glxsb_softc *);
static int glxsb_crypto_newsession(device_t, uint32_t *, struct cryptoini *);
static int glxsb_crypto_freesession(device_t, uint64_t);
static int glxsb_aes(struct glxsb_softc *, uint32_t, uint32_t,
uint32_t, void *, int, void *);
static int glxsb_crypto_encdec(struct cryptop *, struct cryptodesc *,
struct glxsb_session *, struct glxsb_softc *);
static void glxsb_crypto_task(void *, int);
static int glxsb_crypto_process(device_t, struct cryptop *, int);
static device_method_t glxsb_methods[] = {
/* device interface */
DEVMETHOD(device_probe, glxsb_probe),
DEVMETHOD(device_attach, glxsb_attach),
DEVMETHOD(device_detach, glxsb_detach),
/* crypto device methods */
DEVMETHOD(cryptodev_newsession, glxsb_crypto_newsession),
DEVMETHOD(cryptodev_freesession, glxsb_crypto_freesession),
DEVMETHOD(cryptodev_process, glxsb_crypto_process),
{0,0}
};
static driver_t glxsb_driver = {
"glxsb",
glxsb_methods,
sizeof(struct glxsb_softc)
};
static devclass_t glxsb_devclass;
DRIVER_MODULE(glxsb, pci, glxsb_driver, glxsb_devclass, 0, 0);
MODULE_VERSION(glxsb, 1);
MODULE_DEPEND(glxsb, crypto, 1, 1, 1);
static int
glxsb_probe(device_t dev)
{
if (pci_get_vendor(dev) == PCI_VENDOR_AMD &&
pci_get_device(dev) == PCI_PRODUCT_AMD_GEODE_LX_CRYPTO) {
device_set_desc(dev,
"AMD Geode LX Security Block (AES-128-CBC, RNG)");
return (BUS_PROBE_DEFAULT);
}
return (ENXIO);
}
static int
glxsb_attach(device_t dev)
{
struct glxsb_softc *sc = device_get_softc(dev);
uint64_t msr;
sc->sc_dev = dev;
msr = rdmsr(SB_GLD_MSR_CAP);
if ((msr & 0xFFFF00) != 0x130400) {
device_printf(dev, "unknown ID 0x%x\n",
(int)((msr & 0xFFFF00) >> 16));
return (ENXIO);
}
pci_enable_busmaster(dev);
/* Map in the security block configuration/control registers */
sc->sc_rid = PCIR_BAR(0);
sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rid,
RF_ACTIVE);
if (sc->sc_sr == NULL) {
device_printf(dev, "cannot map register space\n");
return (ENXIO);
}
/*
* Configure the Security Block.
*
* We want to enable the noise generator (T_NE), and enable the
* linear feedback shift register and whitener post-processing
* (T_SEL = 3). Also ensure that test mode (deterministic values)
* is disabled.
*/
msr = rdmsr(SB_GLD_MSR_CTRL);
msr &= ~(SB_GMC_T_TM | SB_GMC_T_SEL_MASK);
msr |= SB_GMC_T_NE | SB_GMC_T_SEL3;
#if 0
msr |= SB_GMC_SBI | SB_GMC_SBY; /* for AES, if necessary */
#endif
wrmsr(SB_GLD_MSR_CTRL, msr);
/* Disable interrupts */
bus_write_4(sc->sc_sr, SB_AES_INT, SB_AI_CLEAR_INTR);
/* Allocate a contiguous DMA-able buffer to work in */
if (glxsb_dma_alloc(sc) != 0)
goto fail0;
/* Initialize our task queue */
sc->sc_tq = taskqueue_create("glxsb_taskq", M_NOWAIT | M_ZERO,
taskqueue_thread_enqueue, &sc->sc_tq);
if (sc->sc_tq == NULL) {
device_printf(dev, "cannot create task queue\n");
goto fail0;
}
if (taskqueue_start_threads(&sc->sc_tq, 1, PI_NET, "%s taskq",
device_get_nameunit(dev)) != 0) {
device_printf(dev, "cannot start task queue\n");
goto fail1;
}
TASK_INIT(&sc->sc_cryptotask, 0, glxsb_crypto_task, sc);
/* Initialize crypto */
if (glxsb_crypto_setup(sc) != 0)
goto fail1;
/* Install a periodic collector for the "true" (AMD's word) RNG */
if (hz > 100)
sc->sc_rnghz = hz / 100;
else
sc->sc_rnghz = 1;
- callout_init(&sc->sc_rngco, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_rngco, 1);
glxsb_rnd(sc);
return (0);
fail1:
taskqueue_free(sc->sc_tq);
fail0:
bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rid, sc->sc_sr);
return (ENXIO);
}
static int
glxsb_detach(device_t dev)
{
struct glxsb_softc *sc = device_get_softc(dev);
struct glxsb_session *ses;
rw_wlock(&sc->sc_sessions_lock);
TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) {
if (ses->ses_used) {
rw_wunlock(&sc->sc_sessions_lock);
device_printf(dev,
"cannot detach, sessions still active.\n");
return (EBUSY);
}
}
while (!TAILQ_EMPTY(&sc->sc_sessions)) {
ses = TAILQ_FIRST(&sc->sc_sessions);
TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
free(ses, M_GLXSB);
}
rw_wunlock(&sc->sc_sessions_lock);
crypto_unregister_all(sc->sc_cid);
callout_drain(&sc->sc_rngco);
taskqueue_drain(sc->sc_tq, &sc->sc_cryptotask);
bus_generic_detach(dev);
glxsb_dma_free(sc, &sc->sc_dma);
bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rid, sc->sc_sr);
taskqueue_free(sc->sc_tq);
rw_destroy(&sc->sc_sessions_lock);
mtx_destroy(&sc->sc_task_mtx);
return (0);
}
/*
* callback for bus_dmamap_load()
*/
static void
glxsb_dmamap_cb(void *arg, bus_dma_segment_t *seg, int nseg, int error)
{
bus_addr_t *paddr = (bus_addr_t*) arg;
*paddr = seg[0].ds_addr;
}
static int
glxsb_dma_alloc(struct glxsb_softc *sc)
{
struct glxsb_dma_map *dma = &sc->sc_dma;
int rc;
dma->dma_nsegs = 1;
dma->dma_size = GLXSB_MAX_AES_LEN * 2;
/* Setup DMA descriptor area */
rc = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
SB_AES_ALIGN, 0, /* alignments, bounds */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
dma->dma_size, /* maxsize */
dma->dma_nsegs, /* nsegments */
dma->dma_size, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->sc_dmat);
if (rc != 0) {
device_printf(sc->sc_dev,
"cannot allocate DMA tag (%d)\n", rc);
return (rc);
}
rc = bus_dmamem_alloc(sc->sc_dmat, (void **)&dma->dma_vaddr,
BUS_DMA_NOWAIT, &dma->dma_map);
if (rc != 0) {
device_printf(sc->sc_dev,
"cannot allocate DMA memory of %d bytes (%d)\n",
dma->dma_size, rc);
goto fail0;
}
rc = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
dma->dma_size, glxsb_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
if (rc != 0) {
device_printf(sc->sc_dev,
"cannot load DMA memory for %d bytes (%d)\n",
dma->dma_size, rc);
goto fail1;
}
return (0);
fail1:
bus_dmamem_free(sc->sc_dmat, dma->dma_vaddr, dma->dma_map);
fail0:
bus_dma_tag_destroy(sc->sc_dmat);
return (rc);
}
static void
glxsb_dma_pre_op(struct glxsb_softc *sc, struct glxsb_dma_map *dma)
{
bus_dmamap_sync(sc->sc_dmat, dma->dma_map,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}
static void
glxsb_dma_post_op(struct glxsb_softc *sc, struct glxsb_dma_map *dma)
{
bus_dmamap_sync(sc->sc_dmat, dma->dma_map,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
}
static void
glxsb_dma_free(struct glxsb_softc *sc, struct glxsb_dma_map *dma)
{
bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
bus_dmamem_free(sc->sc_dmat, dma->dma_vaddr, dma->dma_map);
bus_dma_tag_destroy(sc->sc_dmat);
}
static void
glxsb_rnd(void *v)
{
struct glxsb_softc *sc = v;
uint32_t status, value;
status = bus_read_4(sc->sc_sr, SB_RANDOM_NUM_STATUS);
if (status & SB_RNS_TRNG_VALID) {
value = bus_read_4(sc->sc_sr, SB_RANDOM_NUM);
/* feed with one uint32 */
random_harvest(&value, sizeof(value), 32/2, RANDOM_PURE_GLXSB);
}
callout_reset(&sc->sc_rngco, sc->sc_rnghz, glxsb_rnd, sc);
}
static int
glxsb_crypto_setup(struct glxsb_softc *sc)
{
sc->sc_cid = crypto_get_driverid(sc->sc_dev, CRYPTOCAP_F_HARDWARE);
if (sc->sc_cid < 0) {
device_printf(sc->sc_dev, "cannot get crypto driver id\n");
return (ENOMEM);
}
TAILQ_INIT(&sc->sc_sessions);
sc->sc_sid = 1;
rw_init(&sc->sc_sessions_lock, "glxsb_sessions_lock");
mtx_init(&sc->sc_task_mtx, "glxsb_crypto_mtx", NULL, MTX_DEF);
if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
goto crypto_fail;
if (crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0) != 0)
goto crypto_fail;
if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
goto crypto_fail;
if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
goto crypto_fail;
if (crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0) != 0)
goto crypto_fail;
if (crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0) != 0)
goto crypto_fail;
if (crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0) != 0)
goto crypto_fail;
if (crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0) != 0)
goto crypto_fail;
return (0);
crypto_fail:
device_printf(sc->sc_dev, "cannot register crypto\n");
crypto_unregister_all(sc->sc_cid);
rw_destroy(&sc->sc_sessions_lock);
mtx_destroy(&sc->sc_task_mtx);
return (ENOMEM);
}
static int
glxsb_crypto_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
{
struct glxsb_softc *sc = device_get_softc(dev);
struct glxsb_session *ses = NULL;
struct cryptoini *encini, *macini;
int error;
if (sc == NULL || sidp == NULL || cri == NULL)
return (EINVAL);
encini = macini = NULL;
for (; cri != NULL; cri = cri->cri_next) {
switch(cri->cri_alg) {
case CRYPTO_NULL_HMAC:
case CRYPTO_MD5_HMAC:
case CRYPTO_SHA1_HMAC:
case CRYPTO_RIPEMD160_HMAC:
case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:
case CRYPTO_SHA2_512_HMAC:
if (macini != NULL)
return (EINVAL);
macini = cri;
break;
case CRYPTO_AES_CBC:
if (encini != NULL)
return (EINVAL);
encini = cri;
break;
default:
return (EINVAL);
}
}
/*
* We only support HMAC algorithms to be able to work with
* ipsec(4), so if we are asked only for authentication without
* encryption, don't pretend we can accellerate it.
*/
if (encini == NULL)
return (EINVAL);
/*
* Look for a free session
*
* Free sessions goes first, so if first session is used, we need to
* allocate one.
*/
rw_wlock(&sc->sc_sessions_lock);
ses = TAILQ_FIRST(&sc->sc_sessions);
if (ses == NULL || ses->ses_used) {
ses = malloc(sizeof(*ses), M_GLXSB, M_NOWAIT | M_ZERO);
if (ses == NULL) {
rw_wunlock(&sc->sc_sessions_lock);
return (ENOMEM);
}
ses->ses_id = sc->sc_sid++;
} else {
TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
}
ses->ses_used = 1;
TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next);
rw_wunlock(&sc->sc_sessions_lock);
if (encini->cri_alg == CRYPTO_AES_CBC) {
if (encini->cri_klen != 128) {
glxsb_crypto_freesession(sc->sc_dev, ses->ses_id);
return (EINVAL);
}
arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0);
ses->ses_klen = encini->cri_klen;
/* Copy the key (Geode LX wants the primary key only) */
bcopy(encini->cri_key, ses->ses_key, sizeof(ses->ses_key));
}
if (macini != NULL) {
error = glxsb_hash_setup(ses, macini);
if (error != 0) {
glxsb_crypto_freesession(sc->sc_dev, ses->ses_id);
return (error);
}
}
*sidp = ses->ses_id;
return (0);
}
static int
glxsb_crypto_freesession(device_t dev, uint64_t tid)
{
struct glxsb_softc *sc = device_get_softc(dev);
struct glxsb_session *ses = NULL;
uint32_t sid = ((uint32_t)tid) & 0xffffffff;
if (sc == NULL)
return (EINVAL);
rw_wlock(&sc->sc_sessions_lock);
TAILQ_FOREACH_REVERSE(ses, &sc->sc_sessions, ses_head, ses_next) {
if (ses->ses_id == sid)
break;
}
if (ses == NULL) {
rw_wunlock(&sc->sc_sessions_lock);
return (EINVAL);
}
TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
glxsb_hash_free(ses);
bzero(ses, sizeof(*ses));
ses->ses_used = 0;
ses->ses_id = sid;
TAILQ_INSERT_HEAD(&sc->sc_sessions, ses, ses_next);
rw_wunlock(&sc->sc_sessions_lock);
return (0);
}
static int
glxsb_aes(struct glxsb_softc *sc, uint32_t control, uint32_t psrc,
uint32_t pdst, void *key, int len, void *iv)
{
uint32_t status;
int i;
if (len & 0xF) {
device_printf(sc->sc_dev,
"len must be a multiple of 16 (not %d)\n", len);
return (EINVAL);
}
/* Set the source */
bus_write_4(sc->sc_sr, SB_SOURCE_A, psrc);
/* Set the destination address */
bus_write_4(sc->sc_sr, SB_DEST_A, pdst);
/* Set the data length */
bus_write_4(sc->sc_sr, SB_LENGTH_A, len);
/* Set the IV */
if (iv != NULL) {
bus_write_region_4(sc->sc_sr, SB_CBC_IV, iv, 4);
control |= SB_CTL_CBC;
}
/* Set the key */
bus_write_region_4(sc->sc_sr, SB_WKEY, key, 4);
/* Ask the security block to do it */
bus_write_4(sc->sc_sr, SB_CTL_A,
control | SB_CTL_WK | SB_CTL_DC | SB_CTL_SC | SB_CTL_ST);
/*
* Now wait until it is done.
*
* We do a busy wait. Obviously the number of iterations of
* the loop required to perform the AES operation depends upon
* the number of bytes to process.
*
* On a 500 MHz Geode LX we see
*
* length (bytes) typical max iterations
* 16 12
* 64 22
* 256 59
* 1024 212
* 8192 1,537
*
* Since we have a maximum size of operation defined in
* GLXSB_MAX_AES_LEN, we use this constant to decide how long
* to wait. Allow an order of magnitude longer than it should
* really take, just in case.
*/
for (i = 0; i < GLXSB_MAX_AES_LEN * 10; i++) {
status = bus_read_4(sc->sc_sr, SB_CTL_A);
if ((status & SB_CTL_ST) == 0) /* Done */
return (0);
}
device_printf(sc->sc_dev, "operation failed to complete\n");
return (EIO);
}
static int
glxsb_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd,
struct glxsb_session *ses, struct glxsb_softc *sc)
{
char *op_src, *op_dst;
uint32_t op_psrc, op_pdst;
uint8_t op_iv[SB_AES_BLOCK_SIZE], *piv;
int error;
int len, tlen, xlen;
int offset;
uint32_t control;
if (crd == NULL || (crd->crd_len % SB_AES_BLOCK_SIZE) != 0)
return (EINVAL);
/* How much of our buffer will we need to use? */
xlen = crd->crd_len > GLXSB_MAX_AES_LEN ?
GLXSB_MAX_AES_LEN : crd->crd_len;
/*
* XXX Check if we can have input == output on Geode LX.
* XXX In the meantime, use two separate (adjacent) buffers.
*/
op_src = sc->sc_dma.dma_vaddr;
op_dst = (char *)sc->sc_dma.dma_vaddr + xlen;
op_psrc = sc->sc_dma.dma_paddr;
op_pdst = sc->sc_dma.dma_paddr + xlen;
if (crd->crd_flags & CRD_F_ENCRYPT) {
control = SB_CTL_ENC;
if (crd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
else
bcopy(ses->ses_iv, op_iv, sizeof(op_iv));
if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
crypto_copyback(crp->crp_flags, crp->crp_buf,
crd->crd_inject, sizeof(op_iv), op_iv);
}
} else {
control = SB_CTL_DEC;
if (crd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
else {
crypto_copydata(crp->crp_flags, crp->crp_buf,
crd->crd_inject, sizeof(op_iv), op_iv);
}
}
offset = 0;
tlen = crd->crd_len;
piv = op_iv;
/* Process the data in GLXSB_MAX_AES_LEN chunks */
while (tlen > 0) {
len = (tlen > GLXSB_MAX_AES_LEN) ? GLXSB_MAX_AES_LEN : tlen;
crypto_copydata(crp->crp_flags, crp->crp_buf,
crd->crd_skip + offset, len, op_src);
glxsb_dma_pre_op(sc, &sc->sc_dma);
error = glxsb_aes(sc, control, op_psrc, op_pdst, ses->ses_key,
len, op_iv);
glxsb_dma_post_op(sc, &sc->sc_dma);
if (error != 0)
return (error);
crypto_copyback(crp->crp_flags, crp->crp_buf,
crd->crd_skip + offset, len, op_dst);
offset += len;
tlen -= len;
if (tlen <= 0) { /* Ideally, just == 0 */
/* Finished - put the IV in session IV */
piv = ses->ses_iv;
}
/*
* Copy out last block for use as next iteration/session IV.
*
* piv is set to op_iv[] before the loop starts, but is
* set to ses->ses_iv if we're going to exit the loop this
* time.
*/
if (crd->crd_flags & CRD_F_ENCRYPT)
bcopy(op_dst + len - sizeof(op_iv), piv, sizeof(op_iv));
else {
/* Decryption, only need this if another iteration */
if (tlen > 0) {
bcopy(op_src + len - sizeof(op_iv), piv,
sizeof(op_iv));
}
}
} /* while */
/* All AES processing has now been done. */
bzero(sc->sc_dma.dma_vaddr, xlen * 2);
return (0);
}
static void
glxsb_crypto_task(void *arg, int pending)
{
struct glxsb_softc *sc = arg;
struct glxsb_session *ses;
struct cryptop *crp;
struct cryptodesc *enccrd, *maccrd;
int error;
maccrd = sc->sc_to.to_maccrd;
enccrd = sc->sc_to.to_enccrd;
crp = sc->sc_to.to_crp;
ses = sc->sc_to.to_ses;
/* Perform data authentication if requested before encryption */
if (maccrd != NULL && maccrd->crd_next == enccrd) {
error = glxsb_hash_process(ses, maccrd, crp);
if (error != 0)
goto out;
}
error = glxsb_crypto_encdec(crp, enccrd, ses, sc);
if (error != 0)
goto out;
/* Perform data authentication if requested after encryption */
if (maccrd != NULL && enccrd->crd_next == maccrd) {
error = glxsb_hash_process(ses, maccrd, crp);
if (error != 0)
goto out;
}
out:
mtx_lock(&sc->sc_task_mtx);
sc->sc_task_count--;
mtx_unlock(&sc->sc_task_mtx);
crp->crp_etype = error;
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ);
crypto_done(crp);
}
static int
glxsb_crypto_process(device_t dev, struct cryptop *crp, int hint)
{
struct glxsb_softc *sc = device_get_softc(dev);
struct glxsb_session *ses;
struct cryptodesc *crd, *enccrd, *maccrd;
uint32_t sid;
int error = 0;
enccrd = maccrd = NULL;
/* Sanity check. */
if (crp == NULL)
return (EINVAL);
if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
error = EINVAL;
goto fail;
}
for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
switch (crd->crd_alg) {
case CRYPTO_NULL_HMAC:
case CRYPTO_MD5_HMAC:
case CRYPTO_SHA1_HMAC:
case CRYPTO_RIPEMD160_HMAC:
case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:
case CRYPTO_SHA2_512_HMAC:
if (maccrd != NULL) {
error = EINVAL;
goto fail;
}
maccrd = crd;
break;
case CRYPTO_AES_CBC:
if (enccrd != NULL) {
error = EINVAL;
goto fail;
}
enccrd = crd;
break;
default:
error = EINVAL;
goto fail;
}
}
if (enccrd == NULL || enccrd->crd_len % AES_BLOCK_LEN != 0) {
error = EINVAL;
goto fail;
}
sid = crp->crp_sid & 0xffffffff;
rw_rlock(&sc->sc_sessions_lock);
TAILQ_FOREACH_REVERSE(ses, &sc->sc_sessions, ses_head, ses_next) {
if (ses->ses_id == sid)
break;
}
rw_runlock(&sc->sc_sessions_lock);
if (ses == NULL || !ses->ses_used) {
error = EINVAL;
goto fail;
}
mtx_lock(&sc->sc_task_mtx);
if (sc->sc_task_count != 0) {
mtx_unlock(&sc->sc_task_mtx);
return (ERESTART);
}
sc->sc_task_count++;
sc->sc_to.to_maccrd = maccrd;
sc->sc_to.to_enccrd = enccrd;
sc->sc_to.to_crp = crp;
sc->sc_to.to_ses = ses;
mtx_unlock(&sc->sc_task_mtx);
taskqueue_enqueue(sc->sc_tq, &sc->sc_cryptotask);
return(0);
fail:
crp->crp_etype = error;
crypto_done(crp);
return (error);
}
Index: head/sys/dev/gxemul/cons/gxemul_cons.c
===================================================================
--- head/sys/dev/gxemul/cons/gxemul_cons.c (revision 283290)
+++ head/sys/dev/gxemul/cons/gxemul_cons.c (revision 283291)
@@ -1,333 +1,333 @@
/*-
* Copyright (c) 2011-2012 Robert N. M. Watson
* All rights reserved.
*
* This software was developed by SRI International and the University of
* Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
* ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/cons.h>
#include <sys/endian.h>
#include <sys/kdb.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/reboot.h>
#include <sys/tty.h>
#include <ddb/ddb.h>
#include <machine/cpuregs.h>
#define GC_LOCK_INIT() mtx_init(&gc_lock, "gc_lock", NULL, MTX_SPIN)
#define GC_LOCK() do { \
if (!kdb_active) \
mtx_lock_spin(&gc_lock); \
} while (0)
#define GC_LOCK_ASSERT() do { \
if (!kdb_active) \
mtx_assert(&gc_lock, MA_OWNED); \
} while (0)
#define GC_UNLOCK() do { \
if (!kdb_active) \
mtx_unlock_spin(&gc_lock); \
} while (0)
static struct mtx gc_lock;
/*
* Low-level console driver functions.
*/
static cn_probe_t gxemul_cons_cnprobe;
static cn_init_t gxemul_cons_cninit;
static cn_term_t gxemul_cons_cnterm;
static cn_getc_t gxemul_cons_cngetc;
static cn_putc_t gxemul_cons_cnputc;
static cn_grab_t gxemul_cons_cngrab;
static cn_ungrab_t gxemul_cons_cnungrab;
/*
* TTY-level fields.
*/
static tsw_outwakeup_t gxemul_cons_outwakeup;
static struct ttydevsw gxemul_cons_ttydevsw = {
.tsw_flags = TF_NOPREFIX,
.tsw_outwakeup = gxemul_cons_outwakeup,
};
static struct callout gxemul_cons_callout;
static u_int gxemul_cons_polltime = 10;
#ifdef KDB
static int gxemul_cons_alt_break_state;
#endif
static void gxemul_cons_timeout(void *);
/*
* I/O routines lifted from Deimos.
*
* XXXRW: Should be using FreeBSD's bus routines here, but they are not
* available until later in the boot.
*/
static inline vm_offset_t
mips_phys_to_uncached(vm_paddr_t phys)
{
return (MIPS_PHYS_TO_DIRECT_UNCACHED(phys));
}
static inline uint8_t
mips_ioread_uint8(vm_offset_t vaddr)
{
uint8_t v;
__asm__ __volatile__ ("lbu %0, 0(%1)" : "=r" (v) : "r" (vaddr));
return (v);
}
static inline void
mips_iowrite_uint8(vm_offset_t vaddr, uint8_t v)
{
__asm__ __volatile__ ("sb %0, 0(%1)" : : "r" (v), "r" (vaddr));
}
/*
* gxemul-specific constants.
*/
#define GXEMUL_CONS_BASE 0x10000000 /* gxemul console device. */
/*
* Routines for interacting with the gxemul test console. Programming details
* are a result of manually inspecting the source code for gxemul's
* dev_cons.cc and dev_cons.h.
*
* Offsets of I/O channels relative to the base.
*/
#define GXEMUL_PUTGETCHAR_OFF 0x00000000
#define GXEMUL_CONS_HALT 0x00000010
/*
* One-byte buffer as we can't check whether the console is readable without
* actually reading from it.
*/
static char buffer_data;
static int buffer_valid;
/*
* Low-level read and write routines.
*/
static inline uint8_t
gxemul_cons_data_read(void)
{
return (mips_ioread_uint8(mips_phys_to_uncached(GXEMUL_CONS_BASE +
GXEMUL_PUTGETCHAR_OFF)));
}
static inline void
gxemul_cons_data_write(uint8_t v)
{
mips_iowrite_uint8(mips_phys_to_uncached(GXEMUL_CONS_BASE +
GXEMUL_PUTGETCHAR_OFF), v);
}
static int
gxemul_cons_writable(void)
{
return (1);
}
static int
gxemul_cons_readable(void)
{
uint32_t v;
GC_LOCK_ASSERT();
if (buffer_valid)
return (1);
v = gxemul_cons_data_read();
if (v != 0) {
buffer_valid = 1;
buffer_data = v;
return (1);
}
return (0);
}
static void
gxemul_cons_write(char ch)
{
GC_LOCK_ASSERT();
while (!gxemul_cons_writable());
gxemul_cons_data_write(ch);
}
static char
gxemul_cons_read(void)
{
GC_LOCK_ASSERT();
while (!gxemul_cons_readable());
buffer_valid = 0;
return (buffer_data);
}
/*
* Implementation of a FreeBSD low-level, polled console driver.
*/
static void
gxemul_cons_cnprobe(struct consdev *cp)
{
sprintf(cp->cn_name, "ttyu0");
cp->cn_pri = (boothowto & RB_SERIAL) ? CN_REMOTE : CN_NORMAL;
}
static void
gxemul_cons_cninit(struct consdev *cp)
{
GC_LOCK_INIT();
}
static void
gxemul_cons_cnterm(struct consdev *cp)
{
}
static int
gxemul_cons_cngetc(struct consdev *cp)
{
int ret;
GC_LOCK();
ret = gxemul_cons_read();
GC_UNLOCK();
return (ret);
}
static void
gxemul_cons_cnputc(struct consdev *cp, int c)
{
GC_LOCK();
gxemul_cons_write(c);
GC_UNLOCK();
}
static void
gxemul_cons_cngrab(struct consdev *cp)
{
}
static void
gxemul_cons_cnungrab(struct consdev *cp)
{
}
CONSOLE_DRIVER(gxemul_cons);
/*
* TTY-level functions for gxemul_cons.
*/
static void
gxemul_cons_ttyinit(void *unused)
{
struct tty *tp;
tp = tty_alloc(&gxemul_cons_ttydevsw, NULL);
tty_init_console(tp, 0);
tty_makedev(tp, NULL, "%s", "ttyu0");
- callout_init(&gxemul_cons_callout, CALLOUT_MPSAFE);
+ callout_init(&gxemul_cons_callout, 1);
callout_reset(&gxemul_cons_callout, gxemul_cons_polltime,
gxemul_cons_timeout, tp);
}
SYSINIT(gxemul_cons_ttyinit, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE,
gxemul_cons_ttyinit, NULL);
static void
gxemul_cons_outwakeup(struct tty *tp)
{
int len;
u_char ch;
/*
* XXXRW: Would be nice not to do blocking writes to the console here,
* rescheduling on our timer tick if work remains to be done..
*/
for (;;) {
len = ttydisc_getc(tp, &ch, sizeof(ch));
if (len == 0)
break;
GC_LOCK();
gxemul_cons_write(ch);
GC_UNLOCK();
}
}
static void
gxemul_cons_timeout(void *v)
{
struct tty *tp;
int c;
tp = v;
tty_lock(tp);
GC_LOCK();
while (gxemul_cons_readable()) {
c = gxemul_cons_read();
GC_UNLOCK();
#ifdef KDB
kdb_alt_break(c, &gxemul_cons_alt_break_state);
#endif
ttydisc_rint(tp, c, 0);
GC_LOCK();
}
GC_UNLOCK();
ttydisc_rint_done(tp);
tty_unlock(tp);
callout_reset(&gxemul_cons_callout, gxemul_cons_polltime,
gxemul_cons_timeout, tp);
}
Index: head/sys/dev/hifn/hifn7751.c
===================================================================
--- head/sys/dev/hifn/hifn7751.c (revision 283290)
+++ head/sys/dev/hifn/hifn7751.c (revision 283291)
@@ -1,2929 +1,2929 @@
/* $OpenBSD: hifn7751.c,v 1.120 2002/05/17 00:33:34 deraadt Exp $ */
/*-
* Invertex AEON / Hifn 7751 driver
* Copyright (c) 1999 Invertex Inc. All rights reserved.
* Copyright (c) 1999 Theo de Raadt
* Copyright (c) 2000-2001 Network Security Technologies, Inc.
* http://www.netsec.net
* Copyright (c) 2003 Hifn Inc.
*
* This driver is based on a previous driver by Invertex, for which they
* requested: Please send any comments, feedback, bug-fixes, or feature
* requests to software@invertex.com.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* Effort sponsored in part by the Defense Advanced Research Projects
* Agency (DARPA) and Air Force Research Laboratory, Air Force
* Materiel Command, USAF, under agreement number F30602-01-2-0537.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* Driver for various Hifn encryption processors.
*/
#include "opt_hifn.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/errno.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/mbuf.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <opencrypto/cryptodev.h>
#include <sys/random.h>
#include <sys/kobj.h>
#include "cryptodev_if.h"
#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
#ifdef HIFN_RNDTEST
#include <dev/rndtest/rndtest.h>
#endif
#include <dev/hifn/hifn7751reg.h>
#include <dev/hifn/hifn7751var.h>
#ifdef HIFN_VULCANDEV
#include <sys/conf.h>
#include <sys/uio.h>
static struct cdevsw vulcanpk_cdevsw; /* forward declaration */
#endif
/*
* Prototypes and count for the pci_device structure
*/
static int hifn_probe(device_t);
static int hifn_attach(device_t);
static int hifn_detach(device_t);
static int hifn_suspend(device_t);
static int hifn_resume(device_t);
static int hifn_shutdown(device_t);
static int hifn_newsession(device_t, u_int32_t *, struct cryptoini *);
static int hifn_freesession(device_t, u_int64_t);
static int hifn_process(device_t, struct cryptop *, int);
static device_method_t hifn_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, hifn_probe),
DEVMETHOD(device_attach, hifn_attach),
DEVMETHOD(device_detach, hifn_detach),
DEVMETHOD(device_suspend, hifn_suspend),
DEVMETHOD(device_resume, hifn_resume),
DEVMETHOD(device_shutdown, hifn_shutdown),
/* crypto device methods */
DEVMETHOD(cryptodev_newsession, hifn_newsession),
DEVMETHOD(cryptodev_freesession,hifn_freesession),
DEVMETHOD(cryptodev_process, hifn_process),
DEVMETHOD_END
};
static driver_t hifn_driver = {
"hifn",
hifn_methods,
sizeof (struct hifn_softc)
};
static devclass_t hifn_devclass;
DRIVER_MODULE(hifn, pci, hifn_driver, hifn_devclass, 0, 0);
MODULE_DEPEND(hifn, crypto, 1, 1, 1);
#ifdef HIFN_RNDTEST
MODULE_DEPEND(hifn, rndtest, 1, 1, 1);
#endif
static void hifn_reset_board(struct hifn_softc *, int);
static void hifn_reset_puc(struct hifn_softc *);
static void hifn_puc_wait(struct hifn_softc *);
static int hifn_enable_crypto(struct hifn_softc *);
static void hifn_set_retry(struct hifn_softc *sc);
static void hifn_init_dma(struct hifn_softc *);
static void hifn_init_pci_registers(struct hifn_softc *);
static int hifn_sramsize(struct hifn_softc *);
static int hifn_dramsize(struct hifn_softc *);
static int hifn_ramtype(struct hifn_softc *);
static void hifn_sessions(struct hifn_softc *);
static void hifn_intr(void *);
static u_int hifn_write_command(struct hifn_command *, u_int8_t *);
static u_int32_t hifn_next_signature(u_int32_t a, u_int cnt);
static void hifn_callback(struct hifn_softc *, struct hifn_command *, u_int8_t *);
static int hifn_crypto(struct hifn_softc *, struct hifn_command *, struct cryptop *, int);
static int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *);
static int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *);
static int hifn_dmamap_load_src(struct hifn_softc *, struct hifn_command *);
static int hifn_dmamap_load_dst(struct hifn_softc *, struct hifn_command *);
static int hifn_init_pubrng(struct hifn_softc *);
static void hifn_rng(void *);
static void hifn_tick(void *);
static void hifn_abort(struct hifn_softc *);
static void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, int *);
static void hifn_write_reg_0(struct hifn_softc *, bus_size_t, u_int32_t);
static void hifn_write_reg_1(struct hifn_softc *, bus_size_t, u_int32_t);
static __inline u_int32_t
READ_REG_0(struct hifn_softc *sc, bus_size_t reg)
{
u_int32_t v = bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg);
sc->sc_bar0_lastreg = (bus_size_t) -1;
return (v);
}
#define WRITE_REG_0(sc, reg, val) hifn_write_reg_0(sc, reg, val)
static __inline u_int32_t
READ_REG_1(struct hifn_softc *sc, bus_size_t reg)
{
u_int32_t v = bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg);
sc->sc_bar1_lastreg = (bus_size_t) -1;
return (v);
}
#define WRITE_REG_1(sc, reg, val) hifn_write_reg_1(sc, reg, val)
static SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0,
"Hifn driver parameters");
#ifdef HIFN_DEBUG
static int hifn_debug = 0;
SYSCTL_INT(_hw_hifn, OID_AUTO, debug, CTLFLAG_RW, &hifn_debug,
0, "control debugging msgs");
#endif
static struct hifn_stats hifnstats;
SYSCTL_STRUCT(_hw_hifn, OID_AUTO, stats, CTLFLAG_RD, &hifnstats,
hifn_stats, "driver statistics");
static int hifn_maxbatch = 1;
SYSCTL_INT(_hw_hifn, OID_AUTO, maxbatch, CTLFLAG_RW, &hifn_maxbatch,
0, "max ops to batch w/o interrupt");
/*
* Probe for a supported device. The PCI vendor and device
* IDs are used to detect devices we know how to handle.
*/
static int
hifn_probe(device_t dev)
{
if (pci_get_vendor(dev) == PCI_VENDOR_INVERTEX &&
pci_get_device(dev) == PCI_PRODUCT_INVERTEX_AEON)
return (BUS_PROBE_DEFAULT);
if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
(pci_get_device(dev) == PCI_PRODUCT_HIFN_7751 ||
pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
pci_get_device(dev) == PCI_PRODUCT_HIFN_7956 ||
pci_get_device(dev) == PCI_PRODUCT_HIFN_7811))
return (BUS_PROBE_DEFAULT);
if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751)
return (BUS_PROBE_DEFAULT);
return (ENXIO);
}
static void
hifn_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
bus_addr_t *paddr = (bus_addr_t*) arg;
*paddr = segs->ds_addr;
}
static const char*
hifn_partname(struct hifn_softc *sc)
{
/* XXX sprintf numbers when not decoded */
switch (pci_get_vendor(sc->sc_dev)) {
case PCI_VENDOR_HIFN:
switch (pci_get_device(sc->sc_dev)) {
case PCI_PRODUCT_HIFN_6500: return "Hifn 6500";
case PCI_PRODUCT_HIFN_7751: return "Hifn 7751";
case PCI_PRODUCT_HIFN_7811: return "Hifn 7811";
case PCI_PRODUCT_HIFN_7951: return "Hifn 7951";
case PCI_PRODUCT_HIFN_7955: return "Hifn 7955";
case PCI_PRODUCT_HIFN_7956: return "Hifn 7956";
}
return "Hifn unknown-part";
case PCI_VENDOR_INVERTEX:
switch (pci_get_device(sc->sc_dev)) {
case PCI_PRODUCT_INVERTEX_AEON: return "Invertex AEON";
}
return "Invertex unknown-part";
case PCI_VENDOR_NETSEC:
switch (pci_get_device(sc->sc_dev)) {
case PCI_PRODUCT_NETSEC_7751: return "NetSec 7751";
}
return "NetSec unknown-part";
}
return "Unknown-vendor unknown-part";
}
static void
default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
{
random_harvest(buf, count, count*NBBY/2, RANDOM_PURE_HIFN);
}
static u_int
checkmaxmin(device_t dev, const char *what, u_int v, u_int min, u_int max)
{
if (v > max) {
device_printf(dev, "Warning, %s %u out of range, "
"using max %u\n", what, v, max);
v = max;
} else if (v < min) {
device_printf(dev, "Warning, %s %u out of range, "
"using min %u\n", what, v, min);
v = min;
}
return v;
}
/*
* Select PLL configuration for 795x parts. This is complicated in
* that we cannot determine the optimal parameters without user input.
* The reference clock is derived from an external clock through a
* multiplier. The external clock is either the host bus (i.e. PCI)
* or an external clock generator. When using the PCI bus we assume
* the clock is either 33 or 66 MHz; for an external source we cannot
* tell the speed.
*
* PLL configuration is done with a string: "pci" for PCI bus, or "ext"
* for an external source, followed by the frequency. We calculate
* the appropriate multiplier and PLL register contents accordingly.
* When no configuration is given we default to "pci66" since that
* always will allow the card to work. If a card is using the PCI
* bus clock and in a 33MHz slot then it will be operating at half
* speed until the correct information is provided.
*
* We use a default setting of "ext66" because according to Mike Ham
* of HiFn, almost every board in existence has an external crystal
* populated at 66Mhz. Using PCI can be a problem on modern motherboards,
* because PCI33 can have clocks from 0 to 33Mhz, and some have
* non-PCI-compliant spread-spectrum clocks, which can confuse the pll.
*/
static void
hifn_getpllconfig(device_t dev, u_int *pll)
{
const char *pllspec;
u_int freq, mul, fl, fh;
u_int32_t pllconfig;
char *nxt;
if (resource_string_value("hifn", device_get_unit(dev),
"pllconfig", &pllspec))
pllspec = "ext66";
fl = 33, fh = 66;
pllconfig = 0;
if (strncmp(pllspec, "ext", 3) == 0) {
pllspec += 3;
pllconfig |= HIFN_PLL_REF_SEL;
switch (pci_get_device(dev)) {
case PCI_PRODUCT_HIFN_7955:
case PCI_PRODUCT_HIFN_7956:
fl = 20, fh = 100;
break;
#ifdef notyet
case PCI_PRODUCT_HIFN_7954:
fl = 20, fh = 66;
break;
#endif
}
} else if (strncmp(pllspec, "pci", 3) == 0)
pllspec += 3;
freq = strtoul(pllspec, &nxt, 10);
if (nxt == pllspec)
freq = 66;
else
freq = checkmaxmin(dev, "frequency", freq, fl, fh);
/*
* Calculate multiplier. We target a Fck of 266 MHz,
* allowing only even values, possibly rounded down.
* Multipliers > 8 must set the charge pump current.
*/
mul = checkmaxmin(dev, "PLL divisor", (266 / freq) &~ 1, 2, 12);
pllconfig |= (mul / 2 - 1) << HIFN_PLL_ND_SHIFT;
if (mul > 8)
pllconfig |= HIFN_PLL_IS;
*pll = pllconfig;
}
/*
* Attach an interface that successfully probed.
*/
static int
hifn_attach(device_t dev)
{
struct hifn_softc *sc = device_get_softc(dev);
caddr_t kva;
int rseg, rid;
char rbase;
u_int16_t ena, rev;
sc->sc_dev = dev;
mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "hifn driver", MTX_DEF);
/* XXX handle power management */
/*
* The 7951 and 795x have a random number generator and
* public key support; note this.
*/
if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
(pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
pci_get_device(dev) == PCI_PRODUCT_HIFN_7956))
sc->sc_flags = HIFN_HAS_RNG | HIFN_HAS_PUBLIC;
/*
* The 7811 has a random number generator and
* we also note it's identity 'cuz of some quirks.
*/
if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
pci_get_device(dev) == PCI_PRODUCT_HIFN_7811)
sc->sc_flags |= HIFN_IS_7811 | HIFN_HAS_RNG;
/*
* The 795x parts support AES.
*/
if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
(pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
pci_get_device(dev) == PCI_PRODUCT_HIFN_7956)) {
sc->sc_flags |= HIFN_IS_7956 | HIFN_HAS_AES;
/*
* Select PLL configuration. This depends on the
* bus and board design and must be manually configured
* if the default setting is unacceptable.
*/
hifn_getpllconfig(dev, &sc->sc_pllconfig);
}
/*
* Setup PCI resources. Note that we record the bus
* tag and handle for each register mapping, this is
* used by the READ_REG_0, WRITE_REG_0, READ_REG_1,
* and WRITE_REG_1 macros throughout the driver.
*/
pci_enable_busmaster(dev);
rid = HIFN_BAR0;
sc->sc_bar0res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (sc->sc_bar0res == NULL) {
device_printf(dev, "cannot map bar%d register space\n", 0);
goto fail_pci;
}
sc->sc_st0 = rman_get_bustag(sc->sc_bar0res);
sc->sc_sh0 = rman_get_bushandle(sc->sc_bar0res);
sc->sc_bar0_lastreg = (bus_size_t) -1;
rid = HIFN_BAR1;
sc->sc_bar1res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (sc->sc_bar1res == NULL) {
device_printf(dev, "cannot map bar%d register space\n", 1);
goto fail_io0;
}
sc->sc_st1 = rman_get_bustag(sc->sc_bar1res);
sc->sc_sh1 = rman_get_bushandle(sc->sc_bar1res);
sc->sc_bar1_lastreg = (bus_size_t) -1;
hifn_set_retry(sc);
/*
* Setup the area where the Hifn DMA's descriptors
* and associated data structures.
*/
if (bus_dma_tag_create(bus_get_dma_tag(dev), /* PCI parent */
1, 0, /* alignment,boundary */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
HIFN_MAX_DMALEN, /* maxsize */
MAX_SCATTER, /* nsegments */
HIFN_MAX_SEGLEN, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, /* lockfunc */
NULL, /* lockarg */
&sc->sc_dmat)) {
device_printf(dev, "cannot allocate DMA tag\n");
goto fail_io1;
}
if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
device_printf(dev, "cannot create dma map\n");
bus_dma_tag_destroy(sc->sc_dmat);
goto fail_io1;
}
if (bus_dmamem_alloc(sc->sc_dmat, (void**) &kva, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
device_printf(dev, "cannot alloc dma buffer\n");
bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
bus_dma_tag_destroy(sc->sc_dmat);
goto fail_io1;
}
if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmamap, kva,
sizeof (*sc->sc_dma),
hifn_dmamap_cb, &sc->sc_dma_physaddr,
BUS_DMA_NOWAIT)) {
device_printf(dev, "cannot load dma map\n");
bus_dmamem_free(sc->sc_dmat, kva, sc->sc_dmamap);
bus_dma_tag_destroy(sc->sc_dmat);
goto fail_io1;
}
sc->sc_dma = (struct hifn_dma *)kva;
bzero(sc->sc_dma, sizeof(*sc->sc_dma));
KASSERT(sc->sc_st0 != 0, ("hifn_attach: null bar0 tag!"));
KASSERT(sc->sc_sh0 != 0, ("hifn_attach: null bar0 handle!"));
KASSERT(sc->sc_st1 != 0, ("hifn_attach: null bar1 tag!"));
KASSERT(sc->sc_sh1 != 0, ("hifn_attach: null bar1 handle!"));
/*
* Reset the board and do the ``secret handshake''
* to enable the crypto support. Then complete the
* initialization procedure by setting up the interrupt
* and hooking in to the system crypto support so we'll
* get used for system services like the crypto device,
* IPsec, RNG device, etc.
*/
hifn_reset_board(sc, 0);
if (hifn_enable_crypto(sc) != 0) {
device_printf(dev, "crypto enabling failed\n");
goto fail_mem;
}
hifn_reset_puc(sc);
hifn_init_dma(sc);
hifn_init_pci_registers(sc);
/* XXX can't dynamically determine ram type for 795x; force dram */
if (sc->sc_flags & HIFN_IS_7956)
sc->sc_drammodel = 1;
else if (hifn_ramtype(sc))
goto fail_mem;
if (sc->sc_drammodel == 0)
hifn_sramsize(sc);
else
hifn_dramsize(sc);
/*
* Workaround for NetSec 7751 rev A: half ram size because two
* of the address lines were left floating
*/
if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751 &&
pci_get_revid(dev) == 0x61) /*XXX???*/
sc->sc_ramsize >>= 1;
/*
* Arrange the interrupt line.
*/
rid = 0;
sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_SHAREABLE|RF_ACTIVE);
if (sc->sc_irq == NULL) {
device_printf(dev, "could not map interrupt\n");
goto fail_mem;
}
/*
* NB: Network code assumes we are blocked with splimp()
* so make sure the IRQ is marked appropriately.
*/
if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
NULL, hifn_intr, sc, &sc->sc_intrhand)) {
device_printf(dev, "could not setup interrupt\n");
goto fail_intr2;
}
hifn_sessions(sc);
/*
* NB: Keep only the low 16 bits; this masks the chip id
* from the 7951.
*/
rev = READ_REG_1(sc, HIFN_1_REVID) & 0xffff;
rseg = sc->sc_ramsize / 1024;
rbase = 'K';
if (sc->sc_ramsize >= (1024 * 1024)) {
rbase = 'M';
rseg /= 1024;
}
device_printf(sc->sc_dev, "%s, rev %u, %d%cB %cram",
hifn_partname(sc), rev,
rseg, rbase, sc->sc_drammodel ? 'd' : 's');
if (sc->sc_flags & HIFN_IS_7956)
printf(", pll=0x%x<%s clk, %ux mult>",
sc->sc_pllconfig,
sc->sc_pllconfig & HIFN_PLL_REF_SEL ? "ext" : "pci",
2 + 2*((sc->sc_pllconfig & HIFN_PLL_ND) >> 11));
printf("\n");
sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
if (sc->sc_cid < 0) {
device_printf(dev, "could not get crypto driver id\n");
goto fail_intr;
}
WRITE_REG_0(sc, HIFN_0_PUCNFG,
READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
switch (ena) {
case HIFN_PUSTAT_ENA_2:
crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0);
if (sc->sc_flags & HIFN_HAS_AES)
crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
/*FALLTHROUGH*/
case HIFN_PUSTAT_ENA_1:
crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
break;
}
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
hifn_init_pubrng(sc);
- callout_init(&sc->sc_tickto, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_tickto, 1);
callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
return (0);
fail_intr:
bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
fail_intr2:
/* XXX don't store rid */
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
fail_mem:
bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
bus_dma_tag_destroy(sc->sc_dmat);
/* Turn off DMA polling */
WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
fail_io1:
bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
fail_io0:
bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
fail_pci:
mtx_destroy(&sc->sc_mtx);
return (ENXIO);
}
/*
* Detach an interface that successfully probed.
*/
static int
hifn_detach(device_t dev)
{
struct hifn_softc *sc = device_get_softc(dev);
KASSERT(sc != NULL, ("hifn_detach: null software carrier!"));
/* disable interrupts */
WRITE_REG_1(sc, HIFN_1_DMA_IER, 0);
/*XXX other resources */
callout_stop(&sc->sc_tickto);
callout_stop(&sc->sc_rngto);
#ifdef HIFN_RNDTEST
if (sc->sc_rndtest)
rndtest_detach(sc->sc_rndtest);
#endif
/* Turn off DMA polling */
WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
crypto_unregister_all(sc->sc_cid);
bus_generic_detach(dev); /*XXX should be no children, right? */
bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
/* XXX don't store rid */
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
bus_dma_tag_destroy(sc->sc_dmat);
bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
mtx_destroy(&sc->sc_mtx);
return (0);
}
/*
* Stop all chip I/O so that the kernel's probe routines don't
* get confused by errant DMAs when rebooting.
*/
static int
hifn_shutdown(device_t dev)
{
#ifdef notyet
hifn_stop(device_get_softc(dev));
#endif
return (0);
}
/*
* Device suspend routine. Stop the interface and save some PCI
* settings in case the BIOS doesn't restore them properly on
* resume.
*/
static int
hifn_suspend(device_t dev)
{
struct hifn_softc *sc = device_get_softc(dev);
#ifdef notyet
hifn_stop(sc);
#endif
sc->sc_suspended = 1;
return (0);
}
/*
* Device resume routine. Restore some PCI settings in case the BIOS
* doesn't, re-enable busmastering, and restart the interface if
* appropriate.
*/
static int
hifn_resume(device_t dev)
{
struct hifn_softc *sc = device_get_softc(dev);
#ifdef notyet
/* reinitialize interface if necessary */
if (ifp->if_flags & IFF_UP)
rl_init(sc);
#endif
sc->sc_suspended = 0;
return (0);
}
static int
hifn_init_pubrng(struct hifn_softc *sc)
{
u_int32_t r;
int i;
#ifdef HIFN_RNDTEST
sc->sc_rndtest = rndtest_attach(sc->sc_dev);
if (sc->sc_rndtest)
sc->sc_harvest = rndtest_harvest;
else
sc->sc_harvest = default_harvest;
#else
sc->sc_harvest = default_harvest;
#endif
if ((sc->sc_flags & HIFN_IS_7811) == 0) {
/* Reset 7951 public key/rng engine */
WRITE_REG_1(sc, HIFN_1_PUB_RESET,
READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
for (i = 0; i < 100; i++) {
DELAY(1000);
if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
HIFN_PUBRST_RESET) == 0)
break;
}
if (i == 100) {
device_printf(sc->sc_dev, "public key init failed\n");
return (1);
}
}
/* Enable the rng, if available */
if (sc->sc_flags & HIFN_HAS_RNG) {
if (sc->sc_flags & HIFN_IS_7811) {
r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
if (r & HIFN_7811_RNGENA_ENA) {
r &= ~HIFN_7811_RNGENA_ENA;
WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
}
WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
HIFN_7811_RNGCFG_DEFL);
r |= HIFN_7811_RNGENA_ENA;
WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
} else
WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
HIFN_RNGCFG_ENA);
sc->sc_rngfirst = 1;
if (hz >= 100)
sc->sc_rnghz = hz / 100;
else
sc->sc_rnghz = 1;
- callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_rngto, 1);
callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
}
/* Enable public key engine, if available */
if (sc->sc_flags & HIFN_HAS_PUBLIC) {
WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
#ifdef HIFN_VULCANDEV
sc->sc_pkdev = make_dev(&vulcanpk_cdevsw, 0,
UID_ROOT, GID_WHEEL, 0666,
"vulcanpk");
sc->sc_pkdev->si_drv1 = sc;
#endif
}
return (0);
}
static void
hifn_rng(void *vsc)
{
#define RANDOM_BITS(n) (n)*sizeof (u_int32_t), (n)*sizeof (u_int32_t)*NBBY, 0
struct hifn_softc *sc = vsc;
u_int32_t sts, num[2];
int i;
if (sc->sc_flags & HIFN_IS_7811) {
/* ONLY VALID ON 7811!!!! */
for (i = 0; i < 5; i++) {
sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
if (sts & HIFN_7811_RNGSTS_UFL) {
device_printf(sc->sc_dev,
"RNG underflow: disabling\n");
return;
}
if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
break;
/*
* There are at least two words in the RNG FIFO
* at this point.
*/
num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
/* NB: discard first data read */
if (sc->sc_rngfirst)
sc->sc_rngfirst = 0;
else
(*sc->sc_harvest)(sc->sc_rndtest,
num, sizeof (num));
}
} else {
num[0] = READ_REG_1(sc, HIFN_1_RNG_DATA);
/* NB: discard first data read */
if (sc->sc_rngfirst)
sc->sc_rngfirst = 0;
else
(*sc->sc_harvest)(sc->sc_rndtest,
num, sizeof (num[0]));
}
callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
#undef RANDOM_BITS
}
static void
hifn_puc_wait(struct hifn_softc *sc)
{
int i;
int reg = HIFN_0_PUCTRL;
if (sc->sc_flags & HIFN_IS_7956) {
reg = HIFN_0_PUCTRL2;
}
for (i = 5000; i > 0; i--) {
DELAY(1);
if (!(READ_REG_0(sc, reg) & HIFN_PUCTRL_RESET))
break;
}
if (!i)
device_printf(sc->sc_dev, "proc unit did not reset\n");
}
/*
* Reset the processing unit.
*/
static void
hifn_reset_puc(struct hifn_softc *sc)
{
/* Reset processing unit */
int reg = HIFN_0_PUCTRL;
if (sc->sc_flags & HIFN_IS_7956) {
reg = HIFN_0_PUCTRL2;
}
WRITE_REG_0(sc, reg, HIFN_PUCTRL_DMAENA);
hifn_puc_wait(sc);
}
/*
* Set the Retry and TRDY registers; note that we set them to
* zero because the 7811 locks up when forced to retry (section
* 3.6 of "Specification Update SU-0014-04". Not clear if we
* should do this for all Hifn parts, but it doesn't seem to hurt.
*/
static void
hifn_set_retry(struct hifn_softc *sc)
{
/* NB: RETRY only responds to 8-bit reads/writes */
pci_write_config(sc->sc_dev, HIFN_RETRY_TIMEOUT, 0, 1);
pci_write_config(sc->sc_dev, HIFN_TRDY_TIMEOUT, 0, 1);
}
/*
* Resets the board. Values in the regesters are left as is
* from the reset (i.e. initial values are assigned elsewhere).
*/
static void
hifn_reset_board(struct hifn_softc *sc, int full)
{
u_int32_t reg;
/*
* Set polling in the DMA configuration register to zero. 0x7 avoids
* resetting the board and zeros out the other fields.
*/
WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
/*
* Now that polling has been disabled, we have to wait 1 ms
* before resetting the board.
*/
DELAY(1000);
/* Reset the DMA unit */
if (full) {
WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
DELAY(1000);
} else {
WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
hifn_reset_puc(sc);
}
KASSERT(sc->sc_dma != NULL, ("hifn_reset_board: null DMA tag!"));
bzero(sc->sc_dma, sizeof(*sc->sc_dma));
/* Bring dma unit out of reset */
WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
hifn_puc_wait(sc);
hifn_set_retry(sc);
if (sc->sc_flags & HIFN_IS_7811) {
for (reg = 0; reg < 1000; reg++) {
if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
HIFN_MIPSRST_CRAMINIT)
break;
DELAY(1000);
}
if (reg == 1000)
printf(": cram init timeout\n");
} else {
/* set up DMA configuration register #2 */
/* turn off all PK and BAR0 swaps */
WRITE_REG_1(sc, HIFN_1_DMA_CNFG2,
(3 << HIFN_DMACNFG2_INIT_WRITE_BURST_SHIFT)|
(3 << HIFN_DMACNFG2_INIT_READ_BURST_SHIFT)|
(2 << HIFN_DMACNFG2_TGT_WRITE_BURST_SHIFT)|
(2 << HIFN_DMACNFG2_TGT_READ_BURST_SHIFT));
}
}
static u_int32_t
hifn_next_signature(u_int32_t a, u_int cnt)
{
int i;
u_int32_t v;
for (i = 0; i < cnt; i++) {
/* get the parity */
v = a & 0x80080125;
v ^= v >> 16;
v ^= v >> 8;
v ^= v >> 4;
v ^= v >> 2;
v ^= v >> 1;
a = (v & 1) ^ (a << 1);
}
return a;
}
struct pci2id {
u_short pci_vendor;
u_short pci_prod;
char card_id[13];
};
static struct pci2id pci2id[] = {
{
PCI_VENDOR_HIFN,
PCI_PRODUCT_HIFN_7951,
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00 }
}, {
PCI_VENDOR_HIFN,
PCI_PRODUCT_HIFN_7955,
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00 }
}, {
PCI_VENDOR_HIFN,
PCI_PRODUCT_HIFN_7956,
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00 }
}, {
PCI_VENDOR_NETSEC,
PCI_PRODUCT_NETSEC_7751,
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00 }
}, {
PCI_VENDOR_INVERTEX,
PCI_PRODUCT_INVERTEX_AEON,
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00 }
}, {
PCI_VENDOR_HIFN,
PCI_PRODUCT_HIFN_7811,
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00 }
}, {
/*
* Other vendors share this PCI ID as well, such as
* http://www.powercrypt.com, and obviously they also
* use the same key.
*/
PCI_VENDOR_HIFN,
PCI_PRODUCT_HIFN_7751,
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00 }
},
};
/*
* Checks to see if crypto is already enabled. If crypto isn't enable,
* "hifn_enable_crypto" is called to enable it. The check is important,
* as enabling crypto twice will lock the board.
*/
static int
hifn_enable_crypto(struct hifn_softc *sc)
{
u_int32_t dmacfg, ramcfg, encl, addr, i;
char *offtbl = NULL;
for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
if (pci2id[i].pci_vendor == pci_get_vendor(sc->sc_dev) &&
pci2id[i].pci_prod == pci_get_device(sc->sc_dev)) {
offtbl = pci2id[i].card_id;
break;
}
}
if (offtbl == NULL) {
device_printf(sc->sc_dev, "Unknown card!\n");
return (1);
}
ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
/*
* The RAM config register's encrypt level bit needs to be set before
* every read performed on the encryption level register.
*/
WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
/*
* Make sure we don't re-unlock. Two unlocks kills chip until the
* next reboot.
*/
if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
#ifdef HIFN_DEBUG
if (hifn_debug)
device_printf(sc->sc_dev,
"Strong crypto already enabled!\n");
#endif
goto report;
}
if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
#ifdef HIFN_DEBUG
if (hifn_debug)
device_printf(sc->sc_dev,
"Unknown encryption level 0x%x\n", encl);
#endif
return 1;
}
WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
DELAY(1000);
addr = READ_REG_1(sc, HIFN_UNLOCK_SECRET1);
DELAY(1000);
WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, 0);
DELAY(1000);
for (i = 0; i <= 12; i++) {
addr = hifn_next_signature(addr, offtbl[i] + 0x101);
WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, addr);
DELAY(1000);
}
WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
#ifdef HIFN_DEBUG
if (hifn_debug) {
if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
device_printf(sc->sc_dev, "Engine is permanently "
"locked until next system reset!\n");
else
device_printf(sc->sc_dev, "Engine enabled "
"successfully!\n");
}
#endif
report:
WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
switch (encl) {
case HIFN_PUSTAT_ENA_1:
case HIFN_PUSTAT_ENA_2:
break;
case HIFN_PUSTAT_ENA_0:
default:
device_printf(sc->sc_dev, "disabled");
break;
}
return 0;
}
/*
* Give initial values to the registers listed in the "Register Space"
* section of the HIFN Software Development reference manual.
*/
static void
hifn_init_pci_registers(struct hifn_softc *sc)
{
/* write fixed values needed by the Initialization registers */
WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
/* write all 4 ring address registers */
WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dma_physaddr +
offsetof(struct hifn_dma, cmdr[0]));
WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dma_physaddr +
offsetof(struct hifn_dma, srcr[0]));
WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dma_physaddr +
offsetof(struct hifn_dma, dstr[0]));
WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dma_physaddr +
offsetof(struct hifn_dma, resr[0]));
DELAY(2000);
/* write status register */
WRITE_REG_1(sc, HIFN_1_DMA_CSR,
HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
HIFN_DMACSR_S_WAIT |
HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
HIFN_DMACSR_C_WAIT |
HIFN_DMACSR_ENGINE |
((sc->sc_flags & HIFN_HAS_PUBLIC) ?
HIFN_DMACSR_PUBDONE : 0) |
((sc->sc_flags & HIFN_IS_7811) ?
HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
((sc->sc_flags & HIFN_IS_7811) ?
HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
if (sc->sc_flags & HIFN_IS_7956) {
u_int32_t pll;
WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
HIFN_PUCNFG_TCALLPHASES |
HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32);
/* turn off the clocks and insure bypass is set */
pll = READ_REG_1(sc, HIFN_1_PLL);
pll = (pll &~ (HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL))
| HIFN_PLL_BP | HIFN_PLL_MBSET;
WRITE_REG_1(sc, HIFN_1_PLL, pll);
DELAY(10*1000); /* 10ms */
/* change configuration */
pll = (pll &~ HIFN_PLL_CONFIG) | sc->sc_pllconfig;
WRITE_REG_1(sc, HIFN_1_PLL, pll);
DELAY(10*1000); /* 10ms */
/* disable bypass */
pll &= ~HIFN_PLL_BP;
WRITE_REG_1(sc, HIFN_1_PLL, pll);
/* enable clocks with new configuration */
pll |= HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL;
WRITE_REG_1(sc, HIFN_1_PLL, pll);
} else {
WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
(sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
}
WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
}
/*
* The maximum number of sessions supported by the card
* is dependent on the amount of context ram, which
* encryption algorithms are enabled, and how compression
* is configured. This should be configured before this
* routine is called.
*/
static void
hifn_sessions(struct hifn_softc *sc)
{
u_int32_t pucnfg;
int ctxsize;
pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
if (pucnfg & HIFN_PUCNFG_COMPSING) {
if (pucnfg & HIFN_PUCNFG_ENCCNFG)
ctxsize = 128;
else
ctxsize = 512;
/*
* 7955/7956 has internal context memory of 32K
*/
if (sc->sc_flags & HIFN_IS_7956)
sc->sc_maxses = 32768 / ctxsize;
else
sc->sc_maxses = 1 +
((sc->sc_ramsize - 32768) / ctxsize);
} else
sc->sc_maxses = sc->sc_ramsize / 16384;
if (sc->sc_maxses > 2048)
sc->sc_maxses = 2048;
}
/*
* Determine ram type (sram or dram). Board should be just out of a reset
* state when this is called.
*/
static int
hifn_ramtype(struct hifn_softc *sc)
{
u_int8_t data[8], dataexpect[8];
int i;
for (i = 0; i < sizeof(data); i++)
data[i] = dataexpect[i] = 0x55;
if (hifn_writeramaddr(sc, 0, data))
return (-1);
if (hifn_readramaddr(sc, 0, data))
return (-1);
if (bcmp(data, dataexpect, sizeof(data)) != 0) {
sc->sc_drammodel = 1;
return (0);
}
for (i = 0; i < sizeof(data); i++)
data[i] = dataexpect[i] = 0xaa;
if (hifn_writeramaddr(sc, 0, data))
return (-1);
if (hifn_readramaddr(sc, 0, data))
return (-1);
if (bcmp(data, dataexpect, sizeof(data)) != 0) {
sc->sc_drammodel = 1;
return (0);
}
return (0);
}
#define HIFN_SRAM_MAX (32 << 20)
#define HIFN_SRAM_STEP_SIZE 16384
#define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
static int
hifn_sramsize(struct hifn_softc *sc)
{
u_int32_t a;
u_int8_t data[8];
u_int8_t dataexpect[sizeof(data)];
int32_t i;
for (i = 0; i < sizeof(data); i++)
data[i] = dataexpect[i] = i ^ 0x5a;
for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) {
a = i * HIFN_SRAM_STEP_SIZE;
bcopy(&i, data, sizeof(i));
hifn_writeramaddr(sc, a, data);
}
for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
a = i * HIFN_SRAM_STEP_SIZE;
bcopy(&i, dataexpect, sizeof(i));
if (hifn_readramaddr(sc, a, data) < 0)
return (0);
if (bcmp(data, dataexpect, sizeof(data)) != 0)
return (0);
sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE;
}
return (0);
}
/*
* XXX For dram boards, one should really try all of the
* HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG
* is already set up correctly.
*/
static int
hifn_dramsize(struct hifn_softc *sc)
{
u_int32_t cnfg;
if (sc->sc_flags & HIFN_IS_7956) {
/*
* 7955/7956 have a fixed internal ram of only 32K.
*/
sc->sc_ramsize = 32768;
} else {
cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
HIFN_PUCNFG_DRAMMASK;
sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
}
return (0);
}
static void
hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, int *resp)
{
struct hifn_dma *dma = sc->sc_dma;
if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
sc->sc_cmdi = 0;
dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
}
*cmdp = sc->sc_cmdi++;
sc->sc_cmdk = sc->sc_cmdi;
if (sc->sc_srci == HIFN_D_SRC_RSIZE) {
sc->sc_srci = 0;
dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
}
*srcp = sc->sc_srci++;
sc->sc_srck = sc->sc_srci;
if (sc->sc_dsti == HIFN_D_DST_RSIZE) {
sc->sc_dsti = 0;
dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
}
*dstp = sc->sc_dsti++;
sc->sc_dstk = sc->sc_dsti;
if (sc->sc_resi == HIFN_D_RES_RSIZE) {
sc->sc_resi = 0;
dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
}
*resp = sc->sc_resi++;
sc->sc_resk = sc->sc_resi;
}
static int
hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
{
struct hifn_dma *dma = sc->sc_dma;
hifn_base_command_t wc;
const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
int r, cmdi, resi, srci, dsti;
wc.masks = htole16(3 << 13);
wc.session_num = htole16(addr >> 14);
wc.total_source_count = htole16(8);
wc.total_dest_count = htole16(addr & 0x3fff);
hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
WRITE_REG_1(sc, HIFN_1_DMA_CSR,
HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
/* build write command */
bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
*(hifn_base_command_t *)dma->command_bufs[cmdi] = wc;
bcopy(data, &dma->test_src, sizeof(dma->test_src));
dma->srcr[srci].p = htole32(sc->sc_dma_physaddr
+ offsetof(struct hifn_dma, test_src));
dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr
+ offsetof(struct hifn_dma, test_dst));
dma->cmdr[cmdi].l = htole32(16 | masks);
dma->srcr[srci].l = htole32(8 | masks);
dma->dstr[dsti].l = htole32(4 | masks);
dma->resr[resi].l = htole32(4 | masks);
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
for (r = 10000; r >= 0; r--) {
DELAY(10);
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
break;
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}
if (r == 0) {
device_printf(sc->sc_dev, "writeramaddr -- "
"result[%d](addr %d) still valid\n", resi, addr);
r = -1;
return (-1);
} else
r = 0;
WRITE_REG_1(sc, HIFN_1_DMA_CSR,
HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
return (r);
}
static int
hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
{
struct hifn_dma *dma = sc->sc_dma;
hifn_base_command_t rc;
const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
int r, cmdi, srci, dsti, resi;
rc.masks = htole16(2 << 13);
rc.session_num = htole16(addr >> 14);
rc.total_source_count = htole16(addr & 0x3fff);
rc.total_dest_count = htole16(8);
hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
WRITE_REG_1(sc, HIFN_1_DMA_CSR,
HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
*(hifn_base_command_t *)dma->command_bufs[cmdi] = rc;
dma->srcr[srci].p = htole32(sc->sc_dma_physaddr +
offsetof(struct hifn_dma, test_src));
dma->test_src = 0;
dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr +
offsetof(struct hifn_dma, test_dst));
dma->test_dst = 0;
dma->cmdr[cmdi].l = htole32(8 | masks);
dma->srcr[srci].l = htole32(8 | masks);
dma->dstr[dsti].l = htole32(8 | masks);
dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
for (r = 10000; r >= 0; r--) {
DELAY(10);
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
break;
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}
if (r == 0) {
device_printf(sc->sc_dev, "readramaddr -- "
"result[%d](addr %d) still valid\n", resi, addr);
r = -1;
} else {
r = 0;
bcopy(&dma->test_dst, data, sizeof(dma->test_dst));
}
WRITE_REG_1(sc, HIFN_1_DMA_CSR,
HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
return (r);
}
/*
* Initialize the descriptor rings.
*/
static void
hifn_init_dma(struct hifn_softc *sc)
{
struct hifn_dma *dma = sc->sc_dma;
int i;
hifn_set_retry(sc);
/* initialize static pointer values */
for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
dma->cmdr[i].p = htole32(sc->sc_dma_physaddr +
offsetof(struct hifn_dma, command_bufs[i][0]));
for (i = 0; i < HIFN_D_RES_RSIZE; i++)
dma->resr[i].p = htole32(sc->sc_dma_physaddr +
offsetof(struct hifn_dma, result_bufs[i][0]));
dma->cmdr[HIFN_D_CMD_RSIZE].p =
htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, cmdr[0]));
dma->srcr[HIFN_D_SRC_RSIZE].p =
htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, srcr[0]));
dma->dstr[HIFN_D_DST_RSIZE].p =
htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, dstr[0]));
dma->resr[HIFN_D_RES_RSIZE].p =
htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, resr[0]));
sc->sc_cmdu = sc->sc_srcu = sc->sc_dstu = sc->sc_resu = 0;
sc->sc_cmdi = sc->sc_srci = sc->sc_dsti = sc->sc_resi = 0;
sc->sc_cmdk = sc->sc_srck = sc->sc_dstk = sc->sc_resk = 0;
}
/*
* Writes out the raw command buffer space. Returns the
* command buffer size.
*/
static u_int
hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
{
u_int8_t *buf_pos;
hifn_base_command_t *base_cmd;
hifn_mac_command_t *mac_cmd;
hifn_crypt_command_t *cry_cmd;
int using_mac, using_crypt, len, ivlen;
u_int32_t dlen, slen;
buf_pos = buf;
using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
base_cmd = (hifn_base_command_t *)buf_pos;
base_cmd->masks = htole16(cmd->base_masks);
slen = cmd->src_mapsize;
if (cmd->sloplen)
dlen = cmd->dst_mapsize - cmd->sloplen + sizeof(u_int32_t);
else
dlen = cmd->dst_mapsize;
base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
dlen >>= 16;
slen >>= 16;
base_cmd->session_num = htole16(
((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
buf_pos += sizeof(hifn_base_command_t);
if (using_mac) {
mac_cmd = (hifn_mac_command_t *)buf_pos;
dlen = cmd->maccrd->crd_len;
mac_cmd->source_count = htole16(dlen & 0xffff);
dlen >>= 16;
mac_cmd->masks = htole16(cmd->mac_masks |
((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
mac_cmd->reserved = 0;
buf_pos += sizeof(hifn_mac_command_t);
}
if (using_crypt) {
cry_cmd = (hifn_crypt_command_t *)buf_pos;
dlen = cmd->enccrd->crd_len;
cry_cmd->source_count = htole16(dlen & 0xffff);
dlen >>= 16;
cry_cmd->masks = htole16(cmd->cry_masks |
((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
cry_cmd->reserved = 0;
buf_pos += sizeof(hifn_crypt_command_t);
}
if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
bcopy(cmd->mac, buf_pos, HIFN_MAC_KEY_LENGTH);
buf_pos += HIFN_MAC_KEY_LENGTH;
}
if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
case HIFN_CRYPT_CMD_ALG_3DES:
bcopy(cmd->ck, buf_pos, HIFN_3DES_KEY_LENGTH);
buf_pos += HIFN_3DES_KEY_LENGTH;
break;
case HIFN_CRYPT_CMD_ALG_DES:
bcopy(cmd->ck, buf_pos, HIFN_DES_KEY_LENGTH);
buf_pos += HIFN_DES_KEY_LENGTH;
break;
case HIFN_CRYPT_CMD_ALG_RC4:
len = 256;
do {
int clen;
clen = MIN(cmd->cklen, len);
bcopy(cmd->ck, buf_pos, clen);
len -= clen;
buf_pos += clen;
} while (len > 0);
bzero(buf_pos, 4);
buf_pos += 4;
break;
case HIFN_CRYPT_CMD_ALG_AES:
/*
* AES keys are variable 128, 192 and
* 256 bits (16, 24 and 32 bytes).
*/
bcopy(cmd->ck, buf_pos, cmd->cklen);
buf_pos += cmd->cklen;
break;
}
}
if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
case HIFN_CRYPT_CMD_ALG_AES:
ivlen = HIFN_AES_IV_LENGTH;
break;
default:
ivlen = HIFN_IV_LENGTH;
break;
}
bcopy(cmd->iv, buf_pos, ivlen);
buf_pos += ivlen;
}
if ((cmd->base_masks & (HIFN_BASE_CMD_MAC|HIFN_BASE_CMD_CRYPT)) == 0) {
bzero(buf_pos, 8);
buf_pos += 8;
}
return (buf_pos - buf);
}
static int
hifn_dmamap_aligned(struct hifn_operand *op)
{
int i;
for (i = 0; i < op->nsegs; i++) {
if (op->segs[i].ds_addr & 3)
return (0);
if ((i != (op->nsegs - 1)) && (op->segs[i].ds_len & 3))
return (0);
}
return (1);
}
static __inline int
hifn_dmamap_dstwrap(struct hifn_softc *sc, int idx)
{
struct hifn_dma *dma = sc->sc_dma;
if (++idx == HIFN_D_DST_RSIZE) {
dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
HIFN_D_MASKDONEIRQ);
HIFN_DSTR_SYNC(sc, idx,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
idx = 0;
}
return (idx);
}
static int
hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
{
struct hifn_dma *dma = sc->sc_dma;
struct hifn_operand *dst = &cmd->dst;
u_int32_t p, l;
int idx, used = 0, i;
idx = sc->sc_dsti;
for (i = 0; i < dst->nsegs - 1; i++) {
dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
dma->dstr[idx].l = htole32(HIFN_D_VALID |
HIFN_D_MASKDONEIRQ | dst->segs[i].ds_len);
HIFN_DSTR_SYNC(sc, idx,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
used++;
idx = hifn_dmamap_dstwrap(sc, idx);
}
if (cmd->sloplen == 0) {
p = dst->segs[i].ds_addr;
l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
dst->segs[i].ds_len;
} else {
p = sc->sc_dma_physaddr +
offsetof(struct hifn_dma, slop[cmd->slopidx]);
l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
sizeof(u_int32_t);
if ((dst->segs[i].ds_len - cmd->sloplen) != 0) {
dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
dma->dstr[idx].l = htole32(HIFN_D_VALID |
HIFN_D_MASKDONEIRQ |
(dst->segs[i].ds_len - cmd->sloplen));
HIFN_DSTR_SYNC(sc, idx,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
used++;
idx = hifn_dmamap_dstwrap(sc, idx);
}
}
dma->dstr[idx].p = htole32(p);
dma->dstr[idx].l = htole32(l);
HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
used++;
idx = hifn_dmamap_dstwrap(sc, idx);
sc->sc_dsti = idx;
sc->sc_dstu += used;
return (idx);
}
static __inline int
hifn_dmamap_srcwrap(struct hifn_softc *sc, int idx)
{
struct hifn_dma *dma = sc->sc_dma;
if (++idx == HIFN_D_SRC_RSIZE) {
dma->srcr[idx].l = htole32(HIFN_D_VALID |
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
idx = 0;
}
return (idx);
}
static int
hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
{
struct hifn_dma *dma = sc->sc_dma;
struct hifn_operand *src = &cmd->src;
int idx, i;
u_int32_t last = 0;
idx = sc->sc_srci;
for (i = 0; i < src->nsegs; i++) {
if (i == src->nsegs - 1)
last = HIFN_D_LAST;
dma->srcr[idx].p = htole32(src->segs[i].ds_addr);
dma->srcr[idx].l = htole32(src->segs[i].ds_len |
HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
HIFN_SRCR_SYNC(sc, idx,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
idx = hifn_dmamap_srcwrap(sc, idx);
}
sc->sc_srci = idx;
sc->sc_srcu += src->nsegs;
return (idx);
}
static void
hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
{
struct hifn_operand *op = arg;
KASSERT(nsegs <= MAX_SCATTER,
("hifn_op_cb: too many DMA segments (%u > %u) "
"returned when mapping operand", nsegs, MAX_SCATTER));
op->mapsize = mapsize;
op->nsegs = nsegs;
bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
}
static int
hifn_crypto(
struct hifn_softc *sc,
struct hifn_command *cmd,
struct cryptop *crp,
int hint)
{
struct hifn_dma *dma = sc->sc_dma;
u_int32_t cmdlen, csr;
int cmdi, resi, err = 0;
/*
* need 1 cmd, and 1 res
*
* NB: check this first since it's easy.
*/
HIFN_LOCK(sc);
if ((sc->sc_cmdu + 1) > HIFN_D_CMD_RSIZE ||
(sc->sc_resu + 1) > HIFN_D_RES_RSIZE) {
#ifdef HIFN_DEBUG
if (hifn_debug) {
device_printf(sc->sc_dev,
"cmd/result exhaustion, cmdu %u resu %u\n",
sc->sc_cmdu, sc->sc_resu);
}
#endif
hifnstats.hst_nomem_cr++;
HIFN_UNLOCK(sc);
return (ERESTART);
}
if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->src_map)) {
hifnstats.hst_nomem_map++;
HIFN_UNLOCK(sc);
return (ENOMEM);
}
if (crp->crp_flags & CRYPTO_F_IMBUF) {
if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
hifnstats.hst_nomem_load++;
err = ENOMEM;
goto err_srcmap1;
}
} else if (crp->crp_flags & CRYPTO_F_IOV) {
if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
hifnstats.hst_nomem_load++;
err = ENOMEM;
goto err_srcmap1;
}
} else {
err = EINVAL;
goto err_srcmap1;
}
if (hifn_dmamap_aligned(&cmd->src)) {
cmd->sloplen = cmd->src_mapsize & 3;
cmd->dst = cmd->src;
} else {
if (crp->crp_flags & CRYPTO_F_IOV) {
err = EINVAL;
goto err_srcmap;
} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
int totlen, len;
struct mbuf *m, *m0, *mlast;
KASSERT(cmd->dst_m == cmd->src_m,
("hifn_crypto: dst_m initialized improperly"));
hifnstats.hst_unaligned++;
/*
* Source is not aligned on a longword boundary.
* Copy the data to insure alignment. If we fail
* to allocate mbufs or clusters while doing this
* we return ERESTART so the operation is requeued
* at the crypto later, but only if there are
* ops already posted to the hardware; otherwise we
* have no guarantee that we'll be re-entered.
*/
totlen = cmd->src_mapsize;
if (cmd->src_m->m_flags & M_PKTHDR) {
len = MHLEN;
MGETHDR(m0, M_NOWAIT, MT_DATA);
if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_NOWAIT)) {
m_free(m0);
m0 = NULL;
}
} else {
len = MLEN;
MGET(m0, M_NOWAIT, MT_DATA);
}
if (m0 == NULL) {
hifnstats.hst_nomem_mbuf++;
err = sc->sc_cmdu ? ERESTART : ENOMEM;
goto err_srcmap;
}
if (totlen >= MINCLSIZE) {
if (!(MCLGET(m0, M_NOWAIT))) {
hifnstats.hst_nomem_mcl++;
err = sc->sc_cmdu ? ERESTART : ENOMEM;
m_freem(m0);
goto err_srcmap;
}
len = MCLBYTES;
}
totlen -= len;
m0->m_pkthdr.len = m0->m_len = len;
mlast = m0;
while (totlen > 0) {
MGET(m, M_NOWAIT, MT_DATA);
if (m == NULL) {
hifnstats.hst_nomem_mbuf++;
err = sc->sc_cmdu ? ERESTART : ENOMEM;
m_freem(m0);
goto err_srcmap;
}
len = MLEN;
if (totlen >= MINCLSIZE) {
if (!(MCLGET(m, M_NOWAIT))) {
hifnstats.hst_nomem_mcl++;
err = sc->sc_cmdu ? ERESTART : ENOMEM;
mlast->m_next = m;
m_freem(m0);
goto err_srcmap;
}
len = MCLBYTES;
}
m->m_len = len;
m0->m_pkthdr.len += len;
totlen -= len;
mlast->m_next = m;
mlast = m;
}
cmd->dst_m = m0;
}
}
if (cmd->dst_map == NULL) {
if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
hifnstats.hst_nomem_map++;
err = ENOMEM;
goto err_srcmap;
}
if (crp->crp_flags & CRYPTO_F_IMBUF) {
if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
hifnstats.hst_nomem_map++;
err = ENOMEM;
goto err_dstmap1;
}
} else if (crp->crp_flags & CRYPTO_F_IOV) {
if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
hifnstats.hst_nomem_load++;
err = ENOMEM;
goto err_dstmap1;
}
}
}
#ifdef HIFN_DEBUG
if (hifn_debug) {
device_printf(sc->sc_dev,
"Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
READ_REG_1(sc, HIFN_1_DMA_CSR),
READ_REG_1(sc, HIFN_1_DMA_IER),
sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu,
cmd->src_nsegs, cmd->dst_nsegs);
}
#endif
if (cmd->src_map == cmd->dst_map) {
bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
} else {
bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
BUS_DMASYNC_PREWRITE);
bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
BUS_DMASYNC_PREREAD);
}
/*
* need N src, and N dst
*/
if ((sc->sc_srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE ||
(sc->sc_dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) {
#ifdef HIFN_DEBUG
if (hifn_debug) {
device_printf(sc->sc_dev,
"src/dst exhaustion, srcu %u+%u dstu %u+%u\n",
sc->sc_srcu, cmd->src_nsegs,
sc->sc_dstu, cmd->dst_nsegs);
}
#endif
hifnstats.hst_nomem_sd++;
err = ERESTART;
goto err_dstmap;
}
if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
sc->sc_cmdi = 0;
dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
}
cmdi = sc->sc_cmdi++;
cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
/* .p for command/result already set */
dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
HIFN_D_MASKDONEIRQ);
HIFN_CMDR_SYNC(sc, cmdi,
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
sc->sc_cmdu++;
/*
* We don't worry about missing an interrupt (which a "command wait"
* interrupt salvages us from), unless there is more than one command
* in the queue.
*/
if (sc->sc_cmdu > 1) {
sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
}
hifnstats.hst_ipackets++;
hifnstats.hst_ibytes += cmd->src_mapsize;
hifn_dmamap_load_src(sc, cmd);
/*
* Unlike other descriptors, we don't mask done interrupt from
* result descriptor.
*/
#ifdef HIFN_DEBUG
if (hifn_debug)
printf("load res\n");
#endif
if (sc->sc_resi == HIFN_D_RES_RSIZE) {
sc->sc_resi = 0;
dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}
resi = sc->sc_resi++;
KASSERT(sc->sc_hifn_commands[resi] == NULL,
("hifn_crypto: command slot %u busy", resi));
sc->sc_hifn_commands[resi] = cmd;
HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) {
dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
sc->sc_curbatch++;
if (sc->sc_curbatch > hifnstats.hst_maxbatch)
hifnstats.hst_maxbatch = sc->sc_curbatch;
hifnstats.hst_totbatch++;
} else {
dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
HIFN_D_VALID | HIFN_D_LAST);
sc->sc_curbatch = 0;
}
HIFN_RESR_SYNC(sc, resi,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
sc->sc_resu++;
if (cmd->sloplen)
cmd->slopidx = resi;
hifn_dmamap_load_dst(sc, cmd);
csr = 0;
if (sc->sc_c_busy == 0) {
csr |= HIFN_DMACSR_C_CTRL_ENA;
sc->sc_c_busy = 1;
}
if (sc->sc_s_busy == 0) {
csr |= HIFN_DMACSR_S_CTRL_ENA;
sc->sc_s_busy = 1;
}
if (sc->sc_r_busy == 0) {
csr |= HIFN_DMACSR_R_CTRL_ENA;
sc->sc_r_busy = 1;
}
if (sc->sc_d_busy == 0) {
csr |= HIFN_DMACSR_D_CTRL_ENA;
sc->sc_d_busy = 1;
}
if (csr)
WRITE_REG_1(sc, HIFN_1_DMA_CSR, csr);
#ifdef HIFN_DEBUG
if (hifn_debug) {
device_printf(sc->sc_dev, "command: stat %8x ier %8x\n",
READ_REG_1(sc, HIFN_1_DMA_CSR),
READ_REG_1(sc, HIFN_1_DMA_IER));
}
#endif
sc->sc_active = 5;
HIFN_UNLOCK(sc);
KASSERT(err == 0, ("hifn_crypto: success with error %u", err));
return (err); /* success */
err_dstmap:
if (cmd->src_map != cmd->dst_map)
bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
err_dstmap1:
if (cmd->src_map != cmd->dst_map)
bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
err_srcmap:
if (crp->crp_flags & CRYPTO_F_IMBUF) {
if (cmd->src_m != cmd->dst_m)
m_freem(cmd->dst_m);
}
bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
err_srcmap1:
bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
HIFN_UNLOCK(sc);
return (err);
}
static void
hifn_tick(void* vsc)
{
struct hifn_softc *sc = vsc;
HIFN_LOCK(sc);
if (sc->sc_active == 0) {
u_int32_t r = 0;
if (sc->sc_cmdu == 0 && sc->sc_c_busy) {
sc->sc_c_busy = 0;
r |= HIFN_DMACSR_C_CTRL_DIS;
}
if (sc->sc_srcu == 0 && sc->sc_s_busy) {
sc->sc_s_busy = 0;
r |= HIFN_DMACSR_S_CTRL_DIS;
}
if (sc->sc_dstu == 0 && sc->sc_d_busy) {
sc->sc_d_busy = 0;
r |= HIFN_DMACSR_D_CTRL_DIS;
}
if (sc->sc_resu == 0 && sc->sc_r_busy) {
sc->sc_r_busy = 0;
r |= HIFN_DMACSR_R_CTRL_DIS;
}
if (r)
WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
} else
sc->sc_active--;
HIFN_UNLOCK(sc);
callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
}
static void
hifn_intr(void *arg)
{
struct hifn_softc *sc = arg;
struct hifn_dma *dma;
u_int32_t dmacsr, restart;
int i, u;
dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
/* Nothing in the DMA unit interrupted */
if ((dmacsr & sc->sc_dmaier) == 0)
return;
HIFN_LOCK(sc);
dma = sc->sc_dma;
#ifdef HIFN_DEBUG
if (hifn_debug) {
device_printf(sc->sc_dev,
"irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n",
dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier,
sc->sc_cmdi, sc->sc_srci, sc->sc_dsti, sc->sc_resi,
sc->sc_cmdk, sc->sc_srck, sc->sc_dstk, sc->sc_resk,
sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
}
#endif
WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
(dmacsr & HIFN_DMACSR_PUBDONE))
WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER);
if (restart)
device_printf(sc->sc_dev, "overrun %x\n", dmacsr);
if (sc->sc_flags & HIFN_IS_7811) {
if (dmacsr & HIFN_DMACSR_ILLR)
device_printf(sc->sc_dev, "illegal read\n");
if (dmacsr & HIFN_DMACSR_ILLW)
device_printf(sc->sc_dev, "illegal write\n");
}
restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
if (restart) {
device_printf(sc->sc_dev, "abort, resetting.\n");
hifnstats.hst_abort++;
hifn_abort(sc);
HIFN_UNLOCK(sc);
return;
}
if ((dmacsr & HIFN_DMACSR_C_WAIT) && (sc->sc_cmdu == 0)) {
/*
* If no slots to process and we receive a "waiting on
* command" interrupt, we disable the "waiting on command"
* (by clearing it).
*/
sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
}
/* clear the rings */
i = sc->sc_resk; u = sc->sc_resu;
while (u != 0) {
HIFN_RESR_SYNC(sc, i,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
HIFN_RESR_SYNC(sc, i,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
break;
}
if (i != HIFN_D_RES_RSIZE) {
struct hifn_command *cmd;
u_int8_t *macbuf = NULL;
HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
cmd = sc->sc_hifn_commands[i];
KASSERT(cmd != NULL,
("hifn_intr: null command slot %u", i));
sc->sc_hifn_commands[i] = NULL;
if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
macbuf = dma->result_bufs[i];
macbuf += 12;
}
hifn_callback(sc, cmd, macbuf);
hifnstats.hst_opackets++;
u--;
}
if (++i == (HIFN_D_RES_RSIZE + 1))
i = 0;
}
sc->sc_resk = i; sc->sc_resu = u;
i = sc->sc_srck; u = sc->sc_srcu;
while (u != 0) {
if (i == HIFN_D_SRC_RSIZE)
i = 0;
HIFN_SRCR_SYNC(sc, i,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
HIFN_SRCR_SYNC(sc, i,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
break;
}
i++, u--;
}
sc->sc_srck = i; sc->sc_srcu = u;
i = sc->sc_cmdk; u = sc->sc_cmdu;
while (u != 0) {
HIFN_CMDR_SYNC(sc, i,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
HIFN_CMDR_SYNC(sc, i,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
break;
}
if (i != HIFN_D_CMD_RSIZE) {
u--;
HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
}
if (++i == (HIFN_D_CMD_RSIZE + 1))
i = 0;
}
sc->sc_cmdk = i; sc->sc_cmdu = u;
HIFN_UNLOCK(sc);
if (sc->sc_needwakeup) { /* XXX check high watermark */
int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
#ifdef HIFN_DEBUG
if (hifn_debug)
device_printf(sc->sc_dev,
"wakeup crypto (%x) u %d/%d/%d/%d\n",
sc->sc_needwakeup,
sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
#endif
sc->sc_needwakeup &= ~wakeup;
crypto_unblock(sc->sc_cid, wakeup);
}
}
/*
* Allocate a new 'session' and return an encoded session id. 'sidp'
* contains our registration id, and should contain an encoded session
* id on successful allocation.
*/
static int
hifn_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
{
struct hifn_softc *sc = device_get_softc(dev);
struct cryptoini *c;
int mac = 0, cry = 0, sesn;
struct hifn_session *ses = NULL;
KASSERT(sc != NULL, ("hifn_newsession: null softc"));
if (sidp == NULL || cri == NULL || sc == NULL)
return (EINVAL);
HIFN_LOCK(sc);
if (sc->sc_sessions == NULL) {
ses = sc->sc_sessions = (struct hifn_session *)malloc(
sizeof(*ses), M_DEVBUF, M_NOWAIT);
if (ses == NULL) {
HIFN_UNLOCK(sc);
return (ENOMEM);
}
sesn = 0;
sc->sc_nsessions = 1;
} else {
for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
if (!sc->sc_sessions[sesn].hs_used) {
ses = &sc->sc_sessions[sesn];
break;
}
}
if (ses == NULL) {
sesn = sc->sc_nsessions;
ses = (struct hifn_session *)malloc((sesn + 1) *
sizeof(*ses), M_DEVBUF, M_NOWAIT);
if (ses == NULL) {
HIFN_UNLOCK(sc);
return (ENOMEM);
}
bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses));
bzero(sc->sc_sessions, sesn * sizeof(*ses));
free(sc->sc_sessions, M_DEVBUF);
sc->sc_sessions = ses;
ses = &sc->sc_sessions[sesn];
sc->sc_nsessions++;
}
}
HIFN_UNLOCK(sc);
bzero(ses, sizeof(*ses));
ses->hs_used = 1;
for (c = cri; c != NULL; c = c->cri_next) {
switch (c->cri_alg) {
case CRYPTO_MD5:
case CRYPTO_SHA1:
case CRYPTO_MD5_HMAC:
case CRYPTO_SHA1_HMAC:
if (mac)
return (EINVAL);
mac = 1;
ses->hs_mlen = c->cri_mlen;
if (ses->hs_mlen == 0) {
switch (c->cri_alg) {
case CRYPTO_MD5:
case CRYPTO_MD5_HMAC:
ses->hs_mlen = 16;
break;
case CRYPTO_SHA1:
case CRYPTO_SHA1_HMAC:
ses->hs_mlen = 20;
break;
}
}
break;
case CRYPTO_DES_CBC:
case CRYPTO_3DES_CBC:
case CRYPTO_AES_CBC:
/* XXX this may read fewer, does it matter? */
read_random(ses->hs_iv,
c->cri_alg == CRYPTO_AES_CBC ?
HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
/*FALLTHROUGH*/
case CRYPTO_ARC4:
if (cry)
return (EINVAL);
cry = 1;
break;
default:
return (EINVAL);
}
}
if (mac == 0 && cry == 0)
return (EINVAL);
*sidp = HIFN_SID(device_get_unit(sc->sc_dev), sesn);
return (0);
}
/*
* Deallocate a session.
* XXX this routine should run a zero'd mac/encrypt key into context ram.
* XXX to blow away any keys already stored there.
*/
static int
hifn_freesession(device_t dev, u_int64_t tid)
{
struct hifn_softc *sc = device_get_softc(dev);
int session, error;
u_int32_t sid = CRYPTO_SESID2LID(tid);
KASSERT(sc != NULL, ("hifn_freesession: null softc"));
if (sc == NULL)
return (EINVAL);
HIFN_LOCK(sc);
session = HIFN_SESSION(sid);
if (session < sc->sc_nsessions) {
bzero(&sc->sc_sessions[session], sizeof(struct hifn_session));
error = 0;
} else
error = EINVAL;
HIFN_UNLOCK(sc);
return (error);
}
static int
hifn_process(device_t dev, struct cryptop *crp, int hint)
{
struct hifn_softc *sc = device_get_softc(dev);
struct hifn_command *cmd = NULL;
int session, err, ivlen;
struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
if (crp == NULL || crp->crp_callback == NULL) {
hifnstats.hst_invalid++;
return (EINVAL);
}
session = HIFN_SESSION(crp->crp_sid);
if (sc == NULL || session >= sc->sc_nsessions) {
err = EINVAL;
goto errout;
}
cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
if (cmd == NULL) {
hifnstats.hst_nomem++;
err = ENOMEM;
goto errout;
}
if (crp->crp_flags & CRYPTO_F_IMBUF) {
cmd->src_m = (struct mbuf *)crp->crp_buf;
cmd->dst_m = (struct mbuf *)crp->crp_buf;
} else if (crp->crp_flags & CRYPTO_F_IOV) {
cmd->src_io = (struct uio *)crp->crp_buf;
cmd->dst_io = (struct uio *)crp->crp_buf;
} else {
err = EINVAL;
goto errout; /* XXX we don't handle contiguous buffers! */
}
crd1 = crp->crp_desc;
if (crd1 == NULL) {
err = EINVAL;
goto errout;
}
crd2 = crd1->crd_next;
if (crd2 == NULL) {
if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC ||
crd1->crd_alg == CRYPTO_SHA1 ||
crd1->crd_alg == CRYPTO_MD5) {
maccrd = crd1;
enccrd = NULL;
} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
crd1->crd_alg == CRYPTO_3DES_CBC ||
crd1->crd_alg == CRYPTO_AES_CBC ||
crd1->crd_alg == CRYPTO_ARC4) {
if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
cmd->base_masks |= HIFN_BASE_CMD_DECODE;
maccrd = NULL;
enccrd = crd1;
} else {
err = EINVAL;
goto errout;
}
} else {
if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC ||
crd1->crd_alg == CRYPTO_MD5 ||
crd1->crd_alg == CRYPTO_SHA1) &&
(crd2->crd_alg == CRYPTO_DES_CBC ||
crd2->crd_alg == CRYPTO_3DES_CBC ||
crd2->crd_alg == CRYPTO_AES_CBC ||
crd2->crd_alg == CRYPTO_ARC4) &&
((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
cmd->base_masks = HIFN_BASE_CMD_DECODE;
maccrd = crd1;
enccrd = crd2;
} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
crd1->crd_alg == CRYPTO_ARC4 ||
crd1->crd_alg == CRYPTO_3DES_CBC ||
crd1->crd_alg == CRYPTO_AES_CBC) &&
(crd2->crd_alg == CRYPTO_MD5_HMAC ||
crd2->crd_alg == CRYPTO_SHA1_HMAC ||
crd2->crd_alg == CRYPTO_MD5 ||
crd2->crd_alg == CRYPTO_SHA1) &&
(crd1->crd_flags & CRD_F_ENCRYPT)) {
enccrd = crd1;
maccrd = crd2;
} else {
/*
* We cannot order the 7751 as requested
*/
err = EINVAL;
goto errout;
}
}
if (enccrd) {
cmd->enccrd = enccrd;
cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
switch (enccrd->crd_alg) {
case CRYPTO_ARC4:
cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
break;
case CRYPTO_DES_CBC:
cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
HIFN_CRYPT_CMD_MODE_CBC |
HIFN_CRYPT_CMD_NEW_IV;
break;
case CRYPTO_3DES_CBC:
cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
HIFN_CRYPT_CMD_MODE_CBC |
HIFN_CRYPT_CMD_NEW_IV;
break;
case CRYPTO_AES_CBC:
cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES |
HIFN_CRYPT_CMD_MODE_CBC |
HIFN_CRYPT_CMD_NEW_IV;
break;
default:
err = EINVAL;
goto errout;
}
if (enccrd->crd_alg != CRYPTO_ARC4) {
ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
if (enccrd->crd_flags & CRD_F_ENCRYPT) {
if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(enccrd->crd_iv, cmd->iv, ivlen);
else
bcopy(sc->sc_sessions[session].hs_iv,
cmd->iv, ivlen);
if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
== 0) {
crypto_copyback(crp->crp_flags,
crp->crp_buf, enccrd->crd_inject,
ivlen, cmd->iv);
}
} else {
if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(enccrd->crd_iv, cmd->iv, ivlen);
else {
crypto_copydata(crp->crp_flags,
crp->crp_buf, enccrd->crd_inject,
ivlen, cmd->iv);
}
}
}
if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
cmd->ck = enccrd->crd_key;
cmd->cklen = enccrd->crd_klen >> 3;
cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
/*
* Need to specify the size for the AES key in the masks.
*/
if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) ==
HIFN_CRYPT_CMD_ALG_AES) {
switch (cmd->cklen) {
case 16:
cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128;
break;
case 24:
cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192;
break;
case 32:
cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256;
break;
default:
err = EINVAL;
goto errout;
}
}
}
if (maccrd) {
cmd->maccrd = maccrd;
cmd->base_masks |= HIFN_BASE_CMD_MAC;
switch (maccrd->crd_alg) {
case CRYPTO_MD5:
cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
HIFN_MAC_CMD_POS_IPSEC;
break;
case CRYPTO_MD5_HMAC:
cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
break;
case CRYPTO_SHA1:
cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
HIFN_MAC_CMD_POS_IPSEC;
break;
case CRYPTO_SHA1_HMAC:
cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
break;
}
if (maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
maccrd->crd_alg == CRYPTO_MD5_HMAC) {
cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
bzero(cmd->mac + (maccrd->crd_klen >> 3),
HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
}
}
cmd->crp = crp;
cmd->session_num = session;
cmd->softc = sc;
err = hifn_crypto(sc, cmd, crp, hint);
if (!err) {
return 0;
} else if (err == ERESTART) {
/*
* There weren't enough resources to dispatch the request
* to the part. Notify the caller so they'll requeue this
* request and resubmit it again soon.
*/
#ifdef HIFN_DEBUG
if (hifn_debug)
device_printf(sc->sc_dev, "requeue request\n");
#endif
free(cmd, M_DEVBUF);
sc->sc_needwakeup |= CRYPTO_SYMQ;
return (err);
}
errout:
if (cmd != NULL)
free(cmd, M_DEVBUF);
if (err == EINVAL)
hifnstats.hst_invalid++;
else
hifnstats.hst_nomem++;
crp->crp_etype = err;
crypto_done(crp);
return (err);
}
static void
hifn_abort(struct hifn_softc *sc)
{
struct hifn_dma *dma = sc->sc_dma;
struct hifn_command *cmd;
struct cryptop *crp;
int i, u;
i = sc->sc_resk; u = sc->sc_resu;
while (u != 0) {
cmd = sc->sc_hifn_commands[i];
KASSERT(cmd != NULL, ("hifn_abort: null command slot %u", i));
sc->sc_hifn_commands[i] = NULL;
crp = cmd->crp;
if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
/* Salvage what we can. */
u_int8_t *macbuf;
if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
macbuf = dma->result_bufs[i];
macbuf += 12;
} else
macbuf = NULL;
hifnstats.hst_opackets++;
hifn_callback(sc, cmd, macbuf);
} else {
if (cmd->src_map == cmd->dst_map) {
bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
} else {
bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
BUS_DMASYNC_POSTREAD);
}
if (cmd->src_m != cmd->dst_m) {
m_freem(cmd->src_m);
crp->crp_buf = (caddr_t)cmd->dst_m;
}
/* non-shared buffers cannot be restarted */
if (cmd->src_map != cmd->dst_map) {
/*
* XXX should be EAGAIN, delayed until
* after the reset.
*/
crp->crp_etype = ENOMEM;
bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
} else
crp->crp_etype = ENOMEM;
bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
free(cmd, M_DEVBUF);
if (crp->crp_etype != EAGAIN)
crypto_done(crp);
}
if (++i == HIFN_D_RES_RSIZE)
i = 0;
u--;
}
sc->sc_resk = i; sc->sc_resu = u;
hifn_reset_board(sc, 1);
hifn_init_dma(sc);
hifn_init_pci_registers(sc);
}
static void
hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
{
struct hifn_dma *dma = sc->sc_dma;
struct cryptop *crp = cmd->crp;
struct cryptodesc *crd;
struct mbuf *m;
int totlen, i, u, ivlen;
if (cmd->src_map == cmd->dst_map) {
bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
} else {
bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
BUS_DMASYNC_POSTREAD);
}
if (crp->crp_flags & CRYPTO_F_IMBUF) {
if (cmd->src_m != cmd->dst_m) {
crp->crp_buf = (caddr_t)cmd->dst_m;
totlen = cmd->src_mapsize;
for (m = cmd->dst_m; m != NULL; m = m->m_next) {
if (totlen < m->m_len) {
m->m_len = totlen;
totlen = 0;
} else
totlen -= m->m_len;
}
cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
m_freem(cmd->src_m);
}
}
if (cmd->sloplen != 0) {
crypto_copyback(crp->crp_flags, crp->crp_buf,
cmd->src_mapsize - cmd->sloplen, cmd->sloplen,
(caddr_t)&dma->slop[cmd->slopidx]);
}
i = sc->sc_dstk; u = sc->sc_dstu;
while (u != 0) {
if (i == HIFN_D_DST_RSIZE)
i = 0;
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
break;
}
i++, u--;
}
sc->sc_dstk = i; sc->sc_dstu = u;
hifnstats.hst_obytes += cmd->dst_mapsize;
if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
HIFN_BASE_CMD_CRYPT) {
for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
if (crd->crd_alg != CRYPTO_DES_CBC &&
crd->crd_alg != CRYPTO_3DES_CBC &&
crd->crd_alg != CRYPTO_AES_CBC)
continue;
ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
crypto_copydata(crp->crp_flags, crp->crp_buf,
crd->crd_skip + crd->crd_len - ivlen, ivlen,
cmd->softc->sc_sessions[cmd->session_num].hs_iv);
break;
}
}
if (macbuf != NULL) {
for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
int len;
if (crd->crd_alg != CRYPTO_MD5 &&
crd->crd_alg != CRYPTO_SHA1 &&
crd->crd_alg != CRYPTO_MD5_HMAC &&
crd->crd_alg != CRYPTO_SHA1_HMAC) {
continue;
}
len = cmd->softc->sc_sessions[cmd->session_num].hs_mlen;
crypto_copyback(crp->crp_flags, crp->crp_buf,
crd->crd_inject, len, macbuf);
break;
}
}
if (cmd->src_map != cmd->dst_map) {
bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
}
bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
free(cmd, M_DEVBUF);
crypto_done(crp);
}
/*
* 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
* and Group 1 registers; avoid conditions that could create
* burst writes by doing a read in between the writes.
*
* NB: The read we interpose is always to the same register;
* we do this because reading from an arbitrary (e.g. last)
* register may not always work.
*/
static void
hifn_write_reg_0(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
{
if (sc->sc_flags & HIFN_IS_7811) {
if (sc->sc_bar0_lastreg == reg - 4)
bus_space_read_4(sc->sc_st0, sc->sc_sh0, HIFN_0_PUCNFG);
sc->sc_bar0_lastreg = reg;
}
bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
}
static void
hifn_write_reg_1(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
{
if (sc->sc_flags & HIFN_IS_7811) {
if (sc->sc_bar1_lastreg == reg - 4)
bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
sc->sc_bar1_lastreg = reg;
}
bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
}
#ifdef HIFN_VULCANDEV
/*
* this code provides support for mapping the PK engine's register
* into a userspace program.
*
*/
static int
vulcanpk_mmap(struct cdev *dev, vm_ooffset_t offset,
vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr)
{
struct hifn_softc *sc;
vm_paddr_t pd;
void *b;
sc = dev->si_drv1;
pd = rman_get_start(sc->sc_bar1res);
b = rman_get_virtual(sc->sc_bar1res);
#if 0
printf("vpk mmap: %p(%016llx) offset=%lld\n", b,
(unsigned long long)pd, offset);
hexdump(b, HIFN_1_PUB_MEMEND, "vpk", 0);
#endif
if (offset == 0) {
*paddr = pd;
return (0);
}
return (-1);
}
static struct cdevsw vulcanpk_cdevsw = {
.d_version = D_VERSION,
.d_mmap = vulcanpk_mmap,
.d_name = "vulcanpk",
};
#endif /* HIFN_VULCANDEV */
Index: head/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c
===================================================================
--- head/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c (revision 283290)
+++ head/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c (revision 283291)
@@ -1,2063 +1,2063 @@
/*-
* Copyright (c) 2009-2012 Microsoft Corp.
* Copyright (c) 2012 NetApp Inc.
* Copyright (c) 2012 Citrix Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice unmodified, this list of conditions, and the following
* disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/**
* StorVSC driver for Hyper-V. This driver presents a SCSI HBA interface
* to the Comman Access Method (CAM) layer. CAM control blocks (CCBs) are
* converted into VSCSI protocol messages which are delivered to the parent
* partition StorVSP driver over the Hyper-V VMBUS.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/proc.h>
#include <sys/condvar.h>
#include <sys/time.h>
#include <sys/systm.h>
#include <sys/sockio.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/kernel.h>
#include <sys/queue.h>
#include <sys/lock.h>
#include <sys/sx.h>
#include <sys/taskqueue.h>
#include <sys/bus.h>
#include <sys/mutex.h>
#include <sys/callout.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/uma.h>
#include <sys/lock.h>
#include <sys/sema.h>
#include <sys/sglist.h>
#include <machine/bus.h>
#include <sys/bus_dma.h>
#include <cam/cam.h>
#include <cam/cam_ccb.h>
#include <cam/cam_periph.h>
#include <cam/cam_sim.h>
#include <cam/cam_xpt_sim.h>
#include <cam/cam_xpt_internal.h>
#include <cam/cam_debug.h>
#include <cam/scsi/scsi_all.h>
#include <cam/scsi/scsi_message.h>
#include <dev/hyperv/include/hyperv.h>
#include "hv_vstorage.h"
#define STORVSC_RINGBUFFER_SIZE (20*PAGE_SIZE)
#define STORVSC_MAX_LUNS_PER_TARGET (64)
#define STORVSC_MAX_IO_REQUESTS (STORVSC_MAX_LUNS_PER_TARGET * 2)
#define BLKVSC_MAX_IDE_DISKS_PER_TARGET (1)
#define BLKVSC_MAX_IO_REQUESTS STORVSC_MAX_IO_REQUESTS
#define STORVSC_MAX_TARGETS (2)
#define STORVSC_WIN7_MAJOR 4
#define STORVSC_WIN7_MINOR 2
#define STORVSC_WIN8_MAJOR 5
#define STORVSC_WIN8_MINOR 1
#define VSTOR_PKT_SIZE (sizeof(struct vstor_packet) - vmscsi_size_delta)
#define HV_ALIGN(x, a) roundup2(x, a)
struct storvsc_softc;
struct hv_sgl_node {
LIST_ENTRY(hv_sgl_node) link;
struct sglist *sgl_data;
};
struct hv_sgl_page_pool{
LIST_HEAD(, hv_sgl_node) in_use_sgl_list;
LIST_HEAD(, hv_sgl_node) free_sgl_list;
boolean_t is_init;
} g_hv_sgl_page_pool;
#define STORVSC_MAX_SG_PAGE_CNT STORVSC_MAX_IO_REQUESTS * HV_MAX_MULTIPAGE_BUFFER_COUNT
enum storvsc_request_type {
WRITE_TYPE,
READ_TYPE,
UNKNOWN_TYPE
};
struct hv_storvsc_request {
LIST_ENTRY(hv_storvsc_request) link;
struct vstor_packet vstor_packet;
hv_vmbus_multipage_buffer data_buf;
void *sense_data;
uint8_t sense_info_len;
uint8_t retries;
union ccb *ccb;
struct storvsc_softc *softc;
struct callout callout;
struct sema synch_sema; /*Synchronize the request/response if needed */
struct sglist *bounce_sgl;
unsigned int bounce_sgl_count;
uint64_t not_aligned_seg_bits;
};
struct storvsc_softc {
struct hv_device *hs_dev;
LIST_HEAD(, hv_storvsc_request) hs_free_list;
struct mtx hs_lock;
struct storvsc_driver_props *hs_drv_props;
int hs_unit;
uint32_t hs_frozen;
struct cam_sim *hs_sim;
struct cam_path *hs_path;
uint32_t hs_num_out_reqs;
boolean_t hs_destroy;
boolean_t hs_drain_notify;
boolean_t hs_open_multi_channel;
struct sema hs_drain_sema;
struct hv_storvsc_request hs_init_req;
struct hv_storvsc_request hs_reset_req;
};
/**
* HyperV storvsc timeout testing cases:
* a. IO returned after first timeout;
* b. IO returned after second timeout and queue freeze;
* c. IO returned while timer handler is running
* The first can be tested by "sg_senddiag -vv /dev/daX",
* and the second and third can be done by
* "sg_wr_mode -v -p 08 -c 0,1a -m 0,ff /dev/daX".
*/
#define HVS_TIMEOUT_TEST 0
/*
* Bus/adapter reset functionality on the Hyper-V host is
* buggy and it will be disabled until
* it can be further tested.
*/
#define HVS_HOST_RESET 0
struct storvsc_driver_props {
char *drv_name;
char *drv_desc;
uint8_t drv_max_luns_per_target;
uint8_t drv_max_ios_per_target;
uint32_t drv_ringbuffer_size;
};
enum hv_storage_type {
DRIVER_BLKVSC,
DRIVER_STORVSC,
DRIVER_UNKNOWN
};
#define HS_MAX_ADAPTERS 10
#define HV_STORAGE_SUPPORTS_MULTI_CHANNEL 0x1
/* {ba6163d9-04a1-4d29-b605-72e2ffb1dc7f} */
static const hv_guid gStorVscDeviceType={
.data = {0xd9, 0x63, 0x61, 0xba, 0xa1, 0x04, 0x29, 0x4d,
0xb6, 0x05, 0x72, 0xe2, 0xff, 0xb1, 0xdc, 0x7f}
};
/* {32412632-86cb-44a2-9b5c-50d1417354f5} */
static const hv_guid gBlkVscDeviceType={
.data = {0x32, 0x26, 0x41, 0x32, 0xcb, 0x86, 0xa2, 0x44,
0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5}
};
static struct storvsc_driver_props g_drv_props_table[] = {
{"blkvsc", "Hyper-V IDE Storage Interface",
BLKVSC_MAX_IDE_DISKS_PER_TARGET, BLKVSC_MAX_IO_REQUESTS,
STORVSC_RINGBUFFER_SIZE},
{"storvsc", "Hyper-V SCSI Storage Interface",
STORVSC_MAX_LUNS_PER_TARGET, STORVSC_MAX_IO_REQUESTS,
STORVSC_RINGBUFFER_SIZE}
};
/*
* Sense buffer size changed in win8; have a run-time
* variable to track the size we should use.
*/
static int sense_buffer_size;
/*
* The size of the vmscsi_request has changed in win8. The
* additional size is for the newly added elements in the
* structure. These elements are valid only when we are talking
* to a win8 host.
* Track the correct size we need to apply.
*/
static int vmscsi_size_delta;
static int storvsc_current_major;
static int storvsc_current_minor;
/* static functions */
static int storvsc_probe(device_t dev);
static int storvsc_attach(device_t dev);
static int storvsc_detach(device_t dev);
static void storvsc_poll(struct cam_sim * sim);
static void storvsc_action(struct cam_sim * sim, union ccb * ccb);
static int create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp);
static void storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp);
static enum hv_storage_type storvsc_get_storage_type(device_t dev);
static void hv_storvsc_rescan_target(struct storvsc_softc *sc);
static void hv_storvsc_on_channel_callback(void *context);
static void hv_storvsc_on_iocompletion( struct storvsc_softc *sc,
struct vstor_packet *vstor_packet,
struct hv_storvsc_request *request);
static int hv_storvsc_connect_vsp(struct hv_device *device);
static void storvsc_io_done(struct hv_storvsc_request *reqp);
static void storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
bus_dma_segment_t *orig_sgl,
unsigned int orig_sgl_count,
uint64_t seg_bits);
void storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
unsigned int dest_sgl_count,
struct sglist* src_sgl,
uint64_t seg_bits);
static device_method_t storvsc_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, storvsc_probe),
DEVMETHOD(device_attach, storvsc_attach),
DEVMETHOD(device_detach, storvsc_detach),
DEVMETHOD(device_shutdown, bus_generic_shutdown),
DEVMETHOD_END
};
static driver_t storvsc_driver = {
"storvsc", storvsc_methods, sizeof(struct storvsc_softc),
};
static devclass_t storvsc_devclass;
DRIVER_MODULE(storvsc, vmbus, storvsc_driver, storvsc_devclass, 0, 0);
MODULE_VERSION(storvsc, 1);
MODULE_DEPEND(storvsc, vmbus, 1, 1, 1);
/**
* The host is capable of sending messages to us that are
* completely unsolicited. So, we need to address the race
* condition where we may be in the process of unloading the
* driver when the host may send us an unsolicited message.
* We address this issue by implementing a sequentially
* consistent protocol:
*
* 1. Channel callback is invoked while holding the the channel lock
* and an unloading driver will reset the channel callback under
* the protection of this channel lock.
*
* 2. To ensure bounded wait time for unloading a driver, we don't
* permit outgoing traffic once the device is marked as being
* destroyed.
*
* 3. Once the device is marked as being destroyed, we only
* permit incoming traffic to properly account for
* packets already sent out.
*/
static inline struct storvsc_softc *
get_stor_device(struct hv_device *device,
boolean_t outbound)
{
struct storvsc_softc *sc;
sc = device_get_softc(device->device);
if (sc == NULL) {
return NULL;
}
if (outbound) {
/*
* Here we permit outgoing I/O only
* if the device is not being destroyed.
*/
if (sc->hs_destroy) {
sc = NULL;
}
} else {
/*
* inbound case; if being destroyed
* only permit to account for
* messages already sent out.
*/
if (sc->hs_destroy && (sc->hs_num_out_reqs == 0)) {
sc = NULL;
}
}
return sc;
}
/**
* @brief Callback handler, will be invoked when receive mutil-channel offer
*
* @param context new multi-channel
*/
static void
storvsc_handle_sc_creation(void *context)
{
hv_vmbus_channel *new_channel;
struct hv_device *device;
struct storvsc_softc *sc;
struct vmstor_chan_props props;
int ret = 0;
new_channel = (hv_vmbus_channel *)context;
device = new_channel->primary_channel->device;
sc = get_stor_device(device, TRUE);
if (sc == NULL)
return;
if (FALSE == sc->hs_open_multi_channel)
return;
memset(&props, 0, sizeof(props));
ret = hv_vmbus_channel_open(new_channel,
sc->hs_drv_props->drv_ringbuffer_size,
sc->hs_drv_props->drv_ringbuffer_size,
(void *)&props,
sizeof(struct vmstor_chan_props),
hv_storvsc_on_channel_callback,
new_channel);
return;
}
/**
* @brief Send multi-channel creation request to host
*
* @param device a Hyper-V device pointer
* @param max_chans the max channels supported by vmbus
*/
static void
storvsc_send_multichannel_request(struct hv_device *dev, int max_chans)
{
struct storvsc_softc *sc;
struct hv_storvsc_request *request;
struct vstor_packet *vstor_packet;
int request_channels_cnt = 0;
int ret;
/* get multichannels count that need to create */
request_channels_cnt = MIN(max_chans, mp_ncpus);
sc = get_stor_device(dev, TRUE);
if (sc == NULL) {
printf("Storvsc_error: get sc failed while send mutilchannel "
"request\n");
return;
}
request = &sc->hs_init_req;
/* Establish a handler for multi-channel */
dev->channel->sc_creation_callback = storvsc_handle_sc_creation;
/* request the host to create multi-channel */
memset(request, 0, sizeof(struct hv_storvsc_request));
sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
vstor_packet = &request->vstor_packet;
vstor_packet->operation = VSTOR_OPERATION_CREATE_MULTI_CHANNELS;
vstor_packet->flags = REQUEST_COMPLETION_FLAG;
vstor_packet->u.multi_channels_cnt = request_channels_cnt;
ret = hv_vmbus_channel_send_packet(
dev->channel,
vstor_packet,
VSTOR_PKT_SIZE,
(uint64_t)(uintptr_t)request,
HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
/* wait for 5 seconds */
ret = sema_timedwait(&request->synch_sema, 5 * hz);
if (ret != 0) {
printf("Storvsc_error: create multi-channel timeout, %d\n",
ret);
return;
}
if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
vstor_packet->status != 0) {
printf("Storvsc_error: create multi-channel invalid operation "
"(%d) or statue (%u)\n",
vstor_packet->operation, vstor_packet->status);
return;
}
sc->hs_open_multi_channel = TRUE;
if (bootverbose)
printf("Storvsc create multi-channel success!\n");
}
/**
* @brief initialize channel connection to parent partition
*
* @param dev a Hyper-V device pointer
* @returns 0 on success, non-zero error on failure
*/
static int
hv_storvsc_channel_init(struct hv_device *dev)
{
int ret = 0;
struct hv_storvsc_request *request;
struct vstor_packet *vstor_packet;
struct storvsc_softc *sc;
uint16_t max_chans = 0;
boolean_t support_multichannel = FALSE;
max_chans = 0;
support_multichannel = FALSE;
sc = get_stor_device(dev, TRUE);
if (sc == NULL)
return (ENODEV);
request = &sc->hs_init_req;
memset(request, 0, sizeof(struct hv_storvsc_request));
vstor_packet = &request->vstor_packet;
request->softc = sc;
/**
* Initiate the vsc/vsp initialization protocol on the open channel
*/
sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
vstor_packet->operation = VSTOR_OPERATION_BEGININITIALIZATION;
vstor_packet->flags = REQUEST_COMPLETION_FLAG;
ret = hv_vmbus_channel_send_packet(
dev->channel,
vstor_packet,
VSTOR_PKT_SIZE,
(uint64_t)(uintptr_t)request,
HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
if (ret != 0)
goto cleanup;
/* wait 5 seconds */
ret = sema_timedwait(&request->synch_sema, 5 * hz);
if (ret != 0)
goto cleanup;
if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
vstor_packet->status != 0) {
goto cleanup;
}
/* reuse the packet for version range supported */
memset(vstor_packet, 0, sizeof(struct vstor_packet));
vstor_packet->operation = VSTOR_OPERATION_QUERYPROTOCOLVERSION;
vstor_packet->flags = REQUEST_COMPLETION_FLAG;
vstor_packet->u.version.major_minor =
VMSTOR_PROTOCOL_VERSION(storvsc_current_major, storvsc_current_minor);
/* revision is only significant for Windows guests */
vstor_packet->u.version.revision = 0;
ret = hv_vmbus_channel_send_packet(
dev->channel,
vstor_packet,
VSTOR_PKT_SIZE,
(uint64_t)(uintptr_t)request,
HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
if (ret != 0)
goto cleanup;
/* wait 5 seconds */
ret = sema_timedwait(&request->synch_sema, 5 * hz);
if (ret)
goto cleanup;
/* TODO: Check returned version */
if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
vstor_packet->status != 0)
goto cleanup;
/**
* Query channel properties
*/
memset(vstor_packet, 0, sizeof(struct vstor_packet));
vstor_packet->operation = VSTOR_OPERATION_QUERYPROPERTIES;
vstor_packet->flags = REQUEST_COMPLETION_FLAG;
ret = hv_vmbus_channel_send_packet(
dev->channel,
vstor_packet,
VSTOR_PKT_SIZE,
(uint64_t)(uintptr_t)request,
HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
if ( ret != 0)
goto cleanup;
/* wait 5 seconds */
ret = sema_timedwait(&request->synch_sema, 5 * hz);
if (ret != 0)
goto cleanup;
/* TODO: Check returned version */
if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
vstor_packet->status != 0) {
goto cleanup;
}
/* multi-channels feature is supported by WIN8 and above version */
max_chans = vstor_packet->u.chan_props.max_channel_cnt;
if ((hv_vmbus_protocal_version != HV_VMBUS_VERSION_WIN7) &&
(hv_vmbus_protocal_version != HV_VMBUS_VERSION_WS2008) &&
(vstor_packet->u.chan_props.flags &
HV_STORAGE_SUPPORTS_MULTI_CHANNEL)) {
support_multichannel = TRUE;
}
memset(vstor_packet, 0, sizeof(struct vstor_packet));
vstor_packet->operation = VSTOR_OPERATION_ENDINITIALIZATION;
vstor_packet->flags = REQUEST_COMPLETION_FLAG;
ret = hv_vmbus_channel_send_packet(
dev->channel,
vstor_packet,
VSTOR_PKT_SIZE,
(uint64_t)(uintptr_t)request,
HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
if (ret != 0) {
goto cleanup;
}
/* wait 5 seconds */
ret = sema_timedwait(&request->synch_sema, 5 * hz);
if (ret != 0)
goto cleanup;
if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
vstor_packet->status != 0)
goto cleanup;
/*
* If multi-channel is supported, send multichannel create
* request to host.
*/
if (support_multichannel)
storvsc_send_multichannel_request(dev, max_chans);
cleanup:
sema_destroy(&request->synch_sema);
return (ret);
}
/**
* @brief Open channel connection to paraent partition StorVSP driver
*
* Open and initialize channel connection to parent partition StorVSP driver.
*
* @param pointer to a Hyper-V device
* @returns 0 on success, non-zero error on failure
*/
static int
hv_storvsc_connect_vsp(struct hv_device *dev)
{
int ret = 0;
struct vmstor_chan_props props;
struct storvsc_softc *sc;
sc = device_get_softc(dev->device);
memset(&props, 0, sizeof(struct vmstor_chan_props));
/*
* Open the channel
*/
ret = hv_vmbus_channel_open(
dev->channel,
sc->hs_drv_props->drv_ringbuffer_size,
sc->hs_drv_props->drv_ringbuffer_size,
(void *)&props,
sizeof(struct vmstor_chan_props),
hv_storvsc_on_channel_callback,
dev->channel);
if (ret != 0) {
return ret;
}
ret = hv_storvsc_channel_init(dev);
return (ret);
}
#if HVS_HOST_RESET
static int
hv_storvsc_host_reset(struct hv_device *dev)
{
int ret = 0;
struct storvsc_softc *sc;
struct hv_storvsc_request *request;
struct vstor_packet *vstor_packet;
sc = get_stor_device(dev, TRUE);
if (sc == NULL) {
return ENODEV;
}
request = &sc->hs_reset_req;
request->softc = sc;
vstor_packet = &request->vstor_packet;
sema_init(&request->synch_sema, 0, "stor synch sema");
vstor_packet->operation = VSTOR_OPERATION_RESETBUS;
vstor_packet->flags = REQUEST_COMPLETION_FLAG;
ret = hv_vmbus_channel_send_packet(dev->channel,
vstor_packet,
VSTOR_PKT_SIZE,
(uint64_t)(uintptr_t)&sc->hs_reset_req,
HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
if (ret != 0) {
goto cleanup;
}
ret = sema_timedwait(&request->synch_sema, 5 * hz); /* KYS 5 seconds */
if (ret) {
goto cleanup;
}
/*
* At this point, all outstanding requests in the adapter
* should have been flushed out and return to us
*/
cleanup:
sema_destroy(&request->synch_sema);
return (ret);
}
#endif /* HVS_HOST_RESET */
/**
* @brief Function to initiate an I/O request
*
* @param device Hyper-V device pointer
* @param request pointer to a request structure
* @returns 0 on success, non-zero error on failure
*/
static int
hv_storvsc_io_request(struct hv_device *device,
struct hv_storvsc_request *request)
{
struct storvsc_softc *sc;
struct vstor_packet *vstor_packet = &request->vstor_packet;
struct hv_vmbus_channel* outgoing_channel = NULL;
int ret = 0;
sc = get_stor_device(device, TRUE);
if (sc == NULL) {
return ENODEV;
}
vstor_packet->flags |= REQUEST_COMPLETION_FLAG;
vstor_packet->u.vm_srb.length = VSTOR_PKT_SIZE;
vstor_packet->u.vm_srb.sense_info_len = sense_buffer_size;
vstor_packet->u.vm_srb.transfer_len = request->data_buf.length;
vstor_packet->operation = VSTOR_OPERATION_EXECUTESRB;
outgoing_channel = vmbus_select_outgoing_channel(device->channel);
mtx_unlock(&request->softc->hs_lock);
if (request->data_buf.length) {
ret = hv_vmbus_channel_send_packet_multipagebuffer(
outgoing_channel,
&request->data_buf,
vstor_packet,
VSTOR_PKT_SIZE,
(uint64_t)(uintptr_t)request);
} else {
ret = hv_vmbus_channel_send_packet(
outgoing_channel,
vstor_packet,
VSTOR_PKT_SIZE,
(uint64_t)(uintptr_t)request,
HV_VMBUS_PACKET_TYPE_DATA_IN_BAND,
HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
}
mtx_lock(&request->softc->hs_lock);
if (ret != 0) {
printf("Unable to send packet %p ret %d", vstor_packet, ret);
} else {
atomic_add_int(&sc->hs_num_out_reqs, 1);
}
return (ret);
}
/**
* Process IO_COMPLETION_OPERATION and ready
* the result to be completed for upper layer
* processing by the CAM layer.
*/
static void
hv_storvsc_on_iocompletion(struct storvsc_softc *sc,
struct vstor_packet *vstor_packet,
struct hv_storvsc_request *request)
{
struct vmscsi_req *vm_srb;
vm_srb = &vstor_packet->u.vm_srb;
if (((vm_srb->scsi_status & 0xFF) == SCSI_STATUS_CHECK_COND) &&
(vm_srb->srb_status & SRB_STATUS_AUTOSENSE_VALID)) {
/* Autosense data available */
KASSERT(vm_srb->sense_info_len <= request->sense_info_len,
("vm_srb->sense_info_len <= "
"request->sense_info_len"));
memcpy(request->sense_data, vm_srb->u.sense_data,
vm_srb->sense_info_len);
request->sense_info_len = vm_srb->sense_info_len;
}
/* Complete request by passing to the CAM layer */
storvsc_io_done(request);
atomic_subtract_int(&sc->hs_num_out_reqs, 1);
if (sc->hs_drain_notify && (sc->hs_num_out_reqs == 0)) {
sema_post(&sc->hs_drain_sema);
}
}
static void
hv_storvsc_rescan_target(struct storvsc_softc *sc)
{
path_id_t pathid;
target_id_t targetid;
union ccb *ccb;
pathid = cam_sim_path(sc->hs_sim);
targetid = CAM_TARGET_WILDCARD;
/*
* Allocate a CCB and schedule a rescan.
*/
ccb = xpt_alloc_ccb_nowait();
if (ccb == NULL) {
printf("unable to alloc CCB for rescan\n");
return;
}
if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid,
CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
printf("unable to create path for rescan, pathid: %d,"
"targetid: %d\n", pathid, targetid);
xpt_free_ccb(ccb);
return;
}
if (targetid == CAM_TARGET_WILDCARD)
ccb->ccb_h.func_code = XPT_SCAN_BUS;
else
ccb->ccb_h.func_code = XPT_SCAN_TGT;
xpt_rescan(ccb);
}
static void
hv_storvsc_on_channel_callback(void *context)
{
int ret = 0;
hv_vmbus_channel *channel = (hv_vmbus_channel *)context;
struct hv_device *device = NULL;
struct storvsc_softc *sc;
uint32_t bytes_recvd;
uint64_t request_id;
uint8_t packet[roundup2(sizeof(struct vstor_packet), 8)];
struct hv_storvsc_request *request;
struct vstor_packet *vstor_packet;
if (channel->primary_channel != NULL){
device = channel->primary_channel->device;
} else {
device = channel->device;
}
KASSERT(device, ("device is NULL"));
sc = get_stor_device(device, FALSE);
if (sc == NULL) {
printf("Storvsc_error: get stor device failed.\n");
return;
}
ret = hv_vmbus_channel_recv_packet(
channel,
packet,
roundup2(VSTOR_PKT_SIZE, 8),
&bytes_recvd,
&request_id);
while ((ret == 0) && (bytes_recvd > 0)) {
request = (struct hv_storvsc_request *)(uintptr_t)request_id;
if ((request == &sc->hs_init_req) ||
(request == &sc->hs_reset_req)) {
memcpy(&request->vstor_packet, packet,
sizeof(struct vstor_packet));
sema_post(&request->synch_sema);
} else {
vstor_packet = (struct vstor_packet *)packet;
switch(vstor_packet->operation) {
case VSTOR_OPERATION_COMPLETEIO:
if (request == NULL)
panic("VMBUS: storvsc received a "
"packet with NULL request id in "
"COMPLETEIO operation.");
hv_storvsc_on_iocompletion(sc,
vstor_packet, request);
break;
case VSTOR_OPERATION_REMOVEDEVICE:
printf("VMBUS: storvsc operation %d not "
"implemented.\n", vstor_packet->operation);
/* TODO: implement */
break;
case VSTOR_OPERATION_ENUMERATE_BUS:
hv_storvsc_rescan_target(sc);
break;
default:
break;
}
}
ret = hv_vmbus_channel_recv_packet(
channel,
packet,
roundup2(VSTOR_PKT_SIZE, 8),
&bytes_recvd,
&request_id);
}
}
/**
* @brief StorVSC probe function
*
* Device probe function. Returns 0 if the input device is a StorVSC
* device. Otherwise, a ENXIO is returned. If the input device is
* for BlkVSC (paravirtual IDE) device and this support is disabled in
* favor of the emulated ATA/IDE device, return ENXIO.
*
* @param a device
* @returns 0 on success, ENXIO if not a matcing StorVSC device
*/
static int
storvsc_probe(device_t dev)
{
int ata_disk_enable = 0;
int ret = ENXIO;
if (hv_vmbus_protocal_version == HV_VMBUS_VERSION_WS2008 ||
hv_vmbus_protocal_version == HV_VMBUS_VERSION_WIN7) {
sense_buffer_size = PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE;
vmscsi_size_delta = sizeof(struct vmscsi_win8_extension);
storvsc_current_major = STORVSC_WIN7_MAJOR;
storvsc_current_minor = STORVSC_WIN7_MINOR;
} else {
sense_buffer_size = POST_WIN7_STORVSC_SENSE_BUFFER_SIZE;
vmscsi_size_delta = 0;
storvsc_current_major = STORVSC_WIN8_MAJOR;
storvsc_current_minor = STORVSC_WIN8_MINOR;
}
switch (storvsc_get_storage_type(dev)) {
case DRIVER_BLKVSC:
if(bootverbose)
device_printf(dev, "DRIVER_BLKVSC-Emulated ATA/IDE probe\n");
if (!getenv_int("hw.ata.disk_enable", &ata_disk_enable)) {
if(bootverbose)
device_printf(dev,
"Enlightened ATA/IDE detected\n");
ret = BUS_PROBE_DEFAULT;
} else if(bootverbose)
device_printf(dev, "Emulated ATA/IDE set (hw.ata.disk_enable set)\n");
break;
case DRIVER_STORVSC:
if(bootverbose)
device_printf(dev, "Enlightened SCSI device detected\n");
ret = BUS_PROBE_DEFAULT;
break;
default:
ret = ENXIO;
}
return (ret);
}
/**
* @brief StorVSC attach function
*
* Function responsible for allocating per-device structures,
* setting up CAM interfaces and scanning for available LUNs to
* be used for SCSI device peripherals.
*
* @param a device
* @returns 0 on success or an error on failure
*/
static int
storvsc_attach(device_t dev)
{
struct hv_device *hv_dev = vmbus_get_devctx(dev);
enum hv_storage_type stor_type;
struct storvsc_softc *sc;
struct cam_devq *devq;
int ret, i, j;
struct hv_storvsc_request *reqp;
struct root_hold_token *root_mount_token = NULL;
struct hv_sgl_node *sgl_node = NULL;
void *tmp_buff = NULL;
/*
* We need to serialize storvsc attach calls.
*/
root_mount_token = root_mount_hold("storvsc");
sc = device_get_softc(dev);
if (sc == NULL) {
ret = ENOMEM;
goto cleanup;
}
stor_type = storvsc_get_storage_type(dev);
if (stor_type == DRIVER_UNKNOWN) {
ret = ENODEV;
goto cleanup;
}
bzero(sc, sizeof(struct storvsc_softc));
/* fill in driver specific properties */
sc->hs_drv_props = &g_drv_props_table[stor_type];
/* fill in device specific properties */
sc->hs_unit = device_get_unit(dev);
sc->hs_dev = hv_dev;
device_set_desc(dev, g_drv_props_table[stor_type].drv_desc);
LIST_INIT(&sc->hs_free_list);
mtx_init(&sc->hs_lock, "hvslck", NULL, MTX_DEF);
for (i = 0; i < sc->hs_drv_props->drv_max_ios_per_target; ++i) {
reqp = malloc(sizeof(struct hv_storvsc_request),
M_DEVBUF, M_WAITOK|M_ZERO);
reqp->softc = sc;
LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
}
/* create sg-list page pool */
if (FALSE == g_hv_sgl_page_pool.is_init) {
g_hv_sgl_page_pool.is_init = TRUE;
LIST_INIT(&g_hv_sgl_page_pool.in_use_sgl_list);
LIST_INIT(&g_hv_sgl_page_pool.free_sgl_list);
/*
* Pre-create SG list, each SG list with
* HV_MAX_MULTIPAGE_BUFFER_COUNT segments, each
* segment has one page buffer
*/
for (i = 0; i < STORVSC_MAX_IO_REQUESTS; i++) {
sgl_node = malloc(sizeof(struct hv_sgl_node),
M_DEVBUF, M_WAITOK|M_ZERO);
sgl_node->sgl_data =
sglist_alloc(HV_MAX_MULTIPAGE_BUFFER_COUNT,
M_WAITOK|M_ZERO);
for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
tmp_buff = malloc(PAGE_SIZE,
M_DEVBUF, M_WAITOK|M_ZERO);
sgl_node->sgl_data->sg_segs[j].ss_paddr =
(vm_paddr_t)tmp_buff;
}
LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list,
sgl_node, link);
}
}
sc->hs_destroy = FALSE;
sc->hs_drain_notify = FALSE;
sc->hs_open_multi_channel = FALSE;
sema_init(&sc->hs_drain_sema, 0, "Store Drain Sema");
ret = hv_storvsc_connect_vsp(hv_dev);
if (ret != 0) {
goto cleanup;
}
/*
* Create the device queue.
* Hyper-V maps each target to one SCSI HBA
*/
devq = cam_simq_alloc(sc->hs_drv_props->drv_max_ios_per_target);
if (devq == NULL) {
device_printf(dev, "Failed to alloc device queue\n");
ret = ENOMEM;
goto cleanup;
}
sc->hs_sim = cam_sim_alloc(storvsc_action,
storvsc_poll,
sc->hs_drv_props->drv_name,
sc,
sc->hs_unit,
&sc->hs_lock, 1,
sc->hs_drv_props->drv_max_ios_per_target,
devq);
if (sc->hs_sim == NULL) {
device_printf(dev, "Failed to alloc sim\n");
cam_simq_free(devq);
ret = ENOMEM;
goto cleanup;
}
mtx_lock(&sc->hs_lock);
/* bus_id is set to 0, need to get it from VMBUS channel query? */
if (xpt_bus_register(sc->hs_sim, dev, 0) != CAM_SUCCESS) {
cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
mtx_unlock(&sc->hs_lock);
device_printf(dev, "Unable to register SCSI bus\n");
ret = ENXIO;
goto cleanup;
}
if (xpt_create_path(&sc->hs_path, /*periph*/NULL,
cam_sim_path(sc->hs_sim),
CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
xpt_bus_deregister(cam_sim_path(sc->hs_sim));
cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
mtx_unlock(&sc->hs_lock);
device_printf(dev, "Unable to create path\n");
ret = ENXIO;
goto cleanup;
}
mtx_unlock(&sc->hs_lock);
root_mount_rel(root_mount_token);
return (0);
cleanup:
root_mount_rel(root_mount_token);
while (!LIST_EMPTY(&sc->hs_free_list)) {
reqp = LIST_FIRST(&sc->hs_free_list);
LIST_REMOVE(reqp, link);
free(reqp, M_DEVBUF);
}
while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
LIST_REMOVE(sgl_node, link);
for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
if (NULL !=
(void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
}
}
sglist_free(sgl_node->sgl_data);
free(sgl_node, M_DEVBUF);
}
return (ret);
}
/**
* @brief StorVSC device detach function
*
* This function is responsible for safely detaching a
* StorVSC device. This includes waiting for inbound responses
* to complete and freeing associated per-device structures.
*
* @param dev a device
* returns 0 on success
*/
static int
storvsc_detach(device_t dev)
{
struct storvsc_softc *sc = device_get_softc(dev);
struct hv_storvsc_request *reqp = NULL;
struct hv_device *hv_device = vmbus_get_devctx(dev);
struct hv_sgl_node *sgl_node = NULL;
int j = 0;
mtx_lock(&hv_device->channel->inbound_lock);
sc->hs_destroy = TRUE;
mtx_unlock(&hv_device->channel->inbound_lock);
/*
* At this point, all outbound traffic should be disabled. We
* only allow inbound traffic (responses) to proceed so that
* outstanding requests can be completed.
*/
sc->hs_drain_notify = TRUE;
sema_wait(&sc->hs_drain_sema);
sc->hs_drain_notify = FALSE;
/*
* Since we have already drained, we don't need to busy wait.
* The call to close the channel will reset the callback
* under the protection of the incoming channel lock.
*/
hv_vmbus_channel_close(hv_device->channel);
mtx_lock(&sc->hs_lock);
while (!LIST_EMPTY(&sc->hs_free_list)) {
reqp = LIST_FIRST(&sc->hs_free_list);
LIST_REMOVE(reqp, link);
free(reqp, M_DEVBUF);
}
mtx_unlock(&sc->hs_lock);
while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
LIST_REMOVE(sgl_node, link);
for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++){
if (NULL !=
(void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
}
}
sglist_free(sgl_node->sgl_data);
free(sgl_node, M_DEVBUF);
}
return (0);
}
#if HVS_TIMEOUT_TEST
/**
* @brief unit test for timed out operations
*
* This function provides unit testing capability to simulate
* timed out operations. Recompilation with HV_TIMEOUT_TEST=1
* is required.
*
* @param reqp pointer to a request structure
* @param opcode SCSI operation being performed
* @param wait if 1, wait for I/O to complete
*/
static void
storvsc_timeout_test(struct hv_storvsc_request *reqp,
uint8_t opcode, int wait)
{
int ret;
union ccb *ccb = reqp->ccb;
struct storvsc_softc *sc = reqp->softc;
if (reqp->vstor_packet.vm_srb.cdb[0] != opcode) {
return;
}
if (wait) {
mtx_lock(&reqp->event.mtx);
}
ret = hv_storvsc_io_request(sc->hs_dev, reqp);
if (ret != 0) {
if (wait) {
mtx_unlock(&reqp->event.mtx);
}
printf("%s: io_request failed with %d.\n",
__func__, ret);
ccb->ccb_h.status = CAM_PROVIDE_FAIL;
mtx_lock(&sc->hs_lock);
storvsc_free_request(sc, reqp);
xpt_done(ccb);
mtx_unlock(&sc->hs_lock);
return;
}
if (wait) {
xpt_print(ccb->ccb_h.path,
"%u: %s: waiting for IO return.\n",
ticks, __func__);
ret = cv_timedwait(&reqp->event.cv, &reqp->event.mtx, 60*hz);
mtx_unlock(&reqp->event.mtx);
xpt_print(ccb->ccb_h.path, "%u: %s: %s.\n",
ticks, __func__, (ret == 0)?
"IO return detected" :
"IO return not detected");
/*
* Now both the timer handler and io done are running
* simultaneously. We want to confirm the io done always
* finishes after the timer handler exits. So reqp used by
* timer handler is not freed or stale. Do busy loop for
* another 1/10 second to make sure io done does
* wait for the timer handler to complete.
*/
DELAY(100*1000);
mtx_lock(&sc->hs_lock);
xpt_print(ccb->ccb_h.path,
"%u: %s: finishing, queue frozen %d, "
"ccb status 0x%x scsi_status 0x%x.\n",
ticks, __func__, sc->hs_frozen,
ccb->ccb_h.status,
ccb->csio.scsi_status);
mtx_unlock(&sc->hs_lock);
}
}
#endif /* HVS_TIMEOUT_TEST */
/**
* @brief timeout handler for requests
*
* This function is called as a result of a callout expiring.
*
* @param arg pointer to a request
*/
static void
storvsc_timeout(void *arg)
{
struct hv_storvsc_request *reqp = arg;
struct storvsc_softc *sc = reqp->softc;
union ccb *ccb = reqp->ccb;
if (reqp->retries == 0) {
mtx_lock(&sc->hs_lock);
xpt_print(ccb->ccb_h.path,
"%u: IO timed out (req=0x%p), wait for another %u secs.\n",
ticks, reqp, ccb->ccb_h.timeout / 1000);
cam_error_print(ccb, CAM_ESF_ALL, CAM_EPF_ALL);
mtx_unlock(&sc->hs_lock);
reqp->retries++;
callout_reset_sbt(&reqp->callout, SBT_1MS * ccb->ccb_h.timeout,
0, storvsc_timeout, reqp, 0);
#if HVS_TIMEOUT_TEST
storvsc_timeout_test(reqp, SEND_DIAGNOSTIC, 0);
#endif
return;
}
mtx_lock(&sc->hs_lock);
xpt_print(ccb->ccb_h.path,
"%u: IO (reqp = 0x%p) did not return for %u seconds, %s.\n",
ticks, reqp, ccb->ccb_h.timeout * (reqp->retries+1) / 1000,
(sc->hs_frozen == 0)?
"freezing the queue" : "the queue is already frozen");
if (sc->hs_frozen == 0) {
sc->hs_frozen = 1;
xpt_freeze_simq(xpt_path_sim(ccb->ccb_h.path), 1);
}
mtx_unlock(&sc->hs_lock);
#if HVS_TIMEOUT_TEST
storvsc_timeout_test(reqp, MODE_SELECT_10, 1);
#endif
}
/**
* @brief StorVSC device poll function
*
* This function is responsible for servicing requests when
* interrupts are disabled (i.e when we are dumping core.)
*
* @param sim a pointer to a CAM SCSI interface module
*/
static void
storvsc_poll(struct cam_sim *sim)
{
struct storvsc_softc *sc = cam_sim_softc(sim);
mtx_assert(&sc->hs_lock, MA_OWNED);
mtx_unlock(&sc->hs_lock);
hv_storvsc_on_channel_callback(sc->hs_dev->channel);
mtx_lock(&sc->hs_lock);
}
/**
* @brief StorVSC device action function
*
* This function is responsible for handling SCSI operations which
* are passed from the CAM layer. The requests are in the form of
* CAM control blocks which indicate the action being performed.
* Not all actions require converting the request to a VSCSI protocol
* message - these actions can be responded to by this driver.
* Requests which are destined for a backend storage device are converted
* to a VSCSI protocol message and sent on the channel connection associated
* with this device.
*
* @param sim pointer to a CAM SCSI interface module
* @param ccb pointer to a CAM control block
*/
static void
storvsc_action(struct cam_sim *sim, union ccb *ccb)
{
struct storvsc_softc *sc = cam_sim_softc(sim);
int res;
mtx_assert(&sc->hs_lock, MA_OWNED);
switch (ccb->ccb_h.func_code) {
case XPT_PATH_INQ: {
struct ccb_pathinq *cpi = &ccb->cpi;
cpi->version_num = 1;
cpi->hba_inquiry = PI_TAG_ABLE|PI_SDTR_ABLE;
cpi->target_sprt = 0;
cpi->hba_misc = PIM_NOBUSRESET;
cpi->hba_eng_cnt = 0;
cpi->max_target = STORVSC_MAX_TARGETS;
cpi->max_lun = sc->hs_drv_props->drv_max_luns_per_target;
cpi->initiator_id = cpi->max_target;
cpi->bus_id = cam_sim_bus(sim);
cpi->base_transfer_speed = 300000;
cpi->transport = XPORT_SAS;
cpi->transport_version = 0;
cpi->protocol = PROTO_SCSI;
cpi->protocol_version = SCSI_REV_SPC2;
strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
strncpy(cpi->hba_vid, sc->hs_drv_props->drv_name, HBA_IDLEN);
strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
cpi->unit_number = cam_sim_unit(sim);
ccb->ccb_h.status = CAM_REQ_CMP;
xpt_done(ccb);
return;
}
case XPT_GET_TRAN_SETTINGS: {
struct ccb_trans_settings *cts = &ccb->cts;
cts->transport = XPORT_SAS;
cts->transport_version = 0;
cts->protocol = PROTO_SCSI;
cts->protocol_version = SCSI_REV_SPC2;
/* enable tag queuing and disconnected mode */
cts->proto_specific.valid = CTS_SCSI_VALID_TQ;
cts->proto_specific.scsi.valid = CTS_SCSI_VALID_TQ;
cts->proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB;
cts->xport_specific.valid = CTS_SPI_VALID_DISC;
cts->xport_specific.spi.flags = CTS_SPI_FLAGS_DISC_ENB;
ccb->ccb_h.status = CAM_REQ_CMP;
xpt_done(ccb);
return;
}
case XPT_SET_TRAN_SETTINGS: {
ccb->ccb_h.status = CAM_REQ_CMP;
xpt_done(ccb);
return;
}
case XPT_CALC_GEOMETRY:{
cam_calc_geometry(&ccb->ccg, 1);
xpt_done(ccb);
return;
}
case XPT_RESET_BUS:
case XPT_RESET_DEV:{
#if HVS_HOST_RESET
if ((res = hv_storvsc_host_reset(sc->hs_dev)) != 0) {
xpt_print(ccb->ccb_h.path,
"hv_storvsc_host_reset failed with %d\n", res);
ccb->ccb_h.status = CAM_PROVIDE_FAIL;
xpt_done(ccb);
return;
}
ccb->ccb_h.status = CAM_REQ_CMP;
xpt_done(ccb);
return;
#else
xpt_print(ccb->ccb_h.path,
"%s reset not supported.\n",
(ccb->ccb_h.func_code == XPT_RESET_BUS)?
"bus" : "dev");
ccb->ccb_h.status = CAM_REQ_INVALID;
xpt_done(ccb);
return;
#endif /* HVS_HOST_RESET */
}
case XPT_SCSI_IO:
case XPT_IMMED_NOTIFY: {
struct hv_storvsc_request *reqp = NULL;
if (ccb->csio.cdb_len == 0) {
panic("cdl_len is 0\n");
}
if (LIST_EMPTY(&sc->hs_free_list)) {
ccb->ccb_h.status = CAM_REQUEUE_REQ;
if (sc->hs_frozen == 0) {
sc->hs_frozen = 1;
xpt_freeze_simq(sim, /* count*/1);
}
xpt_done(ccb);
return;
}
reqp = LIST_FIRST(&sc->hs_free_list);
LIST_REMOVE(reqp, link);
bzero(reqp, sizeof(struct hv_storvsc_request));
reqp->softc = sc;
ccb->ccb_h.status |= CAM_SIM_QUEUED;
if ((res = create_storvsc_request(ccb, reqp)) != 0) {
ccb->ccb_h.status = CAM_REQ_INVALID;
xpt_done(ccb);
return;
}
if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
- callout_init(&reqp->callout, CALLOUT_MPSAFE);
+ callout_init(&reqp->callout, 1);
callout_reset_sbt(&reqp->callout,
SBT_1MS * ccb->ccb_h.timeout, 0,
storvsc_timeout, reqp, 0);
#if HVS_TIMEOUT_TEST
cv_init(&reqp->event.cv, "storvsc timeout cv");
mtx_init(&reqp->event.mtx, "storvsc timeout mutex",
NULL, MTX_DEF);
switch (reqp->vstor_packet.vm_srb.cdb[0]) {
case MODE_SELECT_10:
case SEND_DIAGNOSTIC:
/* To have timer send the request. */
return;
default:
break;
}
#endif /* HVS_TIMEOUT_TEST */
}
if ((res = hv_storvsc_io_request(sc->hs_dev, reqp)) != 0) {
xpt_print(ccb->ccb_h.path,
"hv_storvsc_io_request failed with %d\n", res);
ccb->ccb_h.status = CAM_PROVIDE_FAIL;
storvsc_free_request(sc, reqp);
xpt_done(ccb);
return;
}
return;
}
default:
ccb->ccb_h.status = CAM_REQ_INVALID;
xpt_done(ccb);
return;
}
}
/**
* @brief destroy bounce buffer
*
* This function is responsible for destroy a Scatter/Gather list
* that create by storvsc_create_bounce_buffer()
*
* @param sgl- the Scatter/Gather need be destroy
* @param sg_count- page count of the SG list.
*
*/
static void
storvsc_destroy_bounce_buffer(struct sglist *sgl)
{
struct hv_sgl_node *sgl_node = NULL;
sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.in_use_sgl_list);
LIST_REMOVE(sgl_node, link);
if (NULL == sgl_node) {
printf("storvsc error: not enough in use sgl\n");
return;
}
sgl_node->sgl_data = sgl;
LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list, sgl_node, link);
}
/**
* @brief create bounce buffer
*
* This function is responsible for create a Scatter/Gather list,
* which hold several pages that can be aligned with page size.
*
* @param seg_count- SG-list segments count
* @param write - if WRITE_TYPE, set SG list page used size to 0,
* otherwise set used size to page size.
*
* return NULL if create failed
*/
static struct sglist *
storvsc_create_bounce_buffer(uint16_t seg_count, int write)
{
int i = 0;
struct sglist *bounce_sgl = NULL;
unsigned int buf_len = ((write == WRITE_TYPE) ? 0 : PAGE_SIZE);
struct hv_sgl_node *sgl_node = NULL;
/* get struct sglist from free_sgl_list */
sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
LIST_REMOVE(sgl_node, link);
if (NULL == sgl_node) {
printf("storvsc error: not enough free sgl\n");
return NULL;
}
bounce_sgl = sgl_node->sgl_data;
LIST_INSERT_HEAD(&g_hv_sgl_page_pool.in_use_sgl_list, sgl_node, link);
bounce_sgl->sg_maxseg = seg_count;
if (write == WRITE_TYPE)
bounce_sgl->sg_nseg = 0;
else
bounce_sgl->sg_nseg = seg_count;
for (i = 0; i < seg_count; i++)
bounce_sgl->sg_segs[i].ss_len = buf_len;
return bounce_sgl;
}
/**
* @brief copy data from SG list to bounce buffer
*
* This function is responsible for copy data from one SG list's segments
* to another SG list which used as bounce buffer.
*
* @param bounce_sgl - the destination SG list
* @param orig_sgl - the segment of the source SG list.
* @param orig_sgl_count - the count of segments.
* @param orig_sgl_count - indicate which segment need bounce buffer,
* set 1 means need.
*
*/
static void
storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
bus_dma_segment_t *orig_sgl,
unsigned int orig_sgl_count,
uint64_t seg_bits)
{
int src_sgl_idx = 0;
for (src_sgl_idx = 0; src_sgl_idx < orig_sgl_count; src_sgl_idx++) {
if (seg_bits & (1 << src_sgl_idx)) {
memcpy((void*)bounce_sgl->sg_segs[src_sgl_idx].ss_paddr,
(void*)orig_sgl[src_sgl_idx].ds_addr,
orig_sgl[src_sgl_idx].ds_len);
bounce_sgl->sg_segs[src_sgl_idx].ss_len =
orig_sgl[src_sgl_idx].ds_len;
}
}
}
/**
* @brief copy data from SG list which used as bounce to another SG list
*
* This function is responsible for copy data from one SG list with bounce
* buffer to another SG list's segments.
*
* @param dest_sgl - the destination SG list's segments
* @param dest_sgl_count - the count of destination SG list's segment.
* @param src_sgl - the source SG list.
* @param seg_bits - indicate which segment used bounce buffer of src SG-list.
*
*/
void
storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
unsigned int dest_sgl_count,
struct sglist* src_sgl,
uint64_t seg_bits)
{
int sgl_idx = 0;
for (sgl_idx = 0; sgl_idx < dest_sgl_count; sgl_idx++) {
if (seg_bits & (1 << sgl_idx)) {
memcpy((void*)(dest_sgl[sgl_idx].ds_addr),
(void*)(src_sgl->sg_segs[sgl_idx].ss_paddr),
src_sgl->sg_segs[sgl_idx].ss_len);
}
}
}
/**
* @brief check SG list with bounce buffer or not
*
* This function is responsible for check if need bounce buffer for SG list.
*
* @param sgl - the SG list's segments
* @param sg_count - the count of SG list's segment.
* @param bits - segmengs number that need bounce buffer
*
* return -1 if SG list needless bounce buffer
*/
static int
storvsc_check_bounce_buffer_sgl(bus_dma_segment_t *sgl,
unsigned int sg_count,
uint64_t *bits)
{
int i = 0;
int offset = 0;
uint64_t phys_addr = 0;
uint64_t tmp_bits = 0;
boolean_t found_hole = FALSE;
boolean_t pre_aligned = TRUE;
if (sg_count < 2){
return -1;
}
*bits = 0;
phys_addr = vtophys(sgl[0].ds_addr);
offset = phys_addr - trunc_page(phys_addr);
if (offset != 0) {
pre_aligned = FALSE;
tmp_bits |= 1;
}
for (i = 1; i < sg_count; i++) {
phys_addr = vtophys(sgl[i].ds_addr);
offset = phys_addr - trunc_page(phys_addr);
if (offset == 0) {
if (FALSE == pre_aligned){
/*
* This segment is aligned, if the previous
* one is not aligned, find a hole
*/
found_hole = TRUE;
}
pre_aligned = TRUE;
} else {
tmp_bits |= 1 << i;
if (!pre_aligned) {
if (phys_addr != vtophys(sgl[i-1].ds_addr +
sgl[i-1].ds_len)) {
/*
* Check whether connect to previous
* segment,if not, find the hole
*/
found_hole = TRUE;
}
} else {
found_hole = TRUE;
}
pre_aligned = FALSE;
}
}
if (!found_hole) {
return (-1);
} else {
*bits = tmp_bits;
return 0;
}
}
/**
* @brief Fill in a request structure based on a CAM control block
*
* Fills in a request structure based on the contents of a CAM control
* block. The request structure holds the payload information for
* VSCSI protocol request.
*
* @param ccb pointer to a CAM contorl block
* @param reqp pointer to a request structure
*/
static int
create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
{
struct ccb_scsiio *csio = &ccb->csio;
uint64_t phys_addr;
uint32_t bytes_to_copy = 0;
uint32_t pfn_num = 0;
uint32_t pfn;
uint64_t not_aligned_seg_bits = 0;
/* refer to struct vmscsi_req for meanings of these two fields */
reqp->vstor_packet.u.vm_srb.port =
cam_sim_unit(xpt_path_sim(ccb->ccb_h.path));
reqp->vstor_packet.u.vm_srb.path_id =
cam_sim_bus(xpt_path_sim(ccb->ccb_h.path));
reqp->vstor_packet.u.vm_srb.target_id = ccb->ccb_h.target_id;
reqp->vstor_packet.u.vm_srb.lun = ccb->ccb_h.target_lun;
reqp->vstor_packet.u.vm_srb.cdb_len = csio->cdb_len;
if(ccb->ccb_h.flags & CAM_CDB_POINTER) {
memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_ptr,
csio->cdb_len);
} else {
memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_bytes,
csio->cdb_len);
}
switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
case CAM_DIR_OUT:
reqp->vstor_packet.u.vm_srb.data_in = WRITE_TYPE;
break;
case CAM_DIR_IN:
reqp->vstor_packet.u.vm_srb.data_in = READ_TYPE;
break;
case CAM_DIR_NONE:
reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
break;
default:
reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
break;
}
reqp->sense_data = &csio->sense_data;
reqp->sense_info_len = csio->sense_len;
reqp->ccb = ccb;
if (0 == csio->dxfer_len) {
return (0);
}
reqp->data_buf.length = csio->dxfer_len;
switch (ccb->ccb_h.flags & CAM_DATA_MASK) {
case CAM_DATA_VADDR:
{
bytes_to_copy = csio->dxfer_len;
phys_addr = vtophys(csio->data_ptr);
reqp->data_buf.offset = phys_addr & PAGE_MASK;
while (bytes_to_copy != 0) {
int bytes, page_offset;
phys_addr =
vtophys(&csio->data_ptr[reqp->data_buf.length -
bytes_to_copy]);
pfn = phys_addr >> PAGE_SHIFT;
reqp->data_buf.pfn_array[pfn_num] = pfn;
page_offset = phys_addr & PAGE_MASK;
bytes = min(PAGE_SIZE - page_offset, bytes_to_copy);
bytes_to_copy -= bytes;
pfn_num++;
}
break;
}
case CAM_DATA_SG:
{
int i = 0;
int offset = 0;
int ret;
bus_dma_segment_t *storvsc_sglist =
(bus_dma_segment_t *)ccb->csio.data_ptr;
u_int16_t storvsc_sg_count = ccb->csio.sglist_cnt;
printf("Storvsc: get SG I/O operation, %d\n",
reqp->vstor_packet.u.vm_srb.data_in);
if (storvsc_sg_count > HV_MAX_MULTIPAGE_BUFFER_COUNT){
printf("Storvsc: %d segments is too much, "
"only support %d segments\n",
storvsc_sg_count, HV_MAX_MULTIPAGE_BUFFER_COUNT);
return (EINVAL);
}
/*
* We create our own bounce buffer function currently. Idealy
* we should use BUS_DMA(9) framework. But with current BUS_DMA
* code there is no callback API to check the page alignment of
* middle segments before busdma can decide if a bounce buffer
* is needed for particular segment. There is callback,
* "bus_dma_filter_t *filter", but the parrameters are not
* sufficient for storvsc driver.
* TODO:
* Add page alignment check in BUS_DMA(9) callback. Once
* this is complete, switch the following code to use
* BUS_DMA(9) for storvsc bounce buffer support.
*/
/* check if we need to create bounce buffer */
ret = storvsc_check_bounce_buffer_sgl(storvsc_sglist,
storvsc_sg_count, &not_aligned_seg_bits);
if (ret != -1) {
reqp->bounce_sgl =
storvsc_create_bounce_buffer(storvsc_sg_count,
reqp->vstor_packet.u.vm_srb.data_in);
if (NULL == reqp->bounce_sgl) {
printf("Storvsc_error: "
"create bounce buffer failed.\n");
return (ENOMEM);
}
reqp->bounce_sgl_count = storvsc_sg_count;
reqp->not_aligned_seg_bits = not_aligned_seg_bits;
/*
* if it is write, we need copy the original data
*to bounce buffer
*/
if (WRITE_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
storvsc_copy_sgl_to_bounce_buf(
reqp->bounce_sgl,
storvsc_sglist,
storvsc_sg_count,
reqp->not_aligned_seg_bits);
}
/* transfer virtual address to physical frame number */
if (reqp->not_aligned_seg_bits & 0x1){
phys_addr =
vtophys(reqp->bounce_sgl->sg_segs[0].ss_paddr);
}else{
phys_addr =
vtophys(storvsc_sglist[0].ds_addr);
}
reqp->data_buf.offset = phys_addr & PAGE_MASK;
pfn = phys_addr >> PAGE_SHIFT;
reqp->data_buf.pfn_array[0] = pfn;
for (i = 1; i < storvsc_sg_count; i++) {
if (reqp->not_aligned_seg_bits & (1 << i)) {
phys_addr =
vtophys(reqp->bounce_sgl->sg_segs[i].ss_paddr);
} else {
phys_addr =
vtophys(storvsc_sglist[i].ds_addr);
}
pfn = phys_addr >> PAGE_SHIFT;
reqp->data_buf.pfn_array[i] = pfn;
}
} else {
phys_addr = vtophys(storvsc_sglist[0].ds_addr);
reqp->data_buf.offset = phys_addr & PAGE_MASK;
for (i = 0; i < storvsc_sg_count; i++) {
phys_addr = vtophys(storvsc_sglist[i].ds_addr);
pfn = phys_addr >> PAGE_SHIFT;
reqp->data_buf.pfn_array[i] = pfn;
}
/* check the last segment cross boundary or not */
offset = phys_addr & PAGE_MASK;
if (offset) {
phys_addr =
vtophys(storvsc_sglist[i-1].ds_addr +
PAGE_SIZE - offset);
pfn = phys_addr >> PAGE_SHIFT;
reqp->data_buf.pfn_array[i] = pfn;
}
reqp->bounce_sgl_count = 0;
}
break;
}
default:
printf("Unknow flags: %d\n", ccb->ccb_h.flags);
return(EINVAL);
}
return(0);
}
/**
* @brief completion function before returning to CAM
*
* I/O process has been completed and the result needs
* to be passed to the CAM layer.
* Free resources related to this request.
*
* @param reqp pointer to a request structure
*/
static void
storvsc_io_done(struct hv_storvsc_request *reqp)
{
union ccb *ccb = reqp->ccb;
struct ccb_scsiio *csio = &ccb->csio;
struct storvsc_softc *sc = reqp->softc;
struct vmscsi_req *vm_srb = &reqp->vstor_packet.u.vm_srb;
bus_dma_segment_t *ori_sglist = NULL;
int ori_sg_count = 0;
/* destroy bounce buffer if it is used */
if (reqp->bounce_sgl_count) {
ori_sglist = (bus_dma_segment_t *)ccb->csio.data_ptr;
ori_sg_count = ccb->csio.sglist_cnt;
/*
* If it is READ operation, we should copy back the data
* to original SG list.
*/
if (READ_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
storvsc_copy_from_bounce_buf_to_sgl(ori_sglist,
ori_sg_count,
reqp->bounce_sgl,
reqp->not_aligned_seg_bits);
}
storvsc_destroy_bounce_buffer(reqp->bounce_sgl);
reqp->bounce_sgl_count = 0;
}
if (reqp->retries > 0) {
mtx_lock(&sc->hs_lock);
#if HVS_TIMEOUT_TEST
xpt_print(ccb->ccb_h.path,
"%u: IO returned after timeout, "
"waking up timer handler if any.\n", ticks);
mtx_lock(&reqp->event.mtx);
cv_signal(&reqp->event.cv);
mtx_unlock(&reqp->event.mtx);
#endif
reqp->retries = 0;
xpt_print(ccb->ccb_h.path,
"%u: IO returned after timeout, "
"stopping timer if any.\n", ticks);
mtx_unlock(&sc->hs_lock);
}
/*
* callout_drain() will wait for the timer handler to finish
* if it is running. So we don't need any lock to synchronize
* between this routine and the timer handler.
* Note that we need to make sure reqp is not freed when timer
* handler is using or will use it.
*/
if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
callout_drain(&reqp->callout);
}
ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
ccb->ccb_h.status &= ~CAM_STATUS_MASK;
if (vm_srb->scsi_status == SCSI_STATUS_OK) {
ccb->ccb_h.status |= CAM_REQ_CMP;
} else {
mtx_lock(&sc->hs_lock);
xpt_print(ccb->ccb_h.path,
"srovsc scsi_status = %d\n",
vm_srb->scsi_status);
mtx_unlock(&sc->hs_lock);
ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
}
ccb->csio.scsi_status = (vm_srb->scsi_status & 0xFF);
ccb->csio.resid = ccb->csio.dxfer_len - vm_srb->transfer_len;
if (reqp->sense_info_len != 0) {
csio->sense_resid = csio->sense_len - reqp->sense_info_len;
ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
}
mtx_lock(&sc->hs_lock);
if (reqp->softc->hs_frozen == 1) {
xpt_print(ccb->ccb_h.path,
"%u: storvsc unfreezing softc 0x%p.\n",
ticks, reqp->softc);
ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
reqp->softc->hs_frozen = 0;
}
storvsc_free_request(sc, reqp);
xpt_done(ccb);
mtx_unlock(&sc->hs_lock);
}
/**
* @brief Free a request structure
*
* Free a request structure by returning it to the free list
*
* @param sc pointer to a softc
* @param reqp pointer to a request structure
*/
static void
storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp)
{
LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
}
/**
* @brief Determine type of storage device from GUID
*
* Using the type GUID, determine if this is a StorVSC (paravirtual
* SCSI or BlkVSC (paravirtual IDE) device.
*
* @param dev a device
* returns an enum
*/
static enum hv_storage_type
storvsc_get_storage_type(device_t dev)
{
const char *p = vmbus_get_type(dev);
if (!memcmp(p, &gBlkVscDeviceType, sizeof(hv_guid))) {
return DRIVER_BLKVSC;
} else if (!memcmp(p, &gStorVscDeviceType, sizeof(hv_guid))) {
return DRIVER_STORVSC;
}
return (DRIVER_UNKNOWN);
}
Index: head/sys/dev/if_ndis/if_ndis.c
===================================================================
--- head/sys/dev/if_ndis/if_ndis.c (revision 283290)
+++ head/sys/dev/if_ndis/if_ndis.c (revision 283291)
@@ -1,3395 +1,3395 @@
/*-
* Copyright (c) 2003
* Bill Paul <wpaul@windriver.com>. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by Bill Paul.
* 4. Neither the name of the author nor the names of any co-contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*
* WPA support originally contributed by Arvind Srinivasan <arvind@celar.us>
* then hacked upon mercilessly by my.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sockio.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/endian.h>
#include <sys/priv.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <sys/queue.h>
#include <sys/module.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/kthread.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_arp.h>
#include <net/ethernet.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/if_types.h>
#include <net/route.h>
#include <net/bpf.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <net80211/ieee80211_var.h>
#include <net80211/ieee80211_ioctl.h>
#include <net80211/ieee80211_regdomain.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/usb/usb.h>
#include <dev/usb/usbdi.h>
#include <compat/ndis/pe_var.h>
#include <compat/ndis/cfg_var.h>
#include <compat/ndis/resource_var.h>
#include <compat/ndis/ntoskrnl_var.h>
#include <compat/ndis/hal_var.h>
#include <compat/ndis/ndis_var.h>
#include <compat/ndis/usbd_var.h>
#include <dev/if_ndis/if_ndisvar.h>
#define NDIS_DEBUG
#ifdef NDIS_DEBUG
#define DPRINTF(x) do { if (ndis_debug > 0) printf x; } while (0)
int ndis_debug = 0;
SYSCTL_INT(_debug, OID_AUTO, ndis, CTLFLAG_RW, &ndis_debug, 0,
"if_ndis debug level");
#else
#define DPRINTF(x)
#endif
SYSCTL_DECL(_hw_ndisusb);
int ndisusb_halt = 1;
SYSCTL_INT(_hw_ndisusb, OID_AUTO, halt, CTLFLAG_RW, &ndisusb_halt, 0,
"Halt NDIS USB driver when it's attached");
/* 0 - 30 dBm to mW conversion table */
static const uint16_t dBm2mW[] = {
1, 1, 1, 1, 2, 2, 2, 2, 3, 3,
3, 4, 4, 4, 5, 6, 6, 7, 8, 9,
10, 11, 13, 14, 16, 18, 20, 22, 25, 28,
32, 35, 40, 45, 50, 56, 63, 71, 79, 89,
100, 112, 126, 141, 158, 178, 200, 224, 251, 282,
316, 355, 398, 447, 501, 562, 631, 708, 794, 891,
1000
};
MODULE_DEPEND(ndis, ether, 1, 1, 1);
MODULE_DEPEND(ndis, wlan, 1, 1, 1);
MODULE_DEPEND(ndis, ndisapi, 1, 1, 1);
MODULE_VERSION(ndis, 1);
int ndis_attach (device_t);
int ndis_detach (device_t);
int ndis_suspend (device_t);
int ndis_resume (device_t);
void ndis_shutdown (device_t);
int ndisdrv_modevent (module_t, int, void *);
static void ndis_txeof (ndis_handle, ndis_packet *, ndis_status);
static void ndis_rxeof (ndis_handle, ndis_packet **, uint32_t);
static void ndis_rxeof_eth (ndis_handle, ndis_handle, char *, void *,
uint32_t, void *, uint32_t, uint32_t);
static void ndis_rxeof_done (ndis_handle);
static void ndis_rxeof_xfr (kdpc *, ndis_handle, void *, void *);
static void ndis_rxeof_xfr_done (ndis_handle, ndis_packet *,
uint32_t, uint32_t);
static void ndis_linksts (ndis_handle, ndis_status, void *, uint32_t);
static void ndis_linksts_done (ndis_handle);
/* We need to wrap these functions for amd64. */
static funcptr ndis_txeof_wrap;
static funcptr ndis_rxeof_wrap;
static funcptr ndis_rxeof_eth_wrap;
static funcptr ndis_rxeof_done_wrap;
static funcptr ndis_rxeof_xfr_wrap;
static funcptr ndis_rxeof_xfr_done_wrap;
static funcptr ndis_linksts_wrap;
static funcptr ndis_linksts_done_wrap;
static funcptr ndis_ticktask_wrap;
static funcptr ndis_starttask_wrap;
static funcptr ndis_resettask_wrap;
static funcptr ndis_inputtask_wrap;
static struct ieee80211vap *ndis_vap_create(struct ieee80211com *,
const char [IFNAMSIZ], int, enum ieee80211_opmode, int,
const uint8_t [IEEE80211_ADDR_LEN],
const uint8_t [IEEE80211_ADDR_LEN]);
static void ndis_vap_delete (struct ieee80211vap *);
static void ndis_tick (void *);
static void ndis_ticktask (device_object *, void *);
static int ndis_raw_xmit (struct ieee80211_node *, struct mbuf *,
const struct ieee80211_bpf_params *);
static void ndis_update_mcast (struct ifnet *ifp);
static void ndis_update_promisc (struct ifnet *ifp);
static void ndis_start (struct ifnet *);
static void ndis_starttask (device_object *, void *);
static void ndis_resettask (device_object *, void *);
static void ndis_inputtask (device_object *, void *);
static int ndis_ioctl (struct ifnet *, u_long, caddr_t);
static int ndis_ioctl_80211 (struct ifnet *, u_long, caddr_t);
static int ndis_newstate (struct ieee80211vap *, enum ieee80211_state,
int);
static int ndis_nettype_chan (uint32_t);
static int ndis_nettype_mode (uint32_t);
static void ndis_scan (void *);
static void ndis_scan_results (struct ndis_softc *);
static void ndis_scan_start (struct ieee80211com *);
static void ndis_scan_end (struct ieee80211com *);
static void ndis_set_channel (struct ieee80211com *);
static void ndis_scan_curchan (struct ieee80211_scan_state *, unsigned long);
static void ndis_scan_mindwell (struct ieee80211_scan_state *);
static void ndis_init (void *);
static void ndis_stop (struct ndis_softc *);
static int ndis_ifmedia_upd (struct ifnet *);
static void ndis_ifmedia_sts (struct ifnet *, struct ifmediareq *);
static int ndis_get_bssid_list (struct ndis_softc *,
ndis_80211_bssid_list_ex **);
static int ndis_get_assoc (struct ndis_softc *, ndis_wlan_bssid_ex **);
static int ndis_probe_offload (struct ndis_softc *);
static int ndis_set_offload (struct ndis_softc *);
static void ndis_getstate_80211 (struct ndis_softc *);
static void ndis_setstate_80211 (struct ndis_softc *);
static void ndis_auth_and_assoc (struct ndis_softc *, struct ieee80211vap *);
static void ndis_media_status (struct ifnet *, struct ifmediareq *);
static int ndis_set_cipher (struct ndis_softc *, int);
static int ndis_set_wpa (struct ndis_softc *, void *, int);
static int ndis_add_key (struct ieee80211vap *,
const struct ieee80211_key *, const u_int8_t []);
static int ndis_del_key (struct ieee80211vap *,
const struct ieee80211_key *);
static void ndis_setmulti (struct ndis_softc *);
static void ndis_map_sclist (void *, bus_dma_segment_t *,
int, bus_size_t, int);
static int ndisdrv_loaded = 0;
/*
* This routine should call windrv_load() once for each driver
* image. This will do the relocation and dynalinking for the
* image, and create a Windows driver object which will be
* saved in our driver database.
*/
int
ndisdrv_modevent(mod, cmd, arg)
module_t mod;
int cmd;
void *arg;
{
int error = 0;
switch (cmd) {
case MOD_LOAD:
ndisdrv_loaded++;
if (ndisdrv_loaded > 1)
break;
windrv_wrap((funcptr)ndis_rxeof, &ndis_rxeof_wrap,
3, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_rxeof_eth, &ndis_rxeof_eth_wrap,
8, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_rxeof_done, &ndis_rxeof_done_wrap,
1, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_rxeof_xfr, &ndis_rxeof_xfr_wrap,
4, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_rxeof_xfr_done,
&ndis_rxeof_xfr_done_wrap, 4, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_txeof, &ndis_txeof_wrap,
3, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_linksts, &ndis_linksts_wrap,
4, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_linksts_done,
&ndis_linksts_done_wrap, 1, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_ticktask, &ndis_ticktask_wrap,
2, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_starttask, &ndis_starttask_wrap,
2, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_resettask, &ndis_resettask_wrap,
2, WINDRV_WRAP_STDCALL);
windrv_wrap((funcptr)ndis_inputtask, &ndis_inputtask_wrap,
2, WINDRV_WRAP_STDCALL);
break;
case MOD_UNLOAD:
ndisdrv_loaded--;
if (ndisdrv_loaded > 0)
break;
/* fallthrough */
case MOD_SHUTDOWN:
windrv_unwrap(ndis_rxeof_wrap);
windrv_unwrap(ndis_rxeof_eth_wrap);
windrv_unwrap(ndis_rxeof_done_wrap);
windrv_unwrap(ndis_rxeof_xfr_wrap);
windrv_unwrap(ndis_rxeof_xfr_done_wrap);
windrv_unwrap(ndis_txeof_wrap);
windrv_unwrap(ndis_linksts_wrap);
windrv_unwrap(ndis_linksts_done_wrap);
windrv_unwrap(ndis_ticktask_wrap);
windrv_unwrap(ndis_starttask_wrap);
windrv_unwrap(ndis_resettask_wrap);
windrv_unwrap(ndis_inputtask_wrap);
break;
default:
error = EINVAL;
break;
}
return (error);
}
/*
* Program the 64-bit multicast hash filter.
*/
static void
ndis_setmulti(sc)
struct ndis_softc *sc;
{
struct ifnet *ifp;
struct ifmultiaddr *ifma;
int len, mclistsz, error;
uint8_t *mclist;
ifp = sc->ifp;
if (!NDIS_INITIALIZED(sc))
return;
if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
sc->ndis_filter |= NDIS_PACKET_TYPE_ALL_MULTICAST;
len = sizeof(sc->ndis_filter);
error = ndis_set_info(sc, OID_GEN_CURRENT_PACKET_FILTER,
&sc->ndis_filter, &len);
if (error)
device_printf(sc->ndis_dev,
"set allmulti failed: %d\n", error);
return;
}
if (TAILQ_EMPTY(&ifp->if_multiaddrs))
return;
len = sizeof(mclistsz);
ndis_get_info(sc, OID_802_3_MAXIMUM_LIST_SIZE, &mclistsz, &len);
mclist = malloc(ETHER_ADDR_LEN * mclistsz, M_TEMP, M_NOWAIT|M_ZERO);
if (mclist == NULL) {
sc->ndis_filter |= NDIS_PACKET_TYPE_ALL_MULTICAST;
goto out;
}
sc->ndis_filter |= NDIS_PACKET_TYPE_MULTICAST;
len = 0;
if_maddr_rlock(ifp);
TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
if (ifma->ifma_addr->sa_family != AF_LINK)
continue;
bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
mclist + (ETHER_ADDR_LEN * len), ETHER_ADDR_LEN);
len++;
if (len > mclistsz) {
if_maddr_runlock(ifp);
sc->ndis_filter |= NDIS_PACKET_TYPE_ALL_MULTICAST;
sc->ndis_filter &= ~NDIS_PACKET_TYPE_MULTICAST;
goto out;
}
}
if_maddr_runlock(ifp);
len = len * ETHER_ADDR_LEN;
error = ndis_set_info(sc, OID_802_3_MULTICAST_LIST, mclist, &len);
if (error) {
device_printf(sc->ndis_dev, "set mclist failed: %d\n", error);
sc->ndis_filter |= NDIS_PACKET_TYPE_ALL_MULTICAST;
sc->ndis_filter &= ~NDIS_PACKET_TYPE_MULTICAST;
}
out:
free(mclist, M_TEMP);
len = sizeof(sc->ndis_filter);
error = ndis_set_info(sc, OID_GEN_CURRENT_PACKET_FILTER,
&sc->ndis_filter, &len);
if (error)
device_printf(sc->ndis_dev, "set multi failed: %d\n", error);
}
static int
ndis_set_offload(sc)
struct ndis_softc *sc;
{
ndis_task_offload *nto;
ndis_task_offload_hdr *ntoh;
ndis_task_tcpip_csum *nttc;
struct ifnet *ifp;
int len, error;
ifp = sc->ifp;
if (!NDIS_INITIALIZED(sc))
return (EINVAL);
/* See if there's anything to set. */
error = ndis_probe_offload(sc);
if (error)
return (error);
if (sc->ndis_hwassist == 0 && ifp->if_capabilities == 0)
return (0);
len = sizeof(ndis_task_offload_hdr) + sizeof(ndis_task_offload) +
sizeof(ndis_task_tcpip_csum);
ntoh = malloc(len, M_TEMP, M_NOWAIT|M_ZERO);
if (ntoh == NULL)
return (ENOMEM);
ntoh->ntoh_vers = NDIS_TASK_OFFLOAD_VERSION;
ntoh->ntoh_len = sizeof(ndis_task_offload_hdr);
ntoh->ntoh_offset_firsttask = sizeof(ndis_task_offload_hdr);
ntoh->ntoh_encapfmt.nef_encaphdrlen = sizeof(struct ether_header);
ntoh->ntoh_encapfmt.nef_encap = NDIS_ENCAP_IEEE802_3;
ntoh->ntoh_encapfmt.nef_flags = NDIS_ENCAPFLAG_FIXEDHDRLEN;
nto = (ndis_task_offload *)((char *)ntoh +
ntoh->ntoh_offset_firsttask);
nto->nto_vers = NDIS_TASK_OFFLOAD_VERSION;
nto->nto_len = sizeof(ndis_task_offload);
nto->nto_task = NDIS_TASK_TCPIP_CSUM;
nto->nto_offset_nexttask = 0;
nto->nto_taskbuflen = sizeof(ndis_task_tcpip_csum);
nttc = (ndis_task_tcpip_csum *)nto->nto_taskbuf;
if (ifp->if_capenable & IFCAP_TXCSUM)
nttc->nttc_v4tx = sc->ndis_v4tx;
if (ifp->if_capenable & IFCAP_RXCSUM)
nttc->nttc_v4rx = sc->ndis_v4rx;
error = ndis_set_info(sc, OID_TCP_TASK_OFFLOAD, ntoh, &len);
free(ntoh, M_TEMP);
return (error);
}
static int
ndis_probe_offload(sc)
struct ndis_softc *sc;
{
ndis_task_offload *nto;
ndis_task_offload_hdr *ntoh;
ndis_task_tcpip_csum *nttc = NULL;
struct ifnet *ifp;
int len, error, dummy;
ifp = sc->ifp;
len = sizeof(dummy);
error = ndis_get_info(sc, OID_TCP_TASK_OFFLOAD, &dummy, &len);
if (error != ENOSPC)
return (error);
ntoh = malloc(len, M_TEMP, M_NOWAIT|M_ZERO);
if (ntoh == NULL)
return (ENOMEM);
ntoh->ntoh_vers = NDIS_TASK_OFFLOAD_VERSION;
ntoh->ntoh_len = sizeof(ndis_task_offload_hdr);
ntoh->ntoh_encapfmt.nef_encaphdrlen = sizeof(struct ether_header);
ntoh->ntoh_encapfmt.nef_encap = NDIS_ENCAP_IEEE802_3;
ntoh->ntoh_encapfmt.nef_flags = NDIS_ENCAPFLAG_FIXEDHDRLEN;
error = ndis_get_info(sc, OID_TCP_TASK_OFFLOAD, ntoh, &len);
if (error) {
free(ntoh, M_TEMP);
return (error);
}
if (ntoh->ntoh_vers != NDIS_TASK_OFFLOAD_VERSION) {
free(ntoh, M_TEMP);
return (EINVAL);
}
nto = (ndis_task_offload *)((char *)ntoh +
ntoh->ntoh_offset_firsttask);
while (1) {
switch (nto->nto_task) {
case NDIS_TASK_TCPIP_CSUM:
nttc = (ndis_task_tcpip_csum *)nto->nto_taskbuf;
break;
/* Don't handle these yet. */
case NDIS_TASK_IPSEC:
case NDIS_TASK_TCP_LARGESEND:
default:
break;
}
if (nto->nto_offset_nexttask == 0)
break;
nto = (ndis_task_offload *)((char *)nto +
nto->nto_offset_nexttask);
}
if (nttc == NULL) {
free(ntoh, M_TEMP);
return (ENOENT);
}
sc->ndis_v4tx = nttc->nttc_v4tx;
sc->ndis_v4rx = nttc->nttc_v4rx;
if (nttc->nttc_v4tx & NDIS_TCPSUM_FLAGS_IP_CSUM)
sc->ndis_hwassist |= CSUM_IP;
if (nttc->nttc_v4tx & NDIS_TCPSUM_FLAGS_TCP_CSUM)
sc->ndis_hwassist |= CSUM_TCP;
if (nttc->nttc_v4tx & NDIS_TCPSUM_FLAGS_UDP_CSUM)
sc->ndis_hwassist |= CSUM_UDP;
if (sc->ndis_hwassist)
ifp->if_capabilities |= IFCAP_TXCSUM;
if (nttc->nttc_v4rx & NDIS_TCPSUM_FLAGS_IP_CSUM)
ifp->if_capabilities |= IFCAP_RXCSUM;
if (nttc->nttc_v4rx & NDIS_TCPSUM_FLAGS_TCP_CSUM)
ifp->if_capabilities |= IFCAP_RXCSUM;
if (nttc->nttc_v4rx & NDIS_TCPSUM_FLAGS_UDP_CSUM)
ifp->if_capabilities |= IFCAP_RXCSUM;
free(ntoh, M_TEMP);
return (0);
}
static int
ndis_nettype_chan(uint32_t type)
{
switch (type) {
case NDIS_80211_NETTYPE_11FH: return (IEEE80211_CHAN_FHSS);
case NDIS_80211_NETTYPE_11DS: return (IEEE80211_CHAN_B);
case NDIS_80211_NETTYPE_11OFDM5: return (IEEE80211_CHAN_A);
case NDIS_80211_NETTYPE_11OFDM24: return (IEEE80211_CHAN_G);
}
DPRINTF(("unknown channel nettype %d\n", type));
return (IEEE80211_CHAN_B); /* Default to 11B chan */
}
static int
ndis_nettype_mode(uint32_t type)
{
switch (type) {
case NDIS_80211_NETTYPE_11FH: return (IEEE80211_MODE_FH);
case NDIS_80211_NETTYPE_11DS: return (IEEE80211_MODE_11B);
case NDIS_80211_NETTYPE_11OFDM5: return (IEEE80211_MODE_11A);
case NDIS_80211_NETTYPE_11OFDM24: return (IEEE80211_MODE_11G);
}
DPRINTF(("unknown mode nettype %d\n", type));
return (IEEE80211_MODE_AUTO);
}
/*
* Attach the interface. Allocate softc structures, do ifmedia
* setup and ethernet/BPF attach.
*/
int
ndis_attach(dev)
device_t dev;
{
u_char eaddr[ETHER_ADDR_LEN];
struct ndis_softc *sc;
driver_object *pdrv;
device_object *pdo;
struct ifnet *ifp = NULL;
int error = 0, len, mode;
uint8_t bands = 0;
int i;
sc = device_get_softc(dev);
mtx_init(&sc->ndis_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
MTX_DEF);
KeInitializeSpinLock(&sc->ndis_rxlock);
KeInitializeSpinLock(&sc->ndisusb_tasklock);
KeInitializeSpinLock(&sc->ndisusb_xferdonelock);
InitializeListHead(&sc->ndis_shlist);
InitializeListHead(&sc->ndisusb_tasklist);
InitializeListHead(&sc->ndisusb_xferdonelist);
- callout_init(&sc->ndis_stat_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->ndis_stat_callout, 1);
if (sc->ndis_iftype == PCMCIABus) {
error = ndis_alloc_amem(sc);
if (error) {
device_printf(dev, "failed to allocate "
"attribute memory\n");
goto fail;
}
}
/* Create sysctl registry nodes */
ndis_create_sysctls(sc);
/* Find the PDO for this device instance. */
if (sc->ndis_iftype == PCIBus)
pdrv = windrv_lookup(0, "PCI Bus");
else if (sc->ndis_iftype == PCMCIABus)
pdrv = windrv_lookup(0, "PCCARD Bus");
else
pdrv = windrv_lookup(0, "USB Bus");
pdo = windrv_find_pdo(pdrv, dev);
/*
* Create a new functional device object for this
* device. This is what creates the miniport block
* for this device instance.
*/
if (NdisAddDevice(sc->ndis_dobj, pdo) != STATUS_SUCCESS) {
device_printf(dev, "failed to create FDO!\n");
error = ENXIO;
goto fail;
}
/* Tell the user what version of the API the driver is using. */
device_printf(dev, "NDIS API version: %d.%d\n",
sc->ndis_chars->nmc_version_major,
sc->ndis_chars->nmc_version_minor);
/* Do resource conversion. */
if (sc->ndis_iftype == PCMCIABus || sc->ndis_iftype == PCIBus)
ndis_convert_res(sc);
else
sc->ndis_block->nmb_rlist = NULL;
/* Install our RX and TX interrupt handlers. */
sc->ndis_block->nmb_senddone_func = ndis_txeof_wrap;
sc->ndis_block->nmb_pktind_func = ndis_rxeof_wrap;
sc->ndis_block->nmb_ethrxindicate_func = ndis_rxeof_eth_wrap;
sc->ndis_block->nmb_ethrxdone_func = ndis_rxeof_done_wrap;
sc->ndis_block->nmb_tdcond_func = ndis_rxeof_xfr_done_wrap;
/* Override the status handler so we can detect link changes. */
sc->ndis_block->nmb_status_func = ndis_linksts_wrap;
sc->ndis_block->nmb_statusdone_func = ndis_linksts_done_wrap;
/* Set up work item handlers. */
sc->ndis_tickitem = IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj);
sc->ndis_startitem = IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj);
sc->ndis_resetitem = IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj);
sc->ndis_inputitem = IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj);
sc->ndisusb_xferdoneitem =
IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj);
sc->ndisusb_taskitem =
IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj);
KeInitializeDpc(&sc->ndis_rxdpc, ndis_rxeof_xfr_wrap, sc->ndis_block);
/* Call driver's init routine. */
if (ndis_init_nic(sc)) {
device_printf(dev, "init handler failed\n");
error = ENXIO;
goto fail;
}
/*
* Get station address from the driver.
*/
len = sizeof(eaddr);
ndis_get_info(sc, OID_802_3_CURRENT_ADDRESS, &eaddr, &len);
/*
* Figure out how big to make the TX buffer pool.
*/
len = sizeof(sc->ndis_maxpkts);
if (ndis_get_info(sc, OID_GEN_MAXIMUM_SEND_PACKETS,
&sc->ndis_maxpkts, &len)) {
device_printf(dev, "failed to get max TX packets\n");
error = ENXIO;
goto fail;
}
/*
* If this is a deserialized miniport, we don't have
* to honor the OID_GEN_MAXIMUM_SEND_PACKETS result.
*/
if (!NDIS_SERIALIZED(sc->ndis_block))
sc->ndis_maxpkts = NDIS_TXPKTS;
/* Enforce some sanity, just in case. */
if (sc->ndis_maxpkts == 0)
sc->ndis_maxpkts = 10;
sc->ndis_txarray = malloc(sizeof(ndis_packet *) *
sc->ndis_maxpkts, M_DEVBUF, M_NOWAIT|M_ZERO);
/* Allocate a pool of ndis_packets for TX encapsulation. */
NdisAllocatePacketPool(&i, &sc->ndis_txpool,
sc->ndis_maxpkts, PROTOCOL_RESERVED_SIZE_IN_PACKET);
if (i != NDIS_STATUS_SUCCESS) {
sc->ndis_txpool = NULL;
device_printf(dev, "failed to allocate TX packet pool");
error = ENOMEM;
goto fail;
}
sc->ndis_txpending = sc->ndis_maxpkts;
sc->ndis_oidcnt = 0;
/* Get supported oid list. */
ndis_get_supported_oids(sc, &sc->ndis_oids, &sc->ndis_oidcnt);
/* If the NDIS module requested scatter/gather, init maps. */
if (sc->ndis_sc)
ndis_init_dma(sc);
/*
* See if the OID_802_11_CONFIGURATION OID is
* supported by this driver. If it is, then this an 802.11
* wireless driver, and we should set up media for wireless.
*/
for (i = 0; i < sc->ndis_oidcnt; i++)
if (sc->ndis_oids[i] == OID_802_11_CONFIGURATION) {
sc->ndis_80211++;
break;
}
if (sc->ndis_80211)
ifp = if_alloc(IFT_IEEE80211);
else
ifp = if_alloc(IFT_ETHER);
if (ifp == NULL) {
error = ENOSPC;
goto fail;
}
sc->ifp = ifp;
ifp->if_softc = sc;
/* Check for task offload support. */
ndis_probe_offload(sc);
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = ndis_ioctl;
ifp->if_start = ndis_start;
ifp->if_init = ndis_init;
ifp->if_baudrate = 10000000;
IFQ_SET_MAXLEN(&ifp->if_snd, 50);
ifp->if_snd.ifq_drv_maxlen = 25;
IFQ_SET_READY(&ifp->if_snd);
ifp->if_capenable = ifp->if_capabilities;
ifp->if_hwassist = sc->ndis_hwassist;
/* Do media setup */
if (sc->ndis_80211) {
struct ieee80211com *ic = ifp->if_l2com;
ndis_80211_rates_ex rates;
struct ndis_80211_nettype_list *ntl;
uint32_t arg;
int r;
- callout_init(&sc->ndis_scan_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->ndis_scan_callout, 1);
ifp->if_ioctl = ndis_ioctl_80211;
ic->ic_ifp = ifp;
ic->ic_opmode = IEEE80211_M_STA;
ic->ic_phytype = IEEE80211_T_DS;
ic->ic_caps = IEEE80211_C_8023ENCAP |
IEEE80211_C_STA | IEEE80211_C_IBSS;
setbit(ic->ic_modecaps, IEEE80211_MODE_AUTO);
len = 0;
r = ndis_get_info(sc, OID_802_11_NETWORK_TYPES_SUPPORTED,
NULL, &len);
if (r != ENOSPC)
goto nonettypes;
ntl = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
r = ndis_get_info(sc, OID_802_11_NETWORK_TYPES_SUPPORTED,
ntl, &len);
if (r != 0) {
free(ntl, M_DEVBUF);
goto nonettypes;
}
for (i = 0; i < ntl->ntl_items; i++) {
mode = ndis_nettype_mode(ntl->ntl_type[i]);
if (mode) {
setbit(ic->ic_modecaps, mode);
setbit(&bands, mode);
} else
device_printf(dev, "Unknown nettype %d\n",
ntl->ntl_type[i]);
}
free(ntl, M_DEVBUF);
nonettypes:
/* Default to 11b channels if the card did not supply any */
if (bands == 0) {
setbit(ic->ic_modecaps, IEEE80211_MODE_11B);
setbit(&bands, IEEE80211_MODE_11B);
}
len = sizeof(rates);
bzero((char *)&rates, len);
r = ndis_get_info(sc, OID_802_11_SUPPORTED_RATES,
(void *)rates, &len);
if (r)
device_printf(dev, "get rates failed: 0x%x\n", r);
/*
* Since the supported rates only up to 8 can be supported,
* if this is not 802.11b we're just going to be faking it
* all up to heck.
*/
#define TESTSETRATE(x, y) \
do { \
int i; \
for (i = 0; i < ic->ic_sup_rates[x].rs_nrates; i++) { \
if (ic->ic_sup_rates[x].rs_rates[i] == (y)) \
break; \
} \
if (i == ic->ic_sup_rates[x].rs_nrates) { \
ic->ic_sup_rates[x].rs_rates[i] = (y); \
ic->ic_sup_rates[x].rs_nrates++; \
} \
} while (0)
#define SETRATE(x, y) \
ic->ic_sup_rates[x].rs_rates[ic->ic_sup_rates[x].rs_nrates] = (y)
#define INCRATE(x) \
ic->ic_sup_rates[x].rs_nrates++
ic->ic_curmode = IEEE80211_MODE_AUTO;
if (isset(ic->ic_modecaps, IEEE80211_MODE_11A))
ic->ic_sup_rates[IEEE80211_MODE_11A].rs_nrates = 0;
if (isset(ic->ic_modecaps, IEEE80211_MODE_11B))
ic->ic_sup_rates[IEEE80211_MODE_11B].rs_nrates = 0;
if (isset(ic->ic_modecaps, IEEE80211_MODE_11G))
ic->ic_sup_rates[IEEE80211_MODE_11G].rs_nrates = 0;
for (i = 0; i < len; i++) {
switch (rates[i] & IEEE80211_RATE_VAL) {
case 2:
case 4:
case 11:
case 10:
case 22:
if (isclr(ic->ic_modecaps, IEEE80211_MODE_11B)) {
/* Lazy-init 802.11b. */
setbit(ic->ic_modecaps,
IEEE80211_MODE_11B);
ic->ic_sup_rates[IEEE80211_MODE_11B].
rs_nrates = 0;
}
SETRATE(IEEE80211_MODE_11B, rates[i]);
INCRATE(IEEE80211_MODE_11B);
break;
default:
if (isset(ic->ic_modecaps, IEEE80211_MODE_11A)) {
SETRATE(IEEE80211_MODE_11A, rates[i]);
INCRATE(IEEE80211_MODE_11A);
}
if (isset(ic->ic_modecaps, IEEE80211_MODE_11G)) {
SETRATE(IEEE80211_MODE_11G, rates[i]);
INCRATE(IEEE80211_MODE_11G);
}
break;
}
}
/*
* If the hardware supports 802.11g, it most
* likely supports 802.11b and all of the
* 802.11b and 802.11g speeds, so maybe we can
* just cheat here. Just how in the heck do
* we detect turbo modes, though?
*/
if (isset(ic->ic_modecaps, IEEE80211_MODE_11B)) {
TESTSETRATE(IEEE80211_MODE_11B,
IEEE80211_RATE_BASIC|2);
TESTSETRATE(IEEE80211_MODE_11B,
IEEE80211_RATE_BASIC|4);
TESTSETRATE(IEEE80211_MODE_11B,
IEEE80211_RATE_BASIC|11);
TESTSETRATE(IEEE80211_MODE_11B,
IEEE80211_RATE_BASIC|22);
}
if (isset(ic->ic_modecaps, IEEE80211_MODE_11G)) {
TESTSETRATE(IEEE80211_MODE_11G, 48);
TESTSETRATE(IEEE80211_MODE_11G, 72);
TESTSETRATE(IEEE80211_MODE_11G, 96);
TESTSETRATE(IEEE80211_MODE_11G, 108);
}
if (isset(ic->ic_modecaps, IEEE80211_MODE_11A)) {
TESTSETRATE(IEEE80211_MODE_11A, 48);
TESTSETRATE(IEEE80211_MODE_11A, 72);
TESTSETRATE(IEEE80211_MODE_11A, 96);
TESTSETRATE(IEEE80211_MODE_11A, 108);
}
#undef SETRATE
#undef INCRATE
ieee80211_init_channels(ic, NULL, &bands);
/*
* To test for WPA support, we need to see if we can
* set AUTHENTICATION_MODE to WPA and read it back
* successfully.
*/
i = sizeof(arg);
arg = NDIS_80211_AUTHMODE_WPA;
r = ndis_set_info(sc,
OID_802_11_AUTHENTICATION_MODE, &arg, &i);
if (r == 0) {
r = ndis_get_info(sc,
OID_802_11_AUTHENTICATION_MODE, &arg, &i);
if (r == 0 && arg == NDIS_80211_AUTHMODE_WPA)
ic->ic_caps |= IEEE80211_C_WPA;
}
/*
* To test for supported ciphers, we set each
* available encryption type in descending order.
* If ENC3 works, then we have WEP, TKIP and AES.
* If only ENC2 works, then we have WEP and TKIP.
* If only ENC1 works, then we have just WEP.
*/
i = sizeof(arg);
arg = NDIS_80211_WEPSTAT_ENC3ENABLED;
r = ndis_set_info(sc, OID_802_11_ENCRYPTION_STATUS, &arg, &i);
if (r == 0) {
ic->ic_cryptocaps |= IEEE80211_CRYPTO_WEP
| IEEE80211_CRYPTO_TKIP
| IEEE80211_CRYPTO_AES_CCM;
goto got_crypto;
}
arg = NDIS_80211_WEPSTAT_ENC2ENABLED;
r = ndis_set_info(sc, OID_802_11_ENCRYPTION_STATUS, &arg, &i);
if (r == 0) {
ic->ic_cryptocaps |= IEEE80211_CRYPTO_WEP
| IEEE80211_CRYPTO_TKIP;
goto got_crypto;
}
arg = NDIS_80211_WEPSTAT_ENC1ENABLED;
r = ndis_set_info(sc, OID_802_11_ENCRYPTION_STATUS, &arg, &i);
if (r == 0)
ic->ic_cryptocaps |= IEEE80211_CRYPTO_WEP;
got_crypto:
i = sizeof(arg);
r = ndis_get_info(sc, OID_802_11_POWER_MODE, &arg, &i);
if (r == 0)
ic->ic_caps |= IEEE80211_C_PMGT;
r = ndis_get_info(sc, OID_802_11_TX_POWER_LEVEL, &arg, &i);
if (r == 0)
ic->ic_caps |= IEEE80211_C_TXPMGT;
ieee80211_ifattach(ic, eaddr);
ic->ic_raw_xmit = ndis_raw_xmit;
ic->ic_scan_start = ndis_scan_start;
ic->ic_scan_end = ndis_scan_end;
ic->ic_set_channel = ndis_set_channel;
ic->ic_scan_curchan = ndis_scan_curchan;
ic->ic_scan_mindwell = ndis_scan_mindwell;
ic->ic_bsschan = IEEE80211_CHAN_ANYC;
//ic->ic_bss->ni_chan = ic->ic_bsschan;
ic->ic_vap_create = ndis_vap_create;
ic->ic_vap_delete = ndis_vap_delete;
ic->ic_update_mcast = ndis_update_mcast;
ic->ic_update_promisc = ndis_update_promisc;
if (bootverbose)
ieee80211_announce(ic);
} else {
ifmedia_init(&sc->ifmedia, IFM_IMASK, ndis_ifmedia_upd,
ndis_ifmedia_sts);
ifmedia_add(&sc->ifmedia, IFM_ETHER|IFM_10_T, 0, NULL);
ifmedia_add(&sc->ifmedia, IFM_ETHER|IFM_10_T|IFM_FDX, 0, NULL);
ifmedia_add(&sc->ifmedia, IFM_ETHER|IFM_100_TX, 0, NULL);
ifmedia_add(&sc->ifmedia,
IFM_ETHER|IFM_100_TX|IFM_FDX, 0, NULL);
ifmedia_add(&sc->ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL);
ifmedia_set(&sc->ifmedia, IFM_ETHER|IFM_AUTO);
ether_ifattach(ifp, eaddr);
}
fail:
if (error) {
ndis_detach(dev);
return (error);
}
if (sc->ndis_iftype == PNPBus && ndisusb_halt == 0)
return (error);
DPRINTF(("attach done.\n"));
/* We're done talking to the NIC for now; halt it. */
ndis_halt_nic(sc);
DPRINTF(("halting done.\n"));
return (error);
}
static struct ieee80211vap *
ndis_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
enum ieee80211_opmode opmode, int flags,
const uint8_t bssid[IEEE80211_ADDR_LEN],
const uint8_t mac[IEEE80211_ADDR_LEN])
{
struct ndis_vap *nvp;
struct ieee80211vap *vap;
if (!TAILQ_EMPTY(&ic->ic_vaps)) /* only one at a time */
return NULL;
nvp = (struct ndis_vap *) malloc(sizeof(struct ndis_vap),
M_80211_VAP, M_NOWAIT | M_ZERO);
if (nvp == NULL)
return NULL;
vap = &nvp->vap;
ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid, mac);
/* override with driver methods */
nvp->newstate = vap->iv_newstate;
vap->iv_newstate = ndis_newstate;
/* complete setup */
ieee80211_vap_attach(vap, ieee80211_media_change, ndis_media_status);
ic->ic_opmode = opmode;
/* install key handing routines */
vap->iv_key_set = ndis_add_key;
vap->iv_key_delete = ndis_del_key;
return vap;
}
static void
ndis_vap_delete(struct ieee80211vap *vap)
{
struct ndis_vap *nvp = NDIS_VAP(vap);
struct ieee80211com *ic = vap->iv_ic;
struct ifnet *ifp = ic->ic_ifp;
struct ndis_softc *sc = ifp->if_softc;
ndis_stop(sc);
callout_drain(&sc->ndis_scan_callout);
ieee80211_vap_detach(vap);
free(nvp, M_80211_VAP);
}
/*
* Shutdown hardware and free up resources. This can be called any
* time after the mutex has been initialized. It is called in both
* the error case in attach and the normal detach case so it needs
* to be careful about only freeing resources that have actually been
* allocated.
*/
int
ndis_detach(dev)
device_t dev;
{
struct ndis_softc *sc;
struct ifnet *ifp;
driver_object *drv;
sc = device_get_softc(dev);
NDIS_LOCK(sc);
ifp = sc->ifp;
if (ifp != NULL)
ifp->if_flags &= ~IFF_UP;
if (device_is_attached(dev)) {
NDIS_UNLOCK(sc);
ndis_stop(sc);
if (ifp != NULL) {
if (sc->ndis_80211)
ieee80211_ifdetach(ifp->if_l2com);
else
ether_ifdetach(ifp);
}
} else
NDIS_UNLOCK(sc);
if (sc->ndis_tickitem != NULL)
IoFreeWorkItem(sc->ndis_tickitem);
if (sc->ndis_startitem != NULL)
IoFreeWorkItem(sc->ndis_startitem);
if (sc->ndis_resetitem != NULL)
IoFreeWorkItem(sc->ndis_resetitem);
if (sc->ndis_inputitem != NULL)
IoFreeWorkItem(sc->ndis_inputitem);
if (sc->ndisusb_xferdoneitem != NULL)
IoFreeWorkItem(sc->ndisusb_xferdoneitem);
if (sc->ndisusb_taskitem != NULL)
IoFreeWorkItem(sc->ndisusb_taskitem);
bus_generic_detach(dev);
ndis_unload_driver(sc);
if (sc->ndis_irq)
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->ndis_irq);
if (sc->ndis_res_io)
bus_release_resource(dev, SYS_RES_IOPORT,
sc->ndis_io_rid, sc->ndis_res_io);
if (sc->ndis_res_mem)
bus_release_resource(dev, SYS_RES_MEMORY,
sc->ndis_mem_rid, sc->ndis_res_mem);
if (sc->ndis_res_altmem)
bus_release_resource(dev, SYS_RES_MEMORY,
sc->ndis_altmem_rid, sc->ndis_res_altmem);
if (ifp != NULL)
if_free(ifp);
if (sc->ndis_iftype == PCMCIABus)
ndis_free_amem(sc);
if (sc->ndis_sc)
ndis_destroy_dma(sc);
if (sc->ndis_txarray)
free(sc->ndis_txarray, M_DEVBUF);
if (!sc->ndis_80211)
ifmedia_removeall(&sc->ifmedia);
if (sc->ndis_txpool != NULL)
NdisFreePacketPool(sc->ndis_txpool);
/* Destroy the PDO for this device. */
if (sc->ndis_iftype == PCIBus)
drv = windrv_lookup(0, "PCI Bus");
else if (sc->ndis_iftype == PCMCIABus)
drv = windrv_lookup(0, "PCCARD Bus");
else
drv = windrv_lookup(0, "USB Bus");
if (drv == NULL)
panic("couldn't find driver object");
windrv_destroy_pdo(drv, dev);
if (sc->ndis_iftype == PCIBus)
bus_dma_tag_destroy(sc->ndis_parent_tag);
return (0);
}
int
ndis_suspend(dev)
device_t dev;
{
struct ndis_softc *sc;
struct ifnet *ifp;
sc = device_get_softc(dev);
ifp = sc->ifp;
#ifdef notdef
if (NDIS_INITIALIZED(sc))
ndis_stop(sc);
#endif
return (0);
}
int
ndis_resume(dev)
device_t dev;
{
struct ndis_softc *sc;
struct ifnet *ifp;
sc = device_get_softc(dev);
ifp = sc->ifp;
if (NDIS_INITIALIZED(sc))
ndis_init(sc);
return (0);
}
/*
* The following bunch of routines are here to support drivers that
* use the NdisMEthIndicateReceive()/MiniportTransferData() mechanism.
* The NdisMEthIndicateReceive() handler runs at DISPATCH_LEVEL for
* serialized miniports, or IRQL <= DISPATCH_LEVEL for deserialized
* miniports.
*/
static void
ndis_rxeof_eth(adapter, ctx, addr, hdr, hdrlen, lookahead, lookaheadlen, pktlen)
ndis_handle adapter;
ndis_handle ctx;
char *addr;
void *hdr;
uint32_t hdrlen;
void *lookahead;
uint32_t lookaheadlen;
uint32_t pktlen;
{
ndis_miniport_block *block;
uint8_t irql = 0;
uint32_t status;
ndis_buffer *b;
ndis_packet *p;
struct mbuf *m;
ndis_ethpriv *priv;
block = adapter;
m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
if (m == NULL)
return;
/* Save the data provided to us so far. */
m->m_len = lookaheadlen + hdrlen;
m->m_pkthdr.len = pktlen + hdrlen;
m->m_next = NULL;
m_copyback(m, 0, hdrlen, hdr);
m_copyback(m, hdrlen, lookaheadlen, lookahead);
/* Now create a fake NDIS_PACKET to hold the data */
NdisAllocatePacket(&status, &p, block->nmb_rxpool);
if (status != NDIS_STATUS_SUCCESS) {
m_freem(m);
return;
}
p->np_m0 = m;
b = IoAllocateMdl(m->m_data, m->m_pkthdr.len, FALSE, FALSE, NULL);
if (b == NULL) {
NdisFreePacket(p);
m_freem(m);
return;
}
p->np_private.npp_head = p->np_private.npp_tail = b;
p->np_private.npp_totlen = m->m_pkthdr.len;
/* Save the packet RX context somewhere. */
priv = (ndis_ethpriv *)&p->np_protocolreserved;
priv->nep_ctx = ctx;
if (!NDIS_SERIALIZED(block))
KeAcquireSpinLock(&block->nmb_lock, &irql);
InsertTailList((&block->nmb_packetlist), (&p->np_list));
if (!NDIS_SERIALIZED(block))
KeReleaseSpinLock(&block->nmb_lock, irql);
}
/*
* NdisMEthIndicateReceiveComplete() handler, runs at DISPATCH_LEVEL
* for serialized miniports, or IRQL <= DISPATCH_LEVEL for deserialized
* miniports.
*/
static void
ndis_rxeof_done(adapter)
ndis_handle adapter;
{
struct ndis_softc *sc;
ndis_miniport_block *block;
block = adapter;
/* Schedule transfer/RX of queued packets. */
sc = device_get_softc(block->nmb_physdeviceobj->do_devext);
KeInsertQueueDpc(&sc->ndis_rxdpc, NULL, NULL);
}
/*
* MiniportTransferData() handler, runs at DISPATCH_LEVEL.
*/
static void
ndis_rxeof_xfr(dpc, adapter, sysarg1, sysarg2)
kdpc *dpc;
ndis_handle adapter;
void *sysarg1;
void *sysarg2;
{
ndis_miniport_block *block;
struct ndis_softc *sc;
ndis_packet *p;
list_entry *l;
uint32_t status;
ndis_ethpriv *priv;
struct ifnet *ifp;
struct mbuf *m;
block = adapter;
sc = device_get_softc(block->nmb_physdeviceobj->do_devext);
ifp = sc->ifp;
KeAcquireSpinLockAtDpcLevel(&block->nmb_lock);
l = block->nmb_packetlist.nle_flink;
while(!IsListEmpty(&block->nmb_packetlist)) {
l = RemoveHeadList((&block->nmb_packetlist));
p = CONTAINING_RECORD(l, ndis_packet, np_list);
InitializeListHead((&p->np_list));
priv = (ndis_ethpriv *)&p->np_protocolreserved;
m = p->np_m0;
p->np_softc = sc;
p->np_m0 = NULL;
KeReleaseSpinLockFromDpcLevel(&block->nmb_lock);
status = MSCALL6(sc->ndis_chars->nmc_transferdata_func,
p, &p->np_private.npp_totlen, block, priv->nep_ctx,
m->m_len, m->m_pkthdr.len - m->m_len);
KeAcquireSpinLockAtDpcLevel(&block->nmb_lock);
/*
* If status is NDIS_STATUS_PENDING, do nothing and
* wait for a callback to the ndis_rxeof_xfr_done()
* handler.
*/
m->m_len = m->m_pkthdr.len;
m->m_pkthdr.rcvif = ifp;
if (status == NDIS_STATUS_SUCCESS) {
IoFreeMdl(p->np_private.npp_head);
NdisFreePacket(p);
KeAcquireSpinLockAtDpcLevel(&sc->ndis_rxlock);
_IF_ENQUEUE(&sc->ndis_rxqueue, m);
KeReleaseSpinLockFromDpcLevel(&sc->ndis_rxlock);
IoQueueWorkItem(sc->ndis_inputitem,
(io_workitem_func)ndis_inputtask_wrap,
WORKQUEUE_CRITICAL, ifp);
}
if (status == NDIS_STATUS_FAILURE)
m_freem(m);
/* Advance to next packet */
l = block->nmb_packetlist.nle_flink;
}
KeReleaseSpinLockFromDpcLevel(&block->nmb_lock);
}
/*
* NdisMTransferDataComplete() handler, runs at DISPATCH_LEVEL.
*/
static void
ndis_rxeof_xfr_done(adapter, packet, status, len)
ndis_handle adapter;
ndis_packet *packet;
uint32_t status;
uint32_t len;
{
ndis_miniport_block *block;
struct ndis_softc *sc;
struct ifnet *ifp;
struct mbuf *m;
block = adapter;
sc = device_get_softc(block->nmb_physdeviceobj->do_devext);
ifp = sc->ifp;
m = packet->np_m0;
IoFreeMdl(packet->np_private.npp_head);
NdisFreePacket(packet);
if (status != NDIS_STATUS_SUCCESS) {
m_freem(m);
return;
}
m->m_len = m->m_pkthdr.len;
m->m_pkthdr.rcvif = ifp;
KeAcquireSpinLockAtDpcLevel(&sc->ndis_rxlock);
_IF_ENQUEUE(&sc->ndis_rxqueue, m);
KeReleaseSpinLockFromDpcLevel(&sc->ndis_rxlock);
IoQueueWorkItem(sc->ndis_inputitem,
(io_workitem_func)ndis_inputtask_wrap,
WORKQUEUE_CRITICAL, ifp);
}
/*
* A frame has been uploaded: pass the resulting mbuf chain up to
* the higher level protocols.
*
* When handling received NDIS packets, the 'status' field in the
* out-of-band portion of the ndis_packet has special meaning. In the
* most common case, the underlying NDIS driver will set this field
* to NDIS_STATUS_SUCCESS, which indicates that it's ok for us to
* take posession of it. We then change the status field to
* NDIS_STATUS_PENDING to tell the driver that we now own the packet,
* and that we will return it at some point in the future via the
* return packet handler.
*
* If the driver hands us a packet with a status of NDIS_STATUS_RESOURCES,
* this means the driver is running out of packet/buffer resources and
* wants to maintain ownership of the packet. In this case, we have to
* copy the packet data into local storage and let the driver keep the
* packet.
*/
static void
ndis_rxeof(adapter, packets, pktcnt)
ndis_handle adapter;
ndis_packet **packets;
uint32_t pktcnt;
{
struct ndis_softc *sc;
ndis_miniport_block *block;
ndis_packet *p;
uint32_t s;
ndis_tcpip_csum *csum;
struct ifnet *ifp;
struct mbuf *m0, *m;
int i;
block = (ndis_miniport_block *)adapter;
sc = device_get_softc(block->nmb_physdeviceobj->do_devext);
ifp = sc->ifp;
/*
* There's a slim chance the driver may indicate some packets
* before we're completely ready to handle them. If we detect this,
* we need to return them to the miniport and ignore them.
*/
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
for (i = 0; i < pktcnt; i++) {
p = packets[i];
if (p->np_oob.npo_status == NDIS_STATUS_SUCCESS) {
p->np_refcnt++;
(void)ndis_return_packet(NULL ,p, block);
}
}
return;
}
for (i = 0; i < pktcnt; i++) {
p = packets[i];
/* Stash the softc here so ptom can use it. */
p->np_softc = sc;
if (ndis_ptom(&m0, p)) {
device_printf(sc->ndis_dev, "ptom failed\n");
if (p->np_oob.npo_status == NDIS_STATUS_SUCCESS)
(void)ndis_return_packet(NULL, p, block);
} else {
#ifdef notdef
if (p->np_oob.npo_status == NDIS_STATUS_RESOURCES) {
m = m_dup(m0, M_NOWAIT);
/*
* NOTE: we want to destroy the mbuf here, but
* we don't actually want to return it to the
* driver via the return packet handler. By
* bumping np_refcnt, we can prevent the
* ndis_return_packet() routine from actually
* doing anything.
*/
p->np_refcnt++;
m_freem(m0);
if (m == NULL)
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
else
m0 = m;
} else
p->np_oob.npo_status = NDIS_STATUS_PENDING;
#endif
m = m_dup(m0, M_NOWAIT);
if (p->np_oob.npo_status == NDIS_STATUS_RESOURCES)
p->np_refcnt++;
else
p->np_oob.npo_status = NDIS_STATUS_PENDING;
m_freem(m0);
if (m == NULL) {
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
continue;
}
m0 = m;
m0->m_pkthdr.rcvif = ifp;
/* Deal with checksum offload. */
if (ifp->if_capenable & IFCAP_RXCSUM &&
p->np_ext.npe_info[ndis_tcpipcsum_info] != NULL) {
s = (uintptr_t)
p->np_ext.npe_info[ndis_tcpipcsum_info];
csum = (ndis_tcpip_csum *)&s;
if (csum->u.ntc_rxflags &
NDIS_RXCSUM_IP_PASSED)
m0->m_pkthdr.csum_flags |=
CSUM_IP_CHECKED|CSUM_IP_VALID;
if (csum->u.ntc_rxflags &
(NDIS_RXCSUM_TCP_PASSED |
NDIS_RXCSUM_UDP_PASSED)) {
m0->m_pkthdr.csum_flags |=
CSUM_DATA_VALID|CSUM_PSEUDO_HDR;
m0->m_pkthdr.csum_data = 0xFFFF;
}
}
KeAcquireSpinLockAtDpcLevel(&sc->ndis_rxlock);
_IF_ENQUEUE(&sc->ndis_rxqueue, m0);
KeReleaseSpinLockFromDpcLevel(&sc->ndis_rxlock);
IoQueueWorkItem(sc->ndis_inputitem,
(io_workitem_func)ndis_inputtask_wrap,
WORKQUEUE_CRITICAL, ifp);
}
}
}
/*
* This routine is run at PASSIVE_LEVEL. We use this routine to pass
* packets into the stack in order to avoid calling (*ifp->if_input)()
* with any locks held (at DISPATCH_LEVEL, we'll be holding the
* 'dispatch level' per-cpu sleep lock).
*/
static void
ndis_inputtask(dobj, arg)
device_object *dobj;
void *arg;
{
ndis_miniport_block *block;
struct ifnet *ifp;
struct ndis_softc *sc;
struct mbuf *m;
struct ieee80211com *ic;
struct ieee80211vap *vap;
uint8_t irql;
ifp = arg;
sc = ifp->if_softc;
ic = ifp->if_l2com;
vap = TAILQ_FIRST(&ic->ic_vaps);
block = dobj->do_devext;
KeAcquireSpinLock(&sc->ndis_rxlock, &irql);
while(1) {
_IF_DEQUEUE(&sc->ndis_rxqueue, m);
if (m == NULL)
break;
KeReleaseSpinLock(&sc->ndis_rxlock, irql);
if ((sc->ndis_80211 != 0) && (vap != NULL))
vap->iv_deliver_data(vap, vap->iv_bss, m);
else
(*ifp->if_input)(ifp, m);
KeAcquireSpinLock(&sc->ndis_rxlock, &irql);
}
KeReleaseSpinLock(&sc->ndis_rxlock, irql);
}
/*
* A frame was downloaded to the chip. It's safe for us to clean up
* the list buffers.
*/
static void
ndis_txeof(adapter, packet, status)
ndis_handle adapter;
ndis_packet *packet;
ndis_status status;
{
struct ndis_softc *sc;
ndis_miniport_block *block;
struct ifnet *ifp;
int idx;
struct mbuf *m;
block = (ndis_miniport_block *)adapter;
sc = device_get_softc(block->nmb_physdeviceobj->do_devext);
ifp = sc->ifp;
m = packet->np_m0;
idx = packet->np_txidx;
if (sc->ndis_sc)
bus_dmamap_unload(sc->ndis_ttag, sc->ndis_tmaps[idx]);
ndis_free_packet(packet);
m_freem(m);
NDIS_LOCK(sc);
sc->ndis_txarray[idx] = NULL;
sc->ndis_txpending++;
if (status == NDIS_STATUS_SUCCESS)
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
else
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
sc->ndis_tx_timer = 0;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
NDIS_UNLOCK(sc);
IoQueueWorkItem(sc->ndis_startitem,
(io_workitem_func)ndis_starttask_wrap,
WORKQUEUE_CRITICAL, ifp);
}
static void
ndis_linksts(adapter, status, sbuf, slen)
ndis_handle adapter;
ndis_status status;
void *sbuf;
uint32_t slen;
{
ndis_miniport_block *block;
struct ndis_softc *sc;
block = adapter;
sc = device_get_softc(block->nmb_physdeviceobj->do_devext);
sc->ndis_sts = status;
/* Event list is all full up, drop this one. */
NDIS_LOCK(sc);
if (sc->ndis_evt[sc->ndis_evtpidx].ne_sts) {
NDIS_UNLOCK(sc);
return;
}
/* Cache the event. */
if (slen) {
sc->ndis_evt[sc->ndis_evtpidx].ne_buf = malloc(slen,
M_TEMP, M_NOWAIT);
if (sc->ndis_evt[sc->ndis_evtpidx].ne_buf == NULL) {
NDIS_UNLOCK(sc);
return;
}
bcopy((char *)sbuf,
sc->ndis_evt[sc->ndis_evtpidx].ne_buf, slen);
}
sc->ndis_evt[sc->ndis_evtpidx].ne_sts = status;
sc->ndis_evt[sc->ndis_evtpidx].ne_len = slen;
NDIS_EVTINC(sc->ndis_evtpidx);
NDIS_UNLOCK(sc);
}
static void
ndis_linksts_done(adapter)
ndis_handle adapter;
{
ndis_miniport_block *block;
struct ndis_softc *sc;
struct ifnet *ifp;
block = adapter;
sc = device_get_softc(block->nmb_physdeviceobj->do_devext);
ifp = sc->ifp;
if (!NDIS_INITIALIZED(sc))
return;
switch (sc->ndis_sts) {
case NDIS_STATUS_MEDIA_CONNECT:
IoQueueWorkItem(sc->ndis_tickitem,
(io_workitem_func)ndis_ticktask_wrap,
WORKQUEUE_CRITICAL, sc);
IoQueueWorkItem(sc->ndis_startitem,
(io_workitem_func)ndis_starttask_wrap,
WORKQUEUE_CRITICAL, ifp);
break;
case NDIS_STATUS_MEDIA_DISCONNECT:
if (sc->ndis_link)
IoQueueWorkItem(sc->ndis_tickitem,
(io_workitem_func)ndis_ticktask_wrap,
WORKQUEUE_CRITICAL, sc);
break;
default:
break;
}
}
static void
ndis_tick(xsc)
void *xsc;
{
struct ndis_softc *sc;
sc = xsc;
if (sc->ndis_hang_timer && --sc->ndis_hang_timer == 0) {
IoQueueWorkItem(sc->ndis_tickitem,
(io_workitem_func)ndis_ticktask_wrap,
WORKQUEUE_CRITICAL, sc);
sc->ndis_hang_timer = sc->ndis_block->nmb_checkforhangsecs;
}
if (sc->ndis_tx_timer && --sc->ndis_tx_timer == 0) {
if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1);
device_printf(sc->ndis_dev, "watchdog timeout\n");
IoQueueWorkItem(sc->ndis_resetitem,
(io_workitem_func)ndis_resettask_wrap,
WORKQUEUE_CRITICAL, sc);
IoQueueWorkItem(sc->ndis_startitem,
(io_workitem_func)ndis_starttask_wrap,
WORKQUEUE_CRITICAL, sc->ifp);
}
callout_reset(&sc->ndis_stat_callout, hz, ndis_tick, sc);
}
static void
ndis_ticktask(d, xsc)
device_object *d;
void *xsc;
{
struct ndis_softc *sc;
struct ieee80211com *ic;
struct ieee80211vap *vap;
ndis_checkforhang_handler hangfunc;
uint8_t rval;
sc = xsc;
ic = sc->ifp->if_l2com;
vap = TAILQ_FIRST(&ic->ic_vaps);
NDIS_LOCK(sc);
if (!NDIS_INITIALIZED(sc)) {
NDIS_UNLOCK(sc);
return;
}
NDIS_UNLOCK(sc);
hangfunc = sc->ndis_chars->nmc_checkhang_func;
if (hangfunc != NULL) {
rval = MSCALL1(hangfunc,
sc->ndis_block->nmb_miniportadapterctx);
if (rval == TRUE) {
ndis_reset_nic(sc);
return;
}
}
NDIS_LOCK(sc);
if (sc->ndis_link == 0 &&
sc->ndis_sts == NDIS_STATUS_MEDIA_CONNECT) {
sc->ndis_link = 1;
if ((sc->ndis_80211 != 0) && (vap != NULL)) {
NDIS_UNLOCK(sc);
ndis_getstate_80211(sc);
ieee80211_new_state(vap, IEEE80211_S_RUN, -1);
NDIS_LOCK(sc);
if_link_state_change(vap->iv_ifp, LINK_STATE_UP);
} else
if_link_state_change(sc->ifp, LINK_STATE_UP);
}
if (sc->ndis_link == 1 &&
sc->ndis_sts == NDIS_STATUS_MEDIA_DISCONNECT) {
sc->ndis_link = 0;
if ((sc->ndis_80211 != 0) && (vap != NULL)) {
NDIS_UNLOCK(sc);
ieee80211_new_state(vap, IEEE80211_S_SCAN, 0);
NDIS_LOCK(sc);
if_link_state_change(vap->iv_ifp, LINK_STATE_DOWN);
} else
if_link_state_change(sc->ifp, LINK_STATE_DOWN);
}
NDIS_UNLOCK(sc);
}
static void
ndis_map_sclist(arg, segs, nseg, mapsize, error)
void *arg;
bus_dma_segment_t *segs;
int nseg;
bus_size_t mapsize;
int error;
{
struct ndis_sc_list *sclist;
int i;
if (error || arg == NULL)
return;
sclist = arg;
sclist->nsl_frags = nseg;
for (i = 0; i < nseg; i++) {
sclist->nsl_elements[i].nse_addr.np_quad = segs[i].ds_addr;
sclist->nsl_elements[i].nse_len = segs[i].ds_len;
}
}
static int
ndis_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
const struct ieee80211_bpf_params *params)
{
/* no support; just discard */
m_freem(m);
ieee80211_free_node(ni);
return (0);
}
static void
ndis_update_mcast(struct ifnet *ifp)
{
struct ndis_softc *sc = ifp->if_softc;
ndis_setmulti(sc);
}
static void
ndis_update_promisc(struct ifnet *ifp)
{
/* not supported */
}
static void
ndis_starttask(d, arg)
device_object *d;
void *arg;
{
struct ifnet *ifp;
ifp = arg;
if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
ndis_start(ifp);
}
/*
* Main transmit routine. To make NDIS drivers happy, we need to
* transform mbuf chains into NDIS packets and feed them to the
* send packet routines. Most drivers allow you to send several
* packets at once (up to the maxpkts limit). Unfortunately, rather
* that accepting them in the form of a linked list, they expect
* a contiguous array of pointers to packets.
*
* For those drivers which use the NDIS scatter/gather DMA mechanism,
* we need to perform busdma work here. Those that use map registers
* will do the mapping themselves on a buffer by buffer basis.
*/
static void
ndis_start(ifp)
struct ifnet *ifp;
{
struct ndis_softc *sc;
struct mbuf *m = NULL;
ndis_packet **p0 = NULL, *p = NULL;
ndis_tcpip_csum *csum;
int pcnt = 0, status;
sc = ifp->if_softc;
NDIS_LOCK(sc);
if (!sc->ndis_link || ifp->if_drv_flags & IFF_DRV_OACTIVE) {
NDIS_UNLOCK(sc);
return;
}
p0 = &sc->ndis_txarray[sc->ndis_txidx];
while(sc->ndis_txpending) {
IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
if (m == NULL)
break;
NdisAllocatePacket(&status,
&sc->ndis_txarray[sc->ndis_txidx], sc->ndis_txpool);
if (status != NDIS_STATUS_SUCCESS)
break;
if (ndis_mtop(m, &sc->ndis_txarray[sc->ndis_txidx])) {
IFQ_DRV_PREPEND(&ifp->if_snd, m);
NDIS_UNLOCK(sc);
return;
}
/*
* Save pointer to original mbuf
* so we can free it later.
*/
p = sc->ndis_txarray[sc->ndis_txidx];
p->np_txidx = sc->ndis_txidx;
p->np_m0 = m;
p->np_oob.npo_status = NDIS_STATUS_PENDING;
/*
* Do scatter/gather processing, if driver requested it.
*/
if (sc->ndis_sc) {
bus_dmamap_load_mbuf(sc->ndis_ttag,
sc->ndis_tmaps[sc->ndis_txidx], m,
ndis_map_sclist, &p->np_sclist, BUS_DMA_NOWAIT);
bus_dmamap_sync(sc->ndis_ttag,
sc->ndis_tmaps[sc->ndis_txidx],
BUS_DMASYNC_PREREAD);
p->np_ext.npe_info[ndis_sclist_info] = &p->np_sclist;
}
/* Handle checksum offload. */
if (ifp->if_capenable & IFCAP_TXCSUM &&
m->m_pkthdr.csum_flags) {
csum = (ndis_tcpip_csum *)
&p->np_ext.npe_info[ndis_tcpipcsum_info];
csum->u.ntc_txflags = NDIS_TXCSUM_DO_IPV4;
if (m->m_pkthdr.csum_flags & CSUM_IP)
csum->u.ntc_txflags |= NDIS_TXCSUM_DO_IP;
if (m->m_pkthdr.csum_flags & CSUM_TCP)
csum->u.ntc_txflags |= NDIS_TXCSUM_DO_TCP;
if (m->m_pkthdr.csum_flags & CSUM_UDP)
csum->u.ntc_txflags |= NDIS_TXCSUM_DO_UDP;
p->np_private.npp_flags = NDIS_PROTOCOL_ID_TCP_IP;
}
NDIS_INC(sc);
sc->ndis_txpending--;
pcnt++;
/*
* If there's a BPF listener, bounce a copy of this frame
* to him.
*/
if (!sc->ndis_80211) /* XXX handle 80211 */
BPF_MTAP(ifp, m);
/*
* The array that p0 points to must appear contiguous,
* so we must not wrap past the end of sc->ndis_txarray[].
* If it looks like we're about to wrap, break out here
* so the this batch of packets can be transmitted, then
* wait for txeof to ask us to send the rest.
*/
if (sc->ndis_txidx == 0)
break;
}
if (pcnt == 0) {
NDIS_UNLOCK(sc);
return;
}
if (sc->ndis_txpending == 0)
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
/*
* Set a timeout in case the chip goes out to lunch.
*/
sc->ndis_tx_timer = 5;
NDIS_UNLOCK(sc);
/*
* According to NDIS documentation, if a driver exports
* a MiniportSendPackets() routine, we prefer that over
* a MiniportSend() routine (which sends just a single
* packet).
*/
if (sc->ndis_chars->nmc_sendmulti_func != NULL)
ndis_send_packets(sc, p0, pcnt);
else
ndis_send_packet(sc, p);
return;
}
static void
ndis_init(xsc)
void *xsc;
{
struct ndis_softc *sc = xsc;
struct ifnet *ifp = sc->ifp;
struct ieee80211com *ic = ifp->if_l2com;
int i, len, error;
/*
* Avoid reintializing the link unnecessarily.
* This should be dealt with in a better way by
* fixing the upper layer modules so they don't
* call ifp->if_init() quite as often.
*/
if (sc->ndis_link)
return;
/*
* Cancel pending I/O and free all RX/TX buffers.
*/
ndis_stop(sc);
if (!(sc->ndis_iftype == PNPBus && ndisusb_halt == 0)) {
error = ndis_init_nic(sc);
if (error != 0) {
device_printf(sc->ndis_dev,
"failed to initialize the device: %d\n", error);
return;
}
}
/* Init our MAC address */
/* Program the packet filter */
sc->ndis_filter = NDIS_PACKET_TYPE_DIRECTED;
if (ifp->if_flags & IFF_BROADCAST)
sc->ndis_filter |= NDIS_PACKET_TYPE_BROADCAST;
if (ifp->if_flags & IFF_PROMISC)
sc->ndis_filter |= NDIS_PACKET_TYPE_PROMISCUOUS;
len = sizeof(sc->ndis_filter);
error = ndis_set_info(sc, OID_GEN_CURRENT_PACKET_FILTER,
&sc->ndis_filter, &len);
if (error)
device_printf(sc->ndis_dev, "set filter failed: %d\n", error);
/*
* Set lookahead.
*/
i = ifp->if_mtu;
len = sizeof(i);
ndis_set_info(sc, OID_GEN_CURRENT_LOOKAHEAD, &i, &len);
/*
* Program the multicast filter, if necessary.
*/
ndis_setmulti(sc);
/* Setup task offload. */
ndis_set_offload(sc);
NDIS_LOCK(sc);
sc->ndis_txidx = 0;
sc->ndis_txpending = sc->ndis_maxpkts;
sc->ndis_link = 0;
if_link_state_change(sc->ifp, LINK_STATE_UNKNOWN);
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
sc->ndis_tx_timer = 0;
/*
* Some drivers don't set this value. The NDIS spec says
* the default checkforhang timeout is "approximately 2
* seconds." We use 3 seconds, because it seems for some
* drivers, exactly 2 seconds is too fast.
*/
if (sc->ndis_block->nmb_checkforhangsecs == 0)
sc->ndis_block->nmb_checkforhangsecs = 3;
sc->ndis_hang_timer = sc->ndis_block->nmb_checkforhangsecs;
callout_reset(&sc->ndis_stat_callout, hz, ndis_tick, sc);
NDIS_UNLOCK(sc);
/* XXX force handling */
if (sc->ndis_80211)
ieee80211_start_all(ic); /* start all vap's */
}
/*
* Set media options.
*/
static int
ndis_ifmedia_upd(ifp)
struct ifnet *ifp;
{
struct ndis_softc *sc;
sc = ifp->if_softc;
if (NDIS_INITIALIZED(sc))
ndis_init(sc);
return (0);
}
/*
* Report current media status.
*/
static void
ndis_ifmedia_sts(ifp, ifmr)
struct ifnet *ifp;
struct ifmediareq *ifmr;
{
struct ndis_softc *sc;
uint32_t media_info;
ndis_media_state linkstate;
int len;
ifmr->ifm_status = IFM_AVALID;
ifmr->ifm_active = IFM_ETHER;
sc = ifp->if_softc;
if (!NDIS_INITIALIZED(sc))
return;
len = sizeof(linkstate);
ndis_get_info(sc, OID_GEN_MEDIA_CONNECT_STATUS,
(void *)&linkstate, &len);
len = sizeof(media_info);
ndis_get_info(sc, OID_GEN_LINK_SPEED,
(void *)&media_info, &len);
if (linkstate == nmc_connected)
ifmr->ifm_status |= IFM_ACTIVE;
switch (media_info) {
case 100000:
ifmr->ifm_active |= IFM_10_T;
break;
case 1000000:
ifmr->ifm_active |= IFM_100_TX;
break;
case 10000000:
ifmr->ifm_active |= IFM_1000_T;
break;
default:
device_printf(sc->ndis_dev, "unknown speed: %d\n", media_info);
break;
}
}
static int
ndis_set_cipher(sc, cipher)
struct ndis_softc *sc;
int cipher;
{
struct ieee80211com *ic;
int rval = 0, len;
uint32_t arg, save;
ic = sc->ifp->if_l2com;
len = sizeof(arg);
if (cipher == WPA_CSE_WEP40 || cipher == WPA_CSE_WEP104) {
if (!(ic->ic_cryptocaps & IEEE80211_CRYPTO_WEP))
return (ENOTSUP);
arg = NDIS_80211_WEPSTAT_ENC1ENABLED;
}
if (cipher == WPA_CSE_TKIP) {
if (!(ic->ic_cryptocaps & IEEE80211_CRYPTO_TKIP))
return (ENOTSUP);
arg = NDIS_80211_WEPSTAT_ENC2ENABLED;
}
if (cipher == WPA_CSE_CCMP) {
if (!(ic->ic_cryptocaps & IEEE80211_CRYPTO_AES_CCM))
return (ENOTSUP);
arg = NDIS_80211_WEPSTAT_ENC3ENABLED;
}
DPRINTF(("Setting cipher to %d\n", arg));
save = arg;
rval = ndis_set_info(sc, OID_802_11_ENCRYPTION_STATUS, &arg, &len);
if (rval)
return (rval);
/* Check that the cipher was set correctly. */
len = sizeof(save);
rval = ndis_get_info(sc, OID_802_11_ENCRYPTION_STATUS, &arg, &len);
if (rval != 0 || arg != save)
return (ENODEV);
return (0);
}
/*
* WPA is hairy to set up. Do the work in a separate routine
* so we don't clutter the setstate function too much.
* Important yet undocumented fact: first we have to set the
* authentication mode, _then_ we enable the ciphers. If one
* of the WPA authentication modes isn't enabled, the driver
* might not permit the TKIP or AES ciphers to be selected.
*/
static int
ndis_set_wpa(sc, ie, ielen)
struct ndis_softc *sc;
void *ie;
int ielen;
{
struct ieee80211_ie_wpa *w;
struct ndis_ie *n;
char *pos;
uint32_t arg;
int i;
/*
* Apparently, the only way for us to know what ciphers
* and key management/authentication mode to use is for
* us to inspect the optional information element (IE)
* stored in the 802.11 state machine. This IE should be
* supplied by the WPA supplicant.
*/
w = (struct ieee80211_ie_wpa *)ie;
/* Check for the right kind of IE. */
if (w->wpa_id != IEEE80211_ELEMID_VENDOR) {
DPRINTF(("Incorrect IE type %d\n", w->wpa_id));
return (EINVAL);
}
/* Skip over the ucast cipher OIDs. */
pos = (char *)&w->wpa_uciphers[0];
pos += w->wpa_uciphercnt * sizeof(struct ndis_ie);
/* Skip over the authmode count. */
pos += sizeof(u_int16_t);
/*
* Check for the authentication modes. I'm
* pretty sure there's only supposed to be one.
*/
n = (struct ndis_ie *)pos;
if (n->ni_val == WPA_ASE_NONE)
arg = NDIS_80211_AUTHMODE_WPANONE;
if (n->ni_val == WPA_ASE_8021X_UNSPEC)
arg = NDIS_80211_AUTHMODE_WPA;
if (n->ni_val == WPA_ASE_8021X_PSK)
arg = NDIS_80211_AUTHMODE_WPAPSK;
DPRINTF(("Setting WPA auth mode to %d\n", arg));
i = sizeof(arg);
if (ndis_set_info(sc, OID_802_11_AUTHENTICATION_MODE, &arg, &i))
return (ENOTSUP);
i = sizeof(arg);
ndis_get_info(sc, OID_802_11_AUTHENTICATION_MODE, &arg, &i);
/* Now configure the desired ciphers. */
/* First, set up the multicast group cipher. */
n = (struct ndis_ie *)&w->wpa_mcipher[0];
if (ndis_set_cipher(sc, n->ni_val))
return (ENOTSUP);
/* Now start looking around for the unicast ciphers. */
pos = (char *)&w->wpa_uciphers[0];
n = (struct ndis_ie *)pos;
for (i = 0; i < w->wpa_uciphercnt; i++) {
if (ndis_set_cipher(sc, n->ni_val))
return (ENOTSUP);
n++;
}
return (0);
}
static void
ndis_media_status(struct ifnet *ifp, struct ifmediareq *imr)
{
struct ieee80211vap *vap = ifp->if_softc;
struct ndis_softc *sc = vap->iv_ic->ic_ifp->if_softc;
uint32_t txrate;
int len;
if (!NDIS_INITIALIZED(sc))
return;
len = sizeof(txrate);
if (ndis_get_info(sc, OID_GEN_LINK_SPEED, &txrate, &len) == 0)
vap->iv_bss->ni_txrate = txrate / 5000;
ieee80211_media_status(ifp, imr);
}
static void
ndis_setstate_80211(sc)
struct ndis_softc *sc;
{
struct ieee80211com *ic;
struct ieee80211vap *vap;
ndis_80211_macaddr bssid;
ndis_80211_config config;
int rval = 0, len;
uint32_t arg;
struct ifnet *ifp;
ifp = sc->ifp;
ic = ifp->if_l2com;
vap = TAILQ_FIRST(&ic->ic_vaps);
if (!NDIS_INITIALIZED(sc)) {
DPRINTF(("%s: NDIS not initialized\n", __func__));
return;
}
/* Disassociate and turn off radio. */
len = sizeof(arg);
arg = 1;
ndis_set_info(sc, OID_802_11_DISASSOCIATE, &arg, &len);
/* Set network infrastructure mode. */
len = sizeof(arg);
if (ic->ic_opmode == IEEE80211_M_IBSS)
arg = NDIS_80211_NET_INFRA_IBSS;
else
arg = NDIS_80211_NET_INFRA_BSS;
rval = ndis_set_info(sc, OID_802_11_INFRASTRUCTURE_MODE, &arg, &len);
if (rval)
device_printf (sc->ndis_dev, "set infra failed: %d\n", rval);
/* Set power management */
len = sizeof(arg);
if (vap->iv_flags & IEEE80211_F_PMGTON)
arg = NDIS_80211_POWERMODE_FAST_PSP;
else
arg = NDIS_80211_POWERMODE_CAM;
ndis_set_info(sc, OID_802_11_POWER_MODE, &arg, &len);
/* Set TX power */
if ((ic->ic_caps & IEEE80211_C_TXPMGT) &&
ic->ic_txpowlimit < (sizeof(dBm2mW) / sizeof(dBm2mW[0]))) {
arg = dBm2mW[ic->ic_txpowlimit];
len = sizeof(arg);
ndis_set_info(sc, OID_802_11_TX_POWER_LEVEL, &arg, &len);
}
/*
* Default encryption mode to off, authentication
* to open and privacy to 'accept everything.'
*/
len = sizeof(arg);
arg = NDIS_80211_WEPSTAT_DISABLED;
ndis_set_info(sc, OID_802_11_ENCRYPTION_STATUS, &arg, &len);
len = sizeof(arg);
arg = NDIS_80211_AUTHMODE_OPEN;
ndis_set_info(sc, OID_802_11_AUTHENTICATION_MODE, &arg, &len);
/*
* Note that OID_802_11_PRIVACY_FILTER is optional:
* not all drivers implement it.
*/
len = sizeof(arg);
arg = NDIS_80211_PRIVFILT_8021XWEP;
ndis_set_info(sc, OID_802_11_PRIVACY_FILTER, &arg, &len);
len = sizeof(config);
bzero((char *)&config, len);
config.nc_length = len;
config.nc_fhconfig.ncf_length = sizeof(ndis_80211_config_fh);
rval = ndis_get_info(sc, OID_802_11_CONFIGURATION, &config, &len);
/*
* Some drivers expect us to initialize these values, so
* provide some defaults.
*/
if (config.nc_beaconperiod == 0)
config.nc_beaconperiod = 100;
if (config.nc_atimwin == 0)
config.nc_atimwin = 100;
if (config.nc_fhconfig.ncf_dwelltime == 0)
config.nc_fhconfig.ncf_dwelltime = 200;
if (rval == 0 && ic->ic_bsschan != IEEE80211_CHAN_ANYC) {
int chan, chanflag;
chan = ieee80211_chan2ieee(ic, ic->ic_bsschan);
chanflag = config.nc_dsconfig > 2500000 ? IEEE80211_CHAN_2GHZ :
IEEE80211_CHAN_5GHZ;
if (chan != ieee80211_mhz2ieee(config.nc_dsconfig / 1000, 0)) {
config.nc_dsconfig =
ic->ic_bsschan->ic_freq * 1000;
len = sizeof(config);
config.nc_length = len;
config.nc_fhconfig.ncf_length =
sizeof(ndis_80211_config_fh);
DPRINTF(("Setting channel to %ukHz\n", config.nc_dsconfig));
rval = ndis_set_info(sc, OID_802_11_CONFIGURATION,
&config, &len);
if (rval)
device_printf(sc->ndis_dev, "couldn't change "
"DS config to %ukHz: %d\n",
config.nc_dsconfig, rval);
}
} else if (rval)
device_printf(sc->ndis_dev, "couldn't retrieve "
"channel info: %d\n", rval);
/* Set the BSSID to our value so the driver doesn't associate */
len = IEEE80211_ADDR_LEN;
bcopy(IF_LLADDR(ifp), bssid, len);
DPRINTF(("Setting BSSID to %6D\n", (uint8_t *)&bssid, ":"));
rval = ndis_set_info(sc, OID_802_11_BSSID, &bssid, &len);
if (rval)
device_printf(sc->ndis_dev,
"setting BSSID failed: %d\n", rval);
}
static void
ndis_auth_and_assoc(sc, vap)
struct ndis_softc *sc;
struct ieee80211vap *vap;
{
struct ieee80211com *ic;
struct ieee80211_node *ni;
ndis_80211_ssid ssid;
ndis_80211_macaddr bssid;
ndis_80211_wep wep;
int i, rval = 0, len, error;
uint32_t arg;
struct ifnet *ifp;
ifp = sc->ifp;
ic = ifp->if_l2com;
ni = vap->iv_bss;
if (!NDIS_INITIALIZED(sc)) {
DPRINTF(("%s: NDIS not initialized\n", __func__));
return;
}
/* Initial setup */
ndis_setstate_80211(sc);
/* Set network infrastructure mode. */
len = sizeof(arg);
if (vap->iv_opmode == IEEE80211_M_IBSS)
arg = NDIS_80211_NET_INFRA_IBSS;
else
arg = NDIS_80211_NET_INFRA_BSS;
rval = ndis_set_info(sc, OID_802_11_INFRASTRUCTURE_MODE, &arg, &len);
if (rval)
device_printf (sc->ndis_dev, "set infra failed: %d\n", rval);
/* Set RTS threshold */
len = sizeof(arg);
arg = vap->iv_rtsthreshold;
ndis_set_info(sc, OID_802_11_RTS_THRESHOLD, &arg, &len);
/* Set fragmentation threshold */
len = sizeof(arg);
arg = vap->iv_fragthreshold;
ndis_set_info(sc, OID_802_11_FRAGMENTATION_THRESHOLD, &arg, &len);
/* Set WEP */
if (vap->iv_flags & IEEE80211_F_PRIVACY &&
!(vap->iv_flags & IEEE80211_F_WPA)) {
int keys_set = 0;
if (ni->ni_authmode == IEEE80211_AUTH_SHARED) {
len = sizeof(arg);
arg = NDIS_80211_AUTHMODE_SHARED;
DPRINTF(("Setting shared auth\n"));
ndis_set_info(sc, OID_802_11_AUTHENTICATION_MODE,
&arg, &len);
}
for (i = 0; i < IEEE80211_WEP_NKID; i++) {
if (vap->iv_nw_keys[i].wk_keylen) {
if (vap->iv_nw_keys[i].wk_cipher->ic_cipher !=
IEEE80211_CIPHER_WEP)
continue;
bzero((char *)&wep, sizeof(wep));
wep.nw_keylen = vap->iv_nw_keys[i].wk_keylen;
/*
* 5, 13 and 16 are the only valid
* key lengths. Anything in between
* will be zero padded out to the
* next highest boundary.
*/
if (vap->iv_nw_keys[i].wk_keylen < 5)
wep.nw_keylen = 5;
else if (vap->iv_nw_keys[i].wk_keylen > 5 &&
vap->iv_nw_keys[i].wk_keylen < 13)
wep.nw_keylen = 13;
else if (vap->iv_nw_keys[i].wk_keylen > 13 &&
vap->iv_nw_keys[i].wk_keylen < 16)
wep.nw_keylen = 16;
wep.nw_keyidx = i;
wep.nw_length = (sizeof(uint32_t) * 3)
+ wep.nw_keylen;
if (i == vap->iv_def_txkey)
wep.nw_keyidx |= NDIS_80211_WEPKEY_TX;
bcopy(vap->iv_nw_keys[i].wk_key,
wep.nw_keydata, wep.nw_length);
len = sizeof(wep);
DPRINTF(("Setting WEP key %d\n", i));
rval = ndis_set_info(sc,
OID_802_11_ADD_WEP, &wep, &len);
if (rval)
device_printf(sc->ndis_dev,
"set wepkey failed: %d\n", rval);
keys_set++;
}
}
if (keys_set) {
DPRINTF(("Setting WEP on\n"));
arg = NDIS_80211_WEPSTAT_ENABLED;
len = sizeof(arg);
rval = ndis_set_info(sc,
OID_802_11_WEP_STATUS, &arg, &len);
if (rval)
device_printf(sc->ndis_dev,
"enable WEP failed: %d\n", rval);
if (vap->iv_flags & IEEE80211_F_DROPUNENC)
arg = NDIS_80211_PRIVFILT_8021XWEP;
else
arg = NDIS_80211_PRIVFILT_ACCEPTALL;
len = sizeof(arg);
ndis_set_info(sc,
OID_802_11_PRIVACY_FILTER, &arg, &len);
}
}
/* Set up WPA. */
if ((vap->iv_flags & IEEE80211_F_WPA) &&
vap->iv_appie_assocreq != NULL) {
struct ieee80211_appie *ie = vap->iv_appie_assocreq;
error = ndis_set_wpa(sc, ie->ie_data, ie->ie_len);
if (error != 0)
device_printf(sc->ndis_dev, "WPA setup failed\n");
}
#ifdef notyet
/* Set network type. */
arg = 0;
switch (vap->iv_curmode) {
case IEEE80211_MODE_11A:
arg = NDIS_80211_NETTYPE_11OFDM5;
break;
case IEEE80211_MODE_11B:
arg = NDIS_80211_NETTYPE_11DS;
break;
case IEEE80211_MODE_11G:
arg = NDIS_80211_NETTYPE_11OFDM24;
break;
default:
device_printf(sc->ndis_dev, "unknown mode: %d\n",
vap->iv_curmode);
}
if (arg) {
DPRINTF(("Setting network type to %d\n", arg));
len = sizeof(arg);
rval = ndis_set_info(sc, OID_802_11_NETWORK_TYPE_IN_USE,
&arg, &len);
if (rval)
device_printf(sc->ndis_dev,
"set nettype failed: %d\n", rval);
}
#endif
/*
* If the user selected a specific BSSID, try
* to use that one. This is useful in the case where
* there are several APs in range with the same network
* name. To delete the BSSID, we use the broadcast
* address as the BSSID.
* Note that some drivers seem to allow setting a BSSID
* in ad-hoc mode, which has the effect of forcing the
* NIC to create an ad-hoc cell with a specific BSSID,
* instead of a randomly chosen one. However, the net80211
* code makes the assumtion that the BSSID setting is invalid
* when you're in ad-hoc mode, so we don't allow that here.
*/
len = IEEE80211_ADDR_LEN;
if (vap->iv_flags & IEEE80211_F_DESBSSID &&
vap->iv_opmode != IEEE80211_M_IBSS)
bcopy(ni->ni_bssid, bssid, len);
else
bcopy(ifp->if_broadcastaddr, bssid, len);
DPRINTF(("Setting BSSID to %6D\n", (uint8_t *)&bssid, ":"));
rval = ndis_set_info(sc, OID_802_11_BSSID, &bssid, &len);
if (rval)
device_printf(sc->ndis_dev,
"setting BSSID failed: %d\n", rval);
/* Set SSID -- always do this last. */
#ifdef NDIS_DEBUG
if (ndis_debug > 0) {
printf("Setting ESSID to ");
ieee80211_print_essid(ni->ni_essid, ni->ni_esslen);
printf("\n");
}
#endif
len = sizeof(ssid);
bzero((char *)&ssid, len);
ssid.ns_ssidlen = ni->ni_esslen;
if (ssid.ns_ssidlen == 0) {
ssid.ns_ssidlen = 1;
} else
bcopy(ni->ni_essid, ssid.ns_ssid, ssid.ns_ssidlen);
rval = ndis_set_info(sc, OID_802_11_SSID, &ssid, &len);
if (rval)
device_printf (sc->ndis_dev, "set ssid failed: %d\n", rval);
return;
}
static int
ndis_get_bssid_list(sc, bl)
struct ndis_softc *sc;
ndis_80211_bssid_list_ex **bl;
{
int len, error;
len = sizeof(uint32_t) + (sizeof(ndis_wlan_bssid_ex) * 16);
*bl = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO);
if (*bl == NULL)
return (ENOMEM);
error = ndis_get_info(sc, OID_802_11_BSSID_LIST, *bl, &len);
if (error == ENOSPC) {
free(*bl, M_DEVBUF);
*bl = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO);
if (*bl == NULL)
return (ENOMEM);
error = ndis_get_info(sc, OID_802_11_BSSID_LIST, *bl, &len);
}
if (error) {
DPRINTF(("%s: failed to read\n", __func__));
free(*bl, M_DEVBUF);
return (error);
}
return (0);
}
static int
ndis_get_assoc(sc, assoc)
struct ndis_softc *sc;
ndis_wlan_bssid_ex **assoc;
{
struct ifnet *ifp = sc->ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct ieee80211vap *vap;
struct ieee80211_node *ni;
ndis_80211_bssid_list_ex *bl;
ndis_wlan_bssid_ex *bs;
ndis_80211_macaddr bssid;
int i, len, error;
if (!sc->ndis_link)
return (ENOENT);
len = sizeof(bssid);
error = ndis_get_info(sc, OID_802_11_BSSID, &bssid, &len);
if (error) {
device_printf(sc->ndis_dev, "failed to get bssid\n");
return (ENOENT);
}
vap = TAILQ_FIRST(&ic->ic_vaps);
ni = vap->iv_bss;
error = ndis_get_bssid_list(sc, &bl);
if (error)
return (error);
bs = (ndis_wlan_bssid_ex *)&bl->nblx_bssid[0];
for (i = 0; i < bl->nblx_items; i++) {
if (bcmp(bs->nwbx_macaddr, bssid, sizeof(bssid)) == 0) {
*assoc = malloc(bs->nwbx_len, M_TEMP, M_NOWAIT);
if (*assoc == NULL) {
free(bl, M_TEMP);
return (ENOMEM);
}
bcopy((char *)bs, (char *)*assoc, bs->nwbx_len);
free(bl, M_TEMP);
if (ic->ic_opmode == IEEE80211_M_STA)
ni->ni_associd = 1 | 0xc000; /* fake associd */
return (0);
}
bs = (ndis_wlan_bssid_ex *)((char *)bs + bs->nwbx_len);
}
free(bl, M_TEMP);
return (ENOENT);
}
static void
ndis_getstate_80211(sc)
struct ndis_softc *sc;
{
struct ieee80211com *ic;
struct ieee80211vap *vap;
struct ieee80211_node *ni;
ndis_wlan_bssid_ex *bs;
int rval, len, i = 0;
int chanflag;
uint32_t arg;
struct ifnet *ifp;
ifp = sc->ifp;
ic = ifp->if_l2com;
vap = TAILQ_FIRST(&ic->ic_vaps);
ni = vap->iv_bss;
if (!NDIS_INITIALIZED(sc))
return;
if ((rval = ndis_get_assoc(sc, &bs)) != 0)
return;
/* We're associated, retrieve info on the current bssid. */
ic->ic_curmode = ndis_nettype_mode(bs->nwbx_nettype);
chanflag = ndis_nettype_chan(bs->nwbx_nettype);
IEEE80211_ADDR_COPY(ni->ni_bssid, bs->nwbx_macaddr);
/* Get SSID from current association info. */
bcopy(bs->nwbx_ssid.ns_ssid, ni->ni_essid,
bs->nwbx_ssid.ns_ssidlen);
ni->ni_esslen = bs->nwbx_ssid.ns_ssidlen;
if (ic->ic_caps & IEEE80211_C_PMGT) {
len = sizeof(arg);
rval = ndis_get_info(sc, OID_802_11_POWER_MODE, &arg, &len);
if (rval)
device_printf(sc->ndis_dev,
"get power mode failed: %d\n", rval);
if (arg == NDIS_80211_POWERMODE_CAM)
vap->iv_flags &= ~IEEE80211_F_PMGTON;
else
vap->iv_flags |= IEEE80211_F_PMGTON;
}
/* Get TX power */
if (ic->ic_caps & IEEE80211_C_TXPMGT) {
len = sizeof(arg);
ndis_get_info(sc, OID_802_11_TX_POWER_LEVEL, &arg, &len);
for (i = 0; i < (sizeof(dBm2mW) / sizeof(dBm2mW[0])); i++)
if (dBm2mW[i] >= arg)
break;
ic->ic_txpowlimit = i;
}
/*
* Use the current association information to reflect
* what channel we're on.
*/
ic->ic_curchan = ieee80211_find_channel(ic,
bs->nwbx_config.nc_dsconfig / 1000, chanflag);
if (ic->ic_curchan == NULL)
ic->ic_curchan = &ic->ic_channels[0];
ni->ni_chan = ic->ic_curchan;
ic->ic_bsschan = ic->ic_curchan;
free(bs, M_TEMP);
/*
* Determine current authentication mode.
*/
len = sizeof(arg);
rval = ndis_get_info(sc, OID_802_11_AUTHENTICATION_MODE, &arg, &len);
if (rval)
device_printf(sc->ndis_dev,
"get authmode status failed: %d\n", rval);
else {
vap->iv_flags &= ~IEEE80211_F_WPA;
switch (arg) {
case NDIS_80211_AUTHMODE_OPEN:
ni->ni_authmode = IEEE80211_AUTH_OPEN;
break;
case NDIS_80211_AUTHMODE_SHARED:
ni->ni_authmode = IEEE80211_AUTH_SHARED;
break;
case NDIS_80211_AUTHMODE_AUTO:
ni->ni_authmode = IEEE80211_AUTH_AUTO;
break;
case NDIS_80211_AUTHMODE_WPA:
case NDIS_80211_AUTHMODE_WPAPSK:
case NDIS_80211_AUTHMODE_WPANONE:
ni->ni_authmode = IEEE80211_AUTH_WPA;
vap->iv_flags |= IEEE80211_F_WPA1;
break;
case NDIS_80211_AUTHMODE_WPA2:
case NDIS_80211_AUTHMODE_WPA2PSK:
ni->ni_authmode = IEEE80211_AUTH_WPA;
vap->iv_flags |= IEEE80211_F_WPA2;
break;
default:
ni->ni_authmode = IEEE80211_AUTH_NONE;
break;
}
}
len = sizeof(arg);
rval = ndis_get_info(sc, OID_802_11_WEP_STATUS, &arg, &len);
if (rval)
device_printf(sc->ndis_dev,
"get wep status failed: %d\n", rval);
if (arg == NDIS_80211_WEPSTAT_ENABLED)
vap->iv_flags |= IEEE80211_F_PRIVACY|IEEE80211_F_DROPUNENC;
else
vap->iv_flags &= ~(IEEE80211_F_PRIVACY|IEEE80211_F_DROPUNENC);
}
static int
ndis_ioctl(ifp, command, data)
struct ifnet *ifp;
u_long command;
caddr_t data;
{
struct ndis_softc *sc = ifp->if_softc;
struct ifreq *ifr = (struct ifreq *) data;
int i, error = 0;
/*NDIS_LOCK(sc);*/
switch (command) {
case SIOCSIFFLAGS:
if (ifp->if_flags & IFF_UP) {
if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
ifp->if_flags & IFF_PROMISC &&
!(sc->ndis_if_flags & IFF_PROMISC)) {
sc->ndis_filter |=
NDIS_PACKET_TYPE_PROMISCUOUS;
i = sizeof(sc->ndis_filter);
error = ndis_set_info(sc,
OID_GEN_CURRENT_PACKET_FILTER,
&sc->ndis_filter, &i);
} else if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
!(ifp->if_flags & IFF_PROMISC) &&
sc->ndis_if_flags & IFF_PROMISC) {
sc->ndis_filter &=
~NDIS_PACKET_TYPE_PROMISCUOUS;
i = sizeof(sc->ndis_filter);
error = ndis_set_info(sc,
OID_GEN_CURRENT_PACKET_FILTER,
&sc->ndis_filter, &i);
} else
ndis_init(sc);
} else {
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
ndis_stop(sc);
}
sc->ndis_if_flags = ifp->if_flags;
error = 0;
break;
case SIOCADDMULTI:
case SIOCDELMULTI:
ndis_setmulti(sc);
error = 0;
break;
case SIOCGIFMEDIA:
case SIOCSIFMEDIA:
error = ifmedia_ioctl(ifp, ifr, &sc->ifmedia, command);
break;
case SIOCSIFCAP:
ifp->if_capenable = ifr->ifr_reqcap;
if (ifp->if_capenable & IFCAP_TXCSUM)
ifp->if_hwassist = sc->ndis_hwassist;
else
ifp->if_hwassist = 0;
ndis_set_offload(sc);
break;
default:
error = ether_ioctl(ifp, command, data);
break;
}
/*NDIS_UNLOCK(sc);*/
return(error);
}
static int
ndis_ioctl_80211(ifp, command, data)
struct ifnet *ifp;
u_long command;
caddr_t data;
{
struct ndis_softc *sc = ifp->if_softc;
struct ieee80211com *ic = ifp->if_l2com;
struct ifreq *ifr = (struct ifreq *) data;
struct ndis_oid_data oid;
struct ndis_evt evt;
void *oidbuf;
int error = 0;
switch (command) {
case SIOCSIFFLAGS:
/*NDIS_LOCK(sc);*/
if (ifp->if_flags & IFF_UP) {
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
ndis_init(sc);
} else {
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
ndis_stop(sc);
}
sc->ndis_if_flags = ifp->if_flags;
error = 0;
/*NDIS_UNLOCK(sc);*/
break;
case SIOCGDRVSPEC:
if ((error = priv_check(curthread, PRIV_DRIVER)))
break;
error = copyin(ifr->ifr_data, &oid, sizeof(oid));
if (error)
break;
oidbuf = malloc(oid.len, M_TEMP, M_NOWAIT|M_ZERO);
if (oidbuf == NULL) {
error = ENOMEM;
break;
}
error = copyin(ifr->ifr_data + sizeof(oid), oidbuf, oid.len);
if (error) {
free(oidbuf, M_TEMP);
break;
}
error = ndis_get_info(sc, oid.oid, oidbuf, &oid.len);
if (error) {
free(oidbuf, M_TEMP);
break;
}
error = copyout(&oid, ifr->ifr_data, sizeof(oid));
if (error) {
free(oidbuf, M_TEMP);
break;
}
error = copyout(oidbuf, ifr->ifr_data + sizeof(oid), oid.len);
free(oidbuf, M_TEMP);
break;
case SIOCSDRVSPEC:
if ((error = priv_check(curthread, PRIV_DRIVER)))
break;
error = copyin(ifr->ifr_data, &oid, sizeof(oid));
if (error)
break;
oidbuf = malloc(oid.len, M_TEMP, M_NOWAIT|M_ZERO);
if (oidbuf == NULL) {
error = ENOMEM;
break;
}
error = copyin(ifr->ifr_data + sizeof(oid), oidbuf, oid.len);
if (error) {
free(oidbuf, M_TEMP);
break;
}
error = ndis_set_info(sc, oid.oid, oidbuf, &oid.len);
if (error) {
free(oidbuf, M_TEMP);
break;
}
error = copyout(&oid, ifr->ifr_data, sizeof(oid));
if (error) {
free(oidbuf, M_TEMP);
break;
}
error = copyout(oidbuf, ifr->ifr_data + sizeof(oid), oid.len);
free(oidbuf, M_TEMP);
break;
case SIOCGPRIVATE_0:
if ((error = priv_check(curthread, PRIV_DRIVER)))
break;
NDIS_LOCK(sc);
if (sc->ndis_evt[sc->ndis_evtcidx].ne_sts == 0) {
error = ENOENT;
NDIS_UNLOCK(sc);
break;
}
error = copyin(ifr->ifr_data, &evt, sizeof(evt));
if (error) {
NDIS_UNLOCK(sc);
break;
}
if (evt.ne_len < sc->ndis_evt[sc->ndis_evtcidx].ne_len) {
error = ENOSPC;
NDIS_UNLOCK(sc);
break;
}
error = copyout(&sc->ndis_evt[sc->ndis_evtcidx],
ifr->ifr_data, sizeof(uint32_t) * 2);
if (error) {
NDIS_UNLOCK(sc);
break;
}
if (sc->ndis_evt[sc->ndis_evtcidx].ne_len) {
error = copyout(sc->ndis_evt[sc->ndis_evtcidx].ne_buf,
ifr->ifr_data + (sizeof(uint32_t) * 2),
sc->ndis_evt[sc->ndis_evtcidx].ne_len);
if (error) {
NDIS_UNLOCK(sc);
break;
}
free(sc->ndis_evt[sc->ndis_evtcidx].ne_buf, M_TEMP);
sc->ndis_evt[sc->ndis_evtcidx].ne_buf = NULL;
}
sc->ndis_evt[sc->ndis_evtcidx].ne_len = 0;
sc->ndis_evt[sc->ndis_evtcidx].ne_sts = 0;
NDIS_EVTINC(sc->ndis_evtcidx);
NDIS_UNLOCK(sc);
break;
case SIOCGIFMEDIA:
error = ifmedia_ioctl(ifp, ifr, &ic->ic_media, command);
break;
case SIOCGIFADDR:
error = ether_ioctl(ifp, command, data);
break;
default:
error = EINVAL;
break;
}
return (error);
}
int
ndis_del_key(vap, key)
struct ieee80211vap *vap;
const struct ieee80211_key *key;
{
struct ndis_softc *sc;
ndis_80211_key rkey;
int len, error = 0;
sc = vap->iv_ic->ic_ifp->if_softc;
bzero((char *)&rkey, sizeof(rkey));
len = sizeof(rkey);
rkey.nk_len = len;
rkey.nk_keyidx = key->wk_keyix;
bcopy(vap->iv_ifp->if_broadcastaddr,
rkey.nk_bssid, IEEE80211_ADDR_LEN);
error = ndis_set_info(sc, OID_802_11_REMOVE_KEY, &rkey, &len);
if (error)
return (0);
return (1);
}
/*
* In theory this could be called for any key, but we'll
* only use it for WPA TKIP or AES keys. These need to be
* set after initial authentication with the AP.
*/
static int
ndis_add_key(vap, key, mac)
struct ieee80211vap *vap;
const struct ieee80211_key *key;
const uint8_t mac[IEEE80211_ADDR_LEN];
{
struct ndis_softc *sc;
struct ifnet *ifp;
ndis_80211_key rkey;
int len, error = 0;
ifp = vap->iv_ic->ic_ifp;
sc = ifp->if_softc;
switch (key->wk_cipher->ic_cipher) {
case IEEE80211_CIPHER_TKIP:
len = sizeof(ndis_80211_key);
bzero((char *)&rkey, sizeof(rkey));
rkey.nk_len = len;
rkey.nk_keylen = key->wk_keylen;
if (key->wk_flags & IEEE80211_KEY_SWMIC)
rkey.nk_keylen += 16;
/* key index - gets weird in NDIS */
if (key->wk_keyix != IEEE80211_KEYIX_NONE)
rkey.nk_keyidx = key->wk_keyix;
else
rkey.nk_keyidx = 0;
if (key->wk_flags & IEEE80211_KEY_XMIT)
rkey.nk_keyidx |= 1 << 31;
if (key->wk_flags & IEEE80211_KEY_GROUP) {
bcopy(ifp->if_broadcastaddr,
rkey.nk_bssid, IEEE80211_ADDR_LEN);
} else {
bcopy(vap->iv_bss->ni_bssid,
rkey.nk_bssid, IEEE80211_ADDR_LEN);
/* pairwise key */
rkey.nk_keyidx |= 1 << 30;
}
/* need to set bit 29 based on keyrsc */
rkey.nk_keyrsc = key->wk_keyrsc[0]; /* XXX need tid */
if (rkey.nk_keyrsc)
rkey.nk_keyidx |= 1 << 29;
if (key->wk_flags & IEEE80211_KEY_SWMIC) {
bcopy(key->wk_key, rkey.nk_keydata, 16);
bcopy(key->wk_key + 24, rkey.nk_keydata + 16, 8);
bcopy(key->wk_key + 16, rkey.nk_keydata + 24, 8);
} else
bcopy(key->wk_key, rkey.nk_keydata, key->wk_keylen);
error = ndis_set_info(sc, OID_802_11_ADD_KEY, &rkey, &len);
break;
case IEEE80211_CIPHER_WEP:
error = 0;
break;
/*
* I don't know how to set up keys for the AES
* cipher yet. Is it the same as TKIP?
*/
case IEEE80211_CIPHER_AES_CCM:
default:
error = ENOTTY;
break;
}
/* We need to return 1 for success, 0 for failure. */
if (error)
return (0);
return (1);
}
static void
ndis_resettask(d, arg)
device_object *d;
void *arg;
{
struct ndis_softc *sc;
sc = arg;
ndis_reset_nic(sc);
}
/*
* Stop the adapter and free any mbufs allocated to the
* RX and TX lists.
*/
static void
ndis_stop(sc)
struct ndis_softc *sc;
{
struct ifnet *ifp;
int i;
ifp = sc->ifp;
callout_drain(&sc->ndis_stat_callout);
NDIS_LOCK(sc);
sc->ndis_tx_timer = 0;
sc->ndis_link = 0;
ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
NDIS_UNLOCK(sc);
if (sc->ndis_iftype != PNPBus ||
(sc->ndis_iftype == PNPBus &&
!(sc->ndisusb_status & NDISUSB_STATUS_DETACH) &&
ndisusb_halt != 0))
ndis_halt_nic(sc);
NDIS_LOCK(sc);
for (i = 0; i < NDIS_EVENTS; i++) {
if (sc->ndis_evt[i].ne_sts && sc->ndis_evt[i].ne_buf != NULL) {
free(sc->ndis_evt[i].ne_buf, M_TEMP);
sc->ndis_evt[i].ne_buf = NULL;
}
sc->ndis_evt[i].ne_sts = 0;
sc->ndis_evt[i].ne_len = 0;
}
sc->ndis_evtcidx = 0;
sc->ndis_evtpidx = 0;
NDIS_UNLOCK(sc);
}
/*
* Stop all chip I/O so that the kernel's probe routines don't
* get confused by errant DMAs when rebooting.
*/
void
ndis_shutdown(dev)
device_t dev;
{
struct ndis_softc *sc;
sc = device_get_softc(dev);
ndis_stop(sc);
}
static int
ndis_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
{
struct ndis_vap *nvp = NDIS_VAP(vap);
struct ieee80211com *ic = vap->iv_ic;
struct ifnet *ifp = ic->ic_ifp;
struct ndis_softc *sc = ifp->if_softc;
enum ieee80211_state ostate;
DPRINTF(("%s: %s -> %s\n", __func__,
ieee80211_state_name[vap->iv_state],
ieee80211_state_name[nstate]));
ostate = vap->iv_state;
vap->iv_state = nstate;
switch (nstate) {
/* pass on to net80211 */
case IEEE80211_S_INIT:
case IEEE80211_S_SCAN:
return nvp->newstate(vap, nstate, arg);
case IEEE80211_S_ASSOC:
if (ostate != IEEE80211_S_AUTH) {
IEEE80211_UNLOCK(ic);
ndis_auth_and_assoc(sc, vap);
IEEE80211_LOCK(ic);
}
break;
case IEEE80211_S_AUTH:
IEEE80211_UNLOCK(ic);
ndis_auth_and_assoc(sc, vap);
if (vap->iv_state == IEEE80211_S_AUTH) /* XXX */
ieee80211_new_state(vap, IEEE80211_S_ASSOC, 0);
IEEE80211_LOCK(ic);
break;
default:
break;
}
return (0);
}
static void
ndis_scan(void *arg)
{
struct ieee80211vap *vap = arg;
ieee80211_scan_done(vap);
}
static void
ndis_scan_results(struct ndis_softc *sc)
{
struct ieee80211com *ic;
struct ieee80211vap *vap;
ndis_80211_bssid_list_ex *bl;
ndis_wlan_bssid_ex *wb;
struct ieee80211_scanparams sp;
struct ieee80211_frame wh;
struct ieee80211_channel *saved_chan;
int i, j;
int rssi, noise, freq, chanflag;
uint8_t ssid[2+IEEE80211_NWID_LEN];
uint8_t rates[2+IEEE80211_RATE_MAXSIZE];
uint8_t *frm, *efrm;
ic = sc->ifp->if_l2com;
vap = TAILQ_FIRST(&ic->ic_vaps);
saved_chan = ic->ic_curchan;
noise = -96;
if (ndis_get_bssid_list(sc, &bl))
return;
DPRINTF(("%s: %d results\n", __func__, bl->nblx_items));
wb = &bl->nblx_bssid[0];
for (i = 0; i < bl->nblx_items; i++) {
memset(&sp, 0, sizeof(sp));
memcpy(wh.i_addr2, wb->nwbx_macaddr, sizeof(wh.i_addr2));
memcpy(wh.i_addr3, wb->nwbx_macaddr, sizeof(wh.i_addr3));
rssi = 100 * (wb->nwbx_rssi - noise) / (-32 - noise);
rssi = max(0, min(rssi, 100)); /* limit 0 <= rssi <= 100 */
if (wb->nwbx_privacy)
sp.capinfo |= IEEE80211_CAPINFO_PRIVACY;
sp.bintval = wb->nwbx_config.nc_beaconperiod;
switch (wb->nwbx_netinfra) {
case NDIS_80211_NET_INFRA_IBSS:
sp.capinfo |= IEEE80211_CAPINFO_IBSS;
break;
case NDIS_80211_NET_INFRA_BSS:
sp.capinfo |= IEEE80211_CAPINFO_ESS;
break;
}
sp.rates = &rates[0];
for (j = 0; j < IEEE80211_RATE_MAXSIZE; j++) {
/* XXX - check units */
if (wb->nwbx_supportedrates[j] == 0)
break;
rates[2 + j] =
wb->nwbx_supportedrates[j] & 0x7f;
}
rates[1] = j;
sp.ssid = (uint8_t *)&ssid[0];
memcpy(sp.ssid + 2, &wb->nwbx_ssid.ns_ssid,
wb->nwbx_ssid.ns_ssidlen);
sp.ssid[1] = wb->nwbx_ssid.ns_ssidlen;
chanflag = ndis_nettype_chan(wb->nwbx_nettype);
freq = wb->nwbx_config.nc_dsconfig / 1000;
sp.chan = sp.bchan = ieee80211_mhz2ieee(freq, chanflag);
/* Hack ic->ic_curchan to be in sync with the scan result */
ic->ic_curchan = ieee80211_find_channel(ic, freq, chanflag);
if (ic->ic_curchan == NULL)
ic->ic_curchan = &ic->ic_channels[0];
/* Process extended info from AP */
if (wb->nwbx_len > sizeof(ndis_wlan_bssid)) {
frm = (uint8_t *)&wb->nwbx_ies;
efrm = frm + wb->nwbx_ielen;
if (efrm - frm < 12)
goto done;
sp.tstamp = frm; frm += 8;
sp.bintval = le16toh(*(uint16_t *)frm); frm += 2;
sp.capinfo = le16toh(*(uint16_t *)frm); frm += 2;
sp.ies = frm;
sp.ies_len = efrm - frm;
}
done:
DPRINTF(("scan: bssid %s chan %dMHz (%d/%d) rssi %d\n",
ether_sprintf(wb->nwbx_macaddr), freq, sp.bchan, chanflag,
rssi));
ieee80211_add_scan(vap, ic->ic_curchan, &sp, &wh, 0, rssi, noise);
wb = (ndis_wlan_bssid_ex *)((char *)wb + wb->nwbx_len);
}
free(bl, M_DEVBUF);
/* Restore the channel after messing with it */
ic->ic_curchan = saved_chan;
}
static void
ndis_scan_start(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
struct ndis_softc *sc = ifp->if_softc;
struct ieee80211vap *vap;
struct ieee80211_scan_state *ss;
ndis_80211_ssid ssid;
int error, len;
ss = ic->ic_scan;
vap = TAILQ_FIRST(&ic->ic_vaps);
if (!NDIS_INITIALIZED(sc)) {
DPRINTF(("%s: scan aborted\n", __func__));
ieee80211_cancel_scan(vap);
return;
}
len = sizeof(ssid);
bzero((char *)&ssid, len);
if (ss->ss_nssid == 0)
ssid.ns_ssidlen = 1;
else {
/* Perform a directed scan */
ssid.ns_ssidlen = ss->ss_ssid[0].len;
bcopy(ss->ss_ssid[0].ssid, ssid.ns_ssid, ssid.ns_ssidlen);
}
error = ndis_set_info(sc, OID_802_11_SSID, &ssid, &len);
if (error)
DPRINTF(("%s: set ESSID failed\n", __func__));
len = 0;
error = ndis_set_info(sc, OID_802_11_BSSID_LIST_SCAN, NULL, &len);
if (error) {
DPRINTF(("%s: scan command failed\n", __func__));
ieee80211_cancel_scan(vap);
return;
}
/* Set a timer to collect the results */
callout_reset(&sc->ndis_scan_callout, hz * 3, ndis_scan, vap);
}
static void
ndis_set_channel(struct ieee80211com *ic)
{
/* ignore */
}
static void
ndis_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell)
{
/* ignore */
}
static void
ndis_scan_mindwell(struct ieee80211_scan_state *ss)
{
/* NB: don't try to abort scan; wait for firmware to finish */
}
static void
ndis_scan_end(struct ieee80211com *ic)
{
struct ndis_softc *sc = ic->ic_ifp->if_softc;
ndis_scan_results(sc);
}
Index: head/sys/dev/isci/isci_io_request.c
===================================================================
--- head/sys/dev/isci/isci_io_request.c (revision 283290)
+++ head/sys/dev/isci/isci_io_request.c (revision 283291)
@@ -1,991 +1,991 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <dev/isci/isci.h>
#include <cam/scsi/scsi_all.h>
#include <cam/scsi/scsi_message.h>
#include <dev/isci/scil/intel_sas.h>
#include <dev/isci/scil/sci_util.h>
#include <dev/isci/scil/scif_io_request.h>
#include <dev/isci/scil/scif_controller.h>
#include <dev/isci/scil/scif_remote_device.h>
#include <dev/isci/scil/scif_user_callback.h>
#include <dev/isci/scil/scic_io_request.h>
#include <dev/isci/scil/scic_user_callback.h>
/**
* @brief This user callback will inform the user that an IO request has
* completed.
*
* @param[in] controller This parameter specifies the controller on
* which the IO request is completing.
* @param[in] remote_device This parameter specifies the remote device on
* which this request is completing.
* @param[in] io_request This parameter specifies the IO request that has
* completed.
* @param[in] completion_status This parameter specifies the results of
* the IO request operation. SCI_IO_SUCCESS indicates
* successful completion.
*
* @return none
*/
void
scif_cb_io_request_complete(SCI_CONTROLLER_HANDLE_T scif_controller,
SCI_REMOTE_DEVICE_HANDLE_T remote_device,
SCI_IO_REQUEST_HANDLE_T io_request, SCI_IO_STATUS completion_status)
{
struct ISCI_IO_REQUEST *isci_request =
(struct ISCI_IO_REQUEST *)sci_object_get_association(io_request);
scif_controller_complete_io(scif_controller, remote_device, io_request);
isci_io_request_complete(scif_controller, remote_device, isci_request,
completion_status);
}
void
isci_io_request_complete(SCI_CONTROLLER_HANDLE_T scif_controller,
SCI_REMOTE_DEVICE_HANDLE_T remote_device,
struct ISCI_IO_REQUEST *isci_request, SCI_IO_STATUS completion_status)
{
struct ISCI_CONTROLLER *isci_controller;
struct ISCI_REMOTE_DEVICE *isci_remote_device;
union ccb *ccb;
BOOL complete_ccb;
complete_ccb = TRUE;
isci_controller = (struct ISCI_CONTROLLER *) sci_object_get_association(scif_controller);
isci_remote_device =
(struct ISCI_REMOTE_DEVICE *) sci_object_get_association(remote_device);
ccb = isci_request->ccb;
ccb->ccb_h.status &= ~CAM_STATUS_MASK;
switch (completion_status) {
case SCI_IO_SUCCESS:
case SCI_IO_SUCCESS_COMPLETE_BEFORE_START:
#if __FreeBSD_version >= 900026
if (ccb->ccb_h.func_code == XPT_SMP_IO) {
void *smp_response =
scif_io_request_get_response_iu_address(
isci_request->sci_object);
memcpy(ccb->smpio.smp_response, smp_response,
ccb->smpio.smp_response_len);
}
#endif
ccb->ccb_h.status |= CAM_REQ_CMP;
break;
case SCI_IO_SUCCESS_IO_DONE_EARLY:
ccb->ccb_h.status |= CAM_REQ_CMP;
ccb->csio.resid = ccb->csio.dxfer_len -
scif_io_request_get_number_of_bytes_transferred(
isci_request->sci_object);
break;
case SCI_IO_FAILURE_RESPONSE_VALID:
{
SCI_SSP_RESPONSE_IU_T * response_buffer;
uint32_t sense_length;
int error_code, sense_key, asc, ascq;
struct ccb_scsiio *csio = &ccb->csio;
response_buffer = (SCI_SSP_RESPONSE_IU_T *)
scif_io_request_get_response_iu_address(
isci_request->sci_object);
sense_length = sci_ssp_get_sense_data_length(
response_buffer->sense_data_length);
sense_length = MIN(csio->sense_len, sense_length);
memcpy(&csio->sense_data, response_buffer->data, sense_length);
csio->sense_resid = csio->sense_len - sense_length;
csio->scsi_status = response_buffer->status;
ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
scsi_extract_sense( &csio->sense_data, &error_code, &sense_key,
&asc, &ascq );
isci_log_message(1, "ISCI",
"isci: bus=%x target=%x lun=%x cdb[0]=%x status=%x key=%x asc=%x ascq=%x\n",
ccb->ccb_h.path_id, ccb->ccb_h.target_id,
ccb->ccb_h.target_lun, csio->cdb_io.cdb_bytes[0],
csio->scsi_status, sense_key, asc, ascq);
break;
}
case SCI_IO_FAILURE_REMOTE_DEVICE_RESET_REQUIRED:
isci_remote_device_reset(isci_remote_device, NULL);
ccb->ccb_h.status |= CAM_REQ_TERMIO;
isci_log_message(0, "ISCI",
"isci: bus=%x target=%x lun=%x cdb[0]=%x remote device reset required\n",
ccb->ccb_h.path_id, ccb->ccb_h.target_id,
ccb->ccb_h.target_lun, ccb->csio.cdb_io.cdb_bytes[0]);
break;
case SCI_IO_FAILURE_TERMINATED:
ccb->ccb_h.status |= CAM_REQ_TERMIO;
isci_log_message(0, "ISCI",
"isci: bus=%x target=%x lun=%x cdb[0]=%x terminated\n",
ccb->ccb_h.path_id, ccb->ccb_h.target_id,
ccb->ccb_h.target_lun, ccb->csio.cdb_io.cdb_bytes[0]);
break;
case SCI_IO_FAILURE_INVALID_STATE:
case SCI_IO_FAILURE_INSUFFICIENT_RESOURCES:
complete_ccb = FALSE;
break;
case SCI_IO_FAILURE_INVALID_REMOTE_DEVICE:
ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
break;
case SCI_IO_FAILURE_NO_NCQ_TAG_AVAILABLE:
{
struct ccb_relsim ccb_relsim;
struct cam_path *path;
xpt_create_path(&path, NULL,
cam_sim_path(isci_controller->sim),
isci_remote_device->index, 0);
xpt_setup_ccb(&ccb_relsim.ccb_h, path, 5);
ccb_relsim.ccb_h.func_code = XPT_REL_SIMQ;
ccb_relsim.ccb_h.flags = CAM_DEV_QFREEZE;
ccb_relsim.release_flags = RELSIM_ADJUST_OPENINGS;
ccb_relsim.openings =
scif_remote_device_get_max_queue_depth(remote_device);
xpt_action((union ccb *)&ccb_relsim);
xpt_free_path(path);
complete_ccb = FALSE;
}
break;
case SCI_IO_FAILURE:
case SCI_IO_FAILURE_REQUIRES_SCSI_ABORT:
case SCI_IO_FAILURE_UNSUPPORTED_PROTOCOL:
case SCI_IO_FAILURE_PROTOCOL_VIOLATION:
case SCI_IO_FAILURE_INVALID_PARAMETER_VALUE:
case SCI_IO_FAILURE_CONTROLLER_SPECIFIC_ERR:
default:
isci_log_message(1, "ISCI",
"isci: bus=%x target=%x lun=%x cdb[0]=%x completion status=%x\n",
ccb->ccb_h.path_id, ccb->ccb_h.target_id,
ccb->ccb_h.target_lun, ccb->csio.cdb_io.cdb_bytes[0],
completion_status);
ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
break;
}
callout_stop(&isci_request->parent.timer);
bus_dmamap_sync(isci_request->parent.dma_tag,
isci_request->parent.dma_map,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(isci_request->parent.dma_tag,
isci_request->parent.dma_map);
isci_request->ccb = NULL;
sci_pool_put(isci_controller->request_pool,
(struct ISCI_REQUEST *)isci_request);
if (complete_ccb) {
if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
/* ccb will be completed with some type of non-success
* status. So temporarily freeze the queue until the
* upper layers can act on the status. The
* CAM_DEV_QFRZN flag will then release the queue
* after the status is acted upon.
*/
ccb->ccb_h.status |= CAM_DEV_QFRZN;
xpt_freeze_devq(ccb->ccb_h.path, 1);
}
if (ccb->ccb_h.status & CAM_SIM_QUEUED) {
KASSERT(ccb == isci_remote_device->queued_ccb_in_progress,
("multiple internally queued ccbs in flight"));
TAILQ_REMOVE(&isci_remote_device->queued_ccbs,
&ccb->ccb_h, sim_links.tqe);
ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
/*
* This CCB that was in the queue was completed, so
* set the in_progress pointer to NULL denoting that
* we can retry another CCB from the queue. We only
* allow one CCB at a time from the queue to be
* in progress so that we can effectively maintain
* ordering.
*/
isci_remote_device->queued_ccb_in_progress = NULL;
}
if (isci_remote_device->frozen_lun_mask != 0) {
isci_remote_device_release_device_queue(isci_remote_device);
}
xpt_done(ccb);
if (isci_controller->is_frozen == TRUE) {
isci_controller->is_frozen = FALSE;
xpt_release_simq(isci_controller->sim, TRUE);
}
} else {
isci_remote_device_freeze_lun_queue(isci_remote_device,
ccb->ccb_h.target_lun);
if (ccb->ccb_h.status & CAM_SIM_QUEUED) {
KASSERT(ccb == isci_remote_device->queued_ccb_in_progress,
("multiple internally queued ccbs in flight"));
/*
* Do nothing, CCB is already on the device's queue.
* We leave it on the queue, to be retried again
* next time a CCB on this device completes, or we
* get a ready notification for this device.
*/
isci_log_message(1, "ISCI", "already queued %p %x\n",
ccb, ccb->csio.cdb_io.cdb_bytes[0]);
isci_remote_device->queued_ccb_in_progress = NULL;
} else {
isci_log_message(1, "ISCI", "queue %p %x\n", ccb,
ccb->csio.cdb_io.cdb_bytes[0]);
ccb->ccb_h.status |= CAM_SIM_QUEUED;
TAILQ_INSERT_TAIL(&isci_remote_device->queued_ccbs,
&ccb->ccb_h, sim_links.tqe);
}
}
}
/**
* @brief This callback method asks the user to provide the physical
* address for the supplied virtual address when building an
* io request object.
*
* @param[in] controller This parameter is the core controller object
* handle.
* @param[in] io_request This parameter is the io request object handle
* for which the physical address is being requested.
* @param[in] virtual_address This paramter is the virtual address which
* is to be returned as a physical address.
* @param[out] physical_address The physical address for the supplied virtual
* address.
*
* @return None.
*/
void
scic_cb_io_request_get_physical_address(SCI_CONTROLLER_HANDLE_T controller,
SCI_IO_REQUEST_HANDLE_T io_request, void *virtual_address,
SCI_PHYSICAL_ADDRESS *physical_address)
{
SCI_IO_REQUEST_HANDLE_T scif_request =
sci_object_get_association(io_request);
struct ISCI_REQUEST *isci_request =
sci_object_get_association(scif_request);
if(isci_request != NULL) {
/* isci_request is not NULL, meaning this is a request initiated
* by CAM or the isci layer (i.e. device reset for I/O
* timeout). Therefore we can calculate the physical address
* based on the address we stored in the struct ISCI_REQUEST
* object.
*/
*physical_address = isci_request->physical_address +
(uintptr_t)virtual_address -
(uintptr_t)isci_request;
} else {
/* isci_request is NULL, meaning this is a request generated
* internally by SCIL (i.e. for SMP requests or NCQ error
* recovery). Therefore we calculate the physical address
* based on the controller's uncached controller memory buffer,
* since we know that this is what SCIL uses for internal
* framework requests.
*/
SCI_CONTROLLER_HANDLE_T scif_controller =
(SCI_CONTROLLER_HANDLE_T) sci_object_get_association(controller);
struct ISCI_CONTROLLER *isci_controller =
(struct ISCI_CONTROLLER *)sci_object_get_association(scif_controller);
U64 virt_addr_offset = (uintptr_t)virtual_address -
(U64)isci_controller->uncached_controller_memory.virtual_address;
*physical_address =
isci_controller->uncached_controller_memory.physical_address
+ virt_addr_offset;
}
}
/**
* @brief This callback method asks the user to provide the address for
* the command descriptor block (CDB) associated with this IO request.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
*
* @return This method returns the virtual address of the CDB.
*/
void *
scif_cb_io_request_get_cdb_address(void * scif_user_io_request)
{
struct ISCI_IO_REQUEST *isci_request =
(struct ISCI_IO_REQUEST *)scif_user_io_request;
return (isci_request->ccb->csio.cdb_io.cdb_bytes);
}
/**
* @brief This callback method asks the user to provide the length of
* the command descriptor block (CDB) associated with this IO request.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
*
* @return This method returns the length of the CDB.
*/
uint32_t
scif_cb_io_request_get_cdb_length(void * scif_user_io_request)
{
struct ISCI_IO_REQUEST *isci_request =
(struct ISCI_IO_REQUEST *)scif_user_io_request;
return (isci_request->ccb->csio.cdb_len);
}
/**
* @brief This callback method asks the user to provide the Logical Unit (LUN)
* associated with this IO request.
*
* @note The contents of the value returned from this callback are defined
* by the protocol standard (e.g. T10 SAS specification). Please
* refer to the transport command information unit description
* in the associated standard.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
*
* @return This method returns the LUN associated with this request.
*/
uint32_t
scif_cb_io_request_get_lun(void * scif_user_io_request)
{
struct ISCI_IO_REQUEST *isci_request =
(struct ISCI_IO_REQUEST *)scif_user_io_request;
return (isci_request->ccb->ccb_h.target_lun);
}
/**
* @brief This callback method asks the user to provide the task attribute
* associated with this IO request.
*
* @note The contents of the value returned from this callback are defined
* by the protocol standard (e.g. T10 SAS specification). Please
* refer to the transport command information unit description
* in the associated standard.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
*
* @return This method returns the task attribute associated with this
* IO request.
*/
uint32_t
scif_cb_io_request_get_task_attribute(void * scif_user_io_request)
{
struct ISCI_IO_REQUEST *isci_request =
(struct ISCI_IO_REQUEST *)scif_user_io_request;
uint32_t task_attribute;
if((isci_request->ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0)
switch(isci_request->ccb->csio.tag_action) {
case MSG_HEAD_OF_Q_TAG:
task_attribute = SCI_SAS_HEAD_OF_QUEUE_ATTRIBUTE;
break;
case MSG_ORDERED_Q_TAG:
task_attribute = SCI_SAS_ORDERED_ATTRIBUTE;
break;
case MSG_ACA_TASK:
task_attribute = SCI_SAS_ACA_ATTRIBUTE;
break;
default:
task_attribute = SCI_SAS_SIMPLE_ATTRIBUTE;
break;
}
else
task_attribute = SCI_SAS_SIMPLE_ATTRIBUTE;
return (task_attribute);
}
/**
* @brief This callback method asks the user to provide the command priority
* associated with this IO request.
*
* @note The contents of the value returned from this callback are defined
* by the protocol standard (e.g. T10 SAS specification). Please
* refer to the transport command information unit description
* in the associated standard.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
*
* @return This method returns the command priority associated with this
* IO request.
*/
uint32_t
scif_cb_io_request_get_command_priority(void * scif_user_io_request)
{
return (0);
}
/**
* @brief This method simply returns the virtual address associated
* with the scsi_io and byte_offset supplied parameters.
*
* @note This callback is not utilized in the fast path. The expectation
* is that this method is utilized for items such as SCSI to ATA
* translation for commands like INQUIRY, READ CAPACITY, etc.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
* @param[in] byte_offset This parameter specifies the offset into the data
* buffers pointed to by the SGL. The byte offset starts at 0
* and continues until the last byte pointed to be the last SGL
* element.
*
* @return A virtual address pointer to the location specified by the
* parameters.
*/
uint8_t *
scif_cb_io_request_get_virtual_address_from_sgl(void * scif_user_io_request,
uint32_t byte_offset)
{
struct ISCI_IO_REQUEST *isci_request;
union ccb *ccb;
isci_request = scif_user_io_request;
ccb = isci_request->ccb;
/*
* This callback is only invoked for SCSI/ATA translation of
* PIO commands such as INQUIRY and READ_CAPACITY, to allow
* the driver to write the translated data directly into the
* data buffer. It is never invoked for READ/WRITE commands.
* The driver currently assumes only READ/WRITE commands will
* be unmapped.
*
* As a safeguard against future changes to unmapped commands,
* add an explicit panic here should the DATA_MASK != VADDR.
* Otherwise, we would return some garbage pointer back to the
* caller which would result in a panic or more subtle data
* corruption later on.
*/
if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR)
panic("%s: requesting pointer into unmapped ccb", __func__);
return (ccb->csio.data_ptr + byte_offset);
}
/**
* @brief This callback method asks the user to provide the number of
* bytes to be transfered as part of this request.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
*
* @return This method returns the number of payload data bytes to be
* transfered for this IO request.
*/
uint32_t
scif_cb_io_request_get_transfer_length(void * scif_user_io_request)
{
struct ISCI_IO_REQUEST *isci_request =
(struct ISCI_IO_REQUEST *)scif_user_io_request;
return (isci_request->ccb->csio.dxfer_len);
}
/**
* @brief This callback method asks the user to provide the data direction
* for this request.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
*
* @return This method returns the value of SCI_IO_REQUEST_DATA_OUT,
* SCI_IO_REQUEST_DATA_IN, or SCI_IO_REQUEST_NO_DATA.
*/
SCI_IO_REQUEST_DATA_DIRECTION
scif_cb_io_request_get_data_direction(void * scif_user_io_request)
{
struct ISCI_IO_REQUEST *isci_request =
(struct ISCI_IO_REQUEST *)scif_user_io_request;
switch (isci_request->ccb->ccb_h.flags & CAM_DIR_MASK) {
case CAM_DIR_IN:
return (SCI_IO_REQUEST_DATA_IN);
case CAM_DIR_OUT:
return (SCI_IO_REQUEST_DATA_OUT);
default:
return (SCI_IO_REQUEST_NO_DATA);
}
}
/**
* @brief This callback method asks the user to provide the address
* to where the next Scatter-Gather Element is located.
*
* Details regarding usage:
* - Regarding the first SGE: the user should initialize an index,
* or a pointer, prior to construction of the request that will
* reference the very first scatter-gather element. This is
* important since this method is called for every scatter-gather
* element, including the first element.
* - Regarding the last SGE: the user should return NULL from this
* method when this method is called and the SGL has exhausted
* all elements.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
* @param[in] current_sge_address This parameter specifies the address for
* the current SGE (i.e. the one that has just processed).
* @param[out] next_sge An address specifying the location for the next scatter
* gather element to be processed.
*
* @return None.
*/
void
scif_cb_io_request_get_next_sge(void * scif_user_io_request,
void * current_sge_address, void ** next_sge)
{
struct ISCI_IO_REQUEST *isci_request =
(struct ISCI_IO_REQUEST *)scif_user_io_request;
if (isci_request->current_sge_index == isci_request->num_segments)
*next_sge = NULL;
else {
bus_dma_segment_t *sge =
&isci_request->sge[isci_request->current_sge_index];
isci_request->current_sge_index++;
*next_sge = sge;
}
}
/**
* @brief This callback method asks the user to provide the contents of the
* "address" field in the Scatter-Gather Element.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
* @param[in] sge_address This parameter specifies the address for the
* SGE from which to retrieve the address field.
*
* @return A physical address specifying the contents of the SGE's address
* field.
*/
SCI_PHYSICAL_ADDRESS
scif_cb_sge_get_address_field(void *scif_user_io_request, void *sge_address)
{
bus_dma_segment_t *sge = (bus_dma_segment_t *)sge_address;
return ((SCI_PHYSICAL_ADDRESS)sge->ds_addr);
}
/**
* @brief This callback method asks the user to provide the contents of the
* "length" field in the Scatter-Gather Element.
*
* @param[in] scif_user_io_request This parameter points to the user's
* IO request object. It is a cookie that allows the user to
* provide the necessary information for this callback.
* @param[in] sge_address This parameter specifies the address for the
* SGE from which to retrieve the address field.
*
* @return This method returns the length field specified inside the SGE
* referenced by the sge_address parameter.
*/
uint32_t
scif_cb_sge_get_length_field(void *scif_user_io_request, void *sge_address)
{
bus_dma_segment_t *sge = (bus_dma_segment_t *)sge_address;
return ((uint32_t)sge->ds_len);
}
void
isci_request_construct(struct ISCI_REQUEST *request,
SCI_CONTROLLER_HANDLE_T scif_controller_handle,
bus_dma_tag_t io_buffer_dma_tag, bus_addr_t physical_address)
{
request->controller_handle = scif_controller_handle;
request->dma_tag = io_buffer_dma_tag;
request->physical_address = physical_address;
bus_dmamap_create(request->dma_tag, 0, &request->dma_map);
- callout_init(&request->timer, CALLOUT_MPSAFE);
+ callout_init(&request->timer, 1);
}
static void
isci_io_request_construct(void *arg, bus_dma_segment_t *seg, int nseg,
int error)
{
union ccb *ccb;
struct ISCI_IO_REQUEST *io_request = (struct ISCI_IO_REQUEST *)arg;
SCI_REMOTE_DEVICE_HANDLE_T *device = io_request->parent.remote_device_handle;
SCI_STATUS status;
io_request->num_segments = nseg;
io_request->sge = seg;
ccb = io_request->ccb;
if (error != 0) {
ccb->ccb_h.status = CAM_REQ_INVALID;
xpt_done(ccb);
return;
}
status = scif_io_request_construct(
io_request->parent.controller_handle,
io_request->parent.remote_device_handle,
SCI_CONTROLLER_INVALID_IO_TAG, (void *)io_request,
(void *)((char*)io_request + sizeof(struct ISCI_IO_REQUEST)),
&io_request->sci_object);
if (status != SCI_SUCCESS) {
isci_io_request_complete(io_request->parent.controller_handle,
device, io_request, (SCI_IO_STATUS)status);
return;
}
sci_object_set_association(io_request->sci_object, io_request);
bus_dmamap_sync(io_request->parent.dma_tag, io_request->parent.dma_map,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
status = (SCI_STATUS)scif_controller_start_io(
io_request->parent.controller_handle, device,
io_request->sci_object, SCI_CONTROLLER_INVALID_IO_TAG);
if (status != SCI_SUCCESS) {
isci_io_request_complete(io_request->parent.controller_handle,
device, io_request, (SCI_IO_STATUS)status);
return;
}
if (ccb->ccb_h.timeout != CAM_TIME_INFINITY)
callout_reset_sbt(&io_request->parent.timer,
SBT_1MS * ccb->ccb_h.timeout, 0, isci_io_request_timeout,
io_request, 0);
}
void
isci_io_request_execute_scsi_io(union ccb *ccb,
struct ISCI_CONTROLLER *controller)
{
target_id_t target_id = ccb->ccb_h.target_id;
struct ISCI_REQUEST *request;
struct ISCI_IO_REQUEST *io_request;
struct ISCI_REMOTE_DEVICE *device =
controller->remote_device[target_id];
int error;
if (device == NULL) {
ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
ccb->ccb_h.status &= ~CAM_STATUS_MASK;
ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
xpt_done(ccb);
return;
}
if (sci_pool_empty(controller->request_pool)) {
ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
ccb->ccb_h.status &= ~CAM_STATUS_MASK;
ccb->ccb_h.status |= CAM_REQUEUE_REQ;
xpt_freeze_simq(controller->sim, 1);
controller->is_frozen = TRUE;
xpt_done(ccb);
return;
}
ASSERT(device->is_resetting == FALSE);
sci_pool_get(controller->request_pool, request);
io_request = (struct ISCI_IO_REQUEST *)request;
io_request->ccb = ccb;
io_request->current_sge_index = 0;
io_request->parent.remote_device_handle = device->sci_object;
error = bus_dmamap_load_ccb(io_request->parent.dma_tag,
io_request->parent.dma_map, ccb,
isci_io_request_construct, io_request, 0x0);
/* A resource shortage from BUSDMA will be automatically
* continued at a later point, pushing the CCB processing
* forward, which will in turn unfreeze the simq.
*/
if (error == EINPROGRESS) {
xpt_freeze_simq(controller->sim, 1);
ccb->ccb_h.flags |= CAM_RELEASE_SIMQ;
}
}
void
isci_io_request_timeout(void *arg)
{
struct ISCI_IO_REQUEST *request = (struct ISCI_IO_REQUEST *)arg;
struct ISCI_REMOTE_DEVICE *remote_device = (struct ISCI_REMOTE_DEVICE *)
sci_object_get_association(request->parent.remote_device_handle);
struct ISCI_CONTROLLER *controller = remote_device->domain->controller;
mtx_lock(&controller->lock);
isci_remote_device_reset(remote_device, NULL);
mtx_unlock(&controller->lock);
}
#if __FreeBSD_version >= 900026
/**
* @brief This callback method gets the size of and pointer to the buffer
* (if any) containing the request buffer for an SMP request.
*
* @param[in] core_request This parameter specifies the SCI core's request
* object associated with the SMP request.
* @param[out] smp_request_buffer This parameter returns a pointer to the
* payload portion of the SMP request - i.e. everything after
* the SMP request header.
*
* @return Size of the request buffer in bytes. This does *not* include
* the size of the SMP request header.
*/
static uint32_t
smp_io_request_cb_get_request_buffer(SCI_IO_REQUEST_HANDLE_T core_request,
uint8_t ** smp_request_buffer)
{
struct ISCI_IO_REQUEST *isci_request = (struct ISCI_IO_REQUEST *)
sci_object_get_association(sci_object_get_association(core_request));
*smp_request_buffer = isci_request->ccb->smpio.smp_request +
sizeof(SMP_REQUEST_HEADER_T);
return (isci_request->ccb->smpio.smp_request_len -
sizeof(SMP_REQUEST_HEADER_T));
}
/**
* @brief This callback method gets the SMP function for an SMP request.
*
* @param[in] core_request This parameter specifies the SCI core's request
* object associated with the SMP request.
*
* @return SMP function for the SMP request.
*/
static uint8_t
smp_io_request_cb_get_function(SCI_IO_REQUEST_HANDLE_T core_request)
{
struct ISCI_IO_REQUEST *isci_request = (struct ISCI_IO_REQUEST *)
sci_object_get_association(sci_object_get_association(core_request));
SMP_REQUEST_HEADER_T *header =
(SMP_REQUEST_HEADER_T *)isci_request->ccb->smpio.smp_request;
return (header->function);
}
/**
* @brief This callback method gets the SMP frame type for an SMP request.
*
* @param[in] core_request This parameter specifies the SCI core's request
* object associated with the SMP request.
*
* @return SMP frame type for the SMP request.
*/
static uint8_t
smp_io_request_cb_get_frame_type(SCI_IO_REQUEST_HANDLE_T core_request)
{
struct ISCI_IO_REQUEST *isci_request = (struct ISCI_IO_REQUEST *)
sci_object_get_association(sci_object_get_association(core_request));
SMP_REQUEST_HEADER_T *header =
(SMP_REQUEST_HEADER_T *)isci_request->ccb->smpio.smp_request;
return (header->smp_frame_type);
}
/**
* @brief This callback method gets the allocated response length for an SMP request.
*
* @param[in] core_request This parameter specifies the SCI core's request
* object associated with the SMP request.
*
* @return Allocated response length for the SMP request.
*/
static uint8_t
smp_io_request_cb_get_allocated_response_length(
SCI_IO_REQUEST_HANDLE_T core_request)
{
struct ISCI_IO_REQUEST *isci_request = (struct ISCI_IO_REQUEST *)
sci_object_get_association(sci_object_get_association(core_request));
SMP_REQUEST_HEADER_T *header =
(SMP_REQUEST_HEADER_T *)isci_request->ccb->smpio.smp_request;
return (header->allocated_response_length);
}
static SCI_STATUS
isci_smp_request_construct(struct ISCI_IO_REQUEST *request)
{
SCI_STATUS status;
SCIC_SMP_PASSTHRU_REQUEST_CALLBACKS_T callbacks;
status = scif_request_construct(request->parent.controller_handle,
request->parent.remote_device_handle, SCI_CONTROLLER_INVALID_IO_TAG,
(void *)request,
(void *)((char*)request + sizeof(struct ISCI_IO_REQUEST)),
&request->sci_object);
if (status == SCI_SUCCESS) {
callbacks.scic_cb_smp_passthru_get_request =
&smp_io_request_cb_get_request_buffer;
callbacks.scic_cb_smp_passthru_get_function =
&smp_io_request_cb_get_function;
callbacks.scic_cb_smp_passthru_get_frame_type =
&smp_io_request_cb_get_frame_type;
callbacks.scic_cb_smp_passthru_get_allocated_response_length =
&smp_io_request_cb_get_allocated_response_length;
/* create the smp passthrough part of the io request */
status = scic_io_request_construct_smp_pass_through(
scif_io_request_get_scic_handle(request->sci_object),
&callbacks);
}
return (status);
}
void
isci_io_request_execute_smp_io(union ccb *ccb,
struct ISCI_CONTROLLER *controller)
{
SCI_STATUS status;
target_id_t target_id = ccb->ccb_h.target_id;
struct ISCI_REQUEST *request;
struct ISCI_IO_REQUEST *io_request;
SCI_REMOTE_DEVICE_HANDLE_T smp_device_handle;
struct ISCI_REMOTE_DEVICE *end_device = controller->remote_device[target_id];
/* SMP commands are sent to an end device, because SMP devices are not
* exposed to the kernel. It is our responsibility to use this method
* to get the SMP device that contains the specified end device. If
* the device is direct-attached, the handle will come back NULL, and
* we'll just fail the SMP_IO with DEV_NOT_THERE.
*/
scif_remote_device_get_containing_device(end_device->sci_object,
&smp_device_handle);
if (smp_device_handle == NULL) {
ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
ccb->ccb_h.status &= ~CAM_STATUS_MASK;
ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
xpt_done(ccb);
return;
}
if (sci_pool_empty(controller->request_pool)) {
ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
ccb->ccb_h.status &= ~CAM_STATUS_MASK;
ccb->ccb_h.status |= CAM_REQUEUE_REQ;
xpt_freeze_simq(controller->sim, 1);
controller->is_frozen = TRUE;
xpt_done(ccb);
return;
}
ASSERT(device->is_resetting == FALSE);
sci_pool_get(controller->request_pool, request);
io_request = (struct ISCI_IO_REQUEST *)request;
io_request->ccb = ccb;
io_request->parent.remote_device_handle = smp_device_handle;
status = isci_smp_request_construct(io_request);
if (status != SCI_SUCCESS) {
isci_io_request_complete(controller->scif_controller_handle,
smp_device_handle, io_request, (SCI_IO_STATUS)status);
return;
}
sci_object_set_association(io_request->sci_object, io_request);
status = (SCI_STATUS) scif_controller_start_io(
controller->scif_controller_handle, smp_device_handle,
io_request->sci_object, SCI_CONTROLLER_INVALID_IO_TAG);
if (status != SCI_SUCCESS) {
isci_io_request_complete(controller->scif_controller_handle,
smp_device_handle, io_request, (SCI_IO_STATUS)status);
return;
}
if (ccb->ccb_h.timeout != CAM_TIME_INFINITY)
callout_reset_sbt(&io_request->parent.timer,
SBT_1MS * ccb->ccb_h.timeout, 0, isci_io_request_timeout,
request, 0);
}
#endif
Index: head/sys/dev/mfi/mfi.c
===================================================================
--- head/sys/dev/mfi/mfi.c (revision 283290)
+++ head/sys/dev/mfi/mfi.c (revision 283291)
@@ -1,3803 +1,3803 @@
/*-
* Copyright (c) 2006 IronPort Systems
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*-
* Copyright (c) 2007 LSI Corp.
* Copyright (c) 2007 Rajesh Prabhakaran.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_compat.h"
#include "opt_mfi.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/poll.h>
#include <sys/selinfo.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/eventhandler.h>
#include <sys/rman.h>
#include <sys/bus_dma.h>
#include <sys/bio.h>
#include <sys/ioccom.h>
#include <sys/uio.h>
#include <sys/proc.h>
#include <sys/signalvar.h>
#include <sys/sysent.h>
#include <sys/taskqueue.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <dev/mfi/mfireg.h>
#include <dev/mfi/mfi_ioctl.h>
#include <dev/mfi/mfivar.h>
#include <sys/interrupt.h>
#include <sys/priority.h>
static int mfi_alloc_commands(struct mfi_softc *);
static int mfi_comms_init(struct mfi_softc *);
static int mfi_get_controller_info(struct mfi_softc *);
static int mfi_get_log_state(struct mfi_softc *,
struct mfi_evt_log_state **);
static int mfi_parse_entries(struct mfi_softc *, int, int);
static void mfi_data_cb(void *, bus_dma_segment_t *, int, int);
static void mfi_startup(void *arg);
static void mfi_intr(void *arg);
static void mfi_ldprobe(struct mfi_softc *sc);
static void mfi_syspdprobe(struct mfi_softc *sc);
static void mfi_handle_evt(void *context, int pending);
static int mfi_aen_register(struct mfi_softc *sc, int seq, int locale);
static void mfi_aen_complete(struct mfi_command *);
static int mfi_add_ld(struct mfi_softc *sc, int);
static void mfi_add_ld_complete(struct mfi_command *);
static int mfi_add_sys_pd(struct mfi_softc *sc, int);
static void mfi_add_sys_pd_complete(struct mfi_command *);
static struct mfi_command * mfi_bio_command(struct mfi_softc *);
static void mfi_bio_complete(struct mfi_command *);
static struct mfi_command *mfi_build_ldio(struct mfi_softc *,struct bio*);
static struct mfi_command *mfi_build_syspdio(struct mfi_softc *,struct bio*);
static int mfi_send_frame(struct mfi_softc *, struct mfi_command *);
static int mfi_std_send_frame(struct mfi_softc *, struct mfi_command *);
static int mfi_abort(struct mfi_softc *, struct mfi_command **);
static int mfi_linux_ioctl_int(struct cdev *, u_long, caddr_t, int, struct thread *);
static void mfi_timeout(void *);
static int mfi_user_command(struct mfi_softc *,
struct mfi_ioc_passthru *);
static void mfi_enable_intr_xscale(struct mfi_softc *sc);
static void mfi_enable_intr_ppc(struct mfi_softc *sc);
static int32_t mfi_read_fw_status_xscale(struct mfi_softc *sc);
static int32_t mfi_read_fw_status_ppc(struct mfi_softc *sc);
static int mfi_check_clear_intr_xscale(struct mfi_softc *sc);
static int mfi_check_clear_intr_ppc(struct mfi_softc *sc);
static void mfi_issue_cmd_xscale(struct mfi_softc *sc, bus_addr_t bus_add,
uint32_t frame_cnt);
static void mfi_issue_cmd_ppc(struct mfi_softc *sc, bus_addr_t bus_add,
uint32_t frame_cnt);
static int mfi_config_lock(struct mfi_softc *sc, uint32_t opcode);
static void mfi_config_unlock(struct mfi_softc *sc, int locked);
static int mfi_check_command_pre(struct mfi_softc *sc, struct mfi_command *cm);
static void mfi_check_command_post(struct mfi_softc *sc, struct mfi_command *cm);
static int mfi_check_for_sscd(struct mfi_softc *sc, struct mfi_command *cm);
SYSCTL_NODE(_hw, OID_AUTO, mfi, CTLFLAG_RD, 0, "MFI driver parameters");
static int mfi_event_locale = MFI_EVT_LOCALE_ALL;
SYSCTL_INT(_hw_mfi, OID_AUTO, event_locale, CTLFLAG_RWTUN, &mfi_event_locale,
0, "event message locale");
static int mfi_event_class = MFI_EVT_CLASS_INFO;
SYSCTL_INT(_hw_mfi, OID_AUTO, event_class, CTLFLAG_RWTUN, &mfi_event_class,
0, "event message class");
static int mfi_max_cmds = 128;
SYSCTL_INT(_hw_mfi, OID_AUTO, max_cmds, CTLFLAG_RDTUN, &mfi_max_cmds,
0, "Max commands limit (-1 = controller limit)");
static int mfi_detect_jbod_change = 1;
SYSCTL_INT(_hw_mfi, OID_AUTO, detect_jbod_change, CTLFLAG_RWTUN,
&mfi_detect_jbod_change, 0, "Detect a change to a JBOD");
int mfi_polled_cmd_timeout = MFI_POLL_TIMEOUT_SECS;
SYSCTL_INT(_hw_mfi, OID_AUTO, polled_cmd_timeout, CTLFLAG_RWTUN,
&mfi_polled_cmd_timeout, 0,
"Polled command timeout - used for firmware flash etc (in seconds)");
static int mfi_cmd_timeout = MFI_CMD_TIMEOUT;
SYSCTL_INT(_hw_mfi, OID_AUTO, cmd_timeout, CTLFLAG_RWTUN, &mfi_cmd_timeout,
0, "Command timeout (in seconds)");
/* Management interface */
static d_open_t mfi_open;
static d_close_t mfi_close;
static d_ioctl_t mfi_ioctl;
static d_poll_t mfi_poll;
static struct cdevsw mfi_cdevsw = {
.d_version = D_VERSION,
.d_flags = 0,
.d_open = mfi_open,
.d_close = mfi_close,
.d_ioctl = mfi_ioctl,
.d_poll = mfi_poll,
.d_name = "mfi",
};
MALLOC_DEFINE(M_MFIBUF, "mfibuf", "Buffers for the MFI driver");
#define MFI_INQ_LENGTH SHORT_INQUIRY_LENGTH
struct mfi_skinny_dma_info mfi_skinny;
static void
mfi_enable_intr_xscale(struct mfi_softc *sc)
{
MFI_WRITE4(sc, MFI_OMSK, 0x01);
}
static void
mfi_enable_intr_ppc(struct mfi_softc *sc)
{
if (sc->mfi_flags & MFI_FLAGS_1078) {
MFI_WRITE4(sc, MFI_ODCR0, 0xFFFFFFFF);
MFI_WRITE4(sc, MFI_OMSK, ~MFI_1078_EIM);
}
else if (sc->mfi_flags & MFI_FLAGS_GEN2) {
MFI_WRITE4(sc, MFI_ODCR0, 0xFFFFFFFF);
MFI_WRITE4(sc, MFI_OMSK, ~MFI_GEN2_EIM);
}
else if (sc->mfi_flags & MFI_FLAGS_SKINNY) {
MFI_WRITE4(sc, MFI_OMSK, ~0x00000001);
}
}
static int32_t
mfi_read_fw_status_xscale(struct mfi_softc *sc)
{
return MFI_READ4(sc, MFI_OMSG0);
}
static int32_t
mfi_read_fw_status_ppc(struct mfi_softc *sc)
{
return MFI_READ4(sc, MFI_OSP0);
}
static int
mfi_check_clear_intr_xscale(struct mfi_softc *sc)
{
int32_t status;
status = MFI_READ4(sc, MFI_OSTS);
if ((status & MFI_OSTS_INTR_VALID) == 0)
return 1;
MFI_WRITE4(sc, MFI_OSTS, status);
return 0;
}
static int
mfi_check_clear_intr_ppc(struct mfi_softc *sc)
{
int32_t status;
status = MFI_READ4(sc, MFI_OSTS);
if (sc->mfi_flags & MFI_FLAGS_1078) {
if (!(status & MFI_1078_RM)) {
return 1;
}
}
else if (sc->mfi_flags & MFI_FLAGS_GEN2) {
if (!(status & MFI_GEN2_RM)) {
return 1;
}
}
else if (sc->mfi_flags & MFI_FLAGS_SKINNY) {
if (!(status & MFI_SKINNY_RM)) {
return 1;
}
}
if (sc->mfi_flags & MFI_FLAGS_SKINNY)
MFI_WRITE4(sc, MFI_OSTS, status);
else
MFI_WRITE4(sc, MFI_ODCR0, status);
return 0;
}
static void
mfi_issue_cmd_xscale(struct mfi_softc *sc, bus_addr_t bus_add, uint32_t frame_cnt)
{
MFI_WRITE4(sc, MFI_IQP,(bus_add >>3)|frame_cnt);
}
static void
mfi_issue_cmd_ppc(struct mfi_softc *sc, bus_addr_t bus_add, uint32_t frame_cnt)
{
if (sc->mfi_flags & MFI_FLAGS_SKINNY) {
MFI_WRITE4(sc, MFI_IQPL, (bus_add | frame_cnt <<1)|1 );
MFI_WRITE4(sc, MFI_IQPH, 0x00000000);
} else {
MFI_WRITE4(sc, MFI_IQP, (bus_add | frame_cnt <<1)|1 );
}
}
int
mfi_transition_firmware(struct mfi_softc *sc)
{
uint32_t fw_state, cur_state;
int max_wait, i;
uint32_t cur_abs_reg_val = 0;
uint32_t prev_abs_reg_val = 0;
cur_abs_reg_val = sc->mfi_read_fw_status(sc);
fw_state = cur_abs_reg_val & MFI_FWSTATE_MASK;
while (fw_state != MFI_FWSTATE_READY) {
if (bootverbose)
device_printf(sc->mfi_dev, "Waiting for firmware to "
"become ready\n");
cur_state = fw_state;
switch (fw_state) {
case MFI_FWSTATE_FAULT:
device_printf(sc->mfi_dev, "Firmware fault\n");
return (ENXIO);
case MFI_FWSTATE_WAIT_HANDSHAKE:
if (sc->mfi_flags & MFI_FLAGS_SKINNY || sc->mfi_flags & MFI_FLAGS_TBOLT)
MFI_WRITE4(sc, MFI_SKINNY_IDB, MFI_FWINIT_CLEAR_HANDSHAKE);
else
MFI_WRITE4(sc, MFI_IDB, MFI_FWINIT_CLEAR_HANDSHAKE);
max_wait = MFI_RESET_WAIT_TIME;
break;
case MFI_FWSTATE_OPERATIONAL:
if (sc->mfi_flags & MFI_FLAGS_SKINNY || sc->mfi_flags & MFI_FLAGS_TBOLT)
MFI_WRITE4(sc, MFI_SKINNY_IDB, 7);
else
MFI_WRITE4(sc, MFI_IDB, MFI_FWINIT_READY);
max_wait = MFI_RESET_WAIT_TIME;
break;
case MFI_FWSTATE_UNDEFINED:
case MFI_FWSTATE_BB_INIT:
max_wait = MFI_RESET_WAIT_TIME;
break;
case MFI_FWSTATE_FW_INIT_2:
max_wait = MFI_RESET_WAIT_TIME;
break;
case MFI_FWSTATE_FW_INIT:
case MFI_FWSTATE_FLUSH_CACHE:
max_wait = MFI_RESET_WAIT_TIME;
break;
case MFI_FWSTATE_DEVICE_SCAN:
max_wait = MFI_RESET_WAIT_TIME; /* wait for 180 seconds */
prev_abs_reg_val = cur_abs_reg_val;
break;
case MFI_FWSTATE_BOOT_MESSAGE_PENDING:
if (sc->mfi_flags & MFI_FLAGS_SKINNY || sc->mfi_flags & MFI_FLAGS_TBOLT)
MFI_WRITE4(sc, MFI_SKINNY_IDB, MFI_FWINIT_HOTPLUG);
else
MFI_WRITE4(sc, MFI_IDB, MFI_FWINIT_HOTPLUG);
max_wait = MFI_RESET_WAIT_TIME;
break;
default:
device_printf(sc->mfi_dev, "Unknown firmware state %#x\n",
fw_state);
return (ENXIO);
}
for (i = 0; i < (max_wait * 10); i++) {
cur_abs_reg_val = sc->mfi_read_fw_status(sc);
fw_state = cur_abs_reg_val & MFI_FWSTATE_MASK;
if (fw_state == cur_state)
DELAY(100000);
else
break;
}
if (fw_state == MFI_FWSTATE_DEVICE_SCAN) {
/* Check the device scanning progress */
if (prev_abs_reg_val != cur_abs_reg_val) {
continue;
}
}
if (fw_state == cur_state) {
device_printf(sc->mfi_dev, "Firmware stuck in state "
"%#x\n", fw_state);
return (ENXIO);
}
}
return (0);
}
static void
mfi_addr_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
{
bus_addr_t *addr;
addr = arg;
*addr = segs[0].ds_addr;
}
int
mfi_attach(struct mfi_softc *sc)
{
uint32_t status;
int error, commsz, framessz, sensesz;
int frames, unit, max_fw_sge, max_fw_cmds;
uint32_t tb_mem_size = 0;
if (sc == NULL)
return EINVAL;
device_printf(sc->mfi_dev, "Megaraid SAS driver Ver %s \n",
MEGASAS_VERSION);
mtx_init(&sc->mfi_io_lock, "MFI I/O lock", NULL, MTX_DEF);
sx_init(&sc->mfi_config_lock, "MFI config");
TAILQ_INIT(&sc->mfi_ld_tqh);
TAILQ_INIT(&sc->mfi_syspd_tqh);
TAILQ_INIT(&sc->mfi_ld_pend_tqh);
TAILQ_INIT(&sc->mfi_syspd_pend_tqh);
TAILQ_INIT(&sc->mfi_evt_queue);
TASK_INIT(&sc->mfi_evt_task, 0, mfi_handle_evt, sc);
TASK_INIT(&sc->mfi_map_sync_task, 0, mfi_handle_map_sync, sc);
TAILQ_INIT(&sc->mfi_aen_pids);
TAILQ_INIT(&sc->mfi_cam_ccbq);
mfi_initq_free(sc);
mfi_initq_ready(sc);
mfi_initq_busy(sc);
mfi_initq_bio(sc);
sc->adpreset = 0;
sc->last_seq_num = 0;
sc->disableOnlineCtrlReset = 1;
sc->issuepend_done = 1;
sc->hw_crit_error = 0;
if (sc->mfi_flags & MFI_FLAGS_1064R) {
sc->mfi_enable_intr = mfi_enable_intr_xscale;
sc->mfi_read_fw_status = mfi_read_fw_status_xscale;
sc->mfi_check_clear_intr = mfi_check_clear_intr_xscale;
sc->mfi_issue_cmd = mfi_issue_cmd_xscale;
} else if (sc->mfi_flags & MFI_FLAGS_TBOLT) {
sc->mfi_enable_intr = mfi_tbolt_enable_intr_ppc;
sc->mfi_disable_intr = mfi_tbolt_disable_intr_ppc;
sc->mfi_read_fw_status = mfi_tbolt_read_fw_status_ppc;
sc->mfi_check_clear_intr = mfi_tbolt_check_clear_intr_ppc;
sc->mfi_issue_cmd = mfi_tbolt_issue_cmd_ppc;
sc->mfi_adp_reset = mfi_tbolt_adp_reset;
sc->mfi_tbolt = 1;
TAILQ_INIT(&sc->mfi_cmd_tbolt_tqh);
} else {
sc->mfi_enable_intr = mfi_enable_intr_ppc;
sc->mfi_read_fw_status = mfi_read_fw_status_ppc;
sc->mfi_check_clear_intr = mfi_check_clear_intr_ppc;
sc->mfi_issue_cmd = mfi_issue_cmd_ppc;
}
/* Before we get too far, see if the firmware is working */
if ((error = mfi_transition_firmware(sc)) != 0) {
device_printf(sc->mfi_dev, "Firmware not in READY state, "
"error %d\n", error);
return (ENXIO);
}
/* Start: LSIP200113393 */
if (bus_dma_tag_create( sc->mfi_parent_dmat, /* parent */
1, 0, /* algnmnt, boundary */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
MEGASAS_MAX_NAME*sizeof(bus_addr_t), /* maxsize */
1, /* msegments */
MEGASAS_MAX_NAME*sizeof(bus_addr_t), /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->verbuf_h_dmat)) {
device_printf(sc->mfi_dev, "Cannot allocate verbuf_h_dmat DMA tag\n");
return (ENOMEM);
}
if (bus_dmamem_alloc(sc->verbuf_h_dmat, (void **)&sc->verbuf,
BUS_DMA_NOWAIT, &sc->verbuf_h_dmamap)) {
device_printf(sc->mfi_dev, "Cannot allocate verbuf_h_dmamap memory\n");
return (ENOMEM);
}
bzero(sc->verbuf, MEGASAS_MAX_NAME*sizeof(bus_addr_t));
bus_dmamap_load(sc->verbuf_h_dmat, sc->verbuf_h_dmamap,
sc->verbuf, MEGASAS_MAX_NAME*sizeof(bus_addr_t),
mfi_addr_cb, &sc->verbuf_h_busaddr, 0);
/* End: LSIP200113393 */
/*
* Get information needed for sizing the contiguous memory for the
* frame pool. Size down the sgl parameter since we know that
* we will never need more than what's required for MAXPHYS.
* It would be nice if these constants were available at runtime
* instead of compile time.
*/
status = sc->mfi_read_fw_status(sc);
max_fw_cmds = status & MFI_FWSTATE_MAXCMD_MASK;
if (mfi_max_cmds > 0 && mfi_max_cmds < max_fw_cmds) {
device_printf(sc->mfi_dev, "FW MaxCmds = %d, limiting to %d\n",
max_fw_cmds, mfi_max_cmds);
sc->mfi_max_fw_cmds = mfi_max_cmds;
} else {
sc->mfi_max_fw_cmds = max_fw_cmds;
}
max_fw_sge = (status & MFI_FWSTATE_MAXSGL_MASK) >> 16;
sc->mfi_max_sge = min(max_fw_sge, ((MFI_MAXPHYS / PAGE_SIZE) + 1));
/* ThunderBolt Support get the contiguous memory */
if (sc->mfi_flags & MFI_FLAGS_TBOLT) {
mfi_tbolt_init_globals(sc);
device_printf(sc->mfi_dev, "MaxCmd = %d, Drv MaxCmd = %d, "
"MaxSgl = %d, state = %#x\n", max_fw_cmds,
sc->mfi_max_fw_cmds, sc->mfi_max_sge, status);
tb_mem_size = mfi_tbolt_get_memory_requirement(sc);
if (bus_dma_tag_create( sc->mfi_parent_dmat, /* parent */
1, 0, /* algnmnt, boundary */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
tb_mem_size, /* maxsize */
1, /* msegments */
tb_mem_size, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->mfi_tb_dmat)) {
device_printf(sc->mfi_dev, "Cannot allocate comms DMA tag\n");
return (ENOMEM);
}
if (bus_dmamem_alloc(sc->mfi_tb_dmat, (void **)&sc->request_message_pool,
BUS_DMA_NOWAIT, &sc->mfi_tb_dmamap)) {
device_printf(sc->mfi_dev, "Cannot allocate comms memory\n");
return (ENOMEM);
}
bzero(sc->request_message_pool, tb_mem_size);
bus_dmamap_load(sc->mfi_tb_dmat, sc->mfi_tb_dmamap,
sc->request_message_pool, tb_mem_size, mfi_addr_cb, &sc->mfi_tb_busaddr, 0);
/* For ThunderBolt memory init */
if (bus_dma_tag_create( sc->mfi_parent_dmat, /* parent */
0x100, 0, /* alignmnt, boundary */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
MFI_FRAME_SIZE, /* maxsize */
1, /* msegments */
MFI_FRAME_SIZE, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->mfi_tb_init_dmat)) {
device_printf(sc->mfi_dev, "Cannot allocate init DMA tag\n");
return (ENOMEM);
}
if (bus_dmamem_alloc(sc->mfi_tb_init_dmat, (void **)&sc->mfi_tb_init,
BUS_DMA_NOWAIT, &sc->mfi_tb_init_dmamap)) {
device_printf(sc->mfi_dev, "Cannot allocate init memory\n");
return (ENOMEM);
}
bzero(sc->mfi_tb_init, MFI_FRAME_SIZE);
bus_dmamap_load(sc->mfi_tb_init_dmat, sc->mfi_tb_init_dmamap,
sc->mfi_tb_init, MFI_FRAME_SIZE, mfi_addr_cb,
&sc->mfi_tb_init_busaddr, 0);
if (mfi_tbolt_init_desc_pool(sc, sc->request_message_pool,
tb_mem_size)) {
device_printf(sc->mfi_dev,
"Thunderbolt pool preparation error\n");
return 0;
}
/*
Allocate DMA memory mapping for MPI2 IOC Init descriptor,
we are taking it diffrent from what we have allocated for Request
and reply descriptors to avoid confusion later
*/
tb_mem_size = sizeof(struct MPI2_IOC_INIT_REQUEST);
if (bus_dma_tag_create( sc->mfi_parent_dmat, /* parent */
1, 0, /* algnmnt, boundary */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
tb_mem_size, /* maxsize */
1, /* msegments */
tb_mem_size, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->mfi_tb_ioc_init_dmat)) {
device_printf(sc->mfi_dev,
"Cannot allocate comms DMA tag\n");
return (ENOMEM);
}
if (bus_dmamem_alloc(sc->mfi_tb_ioc_init_dmat,
(void **)&sc->mfi_tb_ioc_init_desc,
BUS_DMA_NOWAIT, &sc->mfi_tb_ioc_init_dmamap)) {
device_printf(sc->mfi_dev, "Cannot allocate comms memory\n");
return (ENOMEM);
}
bzero(sc->mfi_tb_ioc_init_desc, tb_mem_size);
bus_dmamap_load(sc->mfi_tb_ioc_init_dmat, sc->mfi_tb_ioc_init_dmamap,
sc->mfi_tb_ioc_init_desc, tb_mem_size, mfi_addr_cb,
&sc->mfi_tb_ioc_init_busaddr, 0);
}
/*
* Create the dma tag for data buffers. Used both for block I/O
* and for various internal data queries.
*/
if (bus_dma_tag_create( sc->mfi_parent_dmat, /* parent */
1, 0, /* algnmnt, boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
sc->mfi_max_sge, /* nsegments */
BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
busdma_lock_mutex, /* lockfunc */
&sc->mfi_io_lock, /* lockfuncarg */
&sc->mfi_buffer_dmat)) {
device_printf(sc->mfi_dev, "Cannot allocate buffer DMA tag\n");
return (ENOMEM);
}
/*
* Allocate DMA memory for the comms queues. Keep it under 4GB for
* efficiency. The mfi_hwcomms struct includes space for 1 reply queue
* entry, so the calculated size here will be will be 1 more than
* mfi_max_fw_cmds. This is apparently a requirement of the hardware.
*/
commsz = (sizeof(uint32_t) * sc->mfi_max_fw_cmds) +
sizeof(struct mfi_hwcomms);
if (bus_dma_tag_create( sc->mfi_parent_dmat, /* parent */
1, 0, /* algnmnt, boundary */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
commsz, /* maxsize */
1, /* msegments */
commsz, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->mfi_comms_dmat)) {
device_printf(sc->mfi_dev, "Cannot allocate comms DMA tag\n");
return (ENOMEM);
}
if (bus_dmamem_alloc(sc->mfi_comms_dmat, (void **)&sc->mfi_comms,
BUS_DMA_NOWAIT, &sc->mfi_comms_dmamap)) {
device_printf(sc->mfi_dev, "Cannot allocate comms memory\n");
return (ENOMEM);
}
bzero(sc->mfi_comms, commsz);
bus_dmamap_load(sc->mfi_comms_dmat, sc->mfi_comms_dmamap,
sc->mfi_comms, commsz, mfi_addr_cb, &sc->mfi_comms_busaddr, 0);
/*
* Allocate DMA memory for the command frames. Keep them in the
* lower 4GB for efficiency. Calculate the size of the commands at
* the same time; each command is one 64 byte frame plus a set of
* additional frames for holding sg lists or other data.
* The assumption here is that the SG list will start at the second
* frame and not use the unused bytes in the first frame. While this
* isn't technically correct, it simplifies the calculation and allows
* for command frames that might be larger than an mfi_io_frame.
*/
if (sizeof(bus_addr_t) == 8) {
sc->mfi_sge_size = sizeof(struct mfi_sg64);
sc->mfi_flags |= MFI_FLAGS_SG64;
} else {
sc->mfi_sge_size = sizeof(struct mfi_sg32);
}
if (sc->mfi_flags & MFI_FLAGS_SKINNY)
sc->mfi_sge_size = sizeof(struct mfi_sg_skinny);
frames = (sc->mfi_sge_size * sc->mfi_max_sge - 1) / MFI_FRAME_SIZE + 2;
sc->mfi_cmd_size = frames * MFI_FRAME_SIZE;
framessz = sc->mfi_cmd_size * sc->mfi_max_fw_cmds;
if (bus_dma_tag_create( sc->mfi_parent_dmat, /* parent */
64, 0, /* algnmnt, boundary */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
framessz, /* maxsize */
1, /* nsegments */
framessz, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->mfi_frames_dmat)) {
device_printf(sc->mfi_dev, "Cannot allocate frame DMA tag\n");
return (ENOMEM);
}
if (bus_dmamem_alloc(sc->mfi_frames_dmat, (void **)&sc->mfi_frames,
BUS_DMA_NOWAIT, &sc->mfi_frames_dmamap)) {
device_printf(sc->mfi_dev, "Cannot allocate frames memory\n");
return (ENOMEM);
}
bzero(sc->mfi_frames, framessz);
bus_dmamap_load(sc->mfi_frames_dmat, sc->mfi_frames_dmamap,
sc->mfi_frames, framessz, mfi_addr_cb, &sc->mfi_frames_busaddr,0);
/*
* Allocate DMA memory for the frame sense data. Keep them in the
* lower 4GB for efficiency
*/
sensesz = sc->mfi_max_fw_cmds * MFI_SENSE_LEN;
if (bus_dma_tag_create( sc->mfi_parent_dmat, /* parent */
4, 0, /* algnmnt, boundary */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
sensesz, /* maxsize */
1, /* nsegments */
sensesz, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->mfi_sense_dmat)) {
device_printf(sc->mfi_dev, "Cannot allocate sense DMA tag\n");
return (ENOMEM);
}
if (bus_dmamem_alloc(sc->mfi_sense_dmat, (void **)&sc->mfi_sense,
BUS_DMA_NOWAIT, &sc->mfi_sense_dmamap)) {
device_printf(sc->mfi_dev, "Cannot allocate sense memory\n");
return (ENOMEM);
}
bus_dmamap_load(sc->mfi_sense_dmat, sc->mfi_sense_dmamap,
sc->mfi_sense, sensesz, mfi_addr_cb, &sc->mfi_sense_busaddr, 0);
if ((error = mfi_alloc_commands(sc)) != 0)
return (error);
/* Before moving the FW to operational state, check whether
* hostmemory is required by the FW or not
*/
/* ThunderBolt MFI_IOC2 INIT */
if (sc->mfi_flags & MFI_FLAGS_TBOLT) {
sc->mfi_disable_intr(sc);
mtx_lock(&sc->mfi_io_lock);
if ((error = mfi_tbolt_init_MFI_queue(sc)) != 0) {
device_printf(sc->mfi_dev,
"TB Init has failed with error %d\n",error);
mtx_unlock(&sc->mfi_io_lock);
return error;
}
mtx_unlock(&sc->mfi_io_lock);
if ((error = mfi_tbolt_alloc_cmd(sc)) != 0)
return error;
if (bus_setup_intr(sc->mfi_dev, sc->mfi_irq,
INTR_MPSAFE|INTR_TYPE_BIO, NULL, mfi_intr_tbolt, sc,
&sc->mfi_intr)) {
device_printf(sc->mfi_dev, "Cannot set up interrupt\n");
return (EINVAL);
}
sc->mfi_intr_ptr = mfi_intr_tbolt;
sc->mfi_enable_intr(sc);
} else {
if ((error = mfi_comms_init(sc)) != 0)
return (error);
if (bus_setup_intr(sc->mfi_dev, sc->mfi_irq,
INTR_MPSAFE|INTR_TYPE_BIO, NULL, mfi_intr, sc, &sc->mfi_intr)) {
device_printf(sc->mfi_dev, "Cannot set up interrupt\n");
return (EINVAL);
}
sc->mfi_intr_ptr = mfi_intr;
sc->mfi_enable_intr(sc);
}
if ((error = mfi_get_controller_info(sc)) != 0)
return (error);
sc->disableOnlineCtrlReset = 0;
/* Register a config hook to probe the bus for arrays */
sc->mfi_ich.ich_func = mfi_startup;
sc->mfi_ich.ich_arg = sc;
if (config_intrhook_establish(&sc->mfi_ich) != 0) {
device_printf(sc->mfi_dev, "Cannot establish configuration "
"hook\n");
return (EINVAL);
}
mtx_lock(&sc->mfi_io_lock);
if ((error = mfi_aen_setup(sc, 0), 0) != 0) {
mtx_unlock(&sc->mfi_io_lock);
return (error);
}
mtx_unlock(&sc->mfi_io_lock);
/*
* Register a shutdown handler.
*/
if ((sc->mfi_eh = EVENTHANDLER_REGISTER(shutdown_final, mfi_shutdown,
sc, SHUTDOWN_PRI_DEFAULT)) == NULL) {
device_printf(sc->mfi_dev, "Warning: shutdown event "
"registration failed\n");
}
/*
* Create the control device for doing management
*/
unit = device_get_unit(sc->mfi_dev);
sc->mfi_cdev = make_dev(&mfi_cdevsw, unit, UID_ROOT, GID_OPERATOR,
0640, "mfi%d", unit);
if (unit == 0)
make_dev_alias(sc->mfi_cdev, "megaraid_sas_ioctl_node");
if (sc->mfi_cdev != NULL)
sc->mfi_cdev->si_drv1 = sc;
SYSCTL_ADD_INT(device_get_sysctl_ctx(sc->mfi_dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(sc->mfi_dev)),
OID_AUTO, "delete_busy_volumes", CTLFLAG_RW,
&sc->mfi_delete_busy_volumes, 0, "Allow removal of busy volumes");
SYSCTL_ADD_INT(device_get_sysctl_ctx(sc->mfi_dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(sc->mfi_dev)),
OID_AUTO, "keep_deleted_volumes", CTLFLAG_RW,
&sc->mfi_keep_deleted_volumes, 0,
"Don't detach the mfid device for a busy volume that is deleted");
device_add_child(sc->mfi_dev, "mfip", -1);
bus_generic_attach(sc->mfi_dev);
/* Start the timeout watchdog */
- callout_init(&sc->mfi_watchdog_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->mfi_watchdog_callout, 1);
callout_reset(&sc->mfi_watchdog_callout, mfi_cmd_timeout * hz,
mfi_timeout, sc);
if (sc->mfi_flags & MFI_FLAGS_TBOLT) {
mtx_lock(&sc->mfi_io_lock);
mfi_tbolt_sync_map_info(sc);
mtx_unlock(&sc->mfi_io_lock);
}
return (0);
}
static int
mfi_alloc_commands(struct mfi_softc *sc)
{
struct mfi_command *cm;
int i, j;
/*
* XXX Should we allocate all the commands up front, or allocate on
* demand later like 'aac' does?
*/
sc->mfi_commands = malloc(sizeof(sc->mfi_commands[0]) *
sc->mfi_max_fw_cmds, M_MFIBUF, M_WAITOK | M_ZERO);
for (i = 0; i < sc->mfi_max_fw_cmds; i++) {
cm = &sc->mfi_commands[i];
cm->cm_frame = (union mfi_frame *)((uintptr_t)sc->mfi_frames +
sc->mfi_cmd_size * i);
cm->cm_frame_busaddr = sc->mfi_frames_busaddr +
sc->mfi_cmd_size * i;
cm->cm_frame->header.context = i;
cm->cm_sense = &sc->mfi_sense[i];
cm->cm_sense_busaddr= sc->mfi_sense_busaddr + MFI_SENSE_LEN * i;
cm->cm_sc = sc;
cm->cm_index = i;
if (bus_dmamap_create(sc->mfi_buffer_dmat, 0,
&cm->cm_dmamap) == 0) {
mtx_lock(&sc->mfi_io_lock);
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
} else {
device_printf(sc->mfi_dev, "Failed to allocate %d "
"command blocks, only allocated %d\n",
sc->mfi_max_fw_cmds, i - 1);
for (j = 0; j < i; j++) {
cm = &sc->mfi_commands[i];
bus_dmamap_destroy(sc->mfi_buffer_dmat,
cm->cm_dmamap);
}
free(sc->mfi_commands, M_MFIBUF);
sc->mfi_commands = NULL;
return (ENOMEM);
}
}
return (0);
}
void
mfi_release_command(struct mfi_command *cm)
{
struct mfi_frame_header *hdr;
uint32_t *hdr_data;
mtx_assert(&cm->cm_sc->mfi_io_lock, MA_OWNED);
/*
* Zero out the important fields of the frame, but make sure the
* context field is preserved. For efficiency, handle the fields
* as 32 bit words. Clear out the first S/G entry too for safety.
*/
hdr = &cm->cm_frame->header;
if (cm->cm_data != NULL && hdr->sg_count) {
cm->cm_sg->sg32[0].len = 0;
cm->cm_sg->sg32[0].addr = 0;
}
/*
* Command may be on other queues e.g. busy queue depending on the
* flow of a previous call to mfi_mapcmd, so ensure its dequeued
* properly
*/
if ((cm->cm_flags & MFI_ON_MFIQ_BUSY) != 0)
mfi_remove_busy(cm);
if ((cm->cm_flags & MFI_ON_MFIQ_READY) != 0)
mfi_remove_ready(cm);
/* We're not expecting it to be on any other queue but check */
if ((cm->cm_flags & MFI_ON_MFIQ_MASK) != 0) {
panic("Command %p is still on another queue, flags = %#x",
cm, cm->cm_flags);
}
/* tbolt cleanup */
if ((cm->cm_flags & MFI_CMD_TBOLT) != 0) {
mfi_tbolt_return_cmd(cm->cm_sc,
cm->cm_sc->mfi_cmd_pool_tbolt[cm->cm_extra_frames - 1],
cm);
}
hdr_data = (uint32_t *)cm->cm_frame;
hdr_data[0] = 0; /* cmd, sense_len, cmd_status, scsi_status */
hdr_data[1] = 0; /* target_id, lun_id, cdb_len, sg_count */
hdr_data[4] = 0; /* flags, timeout */
hdr_data[5] = 0; /* data_len */
cm->cm_extra_frames = 0;
cm->cm_flags = 0;
cm->cm_complete = NULL;
cm->cm_private = NULL;
cm->cm_data = NULL;
cm->cm_sg = 0;
cm->cm_total_frame_size = 0;
cm->retry_for_fw_reset = 0;
mfi_enqueue_free(cm);
}
int
mfi_dcmd_command(struct mfi_softc *sc, struct mfi_command **cmp,
uint32_t opcode, void **bufp, size_t bufsize)
{
struct mfi_command *cm;
struct mfi_dcmd_frame *dcmd;
void *buf = NULL;
uint32_t context = 0;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
cm = mfi_dequeue_free(sc);
if (cm == NULL)
return (EBUSY);
/* Zero out the MFI frame */
context = cm->cm_frame->header.context;
bzero(cm->cm_frame, sizeof(union mfi_frame));
cm->cm_frame->header.context = context;
if ((bufsize > 0) && (bufp != NULL)) {
if (*bufp == NULL) {
buf = malloc(bufsize, M_MFIBUF, M_NOWAIT|M_ZERO);
if (buf == NULL) {
mfi_release_command(cm);
return (ENOMEM);
}
*bufp = buf;
} else {
buf = *bufp;
}
}
dcmd = &cm->cm_frame->dcmd;
bzero(dcmd->mbox, MFI_MBOX_SIZE);
dcmd->header.cmd = MFI_CMD_DCMD;
dcmd->header.timeout = 0;
dcmd->header.flags = 0;
dcmd->header.data_len = bufsize;
dcmd->header.scsi_status = 0;
dcmd->opcode = opcode;
cm->cm_sg = &dcmd->sgl;
cm->cm_total_frame_size = MFI_DCMD_FRAME_SIZE;
cm->cm_flags = 0;
cm->cm_data = buf;
cm->cm_private = buf;
cm->cm_len = bufsize;
*cmp = cm;
if ((bufp != NULL) && (*bufp == NULL) && (buf != NULL))
*bufp = buf;
return (0);
}
static int
mfi_comms_init(struct mfi_softc *sc)
{
struct mfi_command *cm;
struct mfi_init_frame *init;
struct mfi_init_qinfo *qinfo;
int error;
uint32_t context = 0;
mtx_lock(&sc->mfi_io_lock);
if ((cm = mfi_dequeue_free(sc)) == NULL) {
mtx_unlock(&sc->mfi_io_lock);
return (EBUSY);
}
/* Zero out the MFI frame */
context = cm->cm_frame->header.context;
bzero(cm->cm_frame, sizeof(union mfi_frame));
cm->cm_frame->header.context = context;
/*
* Abuse the SG list area of the frame to hold the init_qinfo
* object;
*/
init = &cm->cm_frame->init;
qinfo = (struct mfi_init_qinfo *)((uintptr_t)init + MFI_FRAME_SIZE);
bzero(qinfo, sizeof(struct mfi_init_qinfo));
qinfo->rq_entries = sc->mfi_max_fw_cmds + 1;
qinfo->rq_addr_lo = sc->mfi_comms_busaddr +
offsetof(struct mfi_hwcomms, hw_reply_q);
qinfo->pi_addr_lo = sc->mfi_comms_busaddr +
offsetof(struct mfi_hwcomms, hw_pi);
qinfo->ci_addr_lo = sc->mfi_comms_busaddr +
offsetof(struct mfi_hwcomms, hw_ci);
init->header.cmd = MFI_CMD_INIT;
init->header.data_len = sizeof(struct mfi_init_qinfo);
init->qinfo_new_addr_lo = cm->cm_frame_busaddr + MFI_FRAME_SIZE;
cm->cm_data = NULL;
cm->cm_flags = MFI_CMD_POLLED;
if ((error = mfi_mapcmd(sc, cm)) != 0)
device_printf(sc->mfi_dev, "failed to send init command\n");
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
return (error);
}
static int
mfi_get_controller_info(struct mfi_softc *sc)
{
struct mfi_command *cm = NULL;
struct mfi_ctrl_info *ci = NULL;
uint32_t max_sectors_1, max_sectors_2;
int error;
mtx_lock(&sc->mfi_io_lock);
error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_GETINFO,
(void **)&ci, sizeof(*ci));
if (error)
goto out;
cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_POLLED;
if ((error = mfi_mapcmd(sc, cm)) != 0) {
device_printf(sc->mfi_dev, "Failed to get controller info\n");
sc->mfi_max_io = (sc->mfi_max_sge - 1) * PAGE_SIZE /
MFI_SECTOR_LEN;
error = 0;
goto out;
}
bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
max_sectors_1 = (1 << ci->stripe_sz_ops.max) * ci->max_strips_per_io;
max_sectors_2 = ci->max_request_size;
sc->mfi_max_io = min(max_sectors_1, max_sectors_2);
sc->disableOnlineCtrlReset =
ci->properties.OnOffProperties.disableOnlineCtrlReset;
out:
if (ci)
free(ci, M_MFIBUF);
if (cm)
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
return (error);
}
static int
mfi_get_log_state(struct mfi_softc *sc, struct mfi_evt_log_state **log_state)
{
struct mfi_command *cm = NULL;
int error;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_EVENT_GETINFO,
(void **)log_state, sizeof(**log_state));
if (error)
goto out;
cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_POLLED;
if ((error = mfi_mapcmd(sc, cm)) != 0) {
device_printf(sc->mfi_dev, "Failed to get log state\n");
goto out;
}
bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
out:
if (cm)
mfi_release_command(cm);
return (error);
}
int
mfi_aen_setup(struct mfi_softc *sc, uint32_t seq_start)
{
struct mfi_evt_log_state *log_state = NULL;
union mfi_evt class_locale;
int error = 0;
uint32_t seq;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
class_locale.members.reserved = 0;
class_locale.members.locale = mfi_event_locale;
class_locale.members.evt_class = mfi_event_class;
if (seq_start == 0) {
if ((error = mfi_get_log_state(sc, &log_state)) != 0)
goto out;
sc->mfi_boot_seq_num = log_state->boot_seq_num;
/*
* Walk through any events that fired since the last
* shutdown.
*/
if ((error = mfi_parse_entries(sc, log_state->shutdown_seq_num,
log_state->newest_seq_num)) != 0)
goto out;
seq = log_state->newest_seq_num;
} else
seq = seq_start;
error = mfi_aen_register(sc, seq, class_locale.word);
out:
free(log_state, M_MFIBUF);
return (error);
}
int
mfi_wait_command(struct mfi_softc *sc, struct mfi_command *cm)
{
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
cm->cm_complete = NULL;
/*
* MegaCli can issue a DCMD of 0. In this case do nothing
* and return 0 to it as status
*/
if (cm->cm_frame->dcmd.opcode == 0) {
cm->cm_frame->header.cmd_status = MFI_STAT_OK;
cm->cm_error = 0;
return (cm->cm_error);
}
mfi_enqueue_ready(cm);
mfi_startio(sc);
if ((cm->cm_flags & MFI_CMD_COMPLETED) == 0)
msleep(cm, &sc->mfi_io_lock, PRIBIO, "mfiwait", 0);
return (cm->cm_error);
}
void
mfi_free(struct mfi_softc *sc)
{
struct mfi_command *cm;
int i;
callout_drain(&sc->mfi_watchdog_callout);
if (sc->mfi_cdev != NULL)
destroy_dev(sc->mfi_cdev);
if (sc->mfi_commands != NULL) {
for (i = 0; i < sc->mfi_max_fw_cmds; i++) {
cm = &sc->mfi_commands[i];
bus_dmamap_destroy(sc->mfi_buffer_dmat, cm->cm_dmamap);
}
free(sc->mfi_commands, M_MFIBUF);
sc->mfi_commands = NULL;
}
if (sc->mfi_intr)
bus_teardown_intr(sc->mfi_dev, sc->mfi_irq, sc->mfi_intr);
if (sc->mfi_irq != NULL)
bus_release_resource(sc->mfi_dev, SYS_RES_IRQ, sc->mfi_irq_rid,
sc->mfi_irq);
if (sc->mfi_sense_busaddr != 0)
bus_dmamap_unload(sc->mfi_sense_dmat, sc->mfi_sense_dmamap);
if (sc->mfi_sense != NULL)
bus_dmamem_free(sc->mfi_sense_dmat, sc->mfi_sense,
sc->mfi_sense_dmamap);
if (sc->mfi_sense_dmat != NULL)
bus_dma_tag_destroy(sc->mfi_sense_dmat);
if (sc->mfi_frames_busaddr != 0)
bus_dmamap_unload(sc->mfi_frames_dmat, sc->mfi_frames_dmamap);
if (sc->mfi_frames != NULL)
bus_dmamem_free(sc->mfi_frames_dmat, sc->mfi_frames,
sc->mfi_frames_dmamap);
if (sc->mfi_frames_dmat != NULL)
bus_dma_tag_destroy(sc->mfi_frames_dmat);
if (sc->mfi_comms_busaddr != 0)
bus_dmamap_unload(sc->mfi_comms_dmat, sc->mfi_comms_dmamap);
if (sc->mfi_comms != NULL)
bus_dmamem_free(sc->mfi_comms_dmat, sc->mfi_comms,
sc->mfi_comms_dmamap);
if (sc->mfi_comms_dmat != NULL)
bus_dma_tag_destroy(sc->mfi_comms_dmat);
/* ThunderBolt contiguous memory free here */
if (sc->mfi_flags & MFI_FLAGS_TBOLT) {
if (sc->mfi_tb_busaddr != 0)
bus_dmamap_unload(sc->mfi_tb_dmat, sc->mfi_tb_dmamap);
if (sc->request_message_pool != NULL)
bus_dmamem_free(sc->mfi_tb_dmat, sc->request_message_pool,
sc->mfi_tb_dmamap);
if (sc->mfi_tb_dmat != NULL)
bus_dma_tag_destroy(sc->mfi_tb_dmat);
/* Version buffer memory free */
/* Start LSIP200113393 */
if (sc->verbuf_h_busaddr != 0)
bus_dmamap_unload(sc->verbuf_h_dmat, sc->verbuf_h_dmamap);
if (sc->verbuf != NULL)
bus_dmamem_free(sc->verbuf_h_dmat, sc->verbuf,
sc->verbuf_h_dmamap);
if (sc->verbuf_h_dmat != NULL)
bus_dma_tag_destroy(sc->verbuf_h_dmat);
/* End LSIP200113393 */
/* ThunderBolt INIT packet memory Free */
if (sc->mfi_tb_init_busaddr != 0)
bus_dmamap_unload(sc->mfi_tb_init_dmat,
sc->mfi_tb_init_dmamap);
if (sc->mfi_tb_init != NULL)
bus_dmamem_free(sc->mfi_tb_init_dmat, sc->mfi_tb_init,
sc->mfi_tb_init_dmamap);
if (sc->mfi_tb_init_dmat != NULL)
bus_dma_tag_destroy(sc->mfi_tb_init_dmat);
/* ThunderBolt IOC Init Desc memory free here */
if (sc->mfi_tb_ioc_init_busaddr != 0)
bus_dmamap_unload(sc->mfi_tb_ioc_init_dmat,
sc->mfi_tb_ioc_init_dmamap);
if (sc->mfi_tb_ioc_init_desc != NULL)
bus_dmamem_free(sc->mfi_tb_ioc_init_dmat,
sc->mfi_tb_ioc_init_desc,
sc->mfi_tb_ioc_init_dmamap);
if (sc->mfi_tb_ioc_init_dmat != NULL)
bus_dma_tag_destroy(sc->mfi_tb_ioc_init_dmat);
if (sc->mfi_cmd_pool_tbolt != NULL) {
for (int i = 0; i < sc->mfi_max_fw_cmds; i++) {
if (sc->mfi_cmd_pool_tbolt[i] != NULL) {
free(sc->mfi_cmd_pool_tbolt[i],
M_MFIBUF);
sc->mfi_cmd_pool_tbolt[i] = NULL;
}
}
free(sc->mfi_cmd_pool_tbolt, M_MFIBUF);
sc->mfi_cmd_pool_tbolt = NULL;
}
if (sc->request_desc_pool != NULL) {
free(sc->request_desc_pool, M_MFIBUF);
sc->request_desc_pool = NULL;
}
}
if (sc->mfi_buffer_dmat != NULL)
bus_dma_tag_destroy(sc->mfi_buffer_dmat);
if (sc->mfi_parent_dmat != NULL)
bus_dma_tag_destroy(sc->mfi_parent_dmat);
if (mtx_initialized(&sc->mfi_io_lock)) {
mtx_destroy(&sc->mfi_io_lock);
sx_destroy(&sc->mfi_config_lock);
}
return;
}
static void
mfi_startup(void *arg)
{
struct mfi_softc *sc;
sc = (struct mfi_softc *)arg;
config_intrhook_disestablish(&sc->mfi_ich);
sc->mfi_enable_intr(sc);
sx_xlock(&sc->mfi_config_lock);
mtx_lock(&sc->mfi_io_lock);
mfi_ldprobe(sc);
if (sc->mfi_flags & MFI_FLAGS_SKINNY)
mfi_syspdprobe(sc);
mtx_unlock(&sc->mfi_io_lock);
sx_xunlock(&sc->mfi_config_lock);
}
static void
mfi_intr(void *arg)
{
struct mfi_softc *sc;
struct mfi_command *cm;
uint32_t pi, ci, context;
sc = (struct mfi_softc *)arg;
if (sc->mfi_check_clear_intr(sc))
return;
restart:
pi = sc->mfi_comms->hw_pi;
ci = sc->mfi_comms->hw_ci;
mtx_lock(&sc->mfi_io_lock);
while (ci != pi) {
context = sc->mfi_comms->hw_reply_q[ci];
if (context < sc->mfi_max_fw_cmds) {
cm = &sc->mfi_commands[context];
mfi_remove_busy(cm);
cm->cm_error = 0;
mfi_complete(sc, cm);
}
if (++ci == (sc->mfi_max_fw_cmds + 1))
ci = 0;
}
sc->mfi_comms->hw_ci = ci;
/* Give defered I/O a chance to run */
sc->mfi_flags &= ~MFI_FLAGS_QFRZN;
mfi_startio(sc);
mtx_unlock(&sc->mfi_io_lock);
/*
* Dummy read to flush the bus; this ensures that the indexes are up
* to date. Restart processing if more commands have come it.
*/
(void)sc->mfi_read_fw_status(sc);
if (pi != sc->mfi_comms->hw_pi)
goto restart;
return;
}
int
mfi_shutdown(struct mfi_softc *sc)
{
struct mfi_dcmd_frame *dcmd;
struct mfi_command *cm;
int error;
if (sc->mfi_aen_cm != NULL) {
sc->cm_aen_abort = 1;
mfi_abort(sc, &sc->mfi_aen_cm);
}
if (sc->mfi_map_sync_cm != NULL) {
sc->cm_map_abort = 1;
mfi_abort(sc, &sc->mfi_map_sync_cm);
}
mtx_lock(&sc->mfi_io_lock);
error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_SHUTDOWN, NULL, 0);
if (error) {
mtx_unlock(&sc->mfi_io_lock);
return (error);
}
dcmd = &cm->cm_frame->dcmd;
dcmd->header.flags = MFI_FRAME_DIR_NONE;
cm->cm_flags = MFI_CMD_POLLED;
cm->cm_data = NULL;
if ((error = mfi_mapcmd(sc, cm)) != 0)
device_printf(sc->mfi_dev, "Failed to shutdown controller\n");
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
return (error);
}
static void
mfi_syspdprobe(struct mfi_softc *sc)
{
struct mfi_frame_header *hdr;
struct mfi_command *cm = NULL;
struct mfi_pd_list *pdlist = NULL;
struct mfi_system_pd *syspd, *tmp;
struct mfi_system_pending *syspd_pend;
int error, i, found;
sx_assert(&sc->mfi_config_lock, SA_XLOCKED);
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
/* Add SYSTEM PD's */
error = mfi_dcmd_command(sc, &cm, MFI_DCMD_PD_LIST_QUERY,
(void **)&pdlist, sizeof(*pdlist));
if (error) {
device_printf(sc->mfi_dev,
"Error while forming SYSTEM PD list\n");
goto out;
}
cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_POLLED;
cm->cm_frame->dcmd.mbox[0] = MR_PD_QUERY_TYPE_EXPOSED_TO_HOST;
cm->cm_frame->dcmd.mbox[1] = 0;
if (mfi_mapcmd(sc, cm) != 0) {
device_printf(sc->mfi_dev,
"Failed to get syspd device listing\n");
goto out;
}
bus_dmamap_sync(sc->mfi_buffer_dmat,cm->cm_dmamap,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
hdr = &cm->cm_frame->header;
if (hdr->cmd_status != MFI_STAT_OK) {
device_printf(sc->mfi_dev,
"MFI_DCMD_PD_LIST_QUERY failed %x\n", hdr->cmd_status);
goto out;
}
/* Get each PD and add it to the system */
for (i = 0; i < pdlist->count; i++) {
if (pdlist->addr[i].device_id ==
pdlist->addr[i].encl_device_id)
continue;
found = 0;
TAILQ_FOREACH(syspd, &sc->mfi_syspd_tqh, pd_link) {
if (syspd->pd_id == pdlist->addr[i].device_id)
found = 1;
}
TAILQ_FOREACH(syspd_pend, &sc->mfi_syspd_pend_tqh, pd_link) {
if (syspd_pend->pd_id == pdlist->addr[i].device_id)
found = 1;
}
if (found == 0)
mfi_add_sys_pd(sc, pdlist->addr[i].device_id);
}
/* Delete SYSPD's whose state has been changed */
TAILQ_FOREACH_SAFE(syspd, &sc->mfi_syspd_tqh, pd_link, tmp) {
found = 0;
for (i = 0; i < pdlist->count; i++) {
if (syspd->pd_id == pdlist->addr[i].device_id) {
found = 1;
break;
}
}
if (found == 0) {
printf("DELETE\n");
mtx_unlock(&sc->mfi_io_lock);
mtx_lock(&Giant);
device_delete_child(sc->mfi_dev, syspd->pd_dev);
mtx_unlock(&Giant);
mtx_lock(&sc->mfi_io_lock);
}
}
out:
if (pdlist)
free(pdlist, M_MFIBUF);
if (cm)
mfi_release_command(cm);
return;
}
static void
mfi_ldprobe(struct mfi_softc *sc)
{
struct mfi_frame_header *hdr;
struct mfi_command *cm = NULL;
struct mfi_ld_list *list = NULL;
struct mfi_disk *ld;
struct mfi_disk_pending *ld_pend;
int error, i;
sx_assert(&sc->mfi_config_lock, SA_XLOCKED);
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
error = mfi_dcmd_command(sc, &cm, MFI_DCMD_LD_GET_LIST,
(void **)&list, sizeof(*list));
if (error)
goto out;
cm->cm_flags = MFI_CMD_DATAIN;
if (mfi_wait_command(sc, cm) != 0) {
device_printf(sc->mfi_dev, "Failed to get device listing\n");
goto out;
}
hdr = &cm->cm_frame->header;
if (hdr->cmd_status != MFI_STAT_OK) {
device_printf(sc->mfi_dev, "MFI_DCMD_LD_GET_LIST failed %x\n",
hdr->cmd_status);
goto out;
}
for (i = 0; i < list->ld_count; i++) {
TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
if (ld->ld_id == list->ld_list[i].ld.v.target_id)
goto skip_add;
}
TAILQ_FOREACH(ld_pend, &sc->mfi_ld_pend_tqh, ld_link) {
if (ld_pend->ld_id == list->ld_list[i].ld.v.target_id)
goto skip_add;
}
mfi_add_ld(sc, list->ld_list[i].ld.v.target_id);
skip_add:;
}
out:
if (list)
free(list, M_MFIBUF);
if (cm)
mfi_release_command(cm);
return;
}
/*
* The timestamp is the number of seconds since 00:00 Jan 1, 2000. If
* the bits in 24-31 are all set, then it is the number of seconds since
* boot.
*/
static const char *
format_timestamp(uint32_t timestamp)
{
static char buffer[32];
if ((timestamp & 0xff000000) == 0xff000000)
snprintf(buffer, sizeof(buffer), "boot + %us", timestamp &
0x00ffffff);
else
snprintf(buffer, sizeof(buffer), "%us", timestamp);
return (buffer);
}
static const char *
format_class(int8_t class)
{
static char buffer[6];
switch (class) {
case MFI_EVT_CLASS_DEBUG:
return ("debug");
case MFI_EVT_CLASS_PROGRESS:
return ("progress");
case MFI_EVT_CLASS_INFO:
return ("info");
case MFI_EVT_CLASS_WARNING:
return ("WARN");
case MFI_EVT_CLASS_CRITICAL:
return ("CRIT");
case MFI_EVT_CLASS_FATAL:
return ("FATAL");
case MFI_EVT_CLASS_DEAD:
return ("DEAD");
default:
snprintf(buffer, sizeof(buffer), "%d", class);
return (buffer);
}
}
static void
mfi_decode_evt(struct mfi_softc *sc, struct mfi_evt_detail *detail)
{
struct mfi_system_pd *syspd = NULL;
device_printf(sc->mfi_dev, "%d (%s/0x%04x/%s) - %s\n", detail->seq,
format_timestamp(detail->time), detail->evt_class.members.locale,
format_class(detail->evt_class.members.evt_class),
detail->description);
/* Don't act on old AEN's or while shutting down */
if (detail->seq < sc->mfi_boot_seq_num || sc->mfi_detaching)
return;
switch (detail->arg_type) {
case MR_EVT_ARGS_NONE:
if (detail->code == MR_EVT_CTRL_HOST_BUS_SCAN_REQUESTED) {
device_printf(sc->mfi_dev, "HostBus scan raised\n");
if (mfi_detect_jbod_change) {
/*
* Probe for new SYSPD's and Delete
* invalid SYSPD's
*/
sx_xlock(&sc->mfi_config_lock);
mtx_lock(&sc->mfi_io_lock);
mfi_syspdprobe(sc);
mtx_unlock(&sc->mfi_io_lock);
sx_xunlock(&sc->mfi_config_lock);
}
}
break;
case MR_EVT_ARGS_LD_STATE:
/* During load time driver reads all the events starting
* from the one that has been logged after shutdown. Avoid
* these old events.
*/
if (detail->args.ld_state.new_state == MFI_LD_STATE_OFFLINE ) {
/* Remove the LD */
struct mfi_disk *ld;
TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
if (ld->ld_id ==
detail->args.ld_state.ld.target_id)
break;
}
/*
Fix: for kernel panics when SSCD is removed
KASSERT(ld != NULL, ("volume dissappeared"));
*/
if (ld != NULL) {
mtx_lock(&Giant);
device_delete_child(sc->mfi_dev, ld->ld_dev);
mtx_unlock(&Giant);
}
}
break;
case MR_EVT_ARGS_PD:
if (detail->code == MR_EVT_PD_REMOVED) {
if (mfi_detect_jbod_change) {
/*
* If the removed device is a SYSPD then
* delete it
*/
TAILQ_FOREACH(syspd, &sc->mfi_syspd_tqh,
pd_link) {
if (syspd->pd_id ==
detail->args.pd.device_id) {
mtx_lock(&Giant);
device_delete_child(
sc->mfi_dev,
syspd->pd_dev);
mtx_unlock(&Giant);
break;
}
}
}
}
if (detail->code == MR_EVT_PD_INSERTED) {
if (mfi_detect_jbod_change) {
/* Probe for new SYSPD's */
sx_xlock(&sc->mfi_config_lock);
mtx_lock(&sc->mfi_io_lock);
mfi_syspdprobe(sc);
mtx_unlock(&sc->mfi_io_lock);
sx_xunlock(&sc->mfi_config_lock);
}
}
if (sc->mfi_cam_rescan_cb != NULL &&
(detail->code == MR_EVT_PD_INSERTED ||
detail->code == MR_EVT_PD_REMOVED)) {
sc->mfi_cam_rescan_cb(sc, detail->args.pd.device_id);
}
break;
}
}
static void
mfi_queue_evt(struct mfi_softc *sc, struct mfi_evt_detail *detail)
{
struct mfi_evt_queue_elm *elm;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
elm = malloc(sizeof(*elm), M_MFIBUF, M_NOWAIT|M_ZERO);
if (elm == NULL)
return;
memcpy(&elm->detail, detail, sizeof(*detail));
TAILQ_INSERT_TAIL(&sc->mfi_evt_queue, elm, link);
taskqueue_enqueue(taskqueue_swi, &sc->mfi_evt_task);
}
static void
mfi_handle_evt(void *context, int pending)
{
TAILQ_HEAD(,mfi_evt_queue_elm) queue;
struct mfi_softc *sc;
struct mfi_evt_queue_elm *elm;
sc = context;
TAILQ_INIT(&queue);
mtx_lock(&sc->mfi_io_lock);
TAILQ_CONCAT(&queue, &sc->mfi_evt_queue, link);
mtx_unlock(&sc->mfi_io_lock);
while ((elm = TAILQ_FIRST(&queue)) != NULL) {
TAILQ_REMOVE(&queue, elm, link);
mfi_decode_evt(sc, &elm->detail);
free(elm, M_MFIBUF);
}
}
static int
mfi_aen_register(struct mfi_softc *sc, int seq, int locale)
{
struct mfi_command *cm;
struct mfi_dcmd_frame *dcmd;
union mfi_evt current_aen, prior_aen;
struct mfi_evt_detail *ed = NULL;
int error = 0;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
current_aen.word = locale;
if (sc->mfi_aen_cm != NULL) {
prior_aen.word =
((uint32_t *)&sc->mfi_aen_cm->cm_frame->dcmd.mbox)[1];
if (prior_aen.members.evt_class <= current_aen.members.evt_class &&
!((prior_aen.members.locale & current_aen.members.locale)
^current_aen.members.locale)) {
return (0);
} else {
prior_aen.members.locale |= current_aen.members.locale;
if (prior_aen.members.evt_class
< current_aen.members.evt_class)
current_aen.members.evt_class =
prior_aen.members.evt_class;
mfi_abort(sc, &sc->mfi_aen_cm);
}
}
error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_EVENT_WAIT,
(void **)&ed, sizeof(*ed));
if (error)
goto out;
dcmd = &cm->cm_frame->dcmd;
((uint32_t *)&dcmd->mbox)[0] = seq;
((uint32_t *)&dcmd->mbox)[1] = locale;
cm->cm_flags = MFI_CMD_DATAIN;
cm->cm_complete = mfi_aen_complete;
sc->last_seq_num = seq;
sc->mfi_aen_cm = cm;
mfi_enqueue_ready(cm);
mfi_startio(sc);
out:
return (error);
}
static void
mfi_aen_complete(struct mfi_command *cm)
{
struct mfi_frame_header *hdr;
struct mfi_softc *sc;
struct mfi_evt_detail *detail;
struct mfi_aen *mfi_aen_entry, *tmp;
int seq = 0, aborted = 0;
sc = cm->cm_sc;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
if (sc->mfi_aen_cm == NULL)
return;
hdr = &cm->cm_frame->header;
if (sc->cm_aen_abort ||
hdr->cmd_status == MFI_STAT_INVALID_STATUS) {
sc->cm_aen_abort = 0;
aborted = 1;
} else {
sc->mfi_aen_triggered = 1;
if (sc->mfi_poll_waiting) {
sc->mfi_poll_waiting = 0;
selwakeup(&sc->mfi_select);
}
detail = cm->cm_data;
mfi_queue_evt(sc, detail);
seq = detail->seq + 1;
TAILQ_FOREACH_SAFE(mfi_aen_entry, &sc->mfi_aen_pids, aen_link,
tmp) {
TAILQ_REMOVE(&sc->mfi_aen_pids, mfi_aen_entry,
aen_link);
PROC_LOCK(mfi_aen_entry->p);
kern_psignal(mfi_aen_entry->p, SIGIO);
PROC_UNLOCK(mfi_aen_entry->p);
free(mfi_aen_entry, M_MFIBUF);
}
}
free(cm->cm_data, M_MFIBUF);
wakeup(&sc->mfi_aen_cm);
sc->mfi_aen_cm = NULL;
mfi_release_command(cm);
/* set it up again so the driver can catch more events */
if (!aborted)
mfi_aen_setup(sc, seq);
}
#define MAX_EVENTS 15
static int
mfi_parse_entries(struct mfi_softc *sc, int start_seq, int stop_seq)
{
struct mfi_command *cm;
struct mfi_dcmd_frame *dcmd;
struct mfi_evt_list *el;
union mfi_evt class_locale;
int error, i, seq, size;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
class_locale.members.reserved = 0;
class_locale.members.locale = mfi_event_locale;
class_locale.members.evt_class = mfi_event_class;
size = sizeof(struct mfi_evt_list) + sizeof(struct mfi_evt_detail)
* (MAX_EVENTS - 1);
el = malloc(size, M_MFIBUF, M_NOWAIT | M_ZERO);
if (el == NULL)
return (ENOMEM);
for (seq = start_seq;;) {
if ((cm = mfi_dequeue_free(sc)) == NULL) {
free(el, M_MFIBUF);
return (EBUSY);
}
dcmd = &cm->cm_frame->dcmd;
bzero(dcmd->mbox, MFI_MBOX_SIZE);
dcmd->header.cmd = MFI_CMD_DCMD;
dcmd->header.timeout = 0;
dcmd->header.data_len = size;
dcmd->opcode = MFI_DCMD_CTRL_EVENT_GET;
((uint32_t *)&dcmd->mbox)[0] = seq;
((uint32_t *)&dcmd->mbox)[1] = class_locale.word;
cm->cm_sg = &dcmd->sgl;
cm->cm_total_frame_size = MFI_DCMD_FRAME_SIZE;
cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_POLLED;
cm->cm_data = el;
cm->cm_len = size;
if ((error = mfi_mapcmd(sc, cm)) != 0) {
device_printf(sc->mfi_dev,
"Failed to get controller entries\n");
mfi_release_command(cm);
break;
}
bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
if (dcmd->header.cmd_status == MFI_STAT_NOT_FOUND) {
mfi_release_command(cm);
break;
}
if (dcmd->header.cmd_status != MFI_STAT_OK) {
device_printf(sc->mfi_dev,
"Error %d fetching controller entries\n",
dcmd->header.cmd_status);
mfi_release_command(cm);
error = EIO;
break;
}
mfi_release_command(cm);
for (i = 0; i < el->count; i++) {
/*
* If this event is newer than 'stop_seq' then
* break out of the loop. Note that the log
* is a circular buffer so we have to handle
* the case that our stop point is earlier in
* the buffer than our start point.
*/
if (el->event[i].seq >= stop_seq) {
if (start_seq <= stop_seq)
break;
else if (el->event[i].seq < start_seq)
break;
}
mfi_queue_evt(sc, &el->event[i]);
}
seq = el->event[el->count - 1].seq + 1;
}
free(el, M_MFIBUF);
return (error);
}
static int
mfi_add_ld(struct mfi_softc *sc, int id)
{
struct mfi_command *cm;
struct mfi_dcmd_frame *dcmd = NULL;
struct mfi_ld_info *ld_info = NULL;
struct mfi_disk_pending *ld_pend;
int error;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
ld_pend = malloc(sizeof(*ld_pend), M_MFIBUF, M_NOWAIT | M_ZERO);
if (ld_pend != NULL) {
ld_pend->ld_id = id;
TAILQ_INSERT_TAIL(&sc->mfi_ld_pend_tqh, ld_pend, ld_link);
}
error = mfi_dcmd_command(sc, &cm, MFI_DCMD_LD_GET_INFO,
(void **)&ld_info, sizeof(*ld_info));
if (error) {
device_printf(sc->mfi_dev,
"Failed to allocate for MFI_DCMD_LD_GET_INFO %d\n", error);
if (ld_info)
free(ld_info, M_MFIBUF);
return (error);
}
cm->cm_flags = MFI_CMD_DATAIN;
dcmd = &cm->cm_frame->dcmd;
dcmd->mbox[0] = id;
if (mfi_wait_command(sc, cm) != 0) {
device_printf(sc->mfi_dev,
"Failed to get logical drive: %d\n", id);
free(ld_info, M_MFIBUF);
return (0);
}
if (ld_info->ld_config.params.isSSCD != 1)
mfi_add_ld_complete(cm);
else {
mfi_release_command(cm);
if (ld_info) /* SSCD drives ld_info free here */
free(ld_info, M_MFIBUF);
}
return (0);
}
static void
mfi_add_ld_complete(struct mfi_command *cm)
{
struct mfi_frame_header *hdr;
struct mfi_ld_info *ld_info;
struct mfi_softc *sc;
device_t child;
sc = cm->cm_sc;
hdr = &cm->cm_frame->header;
ld_info = cm->cm_private;
if (sc->cm_map_abort || hdr->cmd_status != MFI_STAT_OK) {
free(ld_info, M_MFIBUF);
wakeup(&sc->mfi_map_sync_cm);
mfi_release_command(cm);
return;
}
wakeup(&sc->mfi_map_sync_cm);
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
mtx_lock(&Giant);
if ((child = device_add_child(sc->mfi_dev, "mfid", -1)) == NULL) {
device_printf(sc->mfi_dev, "Failed to add logical disk\n");
free(ld_info, M_MFIBUF);
mtx_unlock(&Giant);
mtx_lock(&sc->mfi_io_lock);
return;
}
device_set_ivars(child, ld_info);
device_set_desc(child, "MFI Logical Disk");
bus_generic_attach(sc->mfi_dev);
mtx_unlock(&Giant);
mtx_lock(&sc->mfi_io_lock);
}
static int mfi_add_sys_pd(struct mfi_softc *sc, int id)
{
struct mfi_command *cm;
struct mfi_dcmd_frame *dcmd = NULL;
struct mfi_pd_info *pd_info = NULL;
struct mfi_system_pending *syspd_pend;
int error;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
syspd_pend = malloc(sizeof(*syspd_pend), M_MFIBUF, M_NOWAIT | M_ZERO);
if (syspd_pend != NULL) {
syspd_pend->pd_id = id;
TAILQ_INSERT_TAIL(&sc->mfi_syspd_pend_tqh, syspd_pend, pd_link);
}
error = mfi_dcmd_command(sc, &cm, MFI_DCMD_PD_GET_INFO,
(void **)&pd_info, sizeof(*pd_info));
if (error) {
device_printf(sc->mfi_dev,
"Failed to allocated for MFI_DCMD_PD_GET_INFO %d\n",
error);
if (pd_info)
free(pd_info, M_MFIBUF);
return (error);
}
cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_POLLED;
dcmd = &cm->cm_frame->dcmd;
dcmd->mbox[0]=id;
dcmd->header.scsi_status = 0;
dcmd->header.pad0 = 0;
if ((error = mfi_mapcmd(sc, cm)) != 0) {
device_printf(sc->mfi_dev,
"Failed to get physical drive info %d\n", id);
free(pd_info, M_MFIBUF);
mfi_release_command(cm);
return (error);
}
bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
mfi_add_sys_pd_complete(cm);
return (0);
}
static void
mfi_add_sys_pd_complete(struct mfi_command *cm)
{
struct mfi_frame_header *hdr;
struct mfi_pd_info *pd_info;
struct mfi_softc *sc;
device_t child;
sc = cm->cm_sc;
hdr = &cm->cm_frame->header;
pd_info = cm->cm_private;
if (hdr->cmd_status != MFI_STAT_OK) {
free(pd_info, M_MFIBUF);
mfi_release_command(cm);
return;
}
if (pd_info->fw_state != MFI_PD_STATE_SYSTEM) {
device_printf(sc->mfi_dev, "PD=%x is not SYSTEM PD\n",
pd_info->ref.v.device_id);
free(pd_info, M_MFIBUF);
mfi_release_command(cm);
return;
}
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
mtx_lock(&Giant);
if ((child = device_add_child(sc->mfi_dev, "mfisyspd", -1)) == NULL) {
device_printf(sc->mfi_dev, "Failed to add system pd\n");
free(pd_info, M_MFIBUF);
mtx_unlock(&Giant);
mtx_lock(&sc->mfi_io_lock);
return;
}
device_set_ivars(child, pd_info);
device_set_desc(child, "MFI System PD");
bus_generic_attach(sc->mfi_dev);
mtx_unlock(&Giant);
mtx_lock(&sc->mfi_io_lock);
}
static struct mfi_command *
mfi_bio_command(struct mfi_softc *sc)
{
struct bio *bio;
struct mfi_command *cm = NULL;
/*reserving two commands to avoid starvation for IOCTL*/
if (sc->mfi_qstat[MFIQ_FREE].q_length < 2) {
return (NULL);
}
if ((bio = mfi_dequeue_bio(sc)) == NULL) {
return (NULL);
}
if ((uintptr_t)bio->bio_driver2 == MFI_LD_IO) {
cm = mfi_build_ldio(sc, bio);
} else if ((uintptr_t) bio->bio_driver2 == MFI_SYS_PD_IO) {
cm = mfi_build_syspdio(sc, bio);
}
if (!cm)
mfi_enqueue_bio(sc, bio);
return cm;
}
/*
* mostly copied from cam/scsi/scsi_all.c:scsi_read_write
*/
int
mfi_build_cdb(int readop, uint8_t byte2, u_int64_t lba, u_int32_t block_count, uint8_t *cdb)
{
int cdb_len;
if (((lba & 0x1fffff) == lba)
&& ((block_count & 0xff) == block_count)
&& (byte2 == 0)) {
/* We can fit in a 6 byte cdb */
struct scsi_rw_6 *scsi_cmd;
scsi_cmd = (struct scsi_rw_6 *)cdb;
scsi_cmd->opcode = readop ? READ_6 : WRITE_6;
scsi_ulto3b(lba, scsi_cmd->addr);
scsi_cmd->length = block_count & 0xff;
scsi_cmd->control = 0;
cdb_len = sizeof(*scsi_cmd);
} else if (((block_count & 0xffff) == block_count) && ((lba & 0xffffffff) == lba)) {
/* Need a 10 byte CDB */
struct scsi_rw_10 *scsi_cmd;
scsi_cmd = (struct scsi_rw_10 *)cdb;
scsi_cmd->opcode = readop ? READ_10 : WRITE_10;
scsi_cmd->byte2 = byte2;
scsi_ulto4b(lba, scsi_cmd->addr);
scsi_cmd->reserved = 0;
scsi_ulto2b(block_count, scsi_cmd->length);
scsi_cmd->control = 0;
cdb_len = sizeof(*scsi_cmd);
} else if (((block_count & 0xffffffff) == block_count) &&
((lba & 0xffffffff) == lba)) {
/* Block count is too big for 10 byte CDB use a 12 byte CDB */
struct scsi_rw_12 *scsi_cmd;
scsi_cmd = (struct scsi_rw_12 *)cdb;
scsi_cmd->opcode = readop ? READ_12 : WRITE_12;
scsi_cmd->byte2 = byte2;
scsi_ulto4b(lba, scsi_cmd->addr);
scsi_cmd->reserved = 0;
scsi_ulto4b(block_count, scsi_cmd->length);
scsi_cmd->control = 0;
cdb_len = sizeof(*scsi_cmd);
} else {
/*
* 16 byte CDB. We'll only get here if the LBA is larger
* than 2^32
*/
struct scsi_rw_16 *scsi_cmd;
scsi_cmd = (struct scsi_rw_16 *)cdb;
scsi_cmd->opcode = readop ? READ_16 : WRITE_16;
scsi_cmd->byte2 = byte2;
scsi_u64to8b(lba, scsi_cmd->addr);
scsi_cmd->reserved = 0;
scsi_ulto4b(block_count, scsi_cmd->length);
scsi_cmd->control = 0;
cdb_len = sizeof(*scsi_cmd);
}
return cdb_len;
}
extern char *unmapped_buf;
static struct mfi_command *
mfi_build_syspdio(struct mfi_softc *sc, struct bio *bio)
{
struct mfi_command *cm;
struct mfi_pass_frame *pass;
uint32_t context = 0;
int flags = 0, blkcount = 0, readop;
uint8_t cdb_len;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
if ((cm = mfi_dequeue_free(sc)) == NULL)
return (NULL);
/* Zero out the MFI frame */
context = cm->cm_frame->header.context;
bzero(cm->cm_frame, sizeof(union mfi_frame));
cm->cm_frame->header.context = context;
pass = &cm->cm_frame->pass;
bzero(pass->cdb, 16);
pass->header.cmd = MFI_CMD_PD_SCSI_IO;
switch (bio->bio_cmd & 0x03) {
case BIO_READ:
flags = MFI_CMD_DATAIN | MFI_CMD_BIO;
readop = 1;
break;
case BIO_WRITE:
flags = MFI_CMD_DATAOUT | MFI_CMD_BIO;
readop = 0;
break;
default:
/* TODO: what about BIO_DELETE??? */
panic("Unsupported bio command %x\n", bio->bio_cmd);
}
/* Cheat with the sector length to avoid a non-constant division */
blkcount = (bio->bio_bcount + MFI_SECTOR_LEN - 1) / MFI_SECTOR_LEN;
/* Fill the LBA and Transfer length in CDB */
cdb_len = mfi_build_cdb(readop, 0, bio->bio_pblkno, blkcount,
pass->cdb);
pass->header.target_id = (uintptr_t)bio->bio_driver1;
pass->header.lun_id = 0;
pass->header.timeout = 0;
pass->header.flags = 0;
pass->header.scsi_status = 0;
pass->header.sense_len = MFI_SENSE_LEN;
pass->header.data_len = bio->bio_bcount;
pass->header.cdb_len = cdb_len;
pass->sense_addr_lo = (uint32_t)cm->cm_sense_busaddr;
pass->sense_addr_hi = (uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32);
cm->cm_complete = mfi_bio_complete;
cm->cm_private = bio;
cm->cm_data = unmapped_buf;
cm->cm_len = bio->bio_bcount;
cm->cm_sg = &pass->sgl;
cm->cm_total_frame_size = MFI_PASS_FRAME_SIZE;
cm->cm_flags = flags;
return (cm);
}
static struct mfi_command *
mfi_build_ldio(struct mfi_softc *sc, struct bio *bio)
{
struct mfi_io_frame *io;
struct mfi_command *cm;
int flags;
uint32_t blkcount;
uint32_t context = 0;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
if ((cm = mfi_dequeue_free(sc)) == NULL)
return (NULL);
/* Zero out the MFI frame */
context = cm->cm_frame->header.context;
bzero(cm->cm_frame, sizeof(union mfi_frame));
cm->cm_frame->header.context = context;
io = &cm->cm_frame->io;
switch (bio->bio_cmd & 0x03) {
case BIO_READ:
io->header.cmd = MFI_CMD_LD_READ;
flags = MFI_CMD_DATAIN | MFI_CMD_BIO;
break;
case BIO_WRITE:
io->header.cmd = MFI_CMD_LD_WRITE;
flags = MFI_CMD_DATAOUT | MFI_CMD_BIO;
break;
default:
/* TODO: what about BIO_DELETE??? */
panic("Unsupported bio command %x\n", bio->bio_cmd);
}
/* Cheat with the sector length to avoid a non-constant division */
blkcount = (bio->bio_bcount + MFI_SECTOR_LEN - 1) / MFI_SECTOR_LEN;
io->header.target_id = (uintptr_t)bio->bio_driver1;
io->header.timeout = 0;
io->header.flags = 0;
io->header.scsi_status = 0;
io->header.sense_len = MFI_SENSE_LEN;
io->header.data_len = blkcount;
io->sense_addr_lo = (uint32_t)cm->cm_sense_busaddr;
io->sense_addr_hi = (uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32);
io->lba_hi = (bio->bio_pblkno & 0xffffffff00000000) >> 32;
io->lba_lo = bio->bio_pblkno & 0xffffffff;
cm->cm_complete = mfi_bio_complete;
cm->cm_private = bio;
cm->cm_data = unmapped_buf;
cm->cm_len = bio->bio_bcount;
cm->cm_sg = &io->sgl;
cm->cm_total_frame_size = MFI_IO_FRAME_SIZE;
cm->cm_flags = flags;
return (cm);
}
static void
mfi_bio_complete(struct mfi_command *cm)
{
struct bio *bio;
struct mfi_frame_header *hdr;
struct mfi_softc *sc;
bio = cm->cm_private;
hdr = &cm->cm_frame->header;
sc = cm->cm_sc;
if ((hdr->cmd_status != MFI_STAT_OK) || (hdr->scsi_status != 0)) {
bio->bio_flags |= BIO_ERROR;
bio->bio_error = EIO;
device_printf(sc->mfi_dev, "I/O error, cmd=%p, status=%#x, "
"scsi_status=%#x\n", cm, hdr->cmd_status, hdr->scsi_status);
mfi_print_sense(cm->cm_sc, cm->cm_sense);
} else if (cm->cm_error != 0) {
bio->bio_flags |= BIO_ERROR;
bio->bio_error = cm->cm_error;
device_printf(sc->mfi_dev, "I/O error, cmd=%p, error=%#x\n",
cm, cm->cm_error);
}
mfi_release_command(cm);
mfi_disk_complete(bio);
}
void
mfi_startio(struct mfi_softc *sc)
{
struct mfi_command *cm;
struct ccb_hdr *ccbh;
for (;;) {
/* Don't bother if we're short on resources */
if (sc->mfi_flags & MFI_FLAGS_QFRZN)
break;
/* Try a command that has already been prepared */
cm = mfi_dequeue_ready(sc);
if (cm == NULL) {
if ((ccbh = TAILQ_FIRST(&sc->mfi_cam_ccbq)) != NULL)
cm = sc->mfi_cam_start(ccbh);
}
/* Nope, so look for work on the bioq */
if (cm == NULL)
cm = mfi_bio_command(sc);
/* No work available, so exit */
if (cm == NULL)
break;
/* Send the command to the controller */
if (mfi_mapcmd(sc, cm) != 0) {
device_printf(sc->mfi_dev, "Failed to startio\n");
mfi_requeue_ready(cm);
break;
}
}
}
int
mfi_mapcmd(struct mfi_softc *sc, struct mfi_command *cm)
{
int error, polled;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
if ((cm->cm_data != NULL) && (cm->cm_frame->header.cmd != MFI_CMD_STP )) {
polled = (cm->cm_flags & MFI_CMD_POLLED) ? BUS_DMA_NOWAIT : 0;
if (cm->cm_flags & MFI_CMD_CCB)
error = bus_dmamap_load_ccb(sc->mfi_buffer_dmat,
cm->cm_dmamap, cm->cm_data, mfi_data_cb, cm,
polled);
else if (cm->cm_flags & MFI_CMD_BIO)
error = bus_dmamap_load_bio(sc->mfi_buffer_dmat,
cm->cm_dmamap, cm->cm_private, mfi_data_cb, cm,
polled);
else
error = bus_dmamap_load(sc->mfi_buffer_dmat,
cm->cm_dmamap, cm->cm_data, cm->cm_len,
mfi_data_cb, cm, polled);
if (error == EINPROGRESS) {
sc->mfi_flags |= MFI_FLAGS_QFRZN;
return (0);
}
} else {
error = mfi_send_frame(sc, cm);
}
return (error);
}
static void
mfi_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
{
struct mfi_frame_header *hdr;
struct mfi_command *cm;
union mfi_sgl *sgl;
struct mfi_softc *sc;
int i, j, first, dir;
int sge_size, locked;
cm = (struct mfi_command *)arg;
sc = cm->cm_sc;
hdr = &cm->cm_frame->header;
sgl = cm->cm_sg;
/*
* We need to check if we have the lock as this is async
* callback so even though our caller mfi_mapcmd asserts
* it has the lock, there is no garantee that hasn't been
* dropped if bus_dmamap_load returned prior to our
* completion.
*/
if ((locked = mtx_owned(&sc->mfi_io_lock)) == 0)
mtx_lock(&sc->mfi_io_lock);
if (error) {
printf("error %d in callback\n", error);
cm->cm_error = error;
mfi_complete(sc, cm);
goto out;
}
/* Use IEEE sgl only for IO's on a SKINNY controller
* For other commands on a SKINNY controller use either
* sg32 or sg64 based on the sizeof(bus_addr_t).
* Also calculate the total frame size based on the type
* of SGL used.
*/
if (((cm->cm_frame->header.cmd == MFI_CMD_PD_SCSI_IO) ||
(cm->cm_frame->header.cmd == MFI_CMD_LD_READ) ||
(cm->cm_frame->header.cmd == MFI_CMD_LD_WRITE)) &&
(sc->mfi_flags & MFI_FLAGS_SKINNY)) {
for (i = 0; i < nsegs; i++) {
sgl->sg_skinny[i].addr = segs[i].ds_addr;
sgl->sg_skinny[i].len = segs[i].ds_len;
sgl->sg_skinny[i].flag = 0;
}
hdr->flags |= MFI_FRAME_IEEE_SGL | MFI_FRAME_SGL64;
sge_size = sizeof(struct mfi_sg_skinny);
hdr->sg_count = nsegs;
} else {
j = 0;
if (cm->cm_frame->header.cmd == MFI_CMD_STP) {
first = cm->cm_stp_len;
if ((sc->mfi_flags & MFI_FLAGS_SG64) == 0) {
sgl->sg32[j].addr = segs[0].ds_addr;
sgl->sg32[j++].len = first;
} else {
sgl->sg64[j].addr = segs[0].ds_addr;
sgl->sg64[j++].len = first;
}
} else
first = 0;
if ((sc->mfi_flags & MFI_FLAGS_SG64) == 0) {
for (i = 0; i < nsegs; i++) {
sgl->sg32[j].addr = segs[i].ds_addr + first;
sgl->sg32[j++].len = segs[i].ds_len - first;
first = 0;
}
} else {
for (i = 0; i < nsegs; i++) {
sgl->sg64[j].addr = segs[i].ds_addr + first;
sgl->sg64[j++].len = segs[i].ds_len - first;
first = 0;
}
hdr->flags |= MFI_FRAME_SGL64;
}
hdr->sg_count = j;
sge_size = sc->mfi_sge_size;
}
dir = 0;
if (cm->cm_flags & MFI_CMD_DATAIN) {
dir |= BUS_DMASYNC_PREREAD;
hdr->flags |= MFI_FRAME_DIR_READ;
}
if (cm->cm_flags & MFI_CMD_DATAOUT) {
dir |= BUS_DMASYNC_PREWRITE;
hdr->flags |= MFI_FRAME_DIR_WRITE;
}
bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap, dir);
cm->cm_flags |= MFI_CMD_MAPPED;
/*
* Instead of calculating the total number of frames in the
* compound frame, it's already assumed that there will be at
* least 1 frame, so don't compensate for the modulo of the
* following division.
*/
cm->cm_total_frame_size += (sc->mfi_sge_size * nsegs);
cm->cm_extra_frames = (cm->cm_total_frame_size - 1) / MFI_FRAME_SIZE;
if ((error = mfi_send_frame(sc, cm)) != 0) {
printf("error %d in callback from mfi_send_frame\n", error);
cm->cm_error = error;
mfi_complete(sc, cm);
goto out;
}
out:
/* leave the lock in the state we found it */
if (locked == 0)
mtx_unlock(&sc->mfi_io_lock);
return;
}
static int
mfi_send_frame(struct mfi_softc *sc, struct mfi_command *cm)
{
int error;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
if (sc->MFA_enabled)
error = mfi_tbolt_send_frame(sc, cm);
else
error = mfi_std_send_frame(sc, cm);
if (error != 0 && (cm->cm_flags & MFI_ON_MFIQ_BUSY) != 0)
mfi_remove_busy(cm);
return (error);
}
static int
mfi_std_send_frame(struct mfi_softc *sc, struct mfi_command *cm)
{
struct mfi_frame_header *hdr;
int tm = mfi_polled_cmd_timeout * 1000;
hdr = &cm->cm_frame->header;
if ((cm->cm_flags & MFI_CMD_POLLED) == 0) {
cm->cm_timestamp = time_uptime;
mfi_enqueue_busy(cm);
} else {
hdr->cmd_status = MFI_STAT_INVALID_STATUS;
hdr->flags |= MFI_FRAME_DONT_POST_IN_REPLY_QUEUE;
}
/*
* The bus address of the command is aligned on a 64 byte boundary,
* leaving the least 6 bits as zero. For whatever reason, the
* hardware wants the address shifted right by three, leaving just
* 3 zero bits. These three bits are then used as a prefetching
* hint for the hardware to predict how many frames need to be
* fetched across the bus. If a command has more than 8 frames
* then the 3 bits are set to 0x7 and the firmware uses other
* information in the command to determine the total amount to fetch.
* However, FreeBSD doesn't support I/O larger than 128K, so 8 frames
* is enough for both 32bit and 64bit systems.
*/
if (cm->cm_extra_frames > 7)
cm->cm_extra_frames = 7;
sc->mfi_issue_cmd(sc, cm->cm_frame_busaddr, cm->cm_extra_frames);
if ((cm->cm_flags & MFI_CMD_POLLED) == 0)
return (0);
/* This is a polled command, so busy-wait for it to complete. */
while (hdr->cmd_status == MFI_STAT_INVALID_STATUS) {
DELAY(1000);
tm -= 1;
if (tm <= 0)
break;
}
if (hdr->cmd_status == MFI_STAT_INVALID_STATUS) {
device_printf(sc->mfi_dev, "Frame %p timed out "
"command 0x%X\n", hdr, cm->cm_frame->dcmd.opcode);
return (ETIMEDOUT);
}
return (0);
}
void
mfi_complete(struct mfi_softc *sc, struct mfi_command *cm)
{
int dir;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
if ((cm->cm_flags & MFI_CMD_MAPPED) != 0) {
dir = 0;
if ((cm->cm_flags & MFI_CMD_DATAIN) ||
(cm->cm_frame->header.cmd == MFI_CMD_STP))
dir |= BUS_DMASYNC_POSTREAD;
if (cm->cm_flags & MFI_CMD_DATAOUT)
dir |= BUS_DMASYNC_POSTWRITE;
bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap, dir);
bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
cm->cm_flags &= ~MFI_CMD_MAPPED;
}
cm->cm_flags |= MFI_CMD_COMPLETED;
if (cm->cm_complete != NULL)
cm->cm_complete(cm);
else
wakeup(cm);
}
static int
mfi_abort(struct mfi_softc *sc, struct mfi_command **cm_abort)
{
struct mfi_command *cm;
struct mfi_abort_frame *abort;
int i = 0, error;
uint32_t context = 0;
mtx_lock(&sc->mfi_io_lock);
if ((cm = mfi_dequeue_free(sc)) == NULL) {
mtx_unlock(&sc->mfi_io_lock);
return (EBUSY);
}
/* Zero out the MFI frame */
context = cm->cm_frame->header.context;
bzero(cm->cm_frame, sizeof(union mfi_frame));
cm->cm_frame->header.context = context;
abort = &cm->cm_frame->abort;
abort->header.cmd = MFI_CMD_ABORT;
abort->header.flags = 0;
abort->header.scsi_status = 0;
abort->abort_context = (*cm_abort)->cm_frame->header.context;
abort->abort_mfi_addr_lo = (uint32_t)(*cm_abort)->cm_frame_busaddr;
abort->abort_mfi_addr_hi =
(uint32_t)((uint64_t)(*cm_abort)->cm_frame_busaddr >> 32);
cm->cm_data = NULL;
cm->cm_flags = MFI_CMD_POLLED;
if ((error = mfi_mapcmd(sc, cm)) != 0)
device_printf(sc->mfi_dev, "failed to abort command\n");
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
while (i < 5 && *cm_abort != NULL) {
tsleep(cm_abort, 0, "mfiabort",
5 * hz);
i++;
}
if (*cm_abort != NULL) {
/* Force a complete if command didn't abort */
mtx_lock(&sc->mfi_io_lock);
(*cm_abort)->cm_complete(*cm_abort);
mtx_unlock(&sc->mfi_io_lock);
}
return (error);
}
int
mfi_dump_blocks(struct mfi_softc *sc, int id, uint64_t lba, void *virt,
int len)
{
struct mfi_command *cm;
struct mfi_io_frame *io;
int error;
uint32_t context = 0;
if ((cm = mfi_dequeue_free(sc)) == NULL)
return (EBUSY);
/* Zero out the MFI frame */
context = cm->cm_frame->header.context;
bzero(cm->cm_frame, sizeof(union mfi_frame));
cm->cm_frame->header.context = context;
io = &cm->cm_frame->io;
io->header.cmd = MFI_CMD_LD_WRITE;
io->header.target_id = id;
io->header.timeout = 0;
io->header.flags = 0;
io->header.scsi_status = 0;
io->header.sense_len = MFI_SENSE_LEN;
io->header.data_len = (len + MFI_SECTOR_LEN - 1) / MFI_SECTOR_LEN;
io->sense_addr_lo = (uint32_t)cm->cm_sense_busaddr;
io->sense_addr_hi = (uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32);
io->lba_hi = (lba & 0xffffffff00000000) >> 32;
io->lba_lo = lba & 0xffffffff;
cm->cm_data = virt;
cm->cm_len = len;
cm->cm_sg = &io->sgl;
cm->cm_total_frame_size = MFI_IO_FRAME_SIZE;
cm->cm_flags = MFI_CMD_POLLED | MFI_CMD_DATAOUT;
if ((error = mfi_mapcmd(sc, cm)) != 0)
device_printf(sc->mfi_dev, "failed dump blocks\n");
bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
mfi_release_command(cm);
return (error);
}
int
mfi_dump_syspd_blocks(struct mfi_softc *sc, int id, uint64_t lba, void *virt,
int len)
{
struct mfi_command *cm;
struct mfi_pass_frame *pass;
int error, readop, cdb_len;
uint32_t blkcount;
if ((cm = mfi_dequeue_free(sc)) == NULL)
return (EBUSY);
pass = &cm->cm_frame->pass;
bzero(pass->cdb, 16);
pass->header.cmd = MFI_CMD_PD_SCSI_IO;
readop = 0;
blkcount = (len + MFI_SECTOR_LEN - 1) / MFI_SECTOR_LEN;
cdb_len = mfi_build_cdb(readop, 0, lba, blkcount, pass->cdb);
pass->header.target_id = id;
pass->header.timeout = 0;
pass->header.flags = 0;
pass->header.scsi_status = 0;
pass->header.sense_len = MFI_SENSE_LEN;
pass->header.data_len = len;
pass->header.cdb_len = cdb_len;
pass->sense_addr_lo = (uint32_t)cm->cm_sense_busaddr;
pass->sense_addr_hi = (uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32);
cm->cm_data = virt;
cm->cm_len = len;
cm->cm_sg = &pass->sgl;
cm->cm_total_frame_size = MFI_PASS_FRAME_SIZE;
cm->cm_flags = MFI_CMD_POLLED | MFI_CMD_DATAOUT | MFI_CMD_SCSI;
if ((error = mfi_mapcmd(sc, cm)) != 0)
device_printf(sc->mfi_dev, "failed dump blocks\n");
bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
mfi_release_command(cm);
return (error);
}
static int
mfi_open(struct cdev *dev, int flags, int fmt, struct thread *td)
{
struct mfi_softc *sc;
int error;
sc = dev->si_drv1;
mtx_lock(&sc->mfi_io_lock);
if (sc->mfi_detaching)
error = ENXIO;
else {
sc->mfi_flags |= MFI_FLAGS_OPEN;
error = 0;
}
mtx_unlock(&sc->mfi_io_lock);
return (error);
}
static int
mfi_close(struct cdev *dev, int flags, int fmt, struct thread *td)
{
struct mfi_softc *sc;
struct mfi_aen *mfi_aen_entry, *tmp;
sc = dev->si_drv1;
mtx_lock(&sc->mfi_io_lock);
sc->mfi_flags &= ~MFI_FLAGS_OPEN;
TAILQ_FOREACH_SAFE(mfi_aen_entry, &sc->mfi_aen_pids, aen_link, tmp) {
if (mfi_aen_entry->p == curproc) {
TAILQ_REMOVE(&sc->mfi_aen_pids, mfi_aen_entry,
aen_link);
free(mfi_aen_entry, M_MFIBUF);
}
}
mtx_unlock(&sc->mfi_io_lock);
return (0);
}
static int
mfi_config_lock(struct mfi_softc *sc, uint32_t opcode)
{
switch (opcode) {
case MFI_DCMD_LD_DELETE:
case MFI_DCMD_CFG_ADD:
case MFI_DCMD_CFG_CLEAR:
case MFI_DCMD_CFG_FOREIGN_IMPORT:
sx_xlock(&sc->mfi_config_lock);
return (1);
default:
return (0);
}
}
static void
mfi_config_unlock(struct mfi_softc *sc, int locked)
{
if (locked)
sx_xunlock(&sc->mfi_config_lock);
}
/*
* Perform pre-issue checks on commands from userland and possibly veto
* them.
*/
static int
mfi_check_command_pre(struct mfi_softc *sc, struct mfi_command *cm)
{
struct mfi_disk *ld, *ld2;
int error;
struct mfi_system_pd *syspd = NULL;
uint16_t syspd_id;
uint16_t *mbox;
mtx_assert(&sc->mfi_io_lock, MA_OWNED);
error = 0;
switch (cm->cm_frame->dcmd.opcode) {
case MFI_DCMD_LD_DELETE:
TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
if (ld->ld_id == cm->cm_frame->dcmd.mbox[0])
break;
}
if (ld == NULL)
error = ENOENT;
else
error = mfi_disk_disable(ld);
break;
case MFI_DCMD_CFG_CLEAR:
TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
error = mfi_disk_disable(ld);
if (error)
break;
}
if (error) {
TAILQ_FOREACH(ld2, &sc->mfi_ld_tqh, ld_link) {
if (ld2 == ld)
break;
mfi_disk_enable(ld2);
}
}
break;
case MFI_DCMD_PD_STATE_SET:
mbox = (uint16_t *) cm->cm_frame->dcmd.mbox;
syspd_id = mbox[0];
if (mbox[2] == MFI_PD_STATE_UNCONFIGURED_GOOD) {
TAILQ_FOREACH(syspd, &sc->mfi_syspd_tqh, pd_link) {
if (syspd->pd_id == syspd_id)
break;
}
}
else
break;
if (syspd)
error = mfi_syspd_disable(syspd);
break;
default:
break;
}
return (error);
}
/* Perform post-issue checks on commands from userland. */
static void
mfi_check_command_post(struct mfi_softc *sc, struct mfi_command *cm)
{
struct mfi_disk *ld, *ldn;
struct mfi_system_pd *syspd = NULL;
uint16_t syspd_id;
uint16_t *mbox;
switch (cm->cm_frame->dcmd.opcode) {
case MFI_DCMD_LD_DELETE:
TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
if (ld->ld_id == cm->cm_frame->dcmd.mbox[0])
break;
}
KASSERT(ld != NULL, ("volume dissappeared"));
if (cm->cm_frame->header.cmd_status == MFI_STAT_OK) {
mtx_unlock(&sc->mfi_io_lock);
mtx_lock(&Giant);
device_delete_child(sc->mfi_dev, ld->ld_dev);
mtx_unlock(&Giant);
mtx_lock(&sc->mfi_io_lock);
} else
mfi_disk_enable(ld);
break;
case MFI_DCMD_CFG_CLEAR:
if (cm->cm_frame->header.cmd_status == MFI_STAT_OK) {
mtx_unlock(&sc->mfi_io_lock);
mtx_lock(&Giant);
TAILQ_FOREACH_SAFE(ld, &sc->mfi_ld_tqh, ld_link, ldn) {
device_delete_child(sc->mfi_dev, ld->ld_dev);
}
mtx_unlock(&Giant);
mtx_lock(&sc->mfi_io_lock);
} else {
TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link)
mfi_disk_enable(ld);
}
break;
case MFI_DCMD_CFG_ADD:
mfi_ldprobe(sc);
break;
case MFI_DCMD_CFG_FOREIGN_IMPORT:
mfi_ldprobe(sc);
break;
case MFI_DCMD_PD_STATE_SET:
mbox = (uint16_t *) cm->cm_frame->dcmd.mbox;
syspd_id = mbox[0];
if (mbox[2] == MFI_PD_STATE_UNCONFIGURED_GOOD) {
TAILQ_FOREACH(syspd, &sc->mfi_syspd_tqh,pd_link) {
if (syspd->pd_id == syspd_id)
break;
}
}
else
break;
/* If the transition fails then enable the syspd again */
if (syspd && cm->cm_frame->header.cmd_status != MFI_STAT_OK)
mfi_syspd_enable(syspd);
break;
}
}
static int
mfi_check_for_sscd(struct mfi_softc *sc, struct mfi_command *cm)
{
struct mfi_config_data *conf_data;
struct mfi_command *ld_cm = NULL;
struct mfi_ld_info *ld_info = NULL;
struct mfi_ld_config *ld;
char *p;
int error = 0;
conf_data = (struct mfi_config_data *)cm->cm_data;
if (cm->cm_frame->dcmd.opcode == MFI_DCMD_CFG_ADD) {
p = (char *)conf_data->array;
p += conf_data->array_size * conf_data->array_count;
ld = (struct mfi_ld_config *)p;
if (ld->params.isSSCD == 1)
error = 1;
} else if (cm->cm_frame->dcmd.opcode == MFI_DCMD_LD_DELETE) {
error = mfi_dcmd_command (sc, &ld_cm, MFI_DCMD_LD_GET_INFO,
(void **)&ld_info, sizeof(*ld_info));
if (error) {
device_printf(sc->mfi_dev, "Failed to allocate"
"MFI_DCMD_LD_GET_INFO %d", error);
if (ld_info)
free(ld_info, M_MFIBUF);
return 0;
}
ld_cm->cm_flags = MFI_CMD_DATAIN;
ld_cm->cm_frame->dcmd.mbox[0]= cm->cm_frame->dcmd.mbox[0];
ld_cm->cm_frame->header.target_id = cm->cm_frame->dcmd.mbox[0];
if (mfi_wait_command(sc, ld_cm) != 0) {
device_printf(sc->mfi_dev, "failed to get log drv\n");
mfi_release_command(ld_cm);
free(ld_info, M_MFIBUF);
return 0;
}
if (ld_cm->cm_frame->header.cmd_status != MFI_STAT_OK) {
free(ld_info, M_MFIBUF);
mfi_release_command(ld_cm);
return 0;
}
else
ld_info = (struct mfi_ld_info *)ld_cm->cm_private;
if (ld_info->ld_config.params.isSSCD == 1)
error = 1;
mfi_release_command(ld_cm);
free(ld_info, M_MFIBUF);
}
return error;
}
static int
mfi_stp_cmd(struct mfi_softc *sc, struct mfi_command *cm,caddr_t arg)
{
uint8_t i;
struct mfi_ioc_packet *ioc;
ioc = (struct mfi_ioc_packet *)arg;
int sge_size, error;
struct megasas_sge *kern_sge;
memset(sc->kbuff_arr, 0, sizeof(sc->kbuff_arr));
kern_sge =(struct megasas_sge *) ((uintptr_t)cm->cm_frame + ioc->mfi_sgl_off);
cm->cm_frame->header.sg_count = ioc->mfi_sge_count;
if (sizeof(bus_addr_t) == 8) {
cm->cm_frame->header.flags |= MFI_FRAME_SGL64;
cm->cm_extra_frames = 2;
sge_size = sizeof(struct mfi_sg64);
} else {
cm->cm_extra_frames = (cm->cm_total_frame_size - 1) / MFI_FRAME_SIZE;
sge_size = sizeof(struct mfi_sg32);
}
cm->cm_total_frame_size += (sge_size * ioc->mfi_sge_count);
for (i = 0; i < ioc->mfi_sge_count; i++) {
if (bus_dma_tag_create( sc->mfi_parent_dmat, /* parent */
1, 0, /* algnmnt, boundary */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
ioc->mfi_sgl[i].iov_len,/* maxsize */
2, /* nsegments */
ioc->mfi_sgl[i].iov_len,/* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->mfi_kbuff_arr_dmat[i])) {
device_printf(sc->mfi_dev,
"Cannot allocate mfi_kbuff_arr_dmat tag\n");
return (ENOMEM);
}
if (bus_dmamem_alloc(sc->mfi_kbuff_arr_dmat[i],
(void **)&sc->kbuff_arr[i], BUS_DMA_NOWAIT,
&sc->mfi_kbuff_arr_dmamap[i])) {
device_printf(sc->mfi_dev,
"Cannot allocate mfi_kbuff_arr_dmamap memory\n");
return (ENOMEM);
}
bus_dmamap_load(sc->mfi_kbuff_arr_dmat[i],
sc->mfi_kbuff_arr_dmamap[i], sc->kbuff_arr[i],
ioc->mfi_sgl[i].iov_len, mfi_addr_cb,
&sc->mfi_kbuff_arr_busaddr[i], 0);
if (!sc->kbuff_arr[i]) {
device_printf(sc->mfi_dev,
"Could not allocate memory for kbuff_arr info\n");
return -1;
}
kern_sge[i].phys_addr = sc->mfi_kbuff_arr_busaddr[i];
kern_sge[i].length = ioc->mfi_sgl[i].iov_len;
if (sizeof(bus_addr_t) == 8) {
cm->cm_frame->stp.sgl.sg64[i].addr =
kern_sge[i].phys_addr;
cm->cm_frame->stp.sgl.sg64[i].len =
ioc->mfi_sgl[i].iov_len;
} else {
cm->cm_frame->stp.sgl.sg32[i].addr =
kern_sge[i].phys_addr;
cm->cm_frame->stp.sgl.sg32[i].len =
ioc->mfi_sgl[i].iov_len;
}
error = copyin(ioc->mfi_sgl[i].iov_base,
sc->kbuff_arr[i],
ioc->mfi_sgl[i].iov_len);
if (error != 0) {
device_printf(sc->mfi_dev, "Copy in failed\n");
return error;
}
}
cm->cm_flags |=MFI_CMD_MAPPED;
return 0;
}
static int
mfi_user_command(struct mfi_softc *sc, struct mfi_ioc_passthru *ioc)
{
struct mfi_command *cm;
struct mfi_dcmd_frame *dcmd;
void *ioc_buf = NULL;
uint32_t context;
int error = 0, locked;
if (ioc->buf_size > 0) {
if (ioc->buf_size > 1024 * 1024)
return (ENOMEM);
ioc_buf = malloc(ioc->buf_size, M_MFIBUF, M_WAITOK);
error = copyin(ioc->buf, ioc_buf, ioc->buf_size);
if (error) {
device_printf(sc->mfi_dev, "failed to copyin\n");
free(ioc_buf, M_MFIBUF);
return (error);
}
}
locked = mfi_config_lock(sc, ioc->ioc_frame.opcode);
mtx_lock(&sc->mfi_io_lock);
while ((cm = mfi_dequeue_free(sc)) == NULL)
msleep(mfi_user_command, &sc->mfi_io_lock, 0, "mfiioc", hz);
/* Save context for later */
context = cm->cm_frame->header.context;
dcmd = &cm->cm_frame->dcmd;
bcopy(&ioc->ioc_frame, dcmd, sizeof(struct mfi_dcmd_frame));
cm->cm_sg = &dcmd->sgl;
cm->cm_total_frame_size = MFI_DCMD_FRAME_SIZE;
cm->cm_data = ioc_buf;
cm->cm_len = ioc->buf_size;
/* restore context */
cm->cm_frame->header.context = context;
/* Cheat since we don't know if we're writing or reading */
cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_DATAOUT;
error = mfi_check_command_pre(sc, cm);
if (error)
goto out;
error = mfi_wait_command(sc, cm);
if (error) {
device_printf(sc->mfi_dev, "ioctl failed %d\n", error);
goto out;
}
bcopy(dcmd, &ioc->ioc_frame, sizeof(struct mfi_dcmd_frame));
mfi_check_command_post(sc, cm);
out:
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
mfi_config_unlock(sc, locked);
if (ioc->buf_size > 0)
error = copyout(ioc_buf, ioc->buf, ioc->buf_size);
if (ioc_buf)
free(ioc_buf, M_MFIBUF);
return (error);
}
#define PTRIN(p) ((void *)(uintptr_t)(p))
static int
mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
{
struct mfi_softc *sc;
union mfi_statrequest *ms;
struct mfi_ioc_packet *ioc;
#ifdef COMPAT_FREEBSD32
struct mfi_ioc_packet32 *ioc32;
#endif
struct mfi_ioc_aen *aen;
struct mfi_command *cm = NULL;
uint32_t context = 0;
union mfi_sense_ptr sense_ptr;
uint8_t *data = NULL, *temp, *addr, skip_pre_post = 0;
size_t len;
int i, res;
struct mfi_ioc_passthru *iop = (struct mfi_ioc_passthru *)arg;
#ifdef COMPAT_FREEBSD32
struct mfi_ioc_passthru32 *iop32 = (struct mfi_ioc_passthru32 *)arg;
struct mfi_ioc_passthru iop_swab;
#endif
int error, locked;
union mfi_sgl *sgl;
sc = dev->si_drv1;
error = 0;
if (sc->adpreset)
return EBUSY;
if (sc->hw_crit_error)
return EBUSY;
if (sc->issuepend_done == 0)
return EBUSY;
switch (cmd) {
case MFIIO_STATS:
ms = (union mfi_statrequest *)arg;
switch (ms->ms_item) {
case MFIQ_FREE:
case MFIQ_BIO:
case MFIQ_READY:
case MFIQ_BUSY:
bcopy(&sc->mfi_qstat[ms->ms_item], &ms->ms_qstat,
sizeof(struct mfi_qstat));
break;
default:
error = ENOIOCTL;
break;
}
break;
case MFIIO_QUERY_DISK:
{
struct mfi_query_disk *qd;
struct mfi_disk *ld;
qd = (struct mfi_query_disk *)arg;
mtx_lock(&sc->mfi_io_lock);
TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
if (ld->ld_id == qd->array_id)
break;
}
if (ld == NULL) {
qd->present = 0;
mtx_unlock(&sc->mfi_io_lock);
return (0);
}
qd->present = 1;
if (ld->ld_flags & MFI_DISK_FLAGS_OPEN)
qd->open = 1;
bzero(qd->devname, SPECNAMELEN + 1);
snprintf(qd->devname, SPECNAMELEN, "mfid%d", ld->ld_unit);
mtx_unlock(&sc->mfi_io_lock);
break;
}
case MFI_CMD:
#ifdef COMPAT_FREEBSD32
case MFI_CMD32:
#endif
{
devclass_t devclass;
ioc = (struct mfi_ioc_packet *)arg;
int adapter;
adapter = ioc->mfi_adapter_no;
if (device_get_unit(sc->mfi_dev) == 0 && adapter != 0) {
devclass = devclass_find("mfi");
sc = devclass_get_softc(devclass, adapter);
}
mtx_lock(&sc->mfi_io_lock);
if ((cm = mfi_dequeue_free(sc)) == NULL) {
mtx_unlock(&sc->mfi_io_lock);
return (EBUSY);
}
mtx_unlock(&sc->mfi_io_lock);
locked = 0;
/*
* save off original context since copying from user
* will clobber some data
*/
context = cm->cm_frame->header.context;
cm->cm_frame->header.context = cm->cm_index;
bcopy(ioc->mfi_frame.raw, cm->cm_frame,
2 * MEGAMFI_FRAME_SIZE);
cm->cm_total_frame_size = (sizeof(union mfi_sgl)
* ioc->mfi_sge_count) + ioc->mfi_sgl_off;
cm->cm_frame->header.scsi_status = 0;
cm->cm_frame->header.pad0 = 0;
if (ioc->mfi_sge_count) {
cm->cm_sg =
(union mfi_sgl *)&cm->cm_frame->bytes[ioc->mfi_sgl_off];
}
sgl = cm->cm_sg;
cm->cm_flags = 0;
if (cm->cm_frame->header.flags & MFI_FRAME_DATAIN)
cm->cm_flags |= MFI_CMD_DATAIN;
if (cm->cm_frame->header.flags & MFI_FRAME_DATAOUT)
cm->cm_flags |= MFI_CMD_DATAOUT;
/* Legacy app shim */
if (cm->cm_flags == 0)
cm->cm_flags |= MFI_CMD_DATAIN | MFI_CMD_DATAOUT;
cm->cm_len = cm->cm_frame->header.data_len;
if (cm->cm_frame->header.cmd == MFI_CMD_STP) {
#ifdef COMPAT_FREEBSD32
if (cmd == MFI_CMD) {
#endif
/* Native */
cm->cm_stp_len = ioc->mfi_sgl[0].iov_len;
#ifdef COMPAT_FREEBSD32
} else {
/* 32bit on 64bit */
ioc32 = (struct mfi_ioc_packet32 *)ioc;
cm->cm_stp_len = ioc32->mfi_sgl[0].iov_len;
}
#endif
cm->cm_len += cm->cm_stp_len;
}
if (cm->cm_len &&
(cm->cm_flags & (MFI_CMD_DATAIN | MFI_CMD_DATAOUT))) {
cm->cm_data = data = malloc(cm->cm_len, M_MFIBUF,
M_WAITOK | M_ZERO);
if (cm->cm_data == NULL) {
device_printf(sc->mfi_dev, "Malloc failed\n");
goto out;
}
} else {
cm->cm_data = 0;
}
/* restore header context */
cm->cm_frame->header.context = context;
if (cm->cm_frame->header.cmd == MFI_CMD_STP) {
res = mfi_stp_cmd(sc, cm, arg);
if (res != 0)
goto out;
} else {
temp = data;
if ((cm->cm_flags & MFI_CMD_DATAOUT) ||
(cm->cm_frame->header.cmd == MFI_CMD_STP)) {
for (i = 0; i < ioc->mfi_sge_count; i++) {
#ifdef COMPAT_FREEBSD32
if (cmd == MFI_CMD) {
#endif
/* Native */
addr = ioc->mfi_sgl[i].iov_base;
len = ioc->mfi_sgl[i].iov_len;
#ifdef COMPAT_FREEBSD32
} else {
/* 32bit on 64bit */
ioc32 = (struct mfi_ioc_packet32 *)ioc;
addr = PTRIN(ioc32->mfi_sgl[i].iov_base);
len = ioc32->mfi_sgl[i].iov_len;
}
#endif
error = copyin(addr, temp, len);
if (error != 0) {
device_printf(sc->mfi_dev,
"Copy in failed\n");
goto out;
}
temp = &temp[len];
}
}
}
if (cm->cm_frame->header.cmd == MFI_CMD_DCMD)
locked = mfi_config_lock(sc,
cm->cm_frame->dcmd.opcode);
if (cm->cm_frame->header.cmd == MFI_CMD_PD_SCSI_IO) {
cm->cm_frame->pass.sense_addr_lo =
(uint32_t)cm->cm_sense_busaddr;
cm->cm_frame->pass.sense_addr_hi =
(uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32);
}
mtx_lock(&sc->mfi_io_lock);
skip_pre_post = mfi_check_for_sscd (sc, cm);
if (!skip_pre_post) {
error = mfi_check_command_pre(sc, cm);
if (error) {
mtx_unlock(&sc->mfi_io_lock);
goto out;
}
}
if ((error = mfi_wait_command(sc, cm)) != 0) {
device_printf(sc->mfi_dev,
"Controller polled failed\n");
mtx_unlock(&sc->mfi_io_lock);
goto out;
}
if (!skip_pre_post) {
mfi_check_command_post(sc, cm);
}
mtx_unlock(&sc->mfi_io_lock);
if (cm->cm_frame->header.cmd != MFI_CMD_STP) {
temp = data;
if ((cm->cm_flags & MFI_CMD_DATAIN) ||
(cm->cm_frame->header.cmd == MFI_CMD_STP)) {
for (i = 0; i < ioc->mfi_sge_count; i++) {
#ifdef COMPAT_FREEBSD32
if (cmd == MFI_CMD) {
#endif
/* Native */
addr = ioc->mfi_sgl[i].iov_base;
len = ioc->mfi_sgl[i].iov_len;
#ifdef COMPAT_FREEBSD32
} else {
/* 32bit on 64bit */
ioc32 = (struct mfi_ioc_packet32 *)ioc;
addr = PTRIN(ioc32->mfi_sgl[i].iov_base);
len = ioc32->mfi_sgl[i].iov_len;
}
#endif
error = copyout(temp, addr, len);
if (error != 0) {
device_printf(sc->mfi_dev,
"Copy out failed\n");
goto out;
}
temp = &temp[len];
}
}
}
if (ioc->mfi_sense_len) {
/* get user-space sense ptr then copy out sense */
bcopy(&ioc->mfi_frame.raw[ioc->mfi_sense_off],
&sense_ptr.sense_ptr_data[0],
sizeof(sense_ptr.sense_ptr_data));
#ifdef COMPAT_FREEBSD32
if (cmd != MFI_CMD) {
/*
* not 64bit native so zero out any address
* over 32bit */
sense_ptr.addr.high = 0;
}
#endif
error = copyout(cm->cm_sense, sense_ptr.user_space,
ioc->mfi_sense_len);
if (error != 0) {
device_printf(sc->mfi_dev,
"Copy out failed\n");
goto out;
}
}
ioc->mfi_frame.hdr.cmd_status = cm->cm_frame->header.cmd_status;
out:
mfi_config_unlock(sc, locked);
if (data)
free(data, M_MFIBUF);
if (cm->cm_frame->header.cmd == MFI_CMD_STP) {
for (i = 0; i < 2; i++) {
if (sc->kbuff_arr[i]) {
if (sc->mfi_kbuff_arr_busaddr != 0)
bus_dmamap_unload(
sc->mfi_kbuff_arr_dmat[i],
sc->mfi_kbuff_arr_dmamap[i]
);
if (sc->kbuff_arr[i] != NULL)
bus_dmamem_free(
sc->mfi_kbuff_arr_dmat[i],
sc->kbuff_arr[i],
sc->mfi_kbuff_arr_dmamap[i]
);
if (sc->mfi_kbuff_arr_dmat[i] != NULL)
bus_dma_tag_destroy(
sc->mfi_kbuff_arr_dmat[i]);
}
}
}
if (cm) {
mtx_lock(&sc->mfi_io_lock);
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
}
break;
}
case MFI_SET_AEN:
aen = (struct mfi_ioc_aen *)arg;
mtx_lock(&sc->mfi_io_lock);
error = mfi_aen_register(sc, aen->aen_seq_num,
aen->aen_class_locale);
mtx_unlock(&sc->mfi_io_lock);
break;
case MFI_LINUX_CMD_2: /* Firmware Linux ioctl shim */
{
devclass_t devclass;
struct mfi_linux_ioc_packet l_ioc;
int adapter;
devclass = devclass_find("mfi");
if (devclass == NULL)
return (ENOENT);
error = copyin(arg, &l_ioc, sizeof(l_ioc));
if (error)
return (error);
adapter = l_ioc.lioc_adapter_no;
sc = devclass_get_softc(devclass, adapter);
if (sc == NULL)
return (ENOENT);
return (mfi_linux_ioctl_int(sc->mfi_cdev,
cmd, arg, flag, td));
break;
}
case MFI_LINUX_SET_AEN_2: /* AEN Linux ioctl shim */
{
devclass_t devclass;
struct mfi_linux_ioc_aen l_aen;
int adapter;
devclass = devclass_find("mfi");
if (devclass == NULL)
return (ENOENT);
error = copyin(arg, &l_aen, sizeof(l_aen));
if (error)
return (error);
adapter = l_aen.laen_adapter_no;
sc = devclass_get_softc(devclass, adapter);
if (sc == NULL)
return (ENOENT);
return (mfi_linux_ioctl_int(sc->mfi_cdev,
cmd, arg, flag, td));
break;
}
#ifdef COMPAT_FREEBSD32
case MFIIO_PASSTHRU32:
if (!SV_CURPROC_FLAG(SV_ILP32)) {
error = ENOTTY;
break;
}
iop_swab.ioc_frame = iop32->ioc_frame;
iop_swab.buf_size = iop32->buf_size;
iop_swab.buf = PTRIN(iop32->buf);
iop = &iop_swab;
/* FALLTHROUGH */
#endif
case MFIIO_PASSTHRU:
error = mfi_user_command(sc, iop);
#ifdef COMPAT_FREEBSD32
if (cmd == MFIIO_PASSTHRU32)
iop32->ioc_frame = iop_swab.ioc_frame;
#endif
break;
default:
device_printf(sc->mfi_dev, "IOCTL 0x%lx not handled\n", cmd);
error = ENOTTY;
break;
}
return (error);
}
static int
mfi_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
{
struct mfi_softc *sc;
struct mfi_linux_ioc_packet l_ioc;
struct mfi_linux_ioc_aen l_aen;
struct mfi_command *cm = NULL;
struct mfi_aen *mfi_aen_entry;
union mfi_sense_ptr sense_ptr;
uint32_t context = 0;
uint8_t *data = NULL, *temp;
int i;
int error, locked;
sc = dev->si_drv1;
error = 0;
switch (cmd) {
case MFI_LINUX_CMD_2: /* Firmware Linux ioctl shim */
error = copyin(arg, &l_ioc, sizeof(l_ioc));
if (error != 0)
return (error);
if (l_ioc.lioc_sge_count > MAX_LINUX_IOCTL_SGE) {
return (EINVAL);
}
mtx_lock(&sc->mfi_io_lock);
if ((cm = mfi_dequeue_free(sc)) == NULL) {
mtx_unlock(&sc->mfi_io_lock);
return (EBUSY);
}
mtx_unlock(&sc->mfi_io_lock);
locked = 0;
/*
* save off original context since copying from user
* will clobber some data
*/
context = cm->cm_frame->header.context;
bcopy(l_ioc.lioc_frame.raw, cm->cm_frame,
2 * MFI_DCMD_FRAME_SIZE); /* this isn't quite right */
cm->cm_total_frame_size = (sizeof(union mfi_sgl)
* l_ioc.lioc_sge_count) + l_ioc.lioc_sgl_off;
cm->cm_frame->header.scsi_status = 0;
cm->cm_frame->header.pad0 = 0;
if (l_ioc.lioc_sge_count)
cm->cm_sg =
(union mfi_sgl *)&cm->cm_frame->bytes[l_ioc.lioc_sgl_off];
cm->cm_flags = 0;
if (cm->cm_frame->header.flags & MFI_FRAME_DATAIN)
cm->cm_flags |= MFI_CMD_DATAIN;
if (cm->cm_frame->header.flags & MFI_FRAME_DATAOUT)
cm->cm_flags |= MFI_CMD_DATAOUT;
cm->cm_len = cm->cm_frame->header.data_len;
if (cm->cm_len &&
(cm->cm_flags & (MFI_CMD_DATAIN | MFI_CMD_DATAOUT))) {
cm->cm_data = data = malloc(cm->cm_len, M_MFIBUF,
M_WAITOK | M_ZERO);
if (cm->cm_data == NULL) {
device_printf(sc->mfi_dev, "Malloc failed\n");
goto out;
}
} else {
cm->cm_data = 0;
}
/* restore header context */
cm->cm_frame->header.context = context;
temp = data;
if (cm->cm_flags & MFI_CMD_DATAOUT) {
for (i = 0; i < l_ioc.lioc_sge_count; i++) {
error = copyin(PTRIN(l_ioc.lioc_sgl[i].iov_base),
temp,
l_ioc.lioc_sgl[i].iov_len);
if (error != 0) {
device_printf(sc->mfi_dev,
"Copy in failed\n");
goto out;
}
temp = &temp[l_ioc.lioc_sgl[i].iov_len];
}
}
if (cm->cm_frame->header.cmd == MFI_CMD_DCMD)
locked = mfi_config_lock(sc, cm->cm_frame->dcmd.opcode);
if (cm->cm_frame->header.cmd == MFI_CMD_PD_SCSI_IO) {
cm->cm_frame->pass.sense_addr_lo =
(uint32_t)cm->cm_sense_busaddr;
cm->cm_frame->pass.sense_addr_hi =
(uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32);
}
mtx_lock(&sc->mfi_io_lock);
error = mfi_check_command_pre(sc, cm);
if (error) {
mtx_unlock(&sc->mfi_io_lock);
goto out;
}
if ((error = mfi_wait_command(sc, cm)) != 0) {
device_printf(sc->mfi_dev,
"Controller polled failed\n");
mtx_unlock(&sc->mfi_io_lock);
goto out;
}
mfi_check_command_post(sc, cm);
mtx_unlock(&sc->mfi_io_lock);
temp = data;
if (cm->cm_flags & MFI_CMD_DATAIN) {
for (i = 0; i < l_ioc.lioc_sge_count; i++) {
error = copyout(temp,
PTRIN(l_ioc.lioc_sgl[i].iov_base),
l_ioc.lioc_sgl[i].iov_len);
if (error != 0) {
device_printf(sc->mfi_dev,
"Copy out failed\n");
goto out;
}
temp = &temp[l_ioc.lioc_sgl[i].iov_len];
}
}
if (l_ioc.lioc_sense_len) {
/* get user-space sense ptr then copy out sense */
bcopy(&((struct mfi_linux_ioc_packet*)arg)
->lioc_frame.raw[l_ioc.lioc_sense_off],
&sense_ptr.sense_ptr_data[0],
sizeof(sense_ptr.sense_ptr_data));
#ifdef __amd64__
/*
* only 32bit Linux support so zero out any
* address over 32bit
*/
sense_ptr.addr.high = 0;
#endif
error = copyout(cm->cm_sense, sense_ptr.user_space,
l_ioc.lioc_sense_len);
if (error != 0) {
device_printf(sc->mfi_dev,
"Copy out failed\n");
goto out;
}
}
error = copyout(&cm->cm_frame->header.cmd_status,
&((struct mfi_linux_ioc_packet*)arg)
->lioc_frame.hdr.cmd_status,
1);
if (error != 0) {
device_printf(sc->mfi_dev,
"Copy out failed\n");
goto out;
}
out:
mfi_config_unlock(sc, locked);
if (data)
free(data, M_MFIBUF);
if (cm) {
mtx_lock(&sc->mfi_io_lock);
mfi_release_command(cm);
mtx_unlock(&sc->mfi_io_lock);
}
return (error);
case MFI_LINUX_SET_AEN_2: /* AEN Linux ioctl shim */
error = copyin(arg, &l_aen, sizeof(l_aen));
if (error != 0)
return (error);
printf("AEN IMPLEMENTED for pid %d\n", curproc->p_pid);
mfi_aen_entry = malloc(sizeof(struct mfi_aen), M_MFIBUF,
M_WAITOK);
mtx_lock(&sc->mfi_io_lock);
if (mfi_aen_entry != NULL) {
mfi_aen_entry->p = curproc;
TAILQ_INSERT_TAIL(&sc->mfi_aen_pids, mfi_aen_entry,
aen_link);
}
error = mfi_aen_register(sc, l_aen.laen_seq_num,
l_aen.laen_class_locale);
if (error != 0) {
TAILQ_REMOVE(&sc->mfi_aen_pids, mfi_aen_entry,
aen_link);
free(mfi_aen_entry, M_MFIBUF);
}
mtx_unlock(&sc->mfi_io_lock);
return (error);
default:
device_printf(sc->mfi_dev, "IOCTL 0x%lx not handled\n", cmd);
error = ENOENT;
break;
}
return (error);
}
static int
mfi_poll(struct cdev *dev, int poll_events, struct thread *td)
{
struct mfi_softc *sc;
int revents = 0;
sc = dev->si_drv1;
if (poll_events & (POLLIN | POLLRDNORM)) {
if (sc->mfi_aen_triggered != 0) {
revents |= poll_events & (POLLIN | POLLRDNORM);
sc->mfi_aen_triggered = 0;
}
if (sc->mfi_aen_triggered == 0 && sc->mfi_aen_cm == NULL) {
revents |= POLLERR;
}
}
if (revents == 0) {
if (poll_events & (POLLIN | POLLRDNORM)) {
sc->mfi_poll_waiting = 1;
selrecord(td, &sc->mfi_select);
}
}
return revents;
}
static void
mfi_dump_all(void)
{
struct mfi_softc *sc;
struct mfi_command *cm;
devclass_t dc;
time_t deadline;
int timedout;
int i;
dc = devclass_find("mfi");
if (dc == NULL) {
printf("No mfi dev class\n");
return;
}
for (i = 0; ; i++) {
sc = devclass_get_softc(dc, i);
if (sc == NULL)
break;
device_printf(sc->mfi_dev, "Dumping\n\n");
timedout = 0;
deadline = time_uptime - mfi_cmd_timeout;
mtx_lock(&sc->mfi_io_lock);
TAILQ_FOREACH(cm, &sc->mfi_busy, cm_link) {
if (cm->cm_timestamp <= deadline) {
device_printf(sc->mfi_dev,
"COMMAND %p TIMEOUT AFTER %d SECONDS\n",
cm, (int)(time_uptime - cm->cm_timestamp));
MFI_PRINT_CMD(cm);
timedout++;
}
}
#if 0
if (timedout)
MFI_DUMP_CMDS(sc);
#endif
mtx_unlock(&sc->mfi_io_lock);
}
return;
}
static void
mfi_timeout(void *data)
{
struct mfi_softc *sc = (struct mfi_softc *)data;
struct mfi_command *cm, *tmp;
time_t deadline;
int timedout = 0;
deadline = time_uptime - mfi_cmd_timeout;
if (sc->adpreset == 0) {
if (!mfi_tbolt_reset(sc)) {
callout_reset(&sc->mfi_watchdog_callout,
mfi_cmd_timeout * hz, mfi_timeout, sc);
return;
}
}
mtx_lock(&sc->mfi_io_lock);
TAILQ_FOREACH_SAFE(cm, &sc->mfi_busy, cm_link, tmp) {
if (sc->mfi_aen_cm == cm || sc->mfi_map_sync_cm == cm)
continue;
if (cm->cm_timestamp <= deadline) {
if (sc->adpreset != 0 && sc->issuepend_done == 0) {
cm->cm_timestamp = time_uptime;
} else {
device_printf(sc->mfi_dev,
"COMMAND %p TIMEOUT AFTER %d SECONDS\n",
cm, (int)(time_uptime - cm->cm_timestamp)
);
MFI_PRINT_CMD(cm);
MFI_VALIDATE_CMD(sc, cm);
/*
* While commands can get stuck forever we do
* not fail them as there is no way to tell if
* the controller has actually processed them
* or not.
*
* In addition its very likely that force
* failing a command here would cause a panic
* e.g. in UFS.
*/
timedout++;
}
}
}
#if 0
if (timedout)
MFI_DUMP_CMDS(sc);
#endif
mtx_unlock(&sc->mfi_io_lock);
callout_reset(&sc->mfi_watchdog_callout, mfi_cmd_timeout * hz,
mfi_timeout, sc);
if (0)
mfi_dump_all();
return;
}
Index: head/sys/dev/mwl/if_mwl.c
===================================================================
--- head/sys/dev/mwl/if_mwl.c (revision 283290)
+++ head/sys/dev/mwl/if_mwl.c (revision 283291)
@@ -1,5035 +1,5035 @@
/*-
* Copyright (c) 2007-2009 Sam Leffler, Errno Consulting
* Copyright (c) 2007-2008 Marvell Semiconductor, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer,
* without modification.
* 2. Redistributions in binary form must reproduce at minimum a disclaimer
* similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
* redistribution must be conditioned upon including a substantially
* similar Disclaimer requirement for further binary redistribution.
*
* NO WARRANTY
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
* THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGES.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* Driver for the Marvell 88W8363 Wireless LAN controller.
*/
#include "opt_inet.h"
#include "opt_mwl.h"
#include "opt_wlan.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/errno.h>
#include <sys/callout.h>
#include <sys/bus.h>
#include <sys/endian.h>
#include <sys/kthread.h>
#include <sys/taskqueue.h>
#include <machine/bus.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/if_types.h>
#include <net/if_arp.h>
#include <net/ethernet.h>
#include <net/if_llc.h>
#include <net/bpf.h>
#include <net80211/ieee80211_var.h>
#include <net80211/ieee80211_regdomain.h>
#ifdef INET
#include <netinet/in.h>
#include <netinet/if_ether.h>
#endif /* INET */
#include <dev/mwl/if_mwlvar.h>
#include <dev/mwl/mwldiag.h>
/* idiomatic shorthands: MS = mask+shift, SM = shift+mask */
#define MS(v,x) (((v) & x) >> x##_S)
#define SM(v,x) (((v) << x##_S) & x)
static struct ieee80211vap *mwl_vap_create(struct ieee80211com *,
const char [IFNAMSIZ], int, enum ieee80211_opmode, int,
const uint8_t [IEEE80211_ADDR_LEN],
const uint8_t [IEEE80211_ADDR_LEN]);
static void mwl_vap_delete(struct ieee80211vap *);
static int mwl_setupdma(struct mwl_softc *);
static int mwl_hal_reset(struct mwl_softc *sc);
static int mwl_init_locked(struct mwl_softc *);
static void mwl_init(void *);
static void mwl_stop_locked(struct ifnet *, int);
static int mwl_reset(struct ieee80211vap *, u_long);
static void mwl_stop(struct ifnet *, int);
static void mwl_start(struct ifnet *);
static int mwl_raw_xmit(struct ieee80211_node *, struct mbuf *,
const struct ieee80211_bpf_params *);
static int mwl_media_change(struct ifnet *);
static void mwl_watchdog(void *);
static int mwl_ioctl(struct ifnet *, u_long, caddr_t);
static void mwl_radar_proc(void *, int);
static void mwl_chanswitch_proc(void *, int);
static void mwl_bawatchdog_proc(void *, int);
static int mwl_key_alloc(struct ieee80211vap *,
struct ieee80211_key *,
ieee80211_keyix *, ieee80211_keyix *);
static int mwl_key_delete(struct ieee80211vap *,
const struct ieee80211_key *);
static int mwl_key_set(struct ieee80211vap *, const struct ieee80211_key *,
const uint8_t mac[IEEE80211_ADDR_LEN]);
static int mwl_mode_init(struct mwl_softc *);
static void mwl_update_mcast(struct ifnet *);
static void mwl_update_promisc(struct ifnet *);
static void mwl_updateslot(struct ifnet *);
static int mwl_beacon_setup(struct ieee80211vap *);
static void mwl_beacon_update(struct ieee80211vap *, int);
#ifdef MWL_HOST_PS_SUPPORT
static void mwl_update_ps(struct ieee80211vap *, int);
static int mwl_set_tim(struct ieee80211_node *, int);
#endif
static int mwl_dma_setup(struct mwl_softc *);
static void mwl_dma_cleanup(struct mwl_softc *);
static struct ieee80211_node *mwl_node_alloc(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN]);
static void mwl_node_cleanup(struct ieee80211_node *);
static void mwl_node_drain(struct ieee80211_node *);
static void mwl_node_getsignal(const struct ieee80211_node *,
int8_t *, int8_t *);
static void mwl_node_getmimoinfo(const struct ieee80211_node *,
struct ieee80211_mimo_info *);
static int mwl_rxbuf_init(struct mwl_softc *, struct mwl_rxbuf *);
static void mwl_rx_proc(void *, int);
static void mwl_txq_init(struct mwl_softc *sc, struct mwl_txq *, int);
static int mwl_tx_setup(struct mwl_softc *, int, int);
static int mwl_wme_update(struct ieee80211com *);
static void mwl_tx_cleanupq(struct mwl_softc *, struct mwl_txq *);
static void mwl_tx_cleanup(struct mwl_softc *);
static uint16_t mwl_calcformat(uint8_t rate, const struct ieee80211_node *);
static int mwl_tx_start(struct mwl_softc *, struct ieee80211_node *,
struct mwl_txbuf *, struct mbuf *);
static void mwl_tx_proc(void *, int);
static int mwl_chan_set(struct mwl_softc *, struct ieee80211_channel *);
static void mwl_draintxq(struct mwl_softc *);
static void mwl_cleartxq(struct mwl_softc *, struct ieee80211vap *);
static int mwl_recv_action(struct ieee80211_node *,
const struct ieee80211_frame *,
const uint8_t *, const uint8_t *);
static int mwl_addba_request(struct ieee80211_node *,
struct ieee80211_tx_ampdu *, int dialogtoken,
int baparamset, int batimeout);
static int mwl_addba_response(struct ieee80211_node *,
struct ieee80211_tx_ampdu *, int status,
int baparamset, int batimeout);
static void mwl_addba_stop(struct ieee80211_node *,
struct ieee80211_tx_ampdu *);
static int mwl_startrecv(struct mwl_softc *);
static MWL_HAL_APMODE mwl_getapmode(const struct ieee80211vap *,
struct ieee80211_channel *);
static int mwl_setapmode(struct ieee80211vap *, struct ieee80211_channel*);
static void mwl_scan_start(struct ieee80211com *);
static void mwl_scan_end(struct ieee80211com *);
static void mwl_set_channel(struct ieee80211com *);
static int mwl_peerstadb(struct ieee80211_node *,
int aid, int staid, MWL_HAL_PEERINFO *pi);
static int mwl_localstadb(struct ieee80211vap *);
static int mwl_newstate(struct ieee80211vap *, enum ieee80211_state, int);
static int allocstaid(struct mwl_softc *sc, int aid);
static void delstaid(struct mwl_softc *sc, int staid);
static void mwl_newassoc(struct ieee80211_node *, int);
static void mwl_agestations(void *);
static int mwl_setregdomain(struct ieee80211com *,
struct ieee80211_regdomain *, int,
struct ieee80211_channel []);
static void mwl_getradiocaps(struct ieee80211com *, int, int *,
struct ieee80211_channel []);
static int mwl_getchannels(struct mwl_softc *);
static void mwl_sysctlattach(struct mwl_softc *);
static void mwl_announce(struct mwl_softc *);
SYSCTL_NODE(_hw, OID_AUTO, mwl, CTLFLAG_RD, 0, "Marvell driver parameters");
static int mwl_rxdesc = MWL_RXDESC; /* # rx desc's to allocate */
SYSCTL_INT(_hw_mwl, OID_AUTO, rxdesc, CTLFLAG_RW, &mwl_rxdesc,
0, "rx descriptors allocated");
static int mwl_rxbuf = MWL_RXBUF; /* # rx buffers to allocate */
SYSCTL_INT(_hw_mwl, OID_AUTO, rxbuf, CTLFLAG_RWTUN, &mwl_rxbuf,
0, "rx buffers allocated");
static int mwl_txbuf = MWL_TXBUF; /* # tx buffers to allocate */
SYSCTL_INT(_hw_mwl, OID_AUTO, txbuf, CTLFLAG_RWTUN, &mwl_txbuf,
0, "tx buffers allocated");
static int mwl_txcoalesce = 8; /* # tx packets to q before poking f/w*/
SYSCTL_INT(_hw_mwl, OID_AUTO, txcoalesce, CTLFLAG_RWTUN, &mwl_txcoalesce,
0, "tx buffers to send at once");
static int mwl_rxquota = MWL_RXBUF; /* # max buffers to process */
SYSCTL_INT(_hw_mwl, OID_AUTO, rxquota, CTLFLAG_RWTUN, &mwl_rxquota,
0, "max rx buffers to process per interrupt");
static int mwl_rxdmalow = 3; /* # min buffers for wakeup */
SYSCTL_INT(_hw_mwl, OID_AUTO, rxdmalow, CTLFLAG_RWTUN, &mwl_rxdmalow,
0, "min free rx buffers before restarting traffic");
#ifdef MWL_DEBUG
static int mwl_debug = 0;
SYSCTL_INT(_hw_mwl, OID_AUTO, debug, CTLFLAG_RWTUN, &mwl_debug,
0, "control debugging printfs");
enum {
MWL_DEBUG_XMIT = 0x00000001, /* basic xmit operation */
MWL_DEBUG_XMIT_DESC = 0x00000002, /* xmit descriptors */
MWL_DEBUG_RECV = 0x00000004, /* basic recv operation */
MWL_DEBUG_RECV_DESC = 0x00000008, /* recv descriptors */
MWL_DEBUG_RESET = 0x00000010, /* reset processing */
MWL_DEBUG_BEACON = 0x00000020, /* beacon handling */
MWL_DEBUG_INTR = 0x00000040, /* ISR */
MWL_DEBUG_TX_PROC = 0x00000080, /* tx ISR proc */
MWL_DEBUG_RX_PROC = 0x00000100, /* rx ISR proc */
MWL_DEBUG_KEYCACHE = 0x00000200, /* key cache management */
MWL_DEBUG_STATE = 0x00000400, /* 802.11 state transitions */
MWL_DEBUG_NODE = 0x00000800, /* node management */
MWL_DEBUG_RECV_ALL = 0x00001000, /* trace all frames (beacons) */
MWL_DEBUG_TSO = 0x00002000, /* TSO processing */
MWL_DEBUG_AMPDU = 0x00004000, /* BA stream handling */
MWL_DEBUG_ANY = 0xffffffff
};
#define IS_BEACON(wh) \
((wh->i_fc[0] & (IEEE80211_FC0_TYPE_MASK|IEEE80211_FC0_SUBTYPE_MASK)) == \
(IEEE80211_FC0_TYPE_MGT|IEEE80211_FC0_SUBTYPE_BEACON))
#define IFF_DUMPPKTS_RECV(sc, wh) \
(((sc->sc_debug & MWL_DEBUG_RECV) && \
((sc->sc_debug & MWL_DEBUG_RECV_ALL) || !IS_BEACON(wh))) || \
(sc->sc_ifp->if_flags & (IFF_DEBUG|IFF_LINK2)) == (IFF_DEBUG|IFF_LINK2))
#define IFF_DUMPPKTS_XMIT(sc) \
((sc->sc_debug & MWL_DEBUG_XMIT) || \
(sc->sc_ifp->if_flags & (IFF_DEBUG|IFF_LINK2)) == (IFF_DEBUG|IFF_LINK2))
#define DPRINTF(sc, m, fmt, ...) do { \
if (sc->sc_debug & (m)) \
printf(fmt, __VA_ARGS__); \
} while (0)
#define KEYPRINTF(sc, hk, mac) do { \
if (sc->sc_debug & MWL_DEBUG_KEYCACHE) \
mwl_keyprint(sc, __func__, hk, mac); \
} while (0)
static void mwl_printrxbuf(const struct mwl_rxbuf *bf, u_int ix);
static void mwl_printtxbuf(const struct mwl_txbuf *bf, u_int qnum, u_int ix);
#else
#define IFF_DUMPPKTS_RECV(sc, wh) \
((sc->sc_ifp->if_flags & (IFF_DEBUG|IFF_LINK2)) == (IFF_DEBUG|IFF_LINK2))
#define IFF_DUMPPKTS_XMIT(sc) \
((sc->sc_ifp->if_flags & (IFF_DEBUG|IFF_LINK2)) == (IFF_DEBUG|IFF_LINK2))
#define DPRINTF(sc, m, fmt, ...) do { \
(void) sc; \
} while (0)
#define KEYPRINTF(sc, k, mac) do { \
(void) sc; \
} while (0)
#endif
static MALLOC_DEFINE(M_MWLDEV, "mwldev", "mwl driver dma buffers");
/*
* Each packet has fixed front matter: a 2-byte length
* of the payload, followed by a 4-address 802.11 header
* (regardless of the actual header and always w/o any
* QoS header). The payload then follows.
*/
struct mwltxrec {
uint16_t fwlen;
struct ieee80211_frame_addr4 wh;
} __packed;
/*
* Read/Write shorthands for accesses to BAR 0. Note
* that all BAR 1 operations are done in the "hal" and
* there should be no reference to them here.
*/
#ifdef MWL_DEBUG
static __inline uint32_t
RD4(struct mwl_softc *sc, bus_size_t off)
{
return bus_space_read_4(sc->sc_io0t, sc->sc_io0h, off);
}
#endif
static __inline void
WR4(struct mwl_softc *sc, bus_size_t off, uint32_t val)
{
bus_space_write_4(sc->sc_io0t, sc->sc_io0h, off, val);
}
int
mwl_attach(uint16_t devid, struct mwl_softc *sc)
{
struct ifnet *ifp;
struct ieee80211com *ic;
struct mwl_hal *mh;
int error = 0;
DPRINTF(sc, MWL_DEBUG_ANY, "%s: devid 0x%x\n", __func__, devid);
ifp = sc->sc_ifp = if_alloc(IFT_IEEE80211);
if (ifp == NULL) {
device_printf(sc->sc_dev, "cannot if_alloc()\n");
return ENOSPC;
}
ic = ifp->if_l2com;
/*
* Setup the RX free list lock early, so it can be consistently
* removed.
*/
MWL_RXFREE_INIT(sc);
/* set these up early for if_printf use */
if_initname(ifp, device_get_name(sc->sc_dev),
device_get_unit(sc->sc_dev));
mh = mwl_hal_attach(sc->sc_dev, devid,
sc->sc_io1h, sc->sc_io1t, sc->sc_dmat);
if (mh == NULL) {
if_printf(ifp, "unable to attach HAL\n");
error = EIO;
goto bad;
}
sc->sc_mh = mh;
/*
* Load firmware so we can get setup. We arbitrarily
* pick station firmware; we'll re-load firmware as
* needed so setting up the wrong mode isn't a big deal.
*/
if (mwl_hal_fwload(mh, NULL) != 0) {
if_printf(ifp, "unable to setup builtin firmware\n");
error = EIO;
goto bad1;
}
if (mwl_hal_gethwspecs(mh, &sc->sc_hwspecs) != 0) {
if_printf(ifp, "unable to fetch h/w specs\n");
error = EIO;
goto bad1;
}
error = mwl_getchannels(sc);
if (error != 0)
goto bad1;
sc->sc_txantenna = 0; /* h/w default */
sc->sc_rxantenna = 0; /* h/w default */
sc->sc_invalid = 0; /* ready to go, enable int handling */
sc->sc_ageinterval = MWL_AGEINTERVAL;
/*
* Allocate tx+rx descriptors and populate the lists.
* We immediately push the information to the firmware
* as otherwise it gets upset.
*/
error = mwl_dma_setup(sc);
if (error != 0) {
if_printf(ifp, "failed to setup descriptors: %d\n", error);
goto bad1;
}
error = mwl_setupdma(sc); /* push to firmware */
if (error != 0) /* NB: mwl_setupdma prints msg */
goto bad1;
- callout_init(&sc->sc_timer, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_timer, 1);
callout_init_mtx(&sc->sc_watchdog, &sc->sc_mtx, 0);
sc->sc_tq = taskqueue_create("mwl_taskq", M_NOWAIT,
taskqueue_thread_enqueue, &sc->sc_tq);
taskqueue_start_threads(&sc->sc_tq, 1, PI_NET,
"%s taskq", ifp->if_xname);
TASK_INIT(&sc->sc_rxtask, 0, mwl_rx_proc, sc);
TASK_INIT(&sc->sc_radartask, 0, mwl_radar_proc, sc);
TASK_INIT(&sc->sc_chanswitchtask, 0, mwl_chanswitch_proc, sc);
TASK_INIT(&sc->sc_bawatchdogtask, 0, mwl_bawatchdog_proc, sc);
/* NB: insure BK queue is the lowest priority h/w queue */
if (!mwl_tx_setup(sc, WME_AC_BK, MWL_WME_AC_BK)) {
if_printf(ifp, "unable to setup xmit queue for %s traffic!\n",
ieee80211_wme_acnames[WME_AC_BK]);
error = EIO;
goto bad2;
}
if (!mwl_tx_setup(sc, WME_AC_BE, MWL_WME_AC_BE) ||
!mwl_tx_setup(sc, WME_AC_VI, MWL_WME_AC_VI) ||
!mwl_tx_setup(sc, WME_AC_VO, MWL_WME_AC_VO)) {
/*
* Not enough hardware tx queues to properly do WME;
* just punt and assign them all to the same h/w queue.
* We could do a better job of this if, for example,
* we allocate queues when we switch from station to
* AP mode.
*/
if (sc->sc_ac2q[WME_AC_VI] != NULL)
mwl_tx_cleanupq(sc, sc->sc_ac2q[WME_AC_VI]);
if (sc->sc_ac2q[WME_AC_BE] != NULL)
mwl_tx_cleanupq(sc, sc->sc_ac2q[WME_AC_BE]);
sc->sc_ac2q[WME_AC_BE] = sc->sc_ac2q[WME_AC_BK];
sc->sc_ac2q[WME_AC_VI] = sc->sc_ac2q[WME_AC_BK];
sc->sc_ac2q[WME_AC_VO] = sc->sc_ac2q[WME_AC_BK];
}
TASK_INIT(&sc->sc_txtask, 0, mwl_tx_proc, sc);
ifp->if_softc = sc;
ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST;
ifp->if_start = mwl_start;
ifp->if_ioctl = mwl_ioctl;
ifp->if_init = mwl_init;
IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
IFQ_SET_READY(&ifp->if_snd);
ic->ic_ifp = ifp;
/* XXX not right but it's not used anywhere important */
ic->ic_phytype = IEEE80211_T_OFDM;
ic->ic_opmode = IEEE80211_M_STA;
ic->ic_caps =
IEEE80211_C_STA /* station mode supported */
| IEEE80211_C_HOSTAP /* hostap mode */
| IEEE80211_C_MONITOR /* monitor mode */
#if 0
| IEEE80211_C_IBSS /* ibss, nee adhoc, mode */
| IEEE80211_C_AHDEMO /* adhoc demo mode */
#endif
| IEEE80211_C_MBSS /* mesh point link mode */
| IEEE80211_C_WDS /* WDS supported */
| IEEE80211_C_SHPREAMBLE /* short preamble supported */
| IEEE80211_C_SHSLOT /* short slot time supported */
| IEEE80211_C_WME /* WME/WMM supported */
| IEEE80211_C_BURST /* xmit bursting supported */
| IEEE80211_C_WPA /* capable of WPA1+WPA2 */
| IEEE80211_C_BGSCAN /* capable of bg scanning */
| IEEE80211_C_TXFRAG /* handle tx frags */
| IEEE80211_C_TXPMGT /* capable of txpow mgt */
| IEEE80211_C_DFS /* DFS supported */
;
ic->ic_htcaps =
IEEE80211_HTCAP_SMPS_ENA /* SM PS mode enabled */
| IEEE80211_HTCAP_CHWIDTH40 /* 40MHz channel width */
| IEEE80211_HTCAP_SHORTGI20 /* short GI in 20MHz */
| IEEE80211_HTCAP_SHORTGI40 /* short GI in 40MHz */
| IEEE80211_HTCAP_RXSTBC_2STREAM/* 1-2 spatial streams */
#if MWL_AGGR_SIZE == 7935
| IEEE80211_HTCAP_MAXAMSDU_7935 /* max A-MSDU length */
#else
| IEEE80211_HTCAP_MAXAMSDU_3839 /* max A-MSDU length */
#endif
#if 0
| IEEE80211_HTCAP_PSMP /* PSMP supported */
| IEEE80211_HTCAP_40INTOLERANT /* 40MHz intolerant */
#endif
/* s/w capabilities */
| IEEE80211_HTC_HT /* HT operation */
| IEEE80211_HTC_AMPDU /* tx A-MPDU */
| IEEE80211_HTC_AMSDU /* tx A-MSDU */
| IEEE80211_HTC_SMPS /* SMPS available */
;
/*
* Mark h/w crypto support.
* XXX no way to query h/w support.
*/
ic->ic_cryptocaps |= IEEE80211_CRYPTO_WEP
| IEEE80211_CRYPTO_AES_CCM
| IEEE80211_CRYPTO_TKIP
| IEEE80211_CRYPTO_TKIPMIC
;
/*
* Transmit requires space in the packet for a special
* format transmit record and optional padding between
* this record and the payload. Ask the net80211 layer
* to arrange this when encapsulating packets so we can
* add it efficiently.
*/
ic->ic_headroom = sizeof(struct mwltxrec) -
sizeof(struct ieee80211_frame);
/* call MI attach routine. */
ieee80211_ifattach(ic, sc->sc_hwspecs.macAddr);
ic->ic_setregdomain = mwl_setregdomain;
ic->ic_getradiocaps = mwl_getradiocaps;
/* override default methods */
ic->ic_raw_xmit = mwl_raw_xmit;
ic->ic_newassoc = mwl_newassoc;
ic->ic_updateslot = mwl_updateslot;
ic->ic_update_mcast = mwl_update_mcast;
ic->ic_update_promisc = mwl_update_promisc;
ic->ic_wme.wme_update = mwl_wme_update;
ic->ic_node_alloc = mwl_node_alloc;
sc->sc_node_cleanup = ic->ic_node_cleanup;
ic->ic_node_cleanup = mwl_node_cleanup;
sc->sc_node_drain = ic->ic_node_drain;
ic->ic_node_drain = mwl_node_drain;
ic->ic_node_getsignal = mwl_node_getsignal;
ic->ic_node_getmimoinfo = mwl_node_getmimoinfo;
ic->ic_scan_start = mwl_scan_start;
ic->ic_scan_end = mwl_scan_end;
ic->ic_set_channel = mwl_set_channel;
sc->sc_recv_action = ic->ic_recv_action;
ic->ic_recv_action = mwl_recv_action;
sc->sc_addba_request = ic->ic_addba_request;
ic->ic_addba_request = mwl_addba_request;
sc->sc_addba_response = ic->ic_addba_response;
ic->ic_addba_response = mwl_addba_response;
sc->sc_addba_stop = ic->ic_addba_stop;
ic->ic_addba_stop = mwl_addba_stop;
ic->ic_vap_create = mwl_vap_create;
ic->ic_vap_delete = mwl_vap_delete;
ieee80211_radiotap_attach(ic,
&sc->sc_tx_th.wt_ihdr, sizeof(sc->sc_tx_th),
MWL_TX_RADIOTAP_PRESENT,
&sc->sc_rx_th.wr_ihdr, sizeof(sc->sc_rx_th),
MWL_RX_RADIOTAP_PRESENT);
/*
* Setup dynamic sysctl's now that country code and
* regdomain are available from the hal.
*/
mwl_sysctlattach(sc);
if (bootverbose)
ieee80211_announce(ic);
mwl_announce(sc);
return 0;
bad2:
mwl_dma_cleanup(sc);
bad1:
mwl_hal_detach(mh);
bad:
MWL_RXFREE_DESTROY(sc);
if_free(ifp);
sc->sc_invalid = 1;
return error;
}
int
mwl_detach(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags %x\n",
__func__, ifp->if_flags);
mwl_stop(ifp, 1);
/*
* NB: the order of these is important:
* o call the 802.11 layer before detaching the hal to
* insure callbacks into the driver to delete global
* key cache entries can be handled
* o reclaim the tx queue data structures after calling
* the 802.11 layer as we'll get called back to reclaim
* node state and potentially want to use them
* o to cleanup the tx queues the hal is called, so detach
* it last
* Other than that, it's straightforward...
*/
ieee80211_ifdetach(ic);
callout_drain(&sc->sc_watchdog);
mwl_dma_cleanup(sc);
MWL_RXFREE_DESTROY(sc);
mwl_tx_cleanup(sc);
mwl_hal_detach(sc->sc_mh);
if_free(ifp);
return 0;
}
/*
* MAC address handling for multiple BSS on the same radio.
* The first vap uses the MAC address from the EEPROM. For
* subsequent vap's we set the U/L bit (bit 1) in the MAC
* address and use the next six bits as an index.
*/
static void
assign_address(struct mwl_softc *sc, uint8_t mac[IEEE80211_ADDR_LEN], int clone)
{
int i;
if (clone && mwl_hal_ismbsscapable(sc->sc_mh)) {
/* NB: we only do this if h/w supports multiple bssid */
for (i = 0; i < 32; i++)
if ((sc->sc_bssidmask & (1<<i)) == 0)
break;
if (i != 0)
mac[0] |= (i << 2)|0x2;
} else
i = 0;
sc->sc_bssidmask |= 1<<i;
if (i == 0)
sc->sc_nbssid0++;
}
static void
reclaim_address(struct mwl_softc *sc, uint8_t mac[IEEE80211_ADDR_LEN])
{
int i = mac[0] >> 2;
if (i != 0 || --sc->sc_nbssid0 == 0)
sc->sc_bssidmask &= ~(1<<i);
}
static struct ieee80211vap *
mwl_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
enum ieee80211_opmode opmode, int flags,
const uint8_t bssid[IEEE80211_ADDR_LEN],
const uint8_t mac0[IEEE80211_ADDR_LEN])
{
struct ifnet *ifp = ic->ic_ifp;
struct mwl_softc *sc = ifp->if_softc;
struct mwl_hal *mh = sc->sc_mh;
struct ieee80211vap *vap, *apvap;
struct mwl_hal_vap *hvap;
struct mwl_vap *mvp;
uint8_t mac[IEEE80211_ADDR_LEN];
IEEE80211_ADDR_COPY(mac, mac0);
switch (opmode) {
case IEEE80211_M_HOSTAP:
case IEEE80211_M_MBSS:
if ((flags & IEEE80211_CLONE_MACADDR) == 0)
assign_address(sc, mac, flags & IEEE80211_CLONE_BSSID);
hvap = mwl_hal_newvap(mh, MWL_HAL_AP, mac);
if (hvap == NULL) {
if ((flags & IEEE80211_CLONE_MACADDR) == 0)
reclaim_address(sc, mac);
return NULL;
}
break;
case IEEE80211_M_STA:
if ((flags & IEEE80211_CLONE_MACADDR) == 0)
assign_address(sc, mac, flags & IEEE80211_CLONE_BSSID);
hvap = mwl_hal_newvap(mh, MWL_HAL_STA, mac);
if (hvap == NULL) {
if ((flags & IEEE80211_CLONE_MACADDR) == 0)
reclaim_address(sc, mac);
return NULL;
}
/* no h/w beacon miss support; always use s/w */
flags |= IEEE80211_CLONE_NOBEACONS;
break;
case IEEE80211_M_WDS:
hvap = NULL; /* NB: we use associated AP vap */
if (sc->sc_napvaps == 0)
return NULL; /* no existing AP vap */
break;
case IEEE80211_M_MONITOR:
hvap = NULL;
break;
case IEEE80211_M_IBSS:
case IEEE80211_M_AHDEMO:
default:
return NULL;
}
mvp = (struct mwl_vap *) malloc(sizeof(struct mwl_vap),
M_80211_VAP, M_NOWAIT | M_ZERO);
if (mvp == NULL) {
if (hvap != NULL) {
mwl_hal_delvap(hvap);
if ((flags & IEEE80211_CLONE_MACADDR) == 0)
reclaim_address(sc, mac);
}
/* XXX msg */
return NULL;
}
mvp->mv_hvap = hvap;
if (opmode == IEEE80211_M_WDS) {
/*
* WDS vaps must have an associated AP vap; find one.
* XXX not right.
*/
TAILQ_FOREACH(apvap, &ic->ic_vaps, iv_next)
if (apvap->iv_opmode == IEEE80211_M_HOSTAP) {
mvp->mv_ap_hvap = MWL_VAP(apvap)->mv_hvap;
break;
}
KASSERT(mvp->mv_ap_hvap != NULL, ("no ap vap"));
}
vap = &mvp->mv_vap;
ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid, mac);
if (hvap != NULL)
IEEE80211_ADDR_COPY(vap->iv_myaddr, mac);
/* override with driver methods */
mvp->mv_newstate = vap->iv_newstate;
vap->iv_newstate = mwl_newstate;
vap->iv_max_keyix = 0; /* XXX */
vap->iv_key_alloc = mwl_key_alloc;
vap->iv_key_delete = mwl_key_delete;
vap->iv_key_set = mwl_key_set;
#ifdef MWL_HOST_PS_SUPPORT
if (opmode == IEEE80211_M_HOSTAP || opmode == IEEE80211_M_MBSS) {
vap->iv_update_ps = mwl_update_ps;
mvp->mv_set_tim = vap->iv_set_tim;
vap->iv_set_tim = mwl_set_tim;
}
#endif
vap->iv_reset = mwl_reset;
vap->iv_update_beacon = mwl_beacon_update;
/* override max aid so sta's cannot assoc when we're out of sta id's */
vap->iv_max_aid = MWL_MAXSTAID;
/* override default A-MPDU rx parameters */
vap->iv_ampdu_rxmax = IEEE80211_HTCAP_MAXRXAMPDU_64K;
vap->iv_ampdu_density = IEEE80211_HTCAP_MPDUDENSITY_4;
/* complete setup */
ieee80211_vap_attach(vap, mwl_media_change, ieee80211_media_status);
switch (vap->iv_opmode) {
case IEEE80211_M_HOSTAP:
case IEEE80211_M_MBSS:
case IEEE80211_M_STA:
/*
* Setup sta db entry for local address.
*/
mwl_localstadb(vap);
if (vap->iv_opmode == IEEE80211_M_HOSTAP ||
vap->iv_opmode == IEEE80211_M_MBSS)
sc->sc_napvaps++;
else
sc->sc_nstavaps++;
break;
case IEEE80211_M_WDS:
sc->sc_nwdsvaps++;
break;
default:
break;
}
/*
* Setup overall operating mode.
*/
if (sc->sc_napvaps)
ic->ic_opmode = IEEE80211_M_HOSTAP;
else if (sc->sc_nstavaps)
ic->ic_opmode = IEEE80211_M_STA;
else
ic->ic_opmode = opmode;
return vap;
}
static void
mwl_vap_delete(struct ieee80211vap *vap)
{
struct mwl_vap *mvp = MWL_VAP(vap);
struct ifnet *parent = vap->iv_ic->ic_ifp;
struct mwl_softc *sc = parent->if_softc;
struct mwl_hal *mh = sc->sc_mh;
struct mwl_hal_vap *hvap = mvp->mv_hvap;
enum ieee80211_opmode opmode = vap->iv_opmode;
/* XXX disallow ap vap delete if WDS still present */
if (parent->if_drv_flags & IFF_DRV_RUNNING) {
/* quiesce h/w while we remove the vap */
mwl_hal_intrset(mh, 0); /* disable interrupts */
}
ieee80211_vap_detach(vap);
switch (opmode) {
case IEEE80211_M_HOSTAP:
case IEEE80211_M_MBSS:
case IEEE80211_M_STA:
KASSERT(hvap != NULL, ("no hal vap handle"));
(void) mwl_hal_delstation(hvap, vap->iv_myaddr);
mwl_hal_delvap(hvap);
if (opmode == IEEE80211_M_HOSTAP || opmode == IEEE80211_M_MBSS)
sc->sc_napvaps--;
else
sc->sc_nstavaps--;
/* XXX don't do it for IEEE80211_CLONE_MACADDR */
reclaim_address(sc, vap->iv_myaddr);
break;
case IEEE80211_M_WDS:
sc->sc_nwdsvaps--;
break;
default:
break;
}
mwl_cleartxq(sc, vap);
free(mvp, M_80211_VAP);
if (parent->if_drv_flags & IFF_DRV_RUNNING)
mwl_hal_intrset(mh, sc->sc_imask);
}
void
mwl_suspend(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags %x\n",
__func__, ifp->if_flags);
mwl_stop(ifp, 1);
}
void
mwl_resume(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags %x\n",
__func__, ifp->if_flags);
if (ifp->if_flags & IFF_UP)
mwl_init(sc);
}
void
mwl_shutdown(void *arg)
{
struct mwl_softc *sc = arg;
mwl_stop(sc->sc_ifp, 1);
}
/*
* Interrupt handler. Most of the actual processing is deferred.
*/
void
mwl_intr(void *arg)
{
struct mwl_softc *sc = arg;
struct mwl_hal *mh = sc->sc_mh;
uint32_t status;
if (sc->sc_invalid) {
/*
* The hardware is not ready/present, don't touch anything.
* Note this can happen early on if the IRQ is shared.
*/
DPRINTF(sc, MWL_DEBUG_ANY, "%s: invalid; ignored\n", __func__);
return;
}
/*
* Figure out the reason(s) for the interrupt.
*/
mwl_hal_getisr(mh, &status); /* NB: clears ISR too */
if (status == 0) /* must be a shared irq */
return;
DPRINTF(sc, MWL_DEBUG_INTR, "%s: status 0x%x imask 0x%x\n",
__func__, status, sc->sc_imask);
if (status & MACREG_A2HRIC_BIT_RX_RDY)
taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
if (status & MACREG_A2HRIC_BIT_TX_DONE)
taskqueue_enqueue(sc->sc_tq, &sc->sc_txtask);
if (status & MACREG_A2HRIC_BIT_BA_WATCHDOG)
taskqueue_enqueue(sc->sc_tq, &sc->sc_bawatchdogtask);
if (status & MACREG_A2HRIC_BIT_OPC_DONE)
mwl_hal_cmddone(mh);
if (status & MACREG_A2HRIC_BIT_MAC_EVENT) {
;
}
if (status & MACREG_A2HRIC_BIT_ICV_ERROR) {
/* TKIP ICV error */
sc->sc_stats.mst_rx_badtkipicv++;
}
if (status & MACREG_A2HRIC_BIT_QUEUE_EMPTY) {
/* 11n aggregation queue is empty, re-fill */
;
}
if (status & MACREG_A2HRIC_BIT_QUEUE_FULL) {
;
}
if (status & MACREG_A2HRIC_BIT_RADAR_DETECT) {
/* radar detected, process event */
taskqueue_enqueue(sc->sc_tq, &sc->sc_radartask);
}
if (status & MACREG_A2HRIC_BIT_CHAN_SWITCH) {
/* DFS channel switch */
taskqueue_enqueue(sc->sc_tq, &sc->sc_chanswitchtask);
}
}
static void
mwl_radar_proc(void *arg, int pending)
{
struct mwl_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
DPRINTF(sc, MWL_DEBUG_ANY, "%s: radar detected, pending %u\n",
__func__, pending);
sc->sc_stats.mst_radardetect++;
/* XXX stop h/w BA streams? */
IEEE80211_LOCK(ic);
ieee80211_dfs_notify_radar(ic, ic->ic_curchan);
IEEE80211_UNLOCK(ic);
}
static void
mwl_chanswitch_proc(void *arg, int pending)
{
struct mwl_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
DPRINTF(sc, MWL_DEBUG_ANY, "%s: channel switch notice, pending %u\n",
__func__, pending);
IEEE80211_LOCK(ic);
sc->sc_csapending = 0;
ieee80211_csa_completeswitch(ic);
IEEE80211_UNLOCK(ic);
}
static void
mwl_bawatchdog(const MWL_HAL_BASTREAM *sp)
{
struct ieee80211_node *ni = sp->data[0];
/* send DELBA and drop the stream */
ieee80211_ampdu_stop(ni, sp->data[1], IEEE80211_REASON_UNSPECIFIED);
}
static void
mwl_bawatchdog_proc(void *arg, int pending)
{
struct mwl_softc *sc = arg;
struct mwl_hal *mh = sc->sc_mh;
const MWL_HAL_BASTREAM *sp;
uint8_t bitmap, n;
sc->sc_stats.mst_bawatchdog++;
if (mwl_hal_getwatchdogbitmap(mh, &bitmap) != 0) {
DPRINTF(sc, MWL_DEBUG_AMPDU,
"%s: could not get bitmap\n", __func__);
sc->sc_stats.mst_bawatchdog_failed++;
return;
}
DPRINTF(sc, MWL_DEBUG_AMPDU, "%s: bitmap 0x%x\n", __func__, bitmap);
if (bitmap == 0xff) {
n = 0;
/* disable all ba streams */
for (bitmap = 0; bitmap < 8; bitmap++) {
sp = mwl_hal_bastream_lookup(mh, bitmap);
if (sp != NULL) {
mwl_bawatchdog(sp);
n++;
}
}
if (n == 0) {
DPRINTF(sc, MWL_DEBUG_AMPDU,
"%s: no BA streams found\n", __func__);
sc->sc_stats.mst_bawatchdog_empty++;
}
} else if (bitmap != 0xaa) {
/* disable a single ba stream */
sp = mwl_hal_bastream_lookup(mh, bitmap);
if (sp != NULL) {
mwl_bawatchdog(sp);
} else {
DPRINTF(sc, MWL_DEBUG_AMPDU,
"%s: no BA stream %d\n", __func__, bitmap);
sc->sc_stats.mst_bawatchdog_notfound++;
}
}
}
/*
* Convert net80211 channel to a HAL channel.
*/
static void
mwl_mapchan(MWL_HAL_CHANNEL *hc, const struct ieee80211_channel *chan)
{
hc->channel = chan->ic_ieee;
*(uint32_t *)&hc->channelFlags = 0;
if (IEEE80211_IS_CHAN_2GHZ(chan))
hc->channelFlags.FreqBand = MWL_FREQ_BAND_2DOT4GHZ;
else if (IEEE80211_IS_CHAN_5GHZ(chan))
hc->channelFlags.FreqBand = MWL_FREQ_BAND_5GHZ;
if (IEEE80211_IS_CHAN_HT40(chan)) {
hc->channelFlags.ChnlWidth = MWL_CH_40_MHz_WIDTH;
if (IEEE80211_IS_CHAN_HT40U(chan))
hc->channelFlags.ExtChnlOffset = MWL_EXT_CH_ABOVE_CTRL_CH;
else
hc->channelFlags.ExtChnlOffset = MWL_EXT_CH_BELOW_CTRL_CH;
} else
hc->channelFlags.ChnlWidth = MWL_CH_20_MHz_WIDTH;
/* XXX 10MHz channels */
}
/*
* Inform firmware of our tx/rx dma setup. The BAR 0
* writes below are for compatibility with older firmware.
* For current firmware we send this information with a
* cmd block via mwl_hal_sethwdma.
*/
static int
mwl_setupdma(struct mwl_softc *sc)
{
int error, i;
sc->sc_hwdma.rxDescRead = sc->sc_rxdma.dd_desc_paddr;
WR4(sc, sc->sc_hwspecs.rxDescRead, sc->sc_hwdma.rxDescRead);
WR4(sc, sc->sc_hwspecs.rxDescWrite, sc->sc_hwdma.rxDescRead);
for (i = 0; i < MWL_NUM_TX_QUEUES-MWL_NUM_ACK_QUEUES; i++) {
struct mwl_txq *txq = &sc->sc_txq[i];
sc->sc_hwdma.wcbBase[i] = txq->dma.dd_desc_paddr;
WR4(sc, sc->sc_hwspecs.wcbBase[i], sc->sc_hwdma.wcbBase[i]);
}
sc->sc_hwdma.maxNumTxWcb = mwl_txbuf;
sc->sc_hwdma.maxNumWCB = MWL_NUM_TX_QUEUES-MWL_NUM_ACK_QUEUES;
error = mwl_hal_sethwdma(sc->sc_mh, &sc->sc_hwdma);
if (error != 0) {
device_printf(sc->sc_dev,
"unable to setup tx/rx dma; hal status %u\n", error);
/* XXX */
}
return error;
}
/*
* Inform firmware of tx rate parameters.
* Called after a channel change.
*/
static int
mwl_setcurchanrates(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
const struct ieee80211_rateset *rs;
MWL_HAL_TXRATE rates;
memset(&rates, 0, sizeof(rates));
rs = ieee80211_get_suprates(ic, ic->ic_curchan);
/* rate used to send management frames */
rates.MgtRate = rs->rs_rates[0] & IEEE80211_RATE_VAL;
/* rate used to send multicast frames */
rates.McastRate = rates.MgtRate;
return mwl_hal_settxrate_auto(sc->sc_mh, &rates);
}
/*
* Inform firmware of tx rate parameters. Called whenever
* user-settable params change and after a channel change.
*/
static int
mwl_setrates(struct ieee80211vap *vap)
{
struct mwl_vap *mvp = MWL_VAP(vap);
struct ieee80211_node *ni = vap->iv_bss;
const struct ieee80211_txparam *tp = ni->ni_txparms;
MWL_HAL_TXRATE rates;
KASSERT(vap->iv_state == IEEE80211_S_RUN, ("state %d", vap->iv_state));
/*
* Update the h/w rate map.
* NB: 0x80 for MCS is passed through unchanged
*/
memset(&rates, 0, sizeof(rates));
/* rate used to send management frames */
rates.MgtRate = tp->mgmtrate;
/* rate used to send multicast frames */
rates.McastRate = tp->mcastrate;
/* while here calculate EAPOL fixed rate cookie */
mvp->mv_eapolformat = htole16(mwl_calcformat(rates.MgtRate, ni));
return mwl_hal_settxrate(mvp->mv_hvap,
tp->ucastrate != IEEE80211_FIXED_RATE_NONE ?
RATE_FIXED : RATE_AUTO, &rates);
}
/*
* Setup a fixed xmit rate cookie for EAPOL frames.
*/
static void
mwl_seteapolformat(struct ieee80211vap *vap)
{
struct mwl_vap *mvp = MWL_VAP(vap);
struct ieee80211_node *ni = vap->iv_bss;
enum ieee80211_phymode mode;
uint8_t rate;
KASSERT(vap->iv_state == IEEE80211_S_RUN, ("state %d", vap->iv_state));
mode = ieee80211_chan2mode(ni->ni_chan);
/*
* Use legacy rates when operating a mixed HT+non-HT bss.
* NB: this may violate POLA for sta and wds vap's.
*/
if (mode == IEEE80211_MODE_11NA &&
(vap->iv_flags_ht & IEEE80211_FHT_PUREN) == 0)
rate = vap->iv_txparms[IEEE80211_MODE_11A].mgmtrate;
else if (mode == IEEE80211_MODE_11NG &&
(vap->iv_flags_ht & IEEE80211_FHT_PUREN) == 0)
rate = vap->iv_txparms[IEEE80211_MODE_11G].mgmtrate;
else
rate = vap->iv_txparms[mode].mgmtrate;
mvp->mv_eapolformat = htole16(mwl_calcformat(rate, ni));
}
/*
* Map SKU+country code to region code for radar bin'ing.
*/
static int
mwl_map2regioncode(const struct ieee80211_regdomain *rd)
{
switch (rd->regdomain) {
case SKU_FCC:
case SKU_FCC3:
return DOMAIN_CODE_FCC;
case SKU_CA:
return DOMAIN_CODE_IC;
case SKU_ETSI:
case SKU_ETSI2:
case SKU_ETSI3:
if (rd->country == CTRY_SPAIN)
return DOMAIN_CODE_SPAIN;
if (rd->country == CTRY_FRANCE || rd->country == CTRY_FRANCE2)
return DOMAIN_CODE_FRANCE;
/* XXX force 1.3.1 radar type */
return DOMAIN_CODE_ETSI_131;
case SKU_JAPAN:
return DOMAIN_CODE_MKK;
case SKU_ROW:
return DOMAIN_CODE_DGT; /* Taiwan */
case SKU_APAC:
case SKU_APAC2:
case SKU_APAC3:
return DOMAIN_CODE_AUS; /* Australia */
}
/* XXX KOREA? */
return DOMAIN_CODE_FCC; /* XXX? */
}
static int
mwl_hal_reset(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct mwl_hal *mh = sc->sc_mh;
mwl_hal_setantenna(mh, WL_ANTENNATYPE_RX, sc->sc_rxantenna);
mwl_hal_setantenna(mh, WL_ANTENNATYPE_TX, sc->sc_txantenna);
mwl_hal_setradio(mh, 1, WL_AUTO_PREAMBLE);
mwl_hal_setwmm(sc->sc_mh, (ic->ic_flags & IEEE80211_F_WME) != 0);
mwl_chan_set(sc, ic->ic_curchan);
/* NB: RF/RA performance tuned for indoor mode */
mwl_hal_setrateadaptmode(mh, 0);
mwl_hal_setoptimizationlevel(mh,
(ic->ic_flags & IEEE80211_F_BURST) != 0);
mwl_hal_setregioncode(mh, mwl_map2regioncode(&ic->ic_regdomain));
mwl_hal_setaggampduratemode(mh, 1, 80); /* XXX */
mwl_hal_setcfend(mh, 0); /* XXX */
return 1;
}
static int
mwl_init_locked(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct mwl_hal *mh = sc->sc_mh;
int error = 0;
DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags 0x%x\n",
__func__, ifp->if_flags);
MWL_LOCK_ASSERT(sc);
/*
* Stop anything previously setup. This is safe
* whether this is the first time through or not.
*/
mwl_stop_locked(ifp, 0);
/*
* Push vap-independent state to the firmware.
*/
if (!mwl_hal_reset(sc)) {
if_printf(ifp, "unable to reset hardware\n");
return EIO;
}
/*
* Setup recv (once); transmit is already good to go.
*/
error = mwl_startrecv(sc);
if (error != 0) {
if_printf(ifp, "unable to start recv logic\n");
return error;
}
/*
* Enable interrupts.
*/
sc->sc_imask = MACREG_A2HRIC_BIT_RX_RDY
| MACREG_A2HRIC_BIT_TX_DONE
| MACREG_A2HRIC_BIT_OPC_DONE
#if 0
| MACREG_A2HRIC_BIT_MAC_EVENT
#endif
| MACREG_A2HRIC_BIT_ICV_ERROR
| MACREG_A2HRIC_BIT_RADAR_DETECT
| MACREG_A2HRIC_BIT_CHAN_SWITCH
#if 0
| MACREG_A2HRIC_BIT_QUEUE_EMPTY
#endif
| MACREG_A2HRIC_BIT_BA_WATCHDOG
| MACREQ_A2HRIC_BIT_TX_ACK
;
ifp->if_drv_flags |= IFF_DRV_RUNNING;
mwl_hal_intrset(mh, sc->sc_imask);
callout_reset(&sc->sc_watchdog, hz, mwl_watchdog, sc);
return 0;
}
static void
mwl_init(void *arg)
{
struct mwl_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
int error = 0;
DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags 0x%x\n",
__func__, ifp->if_flags);
MWL_LOCK(sc);
error = mwl_init_locked(sc);
MWL_UNLOCK(sc);
if (error == 0)
ieee80211_start_all(ic); /* start all vap's */
}
static void
mwl_stop_locked(struct ifnet *ifp, int disable)
{
struct mwl_softc *sc = ifp->if_softc;
DPRINTF(sc, MWL_DEBUG_ANY, "%s: invalid %u if_flags 0x%x\n",
__func__, sc->sc_invalid, ifp->if_flags);
MWL_LOCK_ASSERT(sc);
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
/*
* Shutdown the hardware and driver.
*/
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
callout_stop(&sc->sc_watchdog);
sc->sc_tx_timer = 0;
mwl_draintxq(sc);
}
}
static void
mwl_stop(struct ifnet *ifp, int disable)
{
struct mwl_softc *sc = ifp->if_softc;
MWL_LOCK(sc);
mwl_stop_locked(ifp, disable);
MWL_UNLOCK(sc);
}
static int
mwl_reset_vap(struct ieee80211vap *vap, int state)
{
struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
struct ieee80211com *ic = vap->iv_ic;
if (state == IEEE80211_S_RUN)
mwl_setrates(vap);
/* XXX off by 1? */
mwl_hal_setrtsthreshold(hvap, vap->iv_rtsthreshold);
/* XXX auto? 20/40 split? */
mwl_hal_sethtgi(hvap, (vap->iv_flags_ht &
(IEEE80211_FHT_SHORTGI20|IEEE80211_FHT_SHORTGI40)) ? 1 : 0);
mwl_hal_setnprot(hvap, ic->ic_htprotmode == IEEE80211_PROT_NONE ?
HTPROTECT_NONE : HTPROTECT_AUTO);
/* XXX txpower cap */
/* re-setup beacons */
if (state == IEEE80211_S_RUN &&
(vap->iv_opmode == IEEE80211_M_HOSTAP ||
vap->iv_opmode == IEEE80211_M_MBSS ||
vap->iv_opmode == IEEE80211_M_IBSS)) {
mwl_setapmode(vap, vap->iv_bss->ni_chan);
mwl_hal_setnprotmode(hvap,
MS(ic->ic_curhtprotmode, IEEE80211_HTINFO_OPMODE));
return mwl_beacon_setup(vap);
}
return 0;
}
/*
* Reset the hardware w/o losing operational state.
* Used to to reset or reload hardware state for a vap.
*/
static int
mwl_reset(struct ieee80211vap *vap, u_long cmd)
{
struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
int error = 0;
if (hvap != NULL) { /* WDS, MONITOR, etc. */
struct ieee80211com *ic = vap->iv_ic;
struct ifnet *ifp = ic->ic_ifp;
struct mwl_softc *sc = ifp->if_softc;
struct mwl_hal *mh = sc->sc_mh;
/* XXX handle DWDS sta vap change */
/* XXX do we need to disable interrupts? */
mwl_hal_intrset(mh, 0); /* disable interrupts */
error = mwl_reset_vap(vap, vap->iv_state);
mwl_hal_intrset(mh, sc->sc_imask);
}
return error;
}
/*
* Allocate a tx buffer for sending a frame. The
* packet is assumed to have the WME AC stored so
* we can use it to select the appropriate h/w queue.
*/
static struct mwl_txbuf *
mwl_gettxbuf(struct mwl_softc *sc, struct mwl_txq *txq)
{
struct mwl_txbuf *bf;
/*
* Grab a TX buffer and associated resources.
*/
MWL_TXQ_LOCK(txq);
bf = STAILQ_FIRST(&txq->free);
if (bf != NULL) {
STAILQ_REMOVE_HEAD(&txq->free, bf_list);
txq->nfree--;
}
MWL_TXQ_UNLOCK(txq);
if (bf == NULL)
DPRINTF(sc, MWL_DEBUG_XMIT,
"%s: out of xmit buffers on q %d\n", __func__, txq->qnum);
return bf;
}
/*
* Return a tx buffer to the queue it came from. Note there
* are two cases because we must preserve the order of buffers
* as it reflects the fixed order of descriptors in memory
* (the firmware pre-fetches descriptors so we cannot reorder).
*/
static void
mwl_puttxbuf_head(struct mwl_txq *txq, struct mwl_txbuf *bf)
{
bf->bf_m = NULL;
bf->bf_node = NULL;
MWL_TXQ_LOCK(txq);
STAILQ_INSERT_HEAD(&txq->free, bf, bf_list);
txq->nfree++;
MWL_TXQ_UNLOCK(txq);
}
static void
mwl_puttxbuf_tail(struct mwl_txq *txq, struct mwl_txbuf *bf)
{
bf->bf_m = NULL;
bf->bf_node = NULL;
MWL_TXQ_LOCK(txq);
STAILQ_INSERT_TAIL(&txq->free, bf, bf_list);
txq->nfree++;
MWL_TXQ_UNLOCK(txq);
}
static void
mwl_start(struct ifnet *ifp)
{
struct mwl_softc *sc = ifp->if_softc;
struct ieee80211_node *ni;
struct mwl_txbuf *bf;
struct mbuf *m;
struct mwl_txq *txq = NULL; /* XXX silence gcc */
int nqueued;
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid)
return;
nqueued = 0;
for (;;) {
bf = NULL;
IFQ_DEQUEUE(&ifp->if_snd, m);
if (m == NULL)
break;
/*
* Grab the node for the destination.
*/
ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
KASSERT(ni != NULL, ("no node"));
m->m_pkthdr.rcvif = NULL; /* committed, clear ref */
/*
* Grab a TX buffer and associated resources.
* We honor the classification by the 802.11 layer.
*/
txq = sc->sc_ac2q[M_WME_GETAC(m)];
bf = mwl_gettxbuf(sc, txq);
if (bf == NULL) {
m_freem(m);
ieee80211_free_node(ni);
#ifdef MWL_TX_NODROP
sc->sc_stats.mst_tx_qstop++;
/* XXX blocks other traffic */
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
break;
#else
DPRINTF(sc, MWL_DEBUG_XMIT,
"%s: tail drop on q %d\n", __func__, txq->qnum);
sc->sc_stats.mst_tx_qdrop++;
continue;
#endif /* MWL_TX_NODROP */
}
/*
* Pass the frame to the h/w for transmission.
*/
if (mwl_tx_start(sc, ni, bf, m)) {
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
mwl_puttxbuf_head(txq, bf);
ieee80211_free_node(ni);
continue;
}
nqueued++;
if (nqueued >= mwl_txcoalesce) {
/*
* Poke the firmware to process queued frames;
* see below about (lack of) locking.
*/
nqueued = 0;
mwl_hal_txstart(sc->sc_mh, 0/*XXX*/);
}
}
if (nqueued) {
/*
* NB: We don't need to lock against tx done because
* this just prods the firmware to check the transmit
* descriptors. The firmware will also start fetching
* descriptors by itself if it notices new ones are
* present when it goes to deliver a tx done interrupt
* to the host. So if we race with tx done processing
* it's ok. Delivering the kick here rather than in
* mwl_tx_start is an optimization to avoid poking the
* firmware for each packet.
*
* NB: the queue id isn't used so 0 is ok.
*/
mwl_hal_txstart(sc->sc_mh, 0/*XXX*/);
}
}
static int
mwl_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
const struct ieee80211_bpf_params *params)
{
struct ieee80211com *ic = ni->ni_ic;
struct ifnet *ifp = ic->ic_ifp;
struct mwl_softc *sc = ifp->if_softc;
struct mwl_txbuf *bf;
struct mwl_txq *txq;
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid) {
ieee80211_free_node(ni);
m_freem(m);
return ENETDOWN;
}
/*
* Grab a TX buffer and associated resources.
* Note that we depend on the classification
* by the 802.11 layer to get to the right h/w
* queue. Management frames must ALWAYS go on
* queue 1 but we cannot just force that here
* because we may receive non-mgt frames.
*/
txq = sc->sc_ac2q[M_WME_GETAC(m)];
bf = mwl_gettxbuf(sc, txq);
if (bf == NULL) {
sc->sc_stats.mst_tx_qstop++;
/* XXX blocks other traffic */
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
ieee80211_free_node(ni);
m_freem(m);
return ENOBUFS;
}
/*
* Pass the frame to the h/w for transmission.
*/
if (mwl_tx_start(sc, ni, bf, m)) {
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
mwl_puttxbuf_head(txq, bf);
ieee80211_free_node(ni);
return EIO; /* XXX */
}
/*
* NB: We don't need to lock against tx done because
* this just prods the firmware to check the transmit
* descriptors. The firmware will also start fetching
* descriptors by itself if it notices new ones are
* present when it goes to deliver a tx done interrupt
* to the host. So if we race with tx done processing
* it's ok. Delivering the kick here rather than in
* mwl_tx_start is an optimization to avoid poking the
* firmware for each packet.
*
* NB: the queue id isn't used so 0 is ok.
*/
mwl_hal_txstart(sc->sc_mh, 0/*XXX*/);
return 0;
}
static int
mwl_media_change(struct ifnet *ifp)
{
struct ieee80211vap *vap = ifp->if_softc;
int error;
error = ieee80211_media_change(ifp);
/* NB: only the fixed rate can change and that doesn't need a reset */
if (error == ENETRESET) {
mwl_setrates(vap);
error = 0;
}
return error;
}
#ifdef MWL_DEBUG
static void
mwl_keyprint(struct mwl_softc *sc, const char *tag,
const MWL_HAL_KEYVAL *hk, const uint8_t mac[IEEE80211_ADDR_LEN])
{
static const char *ciphers[] = {
"WEP",
"TKIP",
"AES-CCM",
};
int i, n;
printf("%s: [%u] %-7s", tag, hk->keyIndex, ciphers[hk->keyTypeId]);
for (i = 0, n = hk->keyLen; i < n; i++)
printf(" %02x", hk->key.aes[i]);
printf(" mac %s", ether_sprintf(mac));
if (hk->keyTypeId == KEY_TYPE_ID_TKIP) {
printf(" %s", "rxmic");
for (i = 0; i < sizeof(hk->key.tkip.rxMic); i++)
printf(" %02x", hk->key.tkip.rxMic[i]);
printf(" txmic");
for (i = 0; i < sizeof(hk->key.tkip.txMic); i++)
printf(" %02x", hk->key.tkip.txMic[i]);
}
printf(" flags 0x%x\n", hk->keyFlags);
}
#endif
/*
* Allocate a key cache slot for a unicast key. The
* firmware handles key allocation and every station is
* guaranteed key space so we are always successful.
*/
static int
mwl_key_alloc(struct ieee80211vap *vap, struct ieee80211_key *k,
ieee80211_keyix *keyix, ieee80211_keyix *rxkeyix)
{
struct mwl_softc *sc = vap->iv_ic->ic_ifp->if_softc;
if (k->wk_keyix != IEEE80211_KEYIX_NONE ||
(k->wk_flags & IEEE80211_KEY_GROUP)) {
if (!(&vap->iv_nw_keys[0] <= k &&
k < &vap->iv_nw_keys[IEEE80211_WEP_NKID])) {
/* should not happen */
DPRINTF(sc, MWL_DEBUG_KEYCACHE,
"%s: bogus group key\n", __func__);
return 0;
}
/* give the caller what they requested */
*keyix = *rxkeyix = k - vap->iv_nw_keys;
} else {
/*
* Firmware handles key allocation.
*/
*keyix = *rxkeyix = 0;
}
return 1;
}
/*
* Delete a key entry allocated by mwl_key_alloc.
*/
static int
mwl_key_delete(struct ieee80211vap *vap, const struct ieee80211_key *k)
{
struct mwl_softc *sc = vap->iv_ic->ic_ifp->if_softc;
struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
MWL_HAL_KEYVAL hk;
const uint8_t bcastaddr[IEEE80211_ADDR_LEN] =
{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
if (hvap == NULL) {
if (vap->iv_opmode != IEEE80211_M_WDS) {
/* XXX monitor mode? */
DPRINTF(sc, MWL_DEBUG_KEYCACHE,
"%s: no hvap for opmode %d\n", __func__,
vap->iv_opmode);
return 0;
}
hvap = MWL_VAP(vap)->mv_ap_hvap;
}
DPRINTF(sc, MWL_DEBUG_KEYCACHE, "%s: delete key %u\n",
__func__, k->wk_keyix);
memset(&hk, 0, sizeof(hk));
hk.keyIndex = k->wk_keyix;
switch (k->wk_cipher->ic_cipher) {
case IEEE80211_CIPHER_WEP:
hk.keyTypeId = KEY_TYPE_ID_WEP;
break;
case IEEE80211_CIPHER_TKIP:
hk.keyTypeId = KEY_TYPE_ID_TKIP;
break;
case IEEE80211_CIPHER_AES_CCM:
hk.keyTypeId = KEY_TYPE_ID_AES;
break;
default:
/* XXX should not happen */
DPRINTF(sc, MWL_DEBUG_KEYCACHE, "%s: unknown cipher %d\n",
__func__, k->wk_cipher->ic_cipher);
return 0;
}
return (mwl_hal_keyreset(hvap, &hk, bcastaddr) == 0); /*XXX*/
}
static __inline int
addgroupflags(MWL_HAL_KEYVAL *hk, const struct ieee80211_key *k)
{
if (k->wk_flags & IEEE80211_KEY_GROUP) {
if (k->wk_flags & IEEE80211_KEY_XMIT)
hk->keyFlags |= KEY_FLAG_TXGROUPKEY;
if (k->wk_flags & IEEE80211_KEY_RECV)
hk->keyFlags |= KEY_FLAG_RXGROUPKEY;
return 1;
} else
return 0;
}
/*
* Set the key cache contents for the specified key. Key cache
* slot(s) must already have been allocated by mwl_key_alloc.
*/
static int
mwl_key_set(struct ieee80211vap *vap, const struct ieee80211_key *k,
const uint8_t mac[IEEE80211_ADDR_LEN])
{
#define GRPXMIT (IEEE80211_KEY_XMIT | IEEE80211_KEY_GROUP)
/* NB: static wep keys are marked GROUP+tx/rx; GTK will be tx or rx */
#define IEEE80211_IS_STATICKEY(k) \
(((k)->wk_flags & (GRPXMIT|IEEE80211_KEY_RECV)) == \
(GRPXMIT|IEEE80211_KEY_RECV))
struct mwl_softc *sc = vap->iv_ic->ic_ifp->if_softc;
struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
const struct ieee80211_cipher *cip = k->wk_cipher;
const uint8_t *macaddr;
MWL_HAL_KEYVAL hk;
KASSERT((k->wk_flags & IEEE80211_KEY_SWCRYPT) == 0,
("s/w crypto set?"));
if (hvap == NULL) {
if (vap->iv_opmode != IEEE80211_M_WDS) {
/* XXX monitor mode? */
DPRINTF(sc, MWL_DEBUG_KEYCACHE,
"%s: no hvap for opmode %d\n", __func__,
vap->iv_opmode);
return 0;
}
hvap = MWL_VAP(vap)->mv_ap_hvap;
}
memset(&hk, 0, sizeof(hk));
hk.keyIndex = k->wk_keyix;
switch (cip->ic_cipher) {
case IEEE80211_CIPHER_WEP:
hk.keyTypeId = KEY_TYPE_ID_WEP;
hk.keyLen = k->wk_keylen;
if (k->wk_keyix == vap->iv_def_txkey)
hk.keyFlags = KEY_FLAG_WEP_TXKEY;
if (!IEEE80211_IS_STATICKEY(k)) {
/* NB: WEP is never used for the PTK */
(void) addgroupflags(&hk, k);
}
break;
case IEEE80211_CIPHER_TKIP:
hk.keyTypeId = KEY_TYPE_ID_TKIP;
hk.key.tkip.tsc.high = (uint32_t)(k->wk_keytsc >> 16);
hk.key.tkip.tsc.low = (uint16_t)k->wk_keytsc;
hk.keyFlags = KEY_FLAG_TSC_VALID | KEY_FLAG_MICKEY_VALID;
hk.keyLen = k->wk_keylen + IEEE80211_MICBUF_SIZE;
if (!addgroupflags(&hk, k))
hk.keyFlags |= KEY_FLAG_PAIRWISE;
break;
case IEEE80211_CIPHER_AES_CCM:
hk.keyTypeId = KEY_TYPE_ID_AES;
hk.keyLen = k->wk_keylen;
if (!addgroupflags(&hk, k))
hk.keyFlags |= KEY_FLAG_PAIRWISE;
break;
default:
/* XXX should not happen */
DPRINTF(sc, MWL_DEBUG_KEYCACHE, "%s: unknown cipher %d\n",
__func__, k->wk_cipher->ic_cipher);
return 0;
}
/*
* NB: tkip mic keys get copied here too; the layout
* just happens to match that in ieee80211_key.
*/
memcpy(hk.key.aes, k->wk_key, hk.keyLen);
/*
* Locate address of sta db entry for writing key;
* the convention unfortunately is somewhat different
* than how net80211, hostapd, and wpa_supplicant think.
*/
if (vap->iv_opmode == IEEE80211_M_STA) {
/*
* NB: keys plumbed before the sta reaches AUTH state
* will be discarded or written to the wrong sta db
* entry because iv_bss is meaningless. This is ok
* (right now) because we handle deferred plumbing of
* WEP keys when the sta reaches AUTH state.
*/
macaddr = vap->iv_bss->ni_bssid;
if ((k->wk_flags & IEEE80211_KEY_GROUP) == 0) {
/* XXX plumb to local sta db too for static key wep */
mwl_hal_keyset(hvap, &hk, vap->iv_myaddr);
}
} else if (vap->iv_opmode == IEEE80211_M_WDS &&
vap->iv_state != IEEE80211_S_RUN) {
/*
* Prior to RUN state a WDS vap will not it's BSS node
* setup so we will plumb the key to the wrong mac
* address (it'll be our local address). Workaround
* this for the moment by grabbing the correct address.
*/
macaddr = vap->iv_des_bssid;
} else if ((k->wk_flags & GRPXMIT) == GRPXMIT)
macaddr = vap->iv_myaddr;
else
macaddr = mac;
KEYPRINTF(sc, &hk, macaddr);
return (mwl_hal_keyset(hvap, &hk, macaddr) == 0);
#undef IEEE80211_IS_STATICKEY
#undef GRPXMIT
}
/* unaligned little endian access */
#define LE_READ_2(p) \
((uint16_t) \
((((const uint8_t *)(p))[0] ) | \
(((const uint8_t *)(p))[1] << 8)))
#define LE_READ_4(p) \
((uint32_t) \
((((const uint8_t *)(p))[0] ) | \
(((const uint8_t *)(p))[1] << 8) | \
(((const uint8_t *)(p))[2] << 16) | \
(((const uint8_t *)(p))[3] << 24)))
/*
* Set the multicast filter contents into the hardware.
* XXX f/w has no support; just defer to the os.
*/
static void
mwl_setmcastfilter(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
#if 0
struct ether_multi *enm;
struct ether_multistep estep;
uint8_t macs[IEEE80211_ADDR_LEN*MWL_HAL_MCAST_MAX];/* XXX stack use */
uint8_t *mp;
int nmc;
mp = macs;
nmc = 0;
ETHER_FIRST_MULTI(estep, &sc->sc_ec, enm);
while (enm != NULL) {
/* XXX Punt on ranges. */
if (nmc == MWL_HAL_MCAST_MAX ||
!IEEE80211_ADDR_EQ(enm->enm_addrlo, enm->enm_addrhi)) {
ifp->if_flags |= IFF_ALLMULTI;
return;
}
IEEE80211_ADDR_COPY(mp, enm->enm_addrlo);
mp += IEEE80211_ADDR_LEN, nmc++;
ETHER_NEXT_MULTI(estep, enm);
}
ifp->if_flags &= ~IFF_ALLMULTI;
mwl_hal_setmcast(sc->sc_mh, nmc, macs);
#else
/* XXX no mcast filter support; we get everything */
ifp->if_flags |= IFF_ALLMULTI;
#endif
}
static int
mwl_mode_init(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct mwl_hal *mh = sc->sc_mh;
/*
* NB: Ignore promisc in hostap mode; it's set by the
* bridge. This is wrong but we have no way to
* identify internal requests (from the bridge)
* versus external requests such as for tcpdump.
*/
mwl_hal_setpromisc(mh, (ifp->if_flags & IFF_PROMISC) &&
ic->ic_opmode != IEEE80211_M_HOSTAP);
mwl_setmcastfilter(sc);
return 0;
}
/*
* Callback from the 802.11 layer after a multicast state change.
*/
static void
mwl_update_mcast(struct ifnet *ifp)
{
struct mwl_softc *sc = ifp->if_softc;
mwl_setmcastfilter(sc);
}
/*
* Callback from the 802.11 layer after a promiscuous mode change.
* Note this interface does not check the operating mode as this
* is an internal callback and we are expected to honor the current
* state (e.g. this is used for setting the interface in promiscuous
* mode when operating in hostap mode to do ACS).
*/
static void
mwl_update_promisc(struct ifnet *ifp)
{
struct mwl_softc *sc = ifp->if_softc;
mwl_hal_setpromisc(sc->sc_mh, (ifp->if_flags & IFF_PROMISC) != 0);
}
/*
* Callback from the 802.11 layer to update the slot time
* based on the current setting. We use it to notify the
* firmware of ERP changes and the f/w takes care of things
* like slot time and preamble.
*/
static void
mwl_updateslot(struct ifnet *ifp)
{
struct mwl_softc *sc = ifp->if_softc;
struct ieee80211com *ic = ifp->if_l2com;
struct mwl_hal *mh = sc->sc_mh;
int prot;
/* NB: can be called early; suppress needless cmds */
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
return;
/*
* Calculate the ERP flags. The firwmare will use
* this to carry out the appropriate measures.
*/
prot = 0;
if (IEEE80211_IS_CHAN_ANYG(ic->ic_curchan)) {
if ((ic->ic_flags & IEEE80211_F_SHSLOT) == 0)
prot |= IEEE80211_ERP_NON_ERP_PRESENT;
if (ic->ic_flags & IEEE80211_F_USEPROT)
prot |= IEEE80211_ERP_USE_PROTECTION;
if (ic->ic_flags & IEEE80211_F_USEBARKER)
prot |= IEEE80211_ERP_LONG_PREAMBLE;
}
DPRINTF(sc, MWL_DEBUG_RESET,
"%s: chan %u MHz/flags 0x%x %s slot, (prot 0x%x ic_flags 0x%x)\n",
__func__, ic->ic_curchan->ic_freq, ic->ic_curchan->ic_flags,
ic->ic_flags & IEEE80211_F_SHSLOT ? "short" : "long", prot,
ic->ic_flags);
mwl_hal_setgprot(mh, prot);
}
/*
* Setup the beacon frame.
*/
static int
mwl_beacon_setup(struct ieee80211vap *vap)
{
struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
struct ieee80211_node *ni = vap->iv_bss;
struct ieee80211_beacon_offsets bo;
struct mbuf *m;
m = ieee80211_beacon_alloc(ni, &bo);
if (m == NULL)
return ENOBUFS;
mwl_hal_setbeacon(hvap, mtod(m, const void *), m->m_len);
m_free(m);
return 0;
}
/*
* Update the beacon frame in response to a change.
*/
static void
mwl_beacon_update(struct ieee80211vap *vap, int item)
{
struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
struct ieee80211com *ic = vap->iv_ic;
KASSERT(hvap != NULL, ("no beacon"));
switch (item) {
case IEEE80211_BEACON_ERP:
mwl_updateslot(ic->ic_ifp);
break;
case IEEE80211_BEACON_HTINFO:
mwl_hal_setnprotmode(hvap,
MS(ic->ic_curhtprotmode, IEEE80211_HTINFO_OPMODE));
break;
case IEEE80211_BEACON_CAPS:
case IEEE80211_BEACON_WME:
case IEEE80211_BEACON_APPIE:
case IEEE80211_BEACON_CSA:
break;
case IEEE80211_BEACON_TIM:
/* NB: firmware always forms TIM */
return;
}
/* XXX retain beacon frame and update */
mwl_beacon_setup(vap);
}
static void
mwl_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
{
bus_addr_t *paddr = (bus_addr_t*) arg;
KASSERT(error == 0, ("error %u on bus_dma callback", error));
*paddr = segs->ds_addr;
}
#ifdef MWL_HOST_PS_SUPPORT
/*
* Handle power save station occupancy changes.
*/
static void
mwl_update_ps(struct ieee80211vap *vap, int nsta)
{
struct mwl_vap *mvp = MWL_VAP(vap);
if (nsta == 0 || mvp->mv_last_ps_sta == 0)
mwl_hal_setpowersave_bss(mvp->mv_hvap, nsta);
mvp->mv_last_ps_sta = nsta;
}
/*
* Handle associated station power save state changes.
*/
static int
mwl_set_tim(struct ieee80211_node *ni, int set)
{
struct ieee80211vap *vap = ni->ni_vap;
struct mwl_vap *mvp = MWL_VAP(vap);
if (mvp->mv_set_tim(ni, set)) { /* NB: state change */
mwl_hal_setpowersave_sta(mvp->mv_hvap,
IEEE80211_AID(ni->ni_associd), set);
return 1;
} else
return 0;
}
#endif /* MWL_HOST_PS_SUPPORT */
static int
mwl_desc_setup(struct mwl_softc *sc, const char *name,
struct mwl_descdma *dd,
int nbuf, size_t bufsize, int ndesc, size_t descsize)
{
struct ifnet *ifp = sc->sc_ifp;
uint8_t *ds;
int error;
DPRINTF(sc, MWL_DEBUG_RESET,
"%s: %s DMA: %u bufs (%ju) %u desc/buf (%ju)\n",
__func__, name, nbuf, (uintmax_t) bufsize,
ndesc, (uintmax_t) descsize);
dd->dd_name = name;
dd->dd_desc_len = nbuf * ndesc * descsize;
/*
* Setup DMA descriptor area.
*/
error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
PAGE_SIZE, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
dd->dd_desc_len, /* maxsize */
1, /* nsegments */
dd->dd_desc_len, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, /* lockfunc */
NULL, /* lockarg */
&dd->dd_dmat);
if (error != 0) {
if_printf(ifp, "cannot allocate %s DMA tag\n", dd->dd_name);
return error;
}
/* allocate descriptors */
error = bus_dmamem_alloc(dd->dd_dmat, (void**) &dd->dd_desc,
BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
&dd->dd_dmamap);
if (error != 0) {
if_printf(ifp, "unable to alloc memory for %u %s descriptors, "
"error %u\n", nbuf * ndesc, dd->dd_name, error);
goto fail1;
}
error = bus_dmamap_load(dd->dd_dmat, dd->dd_dmamap,
dd->dd_desc, dd->dd_desc_len,
mwl_load_cb, &dd->dd_desc_paddr,
BUS_DMA_NOWAIT);
if (error != 0) {
if_printf(ifp, "unable to map %s descriptors, error %u\n",
dd->dd_name, error);
goto fail2;
}
ds = dd->dd_desc;
memset(ds, 0, dd->dd_desc_len);
DPRINTF(sc, MWL_DEBUG_RESET,
"%s: %s DMA map: %p (%lu) -> 0x%jx (%lu)\n",
__func__, dd->dd_name, ds, (u_long) dd->dd_desc_len,
(uintmax_t) dd->dd_desc_paddr, /*XXX*/ (u_long) dd->dd_desc_len);
return 0;
fail2:
bus_dmamem_free(dd->dd_dmat, dd->dd_desc, dd->dd_dmamap);
fail1:
bus_dma_tag_destroy(dd->dd_dmat);
memset(dd, 0, sizeof(*dd));
return error;
#undef DS2PHYS
}
static void
mwl_desc_cleanup(struct mwl_softc *sc, struct mwl_descdma *dd)
{
bus_dmamap_unload(dd->dd_dmat, dd->dd_dmamap);
bus_dmamem_free(dd->dd_dmat, dd->dd_desc, dd->dd_dmamap);
bus_dma_tag_destroy(dd->dd_dmat);
memset(dd, 0, sizeof(*dd));
}
/*
* Construct a tx q's free list. The order of entries on
* the list must reflect the physical layout of tx descriptors
* because the firmware pre-fetches descriptors.
*
* XXX might be better to use indices into the buffer array.
*/
static void
mwl_txq_reset(struct mwl_softc *sc, struct mwl_txq *txq)
{
struct mwl_txbuf *bf;
int i;
bf = txq->dma.dd_bufptr;
STAILQ_INIT(&txq->free);
for (i = 0; i < mwl_txbuf; i++, bf++)
STAILQ_INSERT_TAIL(&txq->free, bf, bf_list);
txq->nfree = i;
}
#define DS2PHYS(_dd, _ds) \
((_dd)->dd_desc_paddr + ((caddr_t)(_ds) - (caddr_t)(_dd)->dd_desc))
static int
mwl_txdma_setup(struct mwl_softc *sc, struct mwl_txq *txq)
{
struct ifnet *ifp = sc->sc_ifp;
int error, bsize, i;
struct mwl_txbuf *bf;
struct mwl_txdesc *ds;
error = mwl_desc_setup(sc, "tx", &txq->dma,
mwl_txbuf, sizeof(struct mwl_txbuf),
MWL_TXDESC, sizeof(struct mwl_txdesc));
if (error != 0)
return error;
/* allocate and setup tx buffers */
bsize = mwl_txbuf * sizeof(struct mwl_txbuf);
bf = malloc(bsize, M_MWLDEV, M_NOWAIT | M_ZERO);
if (bf == NULL) {
if_printf(ifp, "malloc of %u tx buffers failed\n",
mwl_txbuf);
return ENOMEM;
}
txq->dma.dd_bufptr = bf;
ds = txq->dma.dd_desc;
for (i = 0; i < mwl_txbuf; i++, bf++, ds += MWL_TXDESC) {
bf->bf_desc = ds;
bf->bf_daddr = DS2PHYS(&txq->dma, ds);
error = bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
&bf->bf_dmamap);
if (error != 0) {
if_printf(ifp, "unable to create dmamap for tx "
"buffer %u, error %u\n", i, error);
return error;
}
}
mwl_txq_reset(sc, txq);
return 0;
}
static void
mwl_txdma_cleanup(struct mwl_softc *sc, struct mwl_txq *txq)
{
struct mwl_txbuf *bf;
int i;
bf = txq->dma.dd_bufptr;
for (i = 0; i < mwl_txbuf; i++, bf++) {
KASSERT(bf->bf_m == NULL, ("mbuf on free list"));
KASSERT(bf->bf_node == NULL, ("node on free list"));
if (bf->bf_dmamap != NULL)
bus_dmamap_destroy(sc->sc_dmat, bf->bf_dmamap);
}
STAILQ_INIT(&txq->free);
txq->nfree = 0;
if (txq->dma.dd_bufptr != NULL) {
free(txq->dma.dd_bufptr, M_MWLDEV);
txq->dma.dd_bufptr = NULL;
}
if (txq->dma.dd_desc_len != 0)
mwl_desc_cleanup(sc, &txq->dma);
}
static int
mwl_rxdma_setup(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
int error, jumbosize, bsize, i;
struct mwl_rxbuf *bf;
struct mwl_jumbo *rbuf;
struct mwl_rxdesc *ds;
caddr_t data;
error = mwl_desc_setup(sc, "rx", &sc->sc_rxdma,
mwl_rxdesc, sizeof(struct mwl_rxbuf),
1, sizeof(struct mwl_rxdesc));
if (error != 0)
return error;
/*
* Receive is done to a private pool of jumbo buffers.
* This allows us to attach to mbuf's and avoid re-mapping
* memory on each rx we post. We allocate a large chunk
* of memory and manage it in the driver. The mbuf free
* callback method is used to reclaim frames after sending
* them up the stack. By default we allocate 2x the number of
* rx descriptors configured so we have some slop to hold
* us while frames are processed.
*/
if (mwl_rxbuf < 2*mwl_rxdesc) {
if_printf(ifp,
"too few rx dma buffers (%d); increasing to %d\n",
mwl_rxbuf, 2*mwl_rxdesc);
mwl_rxbuf = 2*mwl_rxdesc;
}
jumbosize = roundup(MWL_AGGR_SIZE, PAGE_SIZE);
sc->sc_rxmemsize = mwl_rxbuf*jumbosize;
error = bus_dma_tag_create(sc->sc_dmat, /* parent */
PAGE_SIZE, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
sc->sc_rxmemsize, /* maxsize */
1, /* nsegments */
sc->sc_rxmemsize, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, /* lockfunc */
NULL, /* lockarg */
&sc->sc_rxdmat);
if (error != 0) {
if_printf(ifp, "could not create rx DMA tag\n");
return error;
}
error = bus_dmamem_alloc(sc->sc_rxdmat, (void**) &sc->sc_rxmem,
BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
&sc->sc_rxmap);
if (error != 0) {
if_printf(ifp, "could not alloc %ju bytes of rx DMA memory\n",
(uintmax_t) sc->sc_rxmemsize);
return error;
}
error = bus_dmamap_load(sc->sc_rxdmat, sc->sc_rxmap,
sc->sc_rxmem, sc->sc_rxmemsize,
mwl_load_cb, &sc->sc_rxmem_paddr,
BUS_DMA_NOWAIT);
if (error != 0) {
if_printf(ifp, "could not load rx DMA map\n");
return error;
}
/*
* Allocate rx buffers and set them up.
*/
bsize = mwl_rxdesc * sizeof(struct mwl_rxbuf);
bf = malloc(bsize, M_MWLDEV, M_NOWAIT | M_ZERO);
if (bf == NULL) {
if_printf(ifp, "malloc of %u rx buffers failed\n", bsize);
return error;
}
sc->sc_rxdma.dd_bufptr = bf;
STAILQ_INIT(&sc->sc_rxbuf);
ds = sc->sc_rxdma.dd_desc;
for (i = 0; i < mwl_rxdesc; i++, bf++, ds++) {
bf->bf_desc = ds;
bf->bf_daddr = DS2PHYS(&sc->sc_rxdma, ds);
/* pre-assign dma buffer */
bf->bf_data = ((uint8_t *)sc->sc_rxmem) + (i*jumbosize);
/* NB: tail is intentional to preserve descriptor order */
STAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
}
/*
* Place remainder of dma memory buffers on the free list.
*/
SLIST_INIT(&sc->sc_rxfree);
for (; i < mwl_rxbuf; i++) {
data = ((uint8_t *)sc->sc_rxmem) + (i*jumbosize);
rbuf = MWL_JUMBO_DATA2BUF(data);
SLIST_INSERT_HEAD(&sc->sc_rxfree, rbuf, next);
sc->sc_nrxfree++;
}
return 0;
}
#undef DS2PHYS
static void
mwl_rxdma_cleanup(struct mwl_softc *sc)
{
if (sc->sc_rxmem_paddr != 0) {
bus_dmamap_unload(sc->sc_rxdmat, sc->sc_rxmap);
sc->sc_rxmem_paddr = 0;
}
if (sc->sc_rxmem != NULL) {
bus_dmamem_free(sc->sc_rxdmat, sc->sc_rxmem, sc->sc_rxmap);
sc->sc_rxmem = NULL;
}
if (sc->sc_rxdma.dd_bufptr != NULL) {
free(sc->sc_rxdma.dd_bufptr, M_MWLDEV);
sc->sc_rxdma.dd_bufptr = NULL;
}
if (sc->sc_rxdma.dd_desc_len != 0)
mwl_desc_cleanup(sc, &sc->sc_rxdma);
}
static int
mwl_dma_setup(struct mwl_softc *sc)
{
int error, i;
error = mwl_rxdma_setup(sc);
if (error != 0) {
mwl_rxdma_cleanup(sc);
return error;
}
for (i = 0; i < MWL_NUM_TX_QUEUES; i++) {
error = mwl_txdma_setup(sc, &sc->sc_txq[i]);
if (error != 0) {
mwl_dma_cleanup(sc);
return error;
}
}
return 0;
}
static void
mwl_dma_cleanup(struct mwl_softc *sc)
{
int i;
for (i = 0; i < MWL_NUM_TX_QUEUES; i++)
mwl_txdma_cleanup(sc, &sc->sc_txq[i]);
mwl_rxdma_cleanup(sc);
}
static struct ieee80211_node *
mwl_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
{
struct ieee80211com *ic = vap->iv_ic;
struct mwl_softc *sc = ic->ic_ifp->if_softc;
const size_t space = sizeof(struct mwl_node);
struct mwl_node *mn;
mn = malloc(space, M_80211_NODE, M_NOWAIT|M_ZERO);
if (mn == NULL) {
/* XXX stat+msg */
return NULL;
}
DPRINTF(sc, MWL_DEBUG_NODE, "%s: mn %p\n", __func__, mn);
return &mn->mn_node;
}
static void
mwl_node_cleanup(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
struct mwl_softc *sc = ic->ic_ifp->if_softc;
struct mwl_node *mn = MWL_NODE(ni);
DPRINTF(sc, MWL_DEBUG_NODE, "%s: ni %p ic %p staid %d\n",
__func__, ni, ni->ni_ic, mn->mn_staid);
if (mn->mn_staid != 0) {
struct ieee80211vap *vap = ni->ni_vap;
if (mn->mn_hvap != NULL) {
if (vap->iv_opmode == IEEE80211_M_STA)
mwl_hal_delstation(mn->mn_hvap, vap->iv_myaddr);
else
mwl_hal_delstation(mn->mn_hvap, ni->ni_macaddr);
}
/*
* NB: legacy WDS peer sta db entry is installed using
* the associate ap's hvap; use it again to delete it.
* XXX can vap be NULL?
*/
else if (vap->iv_opmode == IEEE80211_M_WDS &&
MWL_VAP(vap)->mv_ap_hvap != NULL)
mwl_hal_delstation(MWL_VAP(vap)->mv_ap_hvap,
ni->ni_macaddr);
delstaid(sc, mn->mn_staid);
mn->mn_staid = 0;
}
sc->sc_node_cleanup(ni);
}
/*
* Reclaim rx dma buffers from packets sitting on the ampdu
* reorder queue for a station. We replace buffers with a
* system cluster (if available).
*/
static void
mwl_ampdu_rxdma_reclaim(struct ieee80211_rx_ampdu *rap)
{
#if 0
int i, n, off;
struct mbuf *m;
void *cl;
n = rap->rxa_qframes;
for (i = 0; i < rap->rxa_wnd && n > 0; i++) {
m = rap->rxa_m[i];
if (m == NULL)
continue;
n--;
/* our dma buffers have a well-known free routine */
if ((m->m_flags & M_EXT) == 0 ||
m->m_ext.ext_free != mwl_ext_free)
continue;
/*
* Try to allocate a cluster and move the data.
*/
off = m->m_data - m->m_ext.ext_buf;
if (off + m->m_pkthdr.len > MCLBYTES) {
/* XXX no AMSDU for now */
continue;
}
cl = pool_cache_get_paddr(&mclpool_cache, 0,
&m->m_ext.ext_paddr);
if (cl != NULL) {
/*
* Copy the existing data to the cluster, remove
* the rx dma buffer, and attach the cluster in
* its place. Note we preserve the offset to the
* data so frames being bridged can still prepend
* their headers without adding another mbuf.
*/
memcpy((caddr_t) cl + off, m->m_data, m->m_pkthdr.len);
MEXTREMOVE(m);
MEXTADD(m, cl, MCLBYTES, 0, NULL, &mclpool_cache);
/* setup mbuf like _MCLGET does */
m->m_flags |= M_CLUSTER | M_EXT_RW;
_MOWNERREF(m, M_EXT | M_CLUSTER);
/* NB: m_data is clobbered by MEXTADDR, adjust */
m->m_data += off;
}
}
#endif
}
/*
* Callback to reclaim resources. We first let the
* net80211 layer do it's thing, then if we are still
* blocked by a lack of rx dma buffers we walk the ampdu
* reorder q's to reclaim buffers by copying to a system
* cluster.
*/
static void
mwl_node_drain(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
struct mwl_softc *sc = ic->ic_ifp->if_softc;
struct mwl_node *mn = MWL_NODE(ni);
DPRINTF(sc, MWL_DEBUG_NODE, "%s: ni %p vap %p staid %d\n",
__func__, ni, ni->ni_vap, mn->mn_staid);
/* NB: call up first to age out ampdu q's */
sc->sc_node_drain(ni);
/* XXX better to not check low water mark? */
if (sc->sc_rxblocked && mn->mn_staid != 0 &&
(ni->ni_flags & IEEE80211_NODE_HT)) {
uint8_t tid;
/*
* Walk the reorder q and reclaim rx dma buffers by copying
* the packet contents into clusters.
*/
for (tid = 0; tid < WME_NUM_TID; tid++) {
struct ieee80211_rx_ampdu *rap;
rap = &ni->ni_rx_ampdu[tid];
if ((rap->rxa_flags & IEEE80211_AGGR_XCHGPEND) == 0)
continue;
if (rap->rxa_qframes)
mwl_ampdu_rxdma_reclaim(rap);
}
}
}
static void
mwl_node_getsignal(const struct ieee80211_node *ni, int8_t *rssi, int8_t *noise)
{
*rssi = ni->ni_ic->ic_node_getrssi(ni);
#ifdef MWL_ANT_INFO_SUPPORT
#if 0
/* XXX need to smooth data */
*noise = -MWL_NODE_CONST(ni)->mn_ai.nf;
#else
*noise = -95; /* XXX */
#endif
#else
*noise = -95; /* XXX */
#endif
}
/*
* Convert Hardware per-antenna rssi info to common format:
* Let a1, a2, a3 represent the amplitudes per chain
* Let amax represent max[a1, a2, a3]
* Rssi1_dBm = RSSI_dBm + 20*log10(a1/amax)
* Rssi1_dBm = RSSI_dBm + 20*log10(a1) - 20*log10(amax)
* We store a table that is 4*20*log10(idx) - the extra 4 is to store or
* maintain some extra precision.
*
* Values are stored in .5 db format capped at 127.
*/
static void
mwl_node_getmimoinfo(const struct ieee80211_node *ni,
struct ieee80211_mimo_info *mi)
{
#define CVT(_dst, _src) do { \
(_dst) = rssi + ((logdbtbl[_src] - logdbtbl[rssi_max]) >> 2); \
(_dst) = (_dst) > 64 ? 127 : ((_dst) << 1); \
} while (0)
static const int8_t logdbtbl[32] = {
0, 0, 24, 38, 48, 56, 62, 68,
72, 76, 80, 83, 86, 89, 92, 94,
96, 98, 100, 102, 104, 106, 107, 109,
110, 112, 113, 115, 116, 117, 118, 119
};
const struct mwl_node *mn = MWL_NODE_CONST(ni);
uint8_t rssi = mn->mn_ai.rsvd1/2; /* XXX */
uint32_t rssi_max;
rssi_max = mn->mn_ai.rssi_a;
if (mn->mn_ai.rssi_b > rssi_max)
rssi_max = mn->mn_ai.rssi_b;
if (mn->mn_ai.rssi_c > rssi_max)
rssi_max = mn->mn_ai.rssi_c;
CVT(mi->rssi[0], mn->mn_ai.rssi_a);
CVT(mi->rssi[1], mn->mn_ai.rssi_b);
CVT(mi->rssi[2], mn->mn_ai.rssi_c);
mi->noise[0] = mn->mn_ai.nf_a;
mi->noise[1] = mn->mn_ai.nf_b;
mi->noise[2] = mn->mn_ai.nf_c;
#undef CVT
}
static __inline void *
mwl_getrxdma(struct mwl_softc *sc)
{
struct mwl_jumbo *buf;
void *data;
/*
* Allocate from jumbo pool.
*/
MWL_RXFREE_LOCK(sc);
buf = SLIST_FIRST(&sc->sc_rxfree);
if (buf == NULL) {
DPRINTF(sc, MWL_DEBUG_ANY,
"%s: out of rx dma buffers\n", __func__);
sc->sc_stats.mst_rx_nodmabuf++;
data = NULL;
} else {
SLIST_REMOVE_HEAD(&sc->sc_rxfree, next);
sc->sc_nrxfree--;
data = MWL_JUMBO_BUF2DATA(buf);
}
MWL_RXFREE_UNLOCK(sc);
return data;
}
static __inline void
mwl_putrxdma(struct mwl_softc *sc, void *data)
{
struct mwl_jumbo *buf;
/* XXX bounds check data */
MWL_RXFREE_LOCK(sc);
buf = MWL_JUMBO_DATA2BUF(data);
SLIST_INSERT_HEAD(&sc->sc_rxfree, buf, next);
sc->sc_nrxfree++;
MWL_RXFREE_UNLOCK(sc);
}
static int
mwl_rxbuf_init(struct mwl_softc *sc, struct mwl_rxbuf *bf)
{
struct mwl_rxdesc *ds;
ds = bf->bf_desc;
if (bf->bf_data == NULL) {
bf->bf_data = mwl_getrxdma(sc);
if (bf->bf_data == NULL) {
/* mark descriptor to be skipped */
ds->RxControl = EAGLE_RXD_CTRL_OS_OWN;
/* NB: don't need PREREAD */
MWL_RXDESC_SYNC(sc, ds, BUS_DMASYNC_PREWRITE);
sc->sc_stats.mst_rxbuf_failed++;
return ENOMEM;
}
}
/*
* NB: DMA buffer contents is known to be unmodified
* so there's no need to flush the data cache.
*/
/*
* Setup descriptor.
*/
ds->QosCtrl = 0;
ds->RSSI = 0;
ds->Status = EAGLE_RXD_STATUS_IDLE;
ds->Channel = 0;
ds->PktLen = htole16(MWL_AGGR_SIZE);
ds->SQ2 = 0;
ds->pPhysBuffData = htole32(MWL_JUMBO_DMA_ADDR(sc, bf->bf_data));
/* NB: don't touch pPhysNext, set once */
ds->RxControl = EAGLE_RXD_CTRL_DRIVER_OWN;
MWL_RXDESC_SYNC(sc, ds, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
return 0;
}
static void
mwl_ext_free(struct mbuf *m, void *data, void *arg)
{
struct mwl_softc *sc = arg;
/* XXX bounds check data */
mwl_putrxdma(sc, data);
/*
* If we were previously blocked by a lack of rx dma buffers
* check if we now have enough to restart rx interrupt handling.
* NB: we know we are called at splvm which is above splnet.
*/
if (sc->sc_rxblocked && sc->sc_nrxfree > mwl_rxdmalow) {
sc->sc_rxblocked = 0;
mwl_hal_intrset(sc->sc_mh, sc->sc_imask);
}
}
struct mwl_frame_bar {
u_int8_t i_fc[2];
u_int8_t i_dur[2];
u_int8_t i_ra[IEEE80211_ADDR_LEN];
u_int8_t i_ta[IEEE80211_ADDR_LEN];
/* ctl, seq, FCS */
} __packed;
/*
* Like ieee80211_anyhdrsize, but handles BAR frames
* specially so the logic below to piece the 802.11
* header together works.
*/
static __inline int
mwl_anyhdrsize(const void *data)
{
const struct ieee80211_frame *wh = data;
if ((wh->i_fc[0]&IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_CTL) {
switch (wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK) {
case IEEE80211_FC0_SUBTYPE_CTS:
case IEEE80211_FC0_SUBTYPE_ACK:
return sizeof(struct ieee80211_frame_ack);
case IEEE80211_FC0_SUBTYPE_BAR:
return sizeof(struct mwl_frame_bar);
}
return sizeof(struct ieee80211_frame_min);
} else
return ieee80211_hdrsize(data);
}
static void
mwl_handlemicerror(struct ieee80211com *ic, const uint8_t *data)
{
const struct ieee80211_frame *wh;
struct ieee80211_node *ni;
wh = (const struct ieee80211_frame *)(data + sizeof(uint16_t));
ni = ieee80211_find_rxnode(ic, (const struct ieee80211_frame_min *) wh);
if (ni != NULL) {
ieee80211_notify_michael_failure(ni->ni_vap, wh, 0);
ieee80211_free_node(ni);
}
}
/*
* Convert hardware signal strength to rssi. The value
* provided by the device has the noise floor added in;
* we need to compensate for this but we don't have that
* so we use a fixed value.
*
* The offset of 8 is good for both 2.4 and 5GHz. The LNA
* offset is already set as part of the initial gain. This
* will give at least +/- 3dB for 2.4GHz and +/- 5dB for 5GHz.
*/
static __inline int
cvtrssi(uint8_t ssi)
{
int rssi = (int) ssi + 8;
/* XXX hack guess until we have a real noise floor */
rssi = 2*(87 - rssi); /* NB: .5 dBm units */
return (rssi < 0 ? 0 : rssi > 127 ? 127 : rssi);
}
static void
mwl_rx_proc(void *arg, int npending)
{
#define IEEE80211_DIR_DSTODS(wh) \
((((const struct ieee80211_frame *)wh)->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
struct mwl_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct mwl_rxbuf *bf;
struct mwl_rxdesc *ds;
struct mbuf *m;
struct ieee80211_qosframe *wh;
struct ieee80211_qosframe_addr4 *wh4;
struct ieee80211_node *ni;
struct mwl_node *mn;
int off, len, hdrlen, pktlen, rssi, ntodo;
uint8_t *data, status;
void *newdata;
int16_t nf;
DPRINTF(sc, MWL_DEBUG_RX_PROC, "%s: pending %u rdptr 0x%x wrptr 0x%x\n",
__func__, npending, RD4(sc, sc->sc_hwspecs.rxDescRead),
RD4(sc, sc->sc_hwspecs.rxDescWrite));
nf = -96; /* XXX */
bf = sc->sc_rxnext;
for (ntodo = mwl_rxquota; ntodo > 0; ntodo--) {
if (bf == NULL)
bf = STAILQ_FIRST(&sc->sc_rxbuf);
ds = bf->bf_desc;
data = bf->bf_data;
if (data == NULL) {
/*
* If data allocation failed previously there
* will be no buffer; try again to re-populate it.
* Note the firmware will not advance to the next
* descriptor with a dma buffer so we must mimic
* this or we'll get out of sync.
*/
DPRINTF(sc, MWL_DEBUG_ANY,
"%s: rx buf w/o dma memory\n", __func__);
(void) mwl_rxbuf_init(sc, bf);
sc->sc_stats.mst_rx_dmabufmissing++;
break;
}
MWL_RXDESC_SYNC(sc, ds,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
if (ds->RxControl != EAGLE_RXD_CTRL_DMA_OWN)
break;
#ifdef MWL_DEBUG
if (sc->sc_debug & MWL_DEBUG_RECV_DESC)
mwl_printrxbuf(bf, 0);
#endif
status = ds->Status;
if (status & EAGLE_RXD_STATUS_DECRYPT_ERR_MASK) {
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
sc->sc_stats.mst_rx_crypto++;
/*
* NB: Check EAGLE_RXD_STATUS_GENERAL_DECRYPT_ERR
* for backwards compatibility.
*/
if (status != EAGLE_RXD_STATUS_GENERAL_DECRYPT_ERR &&
(status & EAGLE_RXD_STATUS_TKIP_MIC_DECRYPT_ERR)) {
/*
* MIC error, notify upper layers.
*/
bus_dmamap_sync(sc->sc_rxdmat, sc->sc_rxmap,
BUS_DMASYNC_POSTREAD);
mwl_handlemicerror(ic, data);
sc->sc_stats.mst_rx_tkipmic++;
}
/* XXX too painful to tap packets */
goto rx_next;
}
/*
* Sync the data buffer.
*/
len = le16toh(ds->PktLen);
bus_dmamap_sync(sc->sc_rxdmat, sc->sc_rxmap, BUS_DMASYNC_POSTREAD);
/*
* The 802.11 header is provided all or in part at the front;
* use it to calculate the true size of the header that we'll
* construct below. We use this to figure out where to copy
* payload prior to constructing the header.
*/
hdrlen = mwl_anyhdrsize(data + sizeof(uint16_t));
off = sizeof(uint16_t) + sizeof(struct ieee80211_frame_addr4);
/* calculate rssi early so we can re-use for each aggregate */
rssi = cvtrssi(ds->RSSI);
pktlen = hdrlen + (len - off);
/*
* NB: we know our frame is at least as large as
* IEEE80211_MIN_LEN because there is a 4-address
* frame at the front. Hence there's no need to
* vet the packet length. If the frame in fact
* is too small it should be discarded at the
* net80211 layer.
*/
/*
* Attach dma buffer to an mbuf. We tried
* doing this based on the packet size (i.e.
* copying small packets) but it turns out to
* be a net loss. The tradeoff might be system
* dependent (cache architecture is important).
*/
MGETHDR(m, M_NOWAIT, MT_DATA);
if (m == NULL) {
DPRINTF(sc, MWL_DEBUG_ANY,
"%s: no rx mbuf\n", __func__);
sc->sc_stats.mst_rx_nombuf++;
goto rx_next;
}
/*
* Acquire the replacement dma buffer before
* processing the frame. If we're out of dma
* buffers we disable rx interrupts and wait
* for the free pool to reach mlw_rxdmalow buffers
* before starting to do work again. If the firmware
* runs out of descriptors then it will toss frames
* which is better than our doing it as that can
* starve our processing. It is also important that
* we always process rx'd frames in case they are
* A-MPDU as otherwise the host's view of the BA
* window may get out of sync with the firmware.
*/
newdata = mwl_getrxdma(sc);
if (newdata == NULL) {
/* NB: stat+msg in mwl_getrxdma */
m_free(m);
/* disable RX interrupt and mark state */
mwl_hal_intrset(sc->sc_mh,
sc->sc_imask &~ MACREG_A2HRIC_BIT_RX_RDY);
sc->sc_rxblocked = 1;
ieee80211_drain(ic);
/* XXX check rxblocked and immediately start again? */
goto rx_stop;
}
bf->bf_data = newdata;
/*
* Attach the dma buffer to the mbuf;
* mwl_rxbuf_init will re-setup the rx
* descriptor using the replacement dma
* buffer we just installed above.
*/
MEXTADD(m, data, MWL_AGGR_SIZE, mwl_ext_free,
data, sc, 0, EXT_NET_DRV);
m->m_data += off - hdrlen;
m->m_pkthdr.len = m->m_len = pktlen;
m->m_pkthdr.rcvif = ifp;
/* NB: dma buffer assumed read-only */
/*
* Piece 802.11 header together.
*/
wh = mtod(m, struct ieee80211_qosframe *);
/* NB: don't need to do this sometimes but ... */
/* XXX special case so we can memcpy after m_devget? */
ovbcopy(data + sizeof(uint16_t), wh, hdrlen);
if (IEEE80211_QOS_HAS_SEQ(wh)) {
if (IEEE80211_DIR_DSTODS(wh)) {
wh4 = mtod(m,
struct ieee80211_qosframe_addr4*);
*(uint16_t *)wh4->i_qos = ds->QosCtrl;
} else {
*(uint16_t *)wh->i_qos = ds->QosCtrl;
}
}
/*
* The f/w strips WEP header but doesn't clear
* the WEP bit; mark the packet with M_WEP so
* net80211 will treat the data as decrypted.
* While here also clear the PWR_MGT bit since
* power save is handled by the firmware and
* passing this up will potentially cause the
* upper layer to put a station in power save
* (except when configured with MWL_HOST_PS_SUPPORT).
*/
if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED)
m->m_flags |= M_WEP;
#ifdef MWL_HOST_PS_SUPPORT
wh->i_fc[1] &= ~IEEE80211_FC1_PROTECTED;
#else
wh->i_fc[1] &= ~(IEEE80211_FC1_PROTECTED |
IEEE80211_FC1_PWR_MGT);
#endif
if (ieee80211_radiotap_active(ic)) {
struct mwl_rx_radiotap_header *tap = &sc->sc_rx_th;
tap->wr_flags = 0;
tap->wr_rate = ds->Rate;
tap->wr_antsignal = rssi + nf;
tap->wr_antnoise = nf;
}
if (IFF_DUMPPKTS_RECV(sc, wh)) {
ieee80211_dump_pkt(ic, mtod(m, caddr_t),
len, ds->Rate, rssi);
}
if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
/* dispatch */
ni = ieee80211_find_rxnode(ic,
(const struct ieee80211_frame_min *) wh);
if (ni != NULL) {
mn = MWL_NODE(ni);
#ifdef MWL_ANT_INFO_SUPPORT
mn->mn_ai.rssi_a = ds->ai.rssi_a;
mn->mn_ai.rssi_b = ds->ai.rssi_b;
mn->mn_ai.rssi_c = ds->ai.rssi_c;
mn->mn_ai.rsvd1 = rssi;
#endif
/* tag AMPDU aggregates for reorder processing */
if (ni->ni_flags & IEEE80211_NODE_HT)
m->m_flags |= M_AMPDU;
(void) ieee80211_input(ni, m, rssi, nf);
ieee80211_free_node(ni);
} else
(void) ieee80211_input_all(ic, m, rssi, nf);
rx_next:
/* NB: ignore ENOMEM so we process more descriptors */
(void) mwl_rxbuf_init(sc, bf);
bf = STAILQ_NEXT(bf, bf_list);
}
rx_stop:
sc->sc_rxnext = bf;
if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0 &&
!IFQ_IS_EMPTY(&ifp->if_snd)) {
/* NB: kick fw; the tx thread may have been preempted */
mwl_hal_txstart(sc->sc_mh, 0);
mwl_start(ifp);
}
#undef IEEE80211_DIR_DSTODS
}
static void
mwl_txq_init(struct mwl_softc *sc, struct mwl_txq *txq, int qnum)
{
struct mwl_txbuf *bf, *bn;
struct mwl_txdesc *ds;
MWL_TXQ_LOCK_INIT(sc, txq);
txq->qnum = qnum;
txq->txpri = 0; /* XXX */
#if 0
/* NB: q setup by mwl_txdma_setup XXX */
STAILQ_INIT(&txq->free);
#endif
STAILQ_FOREACH(bf, &txq->free, bf_list) {
bf->bf_txq = txq;
ds = bf->bf_desc;
bn = STAILQ_NEXT(bf, bf_list);
if (bn == NULL)
bn = STAILQ_FIRST(&txq->free);
ds->pPhysNext = htole32(bn->bf_daddr);
}
STAILQ_INIT(&txq->active);
}
/*
* Setup a hardware data transmit queue for the specified
* access control. We record the mapping from ac's
* to h/w queues for use by mwl_tx_start.
*/
static int
mwl_tx_setup(struct mwl_softc *sc, int ac, int mvtype)
{
#define N(a) (sizeof(a)/sizeof(a[0]))
struct mwl_txq *txq;
if (ac >= N(sc->sc_ac2q)) {
device_printf(sc->sc_dev, "AC %u out of range, max %zu!\n",
ac, N(sc->sc_ac2q));
return 0;
}
if (mvtype >= MWL_NUM_TX_QUEUES) {
device_printf(sc->sc_dev, "mvtype %u out of range, max %u!\n",
mvtype, MWL_NUM_TX_QUEUES);
return 0;
}
txq = &sc->sc_txq[mvtype];
mwl_txq_init(sc, txq, mvtype);
sc->sc_ac2q[ac] = txq;
return 1;
#undef N
}
/*
* Update WME parameters for a transmit queue.
*/
static int
mwl_txq_update(struct mwl_softc *sc, int ac)
{
#define MWL_EXPONENT_TO_VALUE(v) ((1<<v)-1)
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct mwl_txq *txq = sc->sc_ac2q[ac];
struct wmeParams *wmep = &ic->ic_wme.wme_chanParams.cap_wmeParams[ac];
struct mwl_hal *mh = sc->sc_mh;
int aifs, cwmin, cwmax, txoplim;
aifs = wmep->wmep_aifsn;
/* XXX in sta mode need to pass log values for cwmin/max */
cwmin = MWL_EXPONENT_TO_VALUE(wmep->wmep_logcwmin);
cwmax = MWL_EXPONENT_TO_VALUE(wmep->wmep_logcwmax);
txoplim = wmep->wmep_txopLimit; /* NB: units of 32us */
if (mwl_hal_setedcaparams(mh, txq->qnum, cwmin, cwmax, aifs, txoplim)) {
device_printf(sc->sc_dev, "unable to update hardware queue "
"parameters for %s traffic!\n",
ieee80211_wme_acnames[ac]);
return 0;
}
return 1;
#undef MWL_EXPONENT_TO_VALUE
}
/*
* Callback from the 802.11 layer to update WME parameters.
*/
static int
mwl_wme_update(struct ieee80211com *ic)
{
struct mwl_softc *sc = ic->ic_ifp->if_softc;
return !mwl_txq_update(sc, WME_AC_BE) ||
!mwl_txq_update(sc, WME_AC_BK) ||
!mwl_txq_update(sc, WME_AC_VI) ||
!mwl_txq_update(sc, WME_AC_VO) ? EIO : 0;
}
/*
* Reclaim resources for a setup queue.
*/
static void
mwl_tx_cleanupq(struct mwl_softc *sc, struct mwl_txq *txq)
{
/* XXX hal work? */
MWL_TXQ_LOCK_DESTROY(txq);
}
/*
* Reclaim all tx queue resources.
*/
static void
mwl_tx_cleanup(struct mwl_softc *sc)
{
int i;
for (i = 0; i < MWL_NUM_TX_QUEUES; i++)
mwl_tx_cleanupq(sc, &sc->sc_txq[i]);
}
static int
mwl_tx_dmasetup(struct mwl_softc *sc, struct mwl_txbuf *bf, struct mbuf *m0)
{
struct mbuf *m;
int error;
/*
* Load the DMA map so any coalescing is done. This
* also calculates the number of descriptors we need.
*/
error = bus_dmamap_load_mbuf_sg(sc->sc_dmat, bf->bf_dmamap, m0,
bf->bf_segs, &bf->bf_nseg,
BUS_DMA_NOWAIT);
if (error == EFBIG) {
/* XXX packet requires too many descriptors */
bf->bf_nseg = MWL_TXDESC+1;
} else if (error != 0) {
sc->sc_stats.mst_tx_busdma++;
m_freem(m0);
return error;
}
/*
* Discard null packets and check for packets that
* require too many TX descriptors. We try to convert
* the latter to a cluster.
*/
if (error == EFBIG) { /* too many desc's, linearize */
sc->sc_stats.mst_tx_linear++;
#if MWL_TXDESC > 1
m = m_collapse(m0, M_NOWAIT, MWL_TXDESC);
#else
m = m_defrag(m0, M_NOWAIT);
#endif
if (m == NULL) {
m_freem(m0);
sc->sc_stats.mst_tx_nombuf++;
return ENOMEM;
}
m0 = m;
error = bus_dmamap_load_mbuf_sg(sc->sc_dmat, bf->bf_dmamap, m0,
bf->bf_segs, &bf->bf_nseg,
BUS_DMA_NOWAIT);
if (error != 0) {
sc->sc_stats.mst_tx_busdma++;
m_freem(m0);
return error;
}
KASSERT(bf->bf_nseg <= MWL_TXDESC,
("too many segments after defrag; nseg %u", bf->bf_nseg));
} else if (bf->bf_nseg == 0) { /* null packet, discard */
sc->sc_stats.mst_tx_nodata++;
m_freem(m0);
return EIO;
}
DPRINTF(sc, MWL_DEBUG_XMIT, "%s: m %p len %u\n",
__func__, m0, m0->m_pkthdr.len);
bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap, BUS_DMASYNC_PREWRITE);
bf->bf_m = m0;
return 0;
}
static __inline int
mwl_cvtlegacyrate(int rate)
{
switch (rate) {
case 2: return 0;
case 4: return 1;
case 11: return 2;
case 22: return 3;
case 44: return 4;
case 12: return 5;
case 18: return 6;
case 24: return 7;
case 36: return 8;
case 48: return 9;
case 72: return 10;
case 96: return 11;
case 108:return 12;
}
return 0;
}
/*
* Calculate fixed tx rate information per client state;
* this value is suitable for writing to the Format field
* of a tx descriptor.
*/
static uint16_t
mwl_calcformat(uint8_t rate, const struct ieee80211_node *ni)
{
uint16_t fmt;
fmt = SM(3, EAGLE_TXD_ANTENNA)
| (IEEE80211_IS_CHAN_HT40D(ni->ni_chan) ?
EAGLE_TXD_EXTCHAN_LO : EAGLE_TXD_EXTCHAN_HI);
if (rate & IEEE80211_RATE_MCS) { /* HT MCS */
fmt |= EAGLE_TXD_FORMAT_HT
/* NB: 0x80 implicitly stripped from ucastrate */
| SM(rate, EAGLE_TXD_RATE);
/* XXX short/long GI may be wrong; re-check */
if (IEEE80211_IS_CHAN_HT40(ni->ni_chan)) {
fmt |= EAGLE_TXD_CHW_40
| (ni->ni_htcap & IEEE80211_HTCAP_SHORTGI40 ?
EAGLE_TXD_GI_SHORT : EAGLE_TXD_GI_LONG);
} else {
fmt |= EAGLE_TXD_CHW_20
| (ni->ni_htcap & IEEE80211_HTCAP_SHORTGI20 ?
EAGLE_TXD_GI_SHORT : EAGLE_TXD_GI_LONG);
}
} else { /* legacy rate */
fmt |= EAGLE_TXD_FORMAT_LEGACY
| SM(mwl_cvtlegacyrate(rate), EAGLE_TXD_RATE)
| EAGLE_TXD_CHW_20
/* XXX iv_flags & IEEE80211_F_SHPREAMBLE? */
| (ni->ni_capinfo & IEEE80211_CAPINFO_SHORT_PREAMBLE ?
EAGLE_TXD_PREAMBLE_SHORT : EAGLE_TXD_PREAMBLE_LONG);
}
return fmt;
}
static int
mwl_tx_start(struct mwl_softc *sc, struct ieee80211_node *ni, struct mwl_txbuf *bf,
struct mbuf *m0)
{
#define IEEE80211_DIR_DSTODS(wh) \
((wh->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct ieee80211vap *vap = ni->ni_vap;
int error, iswep, ismcast;
int hdrlen, copyhdrlen, pktlen;
struct mwl_txdesc *ds;
struct mwl_txq *txq;
struct ieee80211_frame *wh;
struct mwltxrec *tr;
struct mwl_node *mn;
uint16_t qos;
#if MWL_TXDESC > 1
int i;
#endif
wh = mtod(m0, struct ieee80211_frame *);
iswep = wh->i_fc[1] & IEEE80211_FC1_PROTECTED;
ismcast = IEEE80211_IS_MULTICAST(wh->i_addr1);
hdrlen = ieee80211_anyhdrsize(wh);
copyhdrlen = hdrlen;
pktlen = m0->m_pkthdr.len;
if (IEEE80211_QOS_HAS_SEQ(wh)) {
if (IEEE80211_DIR_DSTODS(wh)) {
qos = *(uint16_t *)
(((struct ieee80211_qosframe_addr4 *) wh)->i_qos);
copyhdrlen -= sizeof(qos);
} else
qos = *(uint16_t *)
(((struct ieee80211_qosframe *) wh)->i_qos);
} else
qos = 0;
if (iswep) {
const struct ieee80211_cipher *cip;
struct ieee80211_key *k;
/*
* Construct the 802.11 header+trailer for an encrypted
* frame. The only reason this can fail is because of an
* unknown or unsupported cipher/key type.
*
* NB: we do this even though the firmware will ignore
* what we've done for WEP and TKIP as we need the
* ExtIV filled in for CCMP and this also adjusts
* the headers which simplifies our work below.
*/
k = ieee80211_crypto_encap(ni, m0);
if (k == NULL) {
/*
* This can happen when the key is yanked after the
* frame was queued. Just discard the frame; the
* 802.11 layer counts failures and provides
* debugging/diagnostics.
*/
m_freem(m0);
return EIO;
}
/*
* Adjust the packet length for the crypto additions
* done during encap and any other bits that the f/w
* will add later on.
*/
cip = k->wk_cipher;
pktlen += cip->ic_header + cip->ic_miclen + cip->ic_trailer;
/* packet header may have moved, reset our local pointer */
wh = mtod(m0, struct ieee80211_frame *);
}
if (ieee80211_radiotap_active_vap(vap)) {
sc->sc_tx_th.wt_flags = 0; /* XXX */
if (iswep)
sc->sc_tx_th.wt_flags |= IEEE80211_RADIOTAP_F_WEP;
#if 0
sc->sc_tx_th.wt_rate = ds->DataRate;
#endif
sc->sc_tx_th.wt_txpower = ni->ni_txpower;
sc->sc_tx_th.wt_antenna = sc->sc_txantenna;
ieee80211_radiotap_tx(vap, m0);
}
/*
* Copy up/down the 802.11 header; the firmware requires
* we present a 2-byte payload length followed by a
* 4-address header (w/o QoS), followed (optionally) by
* any WEP/ExtIV header (but only filled in for CCMP).
* We are assured the mbuf has sufficient headroom to
* prepend in-place by the setup of ic_headroom in
* mwl_attach.
*/
if (hdrlen < sizeof(struct mwltxrec)) {
const int space = sizeof(struct mwltxrec) - hdrlen;
if (M_LEADINGSPACE(m0) < space) {
/* NB: should never happen */
device_printf(sc->sc_dev,
"not enough headroom, need %d found %zd, "
"m_flags 0x%x m_len %d\n",
space, M_LEADINGSPACE(m0), m0->m_flags, m0->m_len);
ieee80211_dump_pkt(ic,
mtod(m0, const uint8_t *), m0->m_len, 0, -1);
m_freem(m0);
sc->sc_stats.mst_tx_noheadroom++;
return EIO;
}
M_PREPEND(m0, space, M_NOWAIT);
}
tr = mtod(m0, struct mwltxrec *);
if (wh != (struct ieee80211_frame *) &tr->wh)
ovbcopy(wh, &tr->wh, hdrlen);
/*
* Note: the "firmware length" is actually the length
* of the fully formed "802.11 payload". That is, it's
* everything except for the 802.11 header. In particular
* this includes all crypto material including the MIC!
*/
tr->fwlen = htole16(pktlen - hdrlen);
/*
* Load the DMA map so any coalescing is done. This
* also calculates the number of descriptors we need.
*/
error = mwl_tx_dmasetup(sc, bf, m0);
if (error != 0) {
/* NB: stat collected in mwl_tx_dmasetup */
DPRINTF(sc, MWL_DEBUG_XMIT,
"%s: unable to setup dma\n", __func__);
return error;
}
bf->bf_node = ni; /* NB: held reference */
m0 = bf->bf_m; /* NB: may have changed */
tr = mtod(m0, struct mwltxrec *);
wh = (struct ieee80211_frame *)&tr->wh;
/*
* Formulate tx descriptor.
*/
ds = bf->bf_desc;
txq = bf->bf_txq;
ds->QosCtrl = qos; /* NB: already little-endian */
#if MWL_TXDESC == 1
/*
* NB: multiframes should be zero because the descriptors
* are initialized to zero. This should handle the case
* where the driver is built with MWL_TXDESC=1 but we are
* using firmware with multi-segment support.
*/
ds->PktPtr = htole32(bf->bf_segs[0].ds_addr);
ds->PktLen = htole16(bf->bf_segs[0].ds_len);
#else
ds->multiframes = htole32(bf->bf_nseg);
ds->PktLen = htole16(m0->m_pkthdr.len);
for (i = 0; i < bf->bf_nseg; i++) {
ds->PktPtrArray[i] = htole32(bf->bf_segs[i].ds_addr);
ds->PktLenArray[i] = htole16(bf->bf_segs[i].ds_len);
}
#endif
/* NB: pPhysNext, DataRate, and SapPktInfo setup once, don't touch */
ds->Format = 0;
ds->pad = 0;
ds->ack_wcb_addr = 0;
mn = MWL_NODE(ni);
/*
* Select transmit rate.
*/
switch (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) {
case IEEE80211_FC0_TYPE_MGT:
sc->sc_stats.mst_tx_mgmt++;
/* fall thru... */
case IEEE80211_FC0_TYPE_CTL:
/* NB: assign to BE q to avoid bursting */
ds->TxPriority = MWL_WME_AC_BE;
break;
case IEEE80211_FC0_TYPE_DATA:
if (!ismcast) {
const struct ieee80211_txparam *tp = ni->ni_txparms;
/*
* EAPOL frames get forced to a fixed rate and w/o
* aggregation; otherwise check for any fixed rate
* for the client (may depend on association state).
*/
if (m0->m_flags & M_EAPOL) {
const struct mwl_vap *mvp = MWL_VAP_CONST(vap);
ds->Format = mvp->mv_eapolformat;
ds->pad = htole16(
EAGLE_TXD_FIXED_RATE | EAGLE_TXD_DONT_AGGR);
} else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) {
/* XXX pre-calculate per node */
ds->Format = htole16(
mwl_calcformat(tp->ucastrate, ni));
ds->pad = htole16(EAGLE_TXD_FIXED_RATE);
}
/* NB: EAPOL frames will never have qos set */
if (qos == 0)
ds->TxPriority = txq->qnum;
#if MWL_MAXBA > 3
else if (mwl_bastream_match(&mn->mn_ba[3], qos))
ds->TxPriority = mn->mn_ba[3].txq;
#endif
#if MWL_MAXBA > 2
else if (mwl_bastream_match(&mn->mn_ba[2], qos))
ds->TxPriority = mn->mn_ba[2].txq;
#endif
#if MWL_MAXBA > 1
else if (mwl_bastream_match(&mn->mn_ba[1], qos))
ds->TxPriority = mn->mn_ba[1].txq;
#endif
#if MWL_MAXBA > 0
else if (mwl_bastream_match(&mn->mn_ba[0], qos))
ds->TxPriority = mn->mn_ba[0].txq;
#endif
else
ds->TxPriority = txq->qnum;
} else
ds->TxPriority = txq->qnum;
break;
default:
if_printf(ifp, "bogus frame type 0x%x (%s)\n",
wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK, __func__);
sc->sc_stats.mst_tx_badframetype++;
m_freem(m0);
return EIO;
}
if (IFF_DUMPPKTS_XMIT(sc))
ieee80211_dump_pkt(ic,
mtod(m0, const uint8_t *)+sizeof(uint16_t),
m0->m_len - sizeof(uint16_t), ds->DataRate, -1);
MWL_TXQ_LOCK(txq);
ds->Status = htole32(EAGLE_TXD_STATUS_FW_OWNED);
STAILQ_INSERT_TAIL(&txq->active, bf, bf_list);
MWL_TXDESC_SYNC(txq, ds, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
sc->sc_tx_timer = 5;
MWL_TXQ_UNLOCK(txq);
return 0;
#undef IEEE80211_DIR_DSTODS
}
static __inline int
mwl_cvtlegacyrix(int rix)
{
#define N(x) (sizeof(x)/sizeof(x[0]))
static const int ieeerates[] =
{ 2, 4, 11, 22, 44, 12, 18, 24, 36, 48, 72, 96, 108 };
return (rix < N(ieeerates) ? ieeerates[rix] : 0);
#undef N
}
/*
* Process completed xmit descriptors from the specified queue.
*/
static int
mwl_tx_processq(struct mwl_softc *sc, struct mwl_txq *txq)
{
#define EAGLE_TXD_STATUS_MCAST \
(EAGLE_TXD_STATUS_MULTICAST_TX | EAGLE_TXD_STATUS_BROADCAST_TX)
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
struct mwl_txbuf *bf;
struct mwl_txdesc *ds;
struct ieee80211_node *ni;
struct mwl_node *an;
int nreaped;
uint32_t status;
DPRINTF(sc, MWL_DEBUG_TX_PROC, "%s: tx queue %u\n", __func__, txq->qnum);
for (nreaped = 0;; nreaped++) {
MWL_TXQ_LOCK(txq);
bf = STAILQ_FIRST(&txq->active);
if (bf == NULL) {
MWL_TXQ_UNLOCK(txq);
break;
}
ds = bf->bf_desc;
MWL_TXDESC_SYNC(txq, ds,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
if (ds->Status & htole32(EAGLE_TXD_STATUS_FW_OWNED)) {
MWL_TXQ_UNLOCK(txq);
break;
}
STAILQ_REMOVE_HEAD(&txq->active, bf_list);
MWL_TXQ_UNLOCK(txq);
#ifdef MWL_DEBUG
if (sc->sc_debug & MWL_DEBUG_XMIT_DESC)
mwl_printtxbuf(bf, txq->qnum, nreaped);
#endif
ni = bf->bf_node;
if (ni != NULL) {
an = MWL_NODE(ni);
status = le32toh(ds->Status);
if (status & EAGLE_TXD_STATUS_OK) {
uint16_t Format = le16toh(ds->Format);
uint8_t txant = MS(Format, EAGLE_TXD_ANTENNA);
sc->sc_stats.mst_ant_tx[txant]++;
if (status & EAGLE_TXD_STATUS_OK_RETRY)
sc->sc_stats.mst_tx_retries++;
if (status & EAGLE_TXD_STATUS_OK_MORE_RETRY)
sc->sc_stats.mst_tx_mretries++;
if (txq->qnum >= MWL_WME_AC_VO)
ic->ic_wme.wme_hipri_traffic++;
ni->ni_txrate = MS(Format, EAGLE_TXD_RATE);
if ((Format & EAGLE_TXD_FORMAT_HT) == 0) {
ni->ni_txrate = mwl_cvtlegacyrix(
ni->ni_txrate);
} else
ni->ni_txrate |= IEEE80211_RATE_MCS;
sc->sc_stats.mst_tx_rate = ni->ni_txrate;
} else {
if (status & EAGLE_TXD_STATUS_FAILED_LINK_ERROR)
sc->sc_stats.mst_tx_linkerror++;
if (status & EAGLE_TXD_STATUS_FAILED_XRETRY)
sc->sc_stats.mst_tx_xretries++;
if (status & EAGLE_TXD_STATUS_FAILED_AGING)
sc->sc_stats.mst_tx_aging++;
if (bf->bf_m->m_flags & M_FF)
sc->sc_stats.mst_ff_txerr++;
}
/*
* Do any tx complete callback. Note this must
* be done before releasing the node reference.
* XXX no way to figure out if frame was ACK'd
*/
if (bf->bf_m->m_flags & M_TXCB) {
/* XXX strip fw len in case header inspected */
m_adj(bf->bf_m, sizeof(uint16_t));
ieee80211_process_callback(ni, bf->bf_m,
(status & EAGLE_TXD_STATUS_OK) == 0);
}
/*
* Reclaim reference to node.
*
* NB: the node may be reclaimed here if, for example
* this is a DEAUTH message that was sent and the
* node was timed out due to inactivity.
*/
ieee80211_free_node(ni);
}
ds->Status = htole32(EAGLE_TXD_STATUS_IDLE);
bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
m_freem(bf->bf_m);
mwl_puttxbuf_tail(txq, bf);
}
return nreaped;
#undef EAGLE_TXD_STATUS_MCAST
}
/*
* Deferred processing of transmit interrupt; special-cased
* for four hardware queues, 0-3.
*/
static void
mwl_tx_proc(void *arg, int npending)
{
struct mwl_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
int nreaped;
/*
* Process each active queue.
*/
nreaped = 0;
if (!STAILQ_EMPTY(&sc->sc_txq[0].active))
nreaped += mwl_tx_processq(sc, &sc->sc_txq[0]);
if (!STAILQ_EMPTY(&sc->sc_txq[1].active))
nreaped += mwl_tx_processq(sc, &sc->sc_txq[1]);
if (!STAILQ_EMPTY(&sc->sc_txq[2].active))
nreaped += mwl_tx_processq(sc, &sc->sc_txq[2]);
if (!STAILQ_EMPTY(&sc->sc_txq[3].active))
nreaped += mwl_tx_processq(sc, &sc->sc_txq[3]);
if (nreaped != 0) {
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
sc->sc_tx_timer = 0;
if (!IFQ_IS_EMPTY(&ifp->if_snd)) {
/* NB: kick fw; the tx thread may have been preempted */
mwl_hal_txstart(sc->sc_mh, 0);
mwl_start(ifp);
}
}
}
static void
mwl_tx_draintxq(struct mwl_softc *sc, struct mwl_txq *txq)
{
struct ieee80211_node *ni;
struct mwl_txbuf *bf;
u_int ix;
/*
* NB: this assumes output has been stopped and
* we do not need to block mwl_tx_tasklet
*/
for (ix = 0;; ix++) {
MWL_TXQ_LOCK(txq);
bf = STAILQ_FIRST(&txq->active);
if (bf == NULL) {
MWL_TXQ_UNLOCK(txq);
break;
}
STAILQ_REMOVE_HEAD(&txq->active, bf_list);
MWL_TXQ_UNLOCK(txq);
#ifdef MWL_DEBUG
if (sc->sc_debug & MWL_DEBUG_RESET) {
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
const struct mwltxrec *tr =
mtod(bf->bf_m, const struct mwltxrec *);
mwl_printtxbuf(bf, txq->qnum, ix);
ieee80211_dump_pkt(ic, (const uint8_t *)&tr->wh,
bf->bf_m->m_len - sizeof(tr->fwlen), 0, -1);
}
#endif /* MWL_DEBUG */
bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
ni = bf->bf_node;
if (ni != NULL) {
/*
* Reclaim node reference.
*/
ieee80211_free_node(ni);
}
m_freem(bf->bf_m);
mwl_puttxbuf_tail(txq, bf);
}
}
/*
* Drain the transmit queues and reclaim resources.
*/
static void
mwl_draintxq(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
int i;
for (i = 0; i < MWL_NUM_TX_QUEUES; i++)
mwl_tx_draintxq(sc, &sc->sc_txq[i]);
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
sc->sc_tx_timer = 0;
}
#ifdef MWL_DIAGAPI
/*
* Reset the transmit queues to a pristine state after a fw download.
*/
static void
mwl_resettxq(struct mwl_softc *sc)
{
int i;
for (i = 0; i < MWL_NUM_TX_QUEUES; i++)
mwl_txq_reset(sc, &sc->sc_txq[i]);
}
#endif /* MWL_DIAGAPI */
/*
* Clear the transmit queues of any frames submitted for the
* specified vap. This is done when the vap is deleted so we
* don't potentially reference the vap after it is gone.
* Note we cannot remove the frames; we only reclaim the node
* reference.
*/
static void
mwl_cleartxq(struct mwl_softc *sc, struct ieee80211vap *vap)
{
struct mwl_txq *txq;
struct mwl_txbuf *bf;
int i;
for (i = 0; i < MWL_NUM_TX_QUEUES; i++) {
txq = &sc->sc_txq[i];
MWL_TXQ_LOCK(txq);
STAILQ_FOREACH(bf, &txq->active, bf_list) {
struct ieee80211_node *ni = bf->bf_node;
if (ni != NULL && ni->ni_vap == vap) {
bf->bf_node = NULL;
ieee80211_free_node(ni);
}
}
MWL_TXQ_UNLOCK(txq);
}
}
static int
mwl_recv_action(struct ieee80211_node *ni, const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
struct mwl_softc *sc = ni->ni_ic->ic_ifp->if_softc;
const struct ieee80211_action *ia;
ia = (const struct ieee80211_action *) frm;
if (ia->ia_category == IEEE80211_ACTION_CAT_HT &&
ia->ia_action == IEEE80211_ACTION_HT_MIMOPWRSAVE) {
const struct ieee80211_action_ht_mimopowersave *mps =
(const struct ieee80211_action_ht_mimopowersave *) ia;
mwl_hal_setmimops(sc->sc_mh, ni->ni_macaddr,
mps->am_control & IEEE80211_A_HT_MIMOPWRSAVE_ENA,
MS(mps->am_control, IEEE80211_A_HT_MIMOPWRSAVE_MODE));
return 0;
} else
return sc->sc_recv_action(ni, wh, frm, efrm);
}
static int
mwl_addba_request(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap,
int dialogtoken, int baparamset, int batimeout)
{
struct mwl_softc *sc = ni->ni_ic->ic_ifp->if_softc;
struct ieee80211vap *vap = ni->ni_vap;
struct mwl_node *mn = MWL_NODE(ni);
struct mwl_bastate *bas;
bas = tap->txa_private;
if (bas == NULL) {
const MWL_HAL_BASTREAM *sp;
/*
* Check for a free BA stream slot.
*/
#if MWL_MAXBA > 3
if (mn->mn_ba[3].bastream == NULL)
bas = &mn->mn_ba[3];
else
#endif
#if MWL_MAXBA > 2
if (mn->mn_ba[2].bastream == NULL)
bas = &mn->mn_ba[2];
else
#endif
#if MWL_MAXBA > 1
if (mn->mn_ba[1].bastream == NULL)
bas = &mn->mn_ba[1];
else
#endif
#if MWL_MAXBA > 0
if (mn->mn_ba[0].bastream == NULL)
bas = &mn->mn_ba[0];
else
#endif
{
/* sta already has max BA streams */
/* XXX assign BA stream to highest priority tid */
DPRINTF(sc, MWL_DEBUG_AMPDU,
"%s: already has max bastreams\n", __func__);
sc->sc_stats.mst_ampdu_reject++;
return 0;
}
/* NB: no held reference to ni */
sp = mwl_hal_bastream_alloc(MWL_VAP(vap)->mv_hvap,
(baparamset & IEEE80211_BAPS_POLICY_IMMEDIATE) != 0,
ni->ni_macaddr, tap->txa_tid, ni->ni_htparam,
ni, tap);
if (sp == NULL) {
/*
* No available stream, return 0 so no
* a-mpdu aggregation will be done.
*/
DPRINTF(sc, MWL_DEBUG_AMPDU,
"%s: no bastream available\n", __func__);
sc->sc_stats.mst_ampdu_nostream++;
return 0;
}
DPRINTF(sc, MWL_DEBUG_AMPDU, "%s: alloc bastream %p\n",
__func__, sp);
/* NB: qos is left zero so we won't match in mwl_tx_start */
bas->bastream = sp;
tap->txa_private = bas;
}
/* fetch current seq# from the firmware; if available */
if (mwl_hal_bastream_get_seqno(sc->sc_mh, bas->bastream,
vap->iv_opmode == IEEE80211_M_STA ? vap->iv_myaddr : ni->ni_macaddr,
&tap->txa_start) != 0)
tap->txa_start = 0;
return sc->sc_addba_request(ni, tap, dialogtoken, baparamset, batimeout);
}
static int
mwl_addba_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap,
int code, int baparamset, int batimeout)
{
struct mwl_softc *sc = ni->ni_ic->ic_ifp->if_softc;
struct mwl_bastate *bas;
bas = tap->txa_private;
if (bas == NULL) {
/* XXX should not happen */
DPRINTF(sc, MWL_DEBUG_AMPDU,
"%s: no BA stream allocated, TID %d\n",
__func__, tap->txa_tid);
sc->sc_stats.mst_addba_nostream++;
return 0;
}
if (code == IEEE80211_STATUS_SUCCESS) {
struct ieee80211vap *vap = ni->ni_vap;
int bufsiz, error;
/*
* Tell the firmware to setup the BA stream;
* we know resources are available because we
* pre-allocated one before forming the request.
*/
bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ);
if (bufsiz == 0)
bufsiz = IEEE80211_AGGR_BAWMAX;
error = mwl_hal_bastream_create(MWL_VAP(vap)->mv_hvap,
bas->bastream, bufsiz, bufsiz, tap->txa_start);
if (error != 0) {
/*
* Setup failed, return immediately so no a-mpdu
* aggregation will be done.
*/
mwl_hal_bastream_destroy(sc->sc_mh, bas->bastream);
mwl_bastream_free(bas);
tap->txa_private = NULL;
DPRINTF(sc, MWL_DEBUG_AMPDU,
"%s: create failed, error %d, bufsiz %d TID %d "
"htparam 0x%x\n", __func__, error, bufsiz,
tap->txa_tid, ni->ni_htparam);
sc->sc_stats.mst_bacreate_failed++;
return 0;
}
/* NB: cache txq to avoid ptr indirect */
mwl_bastream_setup(bas, tap->txa_tid, bas->bastream->txq);
DPRINTF(sc, MWL_DEBUG_AMPDU,
"%s: bastream %p assigned to txq %d TID %d bufsiz %d "
"htparam 0x%x\n", __func__, bas->bastream,
bas->txq, tap->txa_tid, bufsiz, ni->ni_htparam);
} else {
/*
* Other side NAK'd us; return the resources.
*/
DPRINTF(sc, MWL_DEBUG_AMPDU,
"%s: request failed with code %d, destroy bastream %p\n",
__func__, code, bas->bastream);
mwl_hal_bastream_destroy(sc->sc_mh, bas->bastream);
mwl_bastream_free(bas);
tap->txa_private = NULL;
}
/* NB: firmware sends BAR so we don't need to */
return sc->sc_addba_response(ni, tap, code, baparamset, batimeout);
}
static void
mwl_addba_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap)
{
struct mwl_softc *sc = ni->ni_ic->ic_ifp->if_softc;
struct mwl_bastate *bas;
bas = tap->txa_private;
if (bas != NULL) {
DPRINTF(sc, MWL_DEBUG_AMPDU, "%s: destroy bastream %p\n",
__func__, bas->bastream);
mwl_hal_bastream_destroy(sc->sc_mh, bas->bastream);
mwl_bastream_free(bas);
tap->txa_private = NULL;
}
sc->sc_addba_stop(ni, tap);
}
/*
* Setup the rx data structures. This should only be
* done once or we may get out of sync with the firmware.
*/
static int
mwl_startrecv(struct mwl_softc *sc)
{
if (!sc->sc_recvsetup) {
struct mwl_rxbuf *bf, *prev;
struct mwl_rxdesc *ds;
prev = NULL;
STAILQ_FOREACH(bf, &sc->sc_rxbuf, bf_list) {
int error = mwl_rxbuf_init(sc, bf);
if (error != 0) {
DPRINTF(sc, MWL_DEBUG_RECV,
"%s: mwl_rxbuf_init failed %d\n",
__func__, error);
return error;
}
if (prev != NULL) {
ds = prev->bf_desc;
ds->pPhysNext = htole32(bf->bf_daddr);
}
prev = bf;
}
if (prev != NULL) {
ds = prev->bf_desc;
ds->pPhysNext =
htole32(STAILQ_FIRST(&sc->sc_rxbuf)->bf_daddr);
}
sc->sc_recvsetup = 1;
}
mwl_mode_init(sc); /* set filters, etc. */
return 0;
}
static MWL_HAL_APMODE
mwl_getapmode(const struct ieee80211vap *vap, struct ieee80211_channel *chan)
{
MWL_HAL_APMODE mode;
if (IEEE80211_IS_CHAN_HT(chan)) {
if (vap->iv_flags_ht & IEEE80211_FHT_PUREN)
mode = AP_MODE_N_ONLY;
else if (IEEE80211_IS_CHAN_5GHZ(chan))
mode = AP_MODE_AandN;
else if (vap->iv_flags & IEEE80211_F_PUREG)
mode = AP_MODE_GandN;
else
mode = AP_MODE_BandGandN;
} else if (IEEE80211_IS_CHAN_ANYG(chan)) {
if (vap->iv_flags & IEEE80211_F_PUREG)
mode = AP_MODE_G_ONLY;
else
mode = AP_MODE_MIXED;
} else if (IEEE80211_IS_CHAN_B(chan))
mode = AP_MODE_B_ONLY;
else if (IEEE80211_IS_CHAN_A(chan))
mode = AP_MODE_A_ONLY;
else
mode = AP_MODE_MIXED; /* XXX should not happen? */
return mode;
}
static int
mwl_setapmode(struct ieee80211vap *vap, struct ieee80211_channel *chan)
{
struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
return mwl_hal_setapmode(hvap, mwl_getapmode(vap, chan));
}
/*
* Set/change channels.
*/
static int
mwl_chan_set(struct mwl_softc *sc, struct ieee80211_channel *chan)
{
struct mwl_hal *mh = sc->sc_mh;
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
MWL_HAL_CHANNEL hchan;
int maxtxpow;
DPRINTF(sc, MWL_DEBUG_RESET, "%s: chan %u MHz/flags 0x%x\n",
__func__, chan->ic_freq, chan->ic_flags);
/*
* Convert to a HAL channel description with
* the flags constrained to reflect the current
* operating mode.
*/
mwl_mapchan(&hchan, chan);
mwl_hal_intrset(mh, 0); /* disable interrupts */
#if 0
mwl_draintxq(sc); /* clear pending tx frames */
#endif
mwl_hal_setchannel(mh, &hchan);
/*
* Tx power is cap'd by the regulatory setting and
* possibly a user-set limit. We pass the min of
* these to the hal to apply them to the cal data
* for this channel.
* XXX min bound?
*/
maxtxpow = 2*chan->ic_maxregpower;
if (maxtxpow > ic->ic_txpowlimit)
maxtxpow = ic->ic_txpowlimit;
mwl_hal_settxpower(mh, &hchan, maxtxpow / 2);
/* NB: potentially change mcast/mgt rates */
mwl_setcurchanrates(sc);
/*
* Update internal state.
*/
sc->sc_tx_th.wt_chan_freq = htole16(chan->ic_freq);
sc->sc_rx_th.wr_chan_freq = htole16(chan->ic_freq);
if (IEEE80211_IS_CHAN_A(chan)) {
sc->sc_tx_th.wt_chan_flags = htole16(IEEE80211_CHAN_A);
sc->sc_rx_th.wr_chan_flags = htole16(IEEE80211_CHAN_A);
} else if (IEEE80211_IS_CHAN_ANYG(chan)) {
sc->sc_tx_th.wt_chan_flags = htole16(IEEE80211_CHAN_G);
sc->sc_rx_th.wr_chan_flags = htole16(IEEE80211_CHAN_G);
} else {
sc->sc_tx_th.wt_chan_flags = htole16(IEEE80211_CHAN_B);
sc->sc_rx_th.wr_chan_flags = htole16(IEEE80211_CHAN_B);
}
sc->sc_curchan = hchan;
mwl_hal_intrset(mh, sc->sc_imask);
return 0;
}
static void
mwl_scan_start(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
struct mwl_softc *sc = ifp->if_softc;
DPRINTF(sc, MWL_DEBUG_STATE, "%s\n", __func__);
}
static void
mwl_scan_end(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
struct mwl_softc *sc = ifp->if_softc;
DPRINTF(sc, MWL_DEBUG_STATE, "%s\n", __func__);
}
static void
mwl_set_channel(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
struct mwl_softc *sc = ifp->if_softc;
(void) mwl_chan_set(sc, ic->ic_curchan);
}
/*
* Handle a channel switch request. We inform the firmware
* and mark the global state to suppress various actions.
* NB: we issue only one request to the fw; we may be called
* multiple times if there are multiple vap's.
*/
static void
mwl_startcsa(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
struct mwl_softc *sc = ic->ic_ifp->if_softc;
MWL_HAL_CHANNEL hchan;
if (sc->sc_csapending)
return;
mwl_mapchan(&hchan, ic->ic_csa_newchan);
/* 1 =>'s quiet channel */
mwl_hal_setchannelswitchie(sc->sc_mh, &hchan, 1, ic->ic_csa_count);
sc->sc_csapending = 1;
}
/*
* Plumb any static WEP key for the station. This is
* necessary as we must propagate the key from the
* global key table of the vap to each sta db entry.
*/
static void
mwl_setanywepkey(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
{
if ((vap->iv_flags & (IEEE80211_F_PRIVACY|IEEE80211_F_WPA)) ==
IEEE80211_F_PRIVACY &&
vap->iv_def_txkey != IEEE80211_KEYIX_NONE &&
vap->iv_nw_keys[vap->iv_def_txkey].wk_keyix != IEEE80211_KEYIX_NONE)
(void) mwl_key_set(vap, &vap->iv_nw_keys[vap->iv_def_txkey], mac);
}
static int
mwl_peerstadb(struct ieee80211_node *ni, int aid, int staid, MWL_HAL_PEERINFO *pi)
{
#define WME(ie) ((const struct ieee80211_wme_info *) ie)
struct ieee80211vap *vap = ni->ni_vap;
struct mwl_hal_vap *hvap;
int error;
if (vap->iv_opmode == IEEE80211_M_WDS) {
/*
* WDS vap's do not have a f/w vap; instead they piggyback
* on an AP vap and we must install the sta db entry and
* crypto state using that AP's handle (the WDS vap has none).
*/
hvap = MWL_VAP(vap)->mv_ap_hvap;
} else
hvap = MWL_VAP(vap)->mv_hvap;
error = mwl_hal_newstation(hvap, ni->ni_macaddr,
aid, staid, pi,
ni->ni_flags & (IEEE80211_NODE_QOS | IEEE80211_NODE_HT),
ni->ni_ies.wme_ie != NULL ? WME(ni->ni_ies.wme_ie)->wme_info : 0);
if (error == 0) {
/*
* Setup security for this station. For sta mode this is
* needed even though do the same thing on transition to
* AUTH state because the call to mwl_hal_newstation
* clobbers the crypto state we setup.
*/
mwl_setanywepkey(vap, ni->ni_macaddr);
}
return error;
#undef WME
}
static void
mwl_setglobalkeys(struct ieee80211vap *vap)
{
struct ieee80211_key *wk;
wk = &vap->iv_nw_keys[0];
for (; wk < &vap->iv_nw_keys[IEEE80211_WEP_NKID]; wk++)
if (wk->wk_keyix != IEEE80211_KEYIX_NONE)
(void) mwl_key_set(vap, wk, vap->iv_myaddr);
}
/*
* Convert a legacy rate set to a firmware bitmask.
*/
static uint32_t
get_rate_bitmap(const struct ieee80211_rateset *rs)
{
uint32_t rates;
int i;
rates = 0;
for (i = 0; i < rs->rs_nrates; i++)
switch (rs->rs_rates[i] & IEEE80211_RATE_VAL) {
case 2: rates |= 0x001; break;
case 4: rates |= 0x002; break;
case 11: rates |= 0x004; break;
case 22: rates |= 0x008; break;
case 44: rates |= 0x010; break;
case 12: rates |= 0x020; break;
case 18: rates |= 0x040; break;
case 24: rates |= 0x080; break;
case 36: rates |= 0x100; break;
case 48: rates |= 0x200; break;
case 72: rates |= 0x400; break;
case 96: rates |= 0x800; break;
case 108: rates |= 0x1000; break;
}
return rates;
}
/*
* Construct an HT firmware bitmask from an HT rate set.
*/
static uint32_t
get_htrate_bitmap(const struct ieee80211_htrateset *rs)
{
uint32_t rates;
int i;
rates = 0;
for (i = 0; i < rs->rs_nrates; i++) {
if (rs->rs_rates[i] < 16)
rates |= 1<<rs->rs_rates[i];
}
return rates;
}
/*
* Craft station database entry for station.
* NB: use host byte order here, the hal handles byte swapping.
*/
static MWL_HAL_PEERINFO *
mkpeerinfo(MWL_HAL_PEERINFO *pi, const struct ieee80211_node *ni)
{
const struct ieee80211vap *vap = ni->ni_vap;
memset(pi, 0, sizeof(*pi));
pi->LegacyRateBitMap = get_rate_bitmap(&ni->ni_rates);
pi->CapInfo = ni->ni_capinfo;
if (ni->ni_flags & IEEE80211_NODE_HT) {
/* HT capabilities, etc */
pi->HTCapabilitiesInfo = ni->ni_htcap;
/* XXX pi.HTCapabilitiesInfo */
pi->MacHTParamInfo = ni->ni_htparam;
pi->HTRateBitMap = get_htrate_bitmap(&ni->ni_htrates);
pi->AddHtInfo.ControlChan = ni->ni_htctlchan;
pi->AddHtInfo.AddChan = ni->ni_ht2ndchan;
pi->AddHtInfo.OpMode = ni->ni_htopmode;
pi->AddHtInfo.stbc = ni->ni_htstbc;
/* constrain according to local configuration */
if ((vap->iv_flags_ht & IEEE80211_FHT_SHORTGI40) == 0)
pi->HTCapabilitiesInfo &= ~IEEE80211_HTCAP_SHORTGI40;
if ((vap->iv_flags_ht & IEEE80211_FHT_SHORTGI20) == 0)
pi->HTCapabilitiesInfo &= ~IEEE80211_HTCAP_SHORTGI20;
if (ni->ni_chw != 40)
pi->HTCapabilitiesInfo &= ~IEEE80211_HTCAP_CHWIDTH40;
}
return pi;
}
/*
* Re-create the local sta db entry for a vap to ensure
* up to date WME state is pushed to the firmware. Because
* this resets crypto state this must be followed by a
* reload of any keys in the global key table.
*/
static int
mwl_localstadb(struct ieee80211vap *vap)
{
#define WME(ie) ((const struct ieee80211_wme_info *) ie)
struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
struct ieee80211_node *bss;
MWL_HAL_PEERINFO pi;
int error;
switch (vap->iv_opmode) {
case IEEE80211_M_STA:
bss = vap->iv_bss;
error = mwl_hal_newstation(hvap, vap->iv_myaddr, 0, 0,
vap->iv_state == IEEE80211_S_RUN ?
mkpeerinfo(&pi, bss) : NULL,
(bss->ni_flags & (IEEE80211_NODE_QOS | IEEE80211_NODE_HT)),
bss->ni_ies.wme_ie != NULL ?
WME(bss->ni_ies.wme_ie)->wme_info : 0);
if (error == 0)
mwl_setglobalkeys(vap);
break;
case IEEE80211_M_HOSTAP:
case IEEE80211_M_MBSS:
error = mwl_hal_newstation(hvap, vap->iv_myaddr,
0, 0, NULL, vap->iv_flags & IEEE80211_F_WME, 0);
if (error == 0)
mwl_setglobalkeys(vap);
break;
default:
error = 0;
break;
}
return error;
#undef WME
}
static int
mwl_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
{
struct mwl_vap *mvp = MWL_VAP(vap);
struct mwl_hal_vap *hvap = mvp->mv_hvap;
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_node *ni = NULL;
struct ifnet *ifp = ic->ic_ifp;
struct mwl_softc *sc = ifp->if_softc;
struct mwl_hal *mh = sc->sc_mh;
enum ieee80211_state ostate = vap->iv_state;
int error;
DPRINTF(sc, MWL_DEBUG_STATE, "%s: %s: %s -> %s\n",
vap->iv_ifp->if_xname, __func__,
ieee80211_state_name[ostate], ieee80211_state_name[nstate]);
callout_stop(&sc->sc_timer);
/*
* Clear current radar detection state.
*/
if (ostate == IEEE80211_S_CAC) {
/* stop quiet mode radar detection */
mwl_hal_setradardetection(mh, DR_CHK_CHANNEL_AVAILABLE_STOP);
} else if (sc->sc_radarena) {
/* stop in-service radar detection */
mwl_hal_setradardetection(mh, DR_DFS_DISABLE);
sc->sc_radarena = 0;
}
/*
* Carry out per-state actions before doing net80211 work.
*/
if (nstate == IEEE80211_S_INIT) {
/* NB: only ap+sta vap's have a fw entity */
if (hvap != NULL)
mwl_hal_stop(hvap);
} else if (nstate == IEEE80211_S_SCAN) {
mwl_hal_start(hvap);
/* NB: this disables beacon frames */
mwl_hal_setinframode(hvap);
} else if (nstate == IEEE80211_S_AUTH) {
/*
* Must create a sta db entry in case a WEP key needs to
* be plumbed. This entry will be overwritten if we
* associate; otherwise it will be reclaimed on node free.
*/
ni = vap->iv_bss;
MWL_NODE(ni)->mn_hvap = hvap;
(void) mwl_peerstadb(ni, 0, 0, NULL);
} else if (nstate == IEEE80211_S_CSA) {
/* XXX move to below? */
if (vap->iv_opmode == IEEE80211_M_HOSTAP ||
vap->iv_opmode == IEEE80211_M_MBSS)
mwl_startcsa(vap);
} else if (nstate == IEEE80211_S_CAC) {
/* XXX move to below? */
/* stop ap xmit and enable quiet mode radar detection */
mwl_hal_setradardetection(mh, DR_CHK_CHANNEL_AVAILABLE_START);
}
/*
* Invoke the parent method to do net80211 work.
*/
error = mvp->mv_newstate(vap, nstate, arg);
/*
* Carry out work that must be done after net80211 runs;
* this work requires up to date state (e.g. iv_bss).
*/
if (error == 0 && nstate == IEEE80211_S_RUN) {
/* NB: collect bss node again, it may have changed */
ni = vap->iv_bss;
DPRINTF(sc, MWL_DEBUG_STATE,
"%s: %s(RUN): iv_flags 0x%08x bintvl %d bssid %s "
"capinfo 0x%04x chan %d\n",
vap->iv_ifp->if_xname, __func__, vap->iv_flags,
ni->ni_intval, ether_sprintf(ni->ni_bssid), ni->ni_capinfo,
ieee80211_chan2ieee(ic, ic->ic_curchan));
/*
* Recreate local sta db entry to update WME/HT state.
*/
mwl_localstadb(vap);
switch (vap->iv_opmode) {
case IEEE80211_M_HOSTAP:
case IEEE80211_M_MBSS:
if (ostate == IEEE80211_S_CAC) {
/* enable in-service radar detection */
mwl_hal_setradardetection(mh,
DR_IN_SERVICE_MONITOR_START);
sc->sc_radarena = 1;
}
/*
* Allocate and setup the beacon frame
* (and related state).
*/
error = mwl_reset_vap(vap, IEEE80211_S_RUN);
if (error != 0) {
DPRINTF(sc, MWL_DEBUG_STATE,
"%s: beacon setup failed, error %d\n",
__func__, error);
goto bad;
}
/* NB: must be after setting up beacon */
mwl_hal_start(hvap);
break;
case IEEE80211_M_STA:
DPRINTF(sc, MWL_DEBUG_STATE, "%s: %s: aid 0x%x\n",
vap->iv_ifp->if_xname, __func__, ni->ni_associd);
/*
* Set state now that we're associated.
*/
mwl_hal_setassocid(hvap, ni->ni_bssid, ni->ni_associd);
mwl_setrates(vap);
mwl_hal_setrtsthreshold(hvap, vap->iv_rtsthreshold);
if ((vap->iv_flags & IEEE80211_F_DWDS) &&
sc->sc_ndwdsvaps++ == 0)
mwl_hal_setdwds(mh, 1);
break;
case IEEE80211_M_WDS:
DPRINTF(sc, MWL_DEBUG_STATE, "%s: %s: bssid %s\n",
vap->iv_ifp->if_xname, __func__,
ether_sprintf(ni->ni_bssid));
mwl_seteapolformat(vap);
break;
default:
break;
}
/*
* Set CS mode according to operating channel;
* this mostly an optimization for 5GHz.
*
* NB: must follow mwl_hal_start which resets csmode
*/
if (IEEE80211_IS_CHAN_5GHZ(ic->ic_bsschan))
mwl_hal_setcsmode(mh, CSMODE_AGGRESSIVE);
else
mwl_hal_setcsmode(mh, CSMODE_AUTO_ENA);
/*
* Start timer to prod firmware.
*/
if (sc->sc_ageinterval != 0)
callout_reset(&sc->sc_timer, sc->sc_ageinterval*hz,
mwl_agestations, sc);
} else if (nstate == IEEE80211_S_SLEEP) {
/* XXX set chip in power save */
} else if ((vap->iv_flags & IEEE80211_F_DWDS) &&
--sc->sc_ndwdsvaps == 0)
mwl_hal_setdwds(mh, 0);
bad:
return error;
}
/*
* Manage station id's; these are separate from AID's
* as AID's may have values out of the range of possible
* station id's acceptable to the firmware.
*/
static int
allocstaid(struct mwl_softc *sc, int aid)
{
int staid;
if (!(0 < aid && aid < MWL_MAXSTAID) || isset(sc->sc_staid, aid)) {
/* NB: don't use 0 */
for (staid = 1; staid < MWL_MAXSTAID; staid++)
if (isclr(sc->sc_staid, staid))
break;
} else
staid = aid;
setbit(sc->sc_staid, staid);
return staid;
}
static void
delstaid(struct mwl_softc *sc, int staid)
{
clrbit(sc->sc_staid, staid);
}
/*
* Setup driver-specific state for a newly associated node.
* Note that we're called also on a re-associate, the isnew
* param tells us if this is the first time or not.
*/
static void
mwl_newassoc(struct ieee80211_node *ni, int isnew)
{
struct ieee80211vap *vap = ni->ni_vap;
struct mwl_softc *sc = vap->iv_ic->ic_ifp->if_softc;
struct mwl_node *mn = MWL_NODE(ni);
MWL_HAL_PEERINFO pi;
uint16_t aid;
int error;
aid = IEEE80211_AID(ni->ni_associd);
if (isnew) {
mn->mn_staid = allocstaid(sc, aid);
mn->mn_hvap = MWL_VAP(vap)->mv_hvap;
} else {
mn = MWL_NODE(ni);
/* XXX reset BA stream? */
}
DPRINTF(sc, MWL_DEBUG_NODE, "%s: mac %s isnew %d aid %d staid %d\n",
__func__, ether_sprintf(ni->ni_macaddr), isnew, aid, mn->mn_staid);
error = mwl_peerstadb(ni, aid, mn->mn_staid, mkpeerinfo(&pi, ni));
if (error != 0) {
DPRINTF(sc, MWL_DEBUG_NODE,
"%s: error %d creating sta db entry\n",
__func__, error);
/* XXX how to deal with error? */
}
}
/*
* Periodically poke the firmware to age out station state
* (power save queues, pending tx aggregates).
*/
static void
mwl_agestations(void *arg)
{
struct mwl_softc *sc = arg;
mwl_hal_setkeepalive(sc->sc_mh);
if (sc->sc_ageinterval != 0) /* NB: catch dynamic changes */
callout_schedule(&sc->sc_timer, sc->sc_ageinterval*hz);
}
static const struct mwl_hal_channel *
findhalchannel(const MWL_HAL_CHANNELINFO *ci, int ieee)
{
int i;
for (i = 0; i < ci->nchannels; i++) {
const struct mwl_hal_channel *hc = &ci->channels[i];
if (hc->ieee == ieee)
return hc;
}
return NULL;
}
static int
mwl_setregdomain(struct ieee80211com *ic, struct ieee80211_regdomain *rd,
int nchan, struct ieee80211_channel chans[])
{
struct mwl_softc *sc = ic->ic_ifp->if_softc;
struct mwl_hal *mh = sc->sc_mh;
const MWL_HAL_CHANNELINFO *ci;
int i;
for (i = 0; i < nchan; i++) {
struct ieee80211_channel *c = &chans[i];
const struct mwl_hal_channel *hc;
if (IEEE80211_IS_CHAN_2GHZ(c)) {
mwl_hal_getchannelinfo(mh, MWL_FREQ_BAND_2DOT4GHZ,
IEEE80211_IS_CHAN_HT40(c) ?
MWL_CH_40_MHz_WIDTH : MWL_CH_20_MHz_WIDTH, &ci);
} else if (IEEE80211_IS_CHAN_5GHZ(c)) {
mwl_hal_getchannelinfo(mh, MWL_FREQ_BAND_5GHZ,
IEEE80211_IS_CHAN_HT40(c) ?
MWL_CH_40_MHz_WIDTH : MWL_CH_20_MHz_WIDTH, &ci);
} else {
if_printf(ic->ic_ifp,
"%s: channel %u freq %u/0x%x not 2.4/5GHz\n",
__func__, c->ic_ieee, c->ic_freq, c->ic_flags);
return EINVAL;
}
/*
* Verify channel has cal data and cap tx power.
*/
hc = findhalchannel(ci, c->ic_ieee);
if (hc != NULL) {
if (c->ic_maxpower > 2*hc->maxTxPow)
c->ic_maxpower = 2*hc->maxTxPow;
goto next;
}
if (IEEE80211_IS_CHAN_HT40(c)) {
/*
* Look for the extension channel since the
* hal table only has the primary channel.
*/
hc = findhalchannel(ci, c->ic_extieee);
if (hc != NULL) {
if (c->ic_maxpower > 2*hc->maxTxPow)
c->ic_maxpower = 2*hc->maxTxPow;
goto next;
}
}
if_printf(ic->ic_ifp,
"%s: no cal data for channel %u ext %u freq %u/0x%x\n",
__func__, c->ic_ieee, c->ic_extieee,
c->ic_freq, c->ic_flags);
return EINVAL;
next:
;
}
return 0;
}
#define IEEE80211_CHAN_HTG (IEEE80211_CHAN_HT|IEEE80211_CHAN_G)
#define IEEE80211_CHAN_HTA (IEEE80211_CHAN_HT|IEEE80211_CHAN_A)
static void
addchan(struct ieee80211_channel *c, int freq, int flags, int ieee, int txpow)
{
c->ic_freq = freq;
c->ic_flags = flags;
c->ic_ieee = ieee;
c->ic_minpower = 0;
c->ic_maxpower = 2*txpow;
c->ic_maxregpower = txpow;
}
static const struct ieee80211_channel *
findchannel(const struct ieee80211_channel chans[], int nchans,
int freq, int flags)
{
const struct ieee80211_channel *c;
int i;
for (i = 0; i < nchans; i++) {
c = &chans[i];
if (c->ic_freq == freq && c->ic_flags == flags)
return c;
}
return NULL;
}
static void
addht40channels(struct ieee80211_channel chans[], int maxchans, int *nchans,
const MWL_HAL_CHANNELINFO *ci, int flags)
{
struct ieee80211_channel *c;
const struct ieee80211_channel *extc;
const struct mwl_hal_channel *hc;
int i;
c = &chans[*nchans];
flags &= ~IEEE80211_CHAN_HT;
for (i = 0; i < ci->nchannels; i++) {
/*
* Each entry defines an HT40 channel pair; find the
* extension channel above and the insert the pair.
*/
hc = &ci->channels[i];
extc = findchannel(chans, *nchans, hc->freq+20,
flags | IEEE80211_CHAN_HT20);
if (extc != NULL) {
if (*nchans >= maxchans)
break;
addchan(c, hc->freq, flags | IEEE80211_CHAN_HT40U,
hc->ieee, hc->maxTxPow);
c->ic_extieee = extc->ic_ieee;
c++, (*nchans)++;
if (*nchans >= maxchans)
break;
addchan(c, extc->ic_freq, flags | IEEE80211_CHAN_HT40D,
extc->ic_ieee, hc->maxTxPow);
c->ic_extieee = hc->ieee;
c++, (*nchans)++;
}
}
}
static void
addchannels(struct ieee80211_channel chans[], int maxchans, int *nchans,
const MWL_HAL_CHANNELINFO *ci, int flags)
{
struct ieee80211_channel *c;
int i;
c = &chans[*nchans];
for (i = 0; i < ci->nchannels; i++) {
const struct mwl_hal_channel *hc;
hc = &ci->channels[i];
if (*nchans >= maxchans)
break;
addchan(c, hc->freq, flags, hc->ieee, hc->maxTxPow);
c++, (*nchans)++;
if (flags == IEEE80211_CHAN_G || flags == IEEE80211_CHAN_HTG) {
/* g channel have a separate b-only entry */
if (*nchans >= maxchans)
break;
c[0] = c[-1];
c[-1].ic_flags = IEEE80211_CHAN_B;
c++, (*nchans)++;
}
if (flags == IEEE80211_CHAN_HTG) {
/* HT g channel have a separate g-only entry */
if (*nchans >= maxchans)
break;
c[-1].ic_flags = IEEE80211_CHAN_G;
c[0] = c[-1];
c[0].ic_flags &= ~IEEE80211_CHAN_HT;
c[0].ic_flags |= IEEE80211_CHAN_HT20; /* HT20 */
c++, (*nchans)++;
}
if (flags == IEEE80211_CHAN_HTA) {
/* HT a channel have a separate a-only entry */
if (*nchans >= maxchans)
break;
c[-1].ic_flags = IEEE80211_CHAN_A;
c[0] = c[-1];
c[0].ic_flags &= ~IEEE80211_CHAN_HT;
c[0].ic_flags |= IEEE80211_CHAN_HT20; /* HT20 */
c++, (*nchans)++;
}
}
}
static void
getchannels(struct mwl_softc *sc, int maxchans, int *nchans,
struct ieee80211_channel chans[])
{
const MWL_HAL_CHANNELINFO *ci;
/*
* Use the channel info from the hal to craft the
* channel list. Note that we pass back an unsorted
* list; the caller is required to sort it for us
* (if desired).
*/
*nchans = 0;
if (mwl_hal_getchannelinfo(sc->sc_mh,
MWL_FREQ_BAND_2DOT4GHZ, MWL_CH_20_MHz_WIDTH, &ci) == 0)
addchannels(chans, maxchans, nchans, ci, IEEE80211_CHAN_HTG);
if (mwl_hal_getchannelinfo(sc->sc_mh,
MWL_FREQ_BAND_5GHZ, MWL_CH_20_MHz_WIDTH, &ci) == 0)
addchannels(chans, maxchans, nchans, ci, IEEE80211_CHAN_HTA);
if (mwl_hal_getchannelinfo(sc->sc_mh,
MWL_FREQ_BAND_2DOT4GHZ, MWL_CH_40_MHz_WIDTH, &ci) == 0)
addht40channels(chans, maxchans, nchans, ci, IEEE80211_CHAN_HTG);
if (mwl_hal_getchannelinfo(sc->sc_mh,
MWL_FREQ_BAND_5GHZ, MWL_CH_40_MHz_WIDTH, &ci) == 0)
addht40channels(chans, maxchans, nchans, ci, IEEE80211_CHAN_HTA);
}
static void
mwl_getradiocaps(struct ieee80211com *ic,
int maxchans, int *nchans, struct ieee80211_channel chans[])
{
struct mwl_softc *sc = ic->ic_ifp->if_softc;
getchannels(sc, maxchans, nchans, chans);
}
static int
mwl_getchannels(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
struct ieee80211com *ic = ifp->if_l2com;
/*
* Use the channel info from the hal to craft the
* channel list for net80211. Note that we pass up
* an unsorted list; net80211 will sort it for us.
*/
memset(ic->ic_channels, 0, sizeof(ic->ic_channels));
ic->ic_nchans = 0;
getchannels(sc, IEEE80211_CHAN_MAX, &ic->ic_nchans, ic->ic_channels);
ic->ic_regdomain.regdomain = SKU_DEBUG;
ic->ic_regdomain.country = CTRY_DEFAULT;
ic->ic_regdomain.location = 'I';
ic->ic_regdomain.isocc[0] = ' '; /* XXX? */
ic->ic_regdomain.isocc[1] = ' ';
return (ic->ic_nchans == 0 ? EIO : 0);
}
#undef IEEE80211_CHAN_HTA
#undef IEEE80211_CHAN_HTG
#ifdef MWL_DEBUG
static void
mwl_printrxbuf(const struct mwl_rxbuf *bf, u_int ix)
{
const struct mwl_rxdesc *ds = bf->bf_desc;
uint32_t status = le32toh(ds->Status);
printf("R[%2u] (DS.V:%p DS.P:0x%jx) NEXT:%08x DATA:%08x RC:%02x%s\n"
" STAT:%02x LEN:%04x RSSI:%02x CHAN:%02x RATE:%02x QOS:%04x HT:%04x\n",
ix, ds, (uintmax_t)bf->bf_daddr, le32toh(ds->pPhysNext),
le32toh(ds->pPhysBuffData), ds->RxControl,
ds->RxControl != EAGLE_RXD_CTRL_DRIVER_OWN ?
"" : (status & EAGLE_RXD_STATUS_OK) ? " *" : " !",
ds->Status, le16toh(ds->PktLen), ds->RSSI, ds->Channel,
ds->Rate, le16toh(ds->QosCtrl), le16toh(ds->HtSig2));
}
static void
mwl_printtxbuf(const struct mwl_txbuf *bf, u_int qnum, u_int ix)
{
const struct mwl_txdesc *ds = bf->bf_desc;
uint32_t status = le32toh(ds->Status);
printf("Q%u[%3u]", qnum, ix);
printf(" (DS.V:%p DS.P:0x%jx)\n", ds, (uintmax_t)bf->bf_daddr);
printf(" NEXT:%08x DATA:%08x LEN:%04x STAT:%08x%s\n",
le32toh(ds->pPhysNext),
le32toh(ds->PktPtr), le16toh(ds->PktLen), status,
status & EAGLE_TXD_STATUS_USED ?
"" : (status & 3) != 0 ? " *" : " !");
printf(" RATE:%02x PRI:%x QOS:%04x SAP:%08x FORMAT:%04x\n",
ds->DataRate, ds->TxPriority, le16toh(ds->QosCtrl),
le32toh(ds->SapPktInfo), le16toh(ds->Format));
#if MWL_TXDESC > 1
printf(" MULTIFRAMES:%u LEN:%04x %04x %04x %04x %04x %04x\n"
, le32toh(ds->multiframes)
, le16toh(ds->PktLenArray[0]), le16toh(ds->PktLenArray[1])
, le16toh(ds->PktLenArray[2]), le16toh(ds->PktLenArray[3])
, le16toh(ds->PktLenArray[4]), le16toh(ds->PktLenArray[5])
);
printf(" DATA:%08x %08x %08x %08x %08x %08x\n"
, le32toh(ds->PktPtrArray[0]), le32toh(ds->PktPtrArray[1])
, le32toh(ds->PktPtrArray[2]), le32toh(ds->PktPtrArray[3])
, le32toh(ds->PktPtrArray[4]), le32toh(ds->PktPtrArray[5])
);
#endif
#if 0
{ const uint8_t *cp = (const uint8_t *) ds;
int i;
for (i = 0; i < sizeof(struct mwl_txdesc); i++) {
printf("%02x ", cp[i]);
if (((i+1) % 16) == 0)
printf("\n");
}
printf("\n");
}
#endif
}
#endif /* MWL_DEBUG */
#if 0
static void
mwl_txq_dump(struct mwl_txq *txq)
{
struct mwl_txbuf *bf;
int i = 0;
MWL_TXQ_LOCK(txq);
STAILQ_FOREACH(bf, &txq->active, bf_list) {
struct mwl_txdesc *ds = bf->bf_desc;
MWL_TXDESC_SYNC(txq, ds,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
#ifdef MWL_DEBUG
mwl_printtxbuf(bf, txq->qnum, i);
#endif
i++;
}
MWL_TXQ_UNLOCK(txq);
}
#endif
static void
mwl_watchdog(void *arg)
{
struct mwl_softc *sc;
struct ifnet *ifp;
sc = arg;
callout_reset(&sc->sc_watchdog, hz, mwl_watchdog, sc);
if (sc->sc_tx_timer == 0 || --sc->sc_tx_timer > 0)
return;
ifp = sc->sc_ifp;
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && !sc->sc_invalid) {
if (mwl_hal_setkeepalive(sc->sc_mh))
if_printf(ifp, "transmit timeout (firmware hung?)\n");
else
if_printf(ifp, "transmit timeout\n");
#if 0
mwl_reset(ifp);
mwl_txq_dump(&sc->sc_txq[0]);/*XXX*/
#endif
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
sc->sc_stats.mst_watchdog++;
}
}
#ifdef MWL_DIAGAPI
/*
* Diagnostic interface to the HAL. This is used by various
* tools to do things like retrieve register contents for
* debugging. The mechanism is intentionally opaque so that
* it can change frequently w/o concern for compatiblity.
*/
static int
mwl_ioctl_diag(struct mwl_softc *sc, struct mwl_diag *md)
{
struct mwl_hal *mh = sc->sc_mh;
u_int id = md->md_id & MWL_DIAG_ID;
void *indata = NULL;
void *outdata = NULL;
u_int32_t insize = md->md_in_size;
u_int32_t outsize = md->md_out_size;
int error = 0;
if (md->md_id & MWL_DIAG_IN) {
/*
* Copy in data.
*/
indata = malloc(insize, M_TEMP, M_NOWAIT);
if (indata == NULL) {
error = ENOMEM;
goto bad;
}
error = copyin(md->md_in_data, indata, insize);
if (error)
goto bad;
}
if (md->md_id & MWL_DIAG_DYN) {
/*
* Allocate a buffer for the results (otherwise the HAL
* returns a pointer to a buffer where we can read the
* results). Note that we depend on the HAL leaving this
* pointer for us to use below in reclaiming the buffer;
* may want to be more defensive.
*/
outdata = malloc(outsize, M_TEMP, M_NOWAIT);
if (outdata == NULL) {
error = ENOMEM;
goto bad;
}
}
if (mwl_hal_getdiagstate(mh, id, indata, insize, &outdata, &outsize)) {
if (outsize < md->md_out_size)
md->md_out_size = outsize;
if (outdata != NULL)
error = copyout(outdata, md->md_out_data,
md->md_out_size);
} else {
error = EINVAL;
}
bad:
if ((md->md_id & MWL_DIAG_IN) && indata != NULL)
free(indata, M_TEMP);
if ((md->md_id & MWL_DIAG_DYN) && outdata != NULL)
free(outdata, M_TEMP);
return error;
}
static int
mwl_ioctl_reset(struct mwl_softc *sc, struct mwl_diag *md)
{
struct mwl_hal *mh = sc->sc_mh;
int error;
MWL_LOCK_ASSERT(sc);
if (md->md_id == 0 && mwl_hal_fwload(mh, NULL) != 0) {
device_printf(sc->sc_dev, "unable to load firmware\n");
return EIO;
}
if (mwl_hal_gethwspecs(mh, &sc->sc_hwspecs) != 0) {
device_printf(sc->sc_dev, "unable to fetch h/w specs\n");
return EIO;
}
error = mwl_setupdma(sc);
if (error != 0) {
/* NB: mwl_setupdma prints a msg */
return error;
}
/*
* Reset tx/rx data structures; after reload we must
* re-start the driver's notion of the next xmit/recv.
*/
mwl_draintxq(sc); /* clear pending frames */
mwl_resettxq(sc); /* rebuild tx q lists */
sc->sc_rxnext = NULL; /* force rx to start at the list head */
return 0;
}
#endif /* MWL_DIAGAPI */
static int
mwl_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
#define IS_RUNNING(ifp) \
((ifp->if_flags & IFF_UP) && (ifp->if_drv_flags & IFF_DRV_RUNNING))
struct mwl_softc *sc = ifp->if_softc;
struct ieee80211com *ic = ifp->if_l2com;
struct ifreq *ifr = (struct ifreq *)data;
int error = 0, startall;
switch (cmd) {
case SIOCSIFFLAGS:
MWL_LOCK(sc);
startall = 0;
if (IS_RUNNING(ifp)) {
/*
* To avoid rescanning another access point,
* do not call mwl_init() here. Instead,
* only reflect promisc mode settings.
*/
mwl_mode_init(sc);
} else if (ifp->if_flags & IFF_UP) {
/*
* Beware of being called during attach/detach
* to reset promiscuous mode. In that case we
* will still be marked UP but not RUNNING.
* However trying to re-init the interface
* is the wrong thing to do as we've already
* torn down much of our state. There's
* probably a better way to deal with this.
*/
if (!sc->sc_invalid) {
mwl_init_locked(sc); /* XXX lose error */
startall = 1;
}
} else
mwl_stop_locked(ifp, 1);
MWL_UNLOCK(sc);
if (startall)
ieee80211_start_all(ic);
break;
case SIOCGMVSTATS:
mwl_hal_gethwstats(sc->sc_mh, &sc->sc_stats.hw_stats);
/* NB: embed these numbers to get a consistent view */
sc->sc_stats.mst_tx_packets =
ifp->if_get_counter(ifp, IFCOUNTER_OPACKETS);
sc->sc_stats.mst_rx_packets =
ifp->if_get_counter(ifp, IFCOUNTER_IPACKETS);
/*
* NB: Drop the softc lock in case of a page fault;
* we'll accept any potential inconsisentcy in the
* statistics. The alternative is to copy the data
* to a local structure.
*/
return copyout(&sc->sc_stats,
ifr->ifr_data, sizeof (sc->sc_stats));
#ifdef MWL_DIAGAPI
case SIOCGMVDIAG:
/* XXX check privs */
return mwl_ioctl_diag(sc, (struct mwl_diag *) ifr);
case SIOCGMVRESET:
/* XXX check privs */
MWL_LOCK(sc);
error = mwl_ioctl_reset(sc,(struct mwl_diag *) ifr);
MWL_UNLOCK(sc);
break;
#endif /* MWL_DIAGAPI */
case SIOCGIFMEDIA:
error = ifmedia_ioctl(ifp, ifr, &ic->ic_media, cmd);
break;
case SIOCGIFADDR:
error = ether_ioctl(ifp, cmd, data);
break;
default:
error = EINVAL;
break;
}
return error;
#undef IS_RUNNING
}
#ifdef MWL_DEBUG
static int
mwl_sysctl_debug(SYSCTL_HANDLER_ARGS)
{
struct mwl_softc *sc = arg1;
int debug, error;
debug = sc->sc_debug | (mwl_hal_getdebug(sc->sc_mh) << 24);
error = sysctl_handle_int(oidp, &debug, 0, req);
if (error || !req->newptr)
return error;
mwl_hal_setdebug(sc->sc_mh, debug >> 24);
sc->sc_debug = debug & 0x00ffffff;
return 0;
}
#endif /* MWL_DEBUG */
static void
mwl_sysctlattach(struct mwl_softc *sc)
{
#ifdef MWL_DEBUG
struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->sc_dev);
struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev);
sc->sc_debug = mwl_debug;
SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
"debug", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
mwl_sysctl_debug, "I", "control debugging printfs");
#endif
}
/*
* Announce various information on device/driver attach.
*/
static void
mwl_announce(struct mwl_softc *sc)
{
struct ifnet *ifp = sc->sc_ifp;
if_printf(ifp, "Rev A%d hardware, v%d.%d.%d.%d firmware (regioncode %d)\n",
sc->sc_hwspecs.hwVersion,
(sc->sc_hwspecs.fwReleaseNumber>>24) & 0xff,
(sc->sc_hwspecs.fwReleaseNumber>>16) & 0xff,
(sc->sc_hwspecs.fwReleaseNumber>>8) & 0xff,
(sc->sc_hwspecs.fwReleaseNumber>>0) & 0xff,
sc->sc_hwspecs.regionCode);
sc->sc_fwrelease = sc->sc_hwspecs.fwReleaseNumber;
if (bootverbose) {
int i;
for (i = 0; i <= WME_AC_VO; i++) {
struct mwl_txq *txq = sc->sc_ac2q[i];
if_printf(ifp, "Use hw queue %u for %s traffic\n",
txq->qnum, ieee80211_wme_acnames[i]);
}
}
if (bootverbose || mwl_rxdesc != MWL_RXDESC)
if_printf(ifp, "using %u rx descriptors\n", mwl_rxdesc);
if (bootverbose || mwl_rxbuf != MWL_RXBUF)
if_printf(ifp, "using %u rx buffers\n", mwl_rxbuf);
if (bootverbose || mwl_txbuf != MWL_TXBUF)
if_printf(ifp, "using %u tx buffers\n", mwl_txbuf);
if (bootverbose && mwl_hal_ismbsscapable(sc->sc_mh))
if_printf(ifp, "multi-bss support\n");
#ifdef MWL_TX_NODROP
if (bootverbose)
if_printf(ifp, "no tx drop\n");
#endif
}
Index: head/sys/dev/nand/nandsim_chip.c
===================================================================
--- head/sys/dev/nand/nandsim_chip.c (revision 283290)
+++ head/sys/dev/nand/nandsim_chip.c (revision 283291)
@@ -1,901 +1,901 @@
/*-
* Copyright (C) 2009-2012 Semihalf
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/kthread.h>
#include <sys/unistd.h>
#include <dev/nand/nand.h>
#include <dev/nand/nandsim_chip.h>
#include <dev/nand/nandsim_log.h>
#include <dev/nand/nandsim_swap.h>
MALLOC_DEFINE(M_NANDSIM, "NANDsim", "NANDsim dynamic data");
#define NANDSIM_CHIP_LOCK(chip) mtx_lock(&(chip)->ns_lock)
#define NANDSIM_CHIP_UNLOCK(chip) mtx_unlock(&(chip)->ns_lock)
static nandsim_evh_t erase_evh;
static nandsim_evh_t idle_evh;
static nandsim_evh_t poweron_evh;
static nandsim_evh_t reset_evh;
static nandsim_evh_t read_evh;
static nandsim_evh_t readid_evh;
static nandsim_evh_t readparam_evh;
static nandsim_evh_t write_evh;
static void nandsim_loop(void *);
static void nandsim_undefined(struct nandsim_chip *, uint8_t);
static void nandsim_bad_address(struct nandsim_chip *, uint8_t *);
static void nandsim_ignore_address(struct nandsim_chip *, uint8_t);
static void nandsim_sm_error(struct nandsim_chip *);
static void nandsim_start_handler(struct nandsim_chip *, nandsim_evh_t);
static void nandsim_callout_eh(void *);
static int nandsim_delay(struct nandsim_chip *, int);
static int nandsim_bbm_init(struct nandsim_chip *, uint32_t, uint32_t *);
static int nandsim_blk_state_init(struct nandsim_chip *, uint32_t, uint32_t);
static void nandsim_blk_state_destroy(struct nandsim_chip *);
static int nandchip_is_block_valid(struct nandsim_chip *, int);
static void nandchip_set_status(struct nandsim_chip *, uint8_t);
static void nandchip_clear_status(struct nandsim_chip *, uint8_t);
struct proc *nandsim_proc;
struct nandsim_chip *
nandsim_chip_init(struct nandsim_softc* sc, uint8_t chip_num,
struct sim_chip *sim_chip)
{
struct nandsim_chip *chip;
struct onfi_params *chip_param;
char swapfile[20];
uint32_t size;
int error;
chip = malloc(sizeof(*chip), M_NANDSIM, M_WAITOK | M_ZERO);
if (!chip)
return (NULL);
mtx_init(&chip->ns_lock, "nandsim lock", NULL, MTX_DEF);
- callout_init(&chip->ns_callout, CALLOUT_MPSAFE);
+ callout_init(&chip->ns_callout, 1);
STAILQ_INIT(&chip->nandsim_events);
chip->chip_num = chip_num;
chip->ctrl_num = sim_chip->ctrl_num;
chip->sc = sc;
if (!sim_chip->is_wp)
nandchip_set_status(chip, NAND_STATUS_WP);
chip_param = &chip->params;
chip->id.dev_id = sim_chip->device_id;
chip->id.man_id = sim_chip->manufact_id;
chip->error_ratio = sim_chip->error_ratio;
chip->wear_level = sim_chip->wear_level;
chip->prog_delay = sim_chip->prog_time;
chip->erase_delay = sim_chip->erase_time;
chip->read_delay = sim_chip->read_time;
chip_param->t_prog = sim_chip->prog_time;
chip_param->t_bers = sim_chip->erase_time;
chip_param->t_r = sim_chip->read_time;
bcopy("onfi", &chip_param->signature, 4);
chip_param->manufacturer_id = sim_chip->manufact_id;
strncpy(chip_param->manufacturer_name, sim_chip->manufacturer, 12);
chip_param->manufacturer_name[11] = 0;
strncpy(chip_param->device_model, sim_chip->device_model, 20);
chip_param->device_model[19] = 0;
chip_param->bytes_per_page = sim_chip->page_size;
chip_param->spare_bytes_per_page = sim_chip->oob_size;
chip_param->pages_per_block = sim_chip->pgs_per_blk;
chip_param->blocks_per_lun = sim_chip->blks_per_lun;
chip_param->luns = sim_chip->luns;
init_chip_geom(&chip->cg, chip_param->luns, chip_param->blocks_per_lun,
chip_param->pages_per_block, chip_param->bytes_per_page,
chip_param->spare_bytes_per_page);
chip_param->address_cycles = sim_chip->row_addr_cycles |
(sim_chip->col_addr_cycles << 4);
chip_param->features = sim_chip->features;
if (sim_chip->width == 16)
chip_param->features |= ONFI_FEAT_16BIT;
size = chip_param->blocks_per_lun * chip_param->luns;
error = nandsim_blk_state_init(chip, size, sim_chip->wear_level);
if (error) {
mtx_destroy(&chip->ns_lock);
free(chip, M_NANDSIM);
return (NULL);
}
error = nandsim_bbm_init(chip, size, sim_chip->bad_block_map);
if (error) {
mtx_destroy(&chip->ns_lock);
nandsim_blk_state_destroy(chip);
free(chip, M_NANDSIM);
return (NULL);
}
nandsim_start_handler(chip, poweron_evh);
nand_debug(NDBG_SIM,"Create thread for chip%d [%8p]", chip->chip_num,
chip);
/* Create chip thread */
error = kproc_kthread_add(nandsim_loop, chip, &nandsim_proc,
&chip->nandsim_td, RFSTOPPED | RFHIGHPID,
0, "nandsim", "chip");
if (error) {
mtx_destroy(&chip->ns_lock);
nandsim_blk_state_destroy(chip);
free(chip, M_NANDSIM);
return (NULL);
}
thread_lock(chip->nandsim_td);
sched_class(chip->nandsim_td, PRI_REALTIME);
sched_add(chip->nandsim_td, SRQ_BORING);
thread_unlock(chip->nandsim_td);
size = (chip_param->bytes_per_page +
chip_param->spare_bytes_per_page) *
chip_param->pages_per_block;
sprintf(swapfile, "chip%d%d.swp", chip->ctrl_num, chip->chip_num);
chip->swap = nandsim_swap_init(swapfile, chip_param->blocks_per_lun *
chip_param->luns, size);
if (!chip->swap)
nandsim_chip_destroy(chip);
/* Wait for new thread to enter main loop */
tsleep(chip->nandsim_td, PWAIT, "ns_chip", 1 * hz);
return (chip);
}
static int
nandsim_blk_state_init(struct nandsim_chip *chip, uint32_t size,
uint32_t wear_lev)
{
int i;
if (!chip || size == 0)
return (-1);
chip->blk_state = malloc(size * sizeof(struct nandsim_block_state),
M_NANDSIM, M_WAITOK | M_ZERO);
if (!chip->blk_state) {
return (-1);
}
for (i = 0; i < size; i++) {
if (wear_lev)
chip->blk_state[i].wear_lev = wear_lev;
else
chip->blk_state[i].wear_lev = -1;
}
return (0);
}
static void
nandsim_blk_state_destroy(struct nandsim_chip *chip)
{
if (chip && chip->blk_state)
free(chip->blk_state, M_NANDSIM);
}
static int
nandsim_bbm_init(struct nandsim_chip *chip, uint32_t size,
uint32_t *sim_bbm)
{
uint32_t index;
int i;
if ((chip == NULL) || (size == 0))
return (-1);
if (chip->blk_state == NULL)
return (-1);
if (sim_bbm == NULL)
return (0);
for (i = 0; i < MAX_BAD_BLOCKS; i++) {
index = sim_bbm[i];
if (index == 0xffffffff)
break;
else if (index > size)
return (-1);
else
chip->blk_state[index].is_bad = 1;
}
return (0);
}
void
nandsim_chip_destroy(struct nandsim_chip *chip)
{
struct nandsim_ev *ev;
ev = create_event(chip, NANDSIM_EV_EXIT, 0);
if (ev)
send_event(ev);
}
void
nandsim_chip_freeze(struct nandsim_chip *chip)
{
chip->flags |= NANDSIM_CHIP_FROZEN;
}
static void
nandsim_loop(void *arg)
{
struct nandsim_chip *chip = (struct nandsim_chip *)arg;
struct nandsim_ev *ev;
nand_debug(NDBG_SIM,"Start main loop for chip%d [%8p]", chip->chip_num,
chip);
for(;;) {
NANDSIM_CHIP_LOCK(chip);
if (!(chip->flags & NANDSIM_CHIP_ACTIVE)) {
chip->flags |= NANDSIM_CHIP_ACTIVE;
wakeup(chip->nandsim_td);
}
if (STAILQ_EMPTY(&chip->nandsim_events)) {
nand_debug(NDBG_SIM,"Chip%d [%8p] going sleep",
chip->chip_num, chip);
msleep(chip, &chip->ns_lock, PRIBIO, "nandev", 0);
}
ev = STAILQ_FIRST(&chip->nandsim_events);
STAILQ_REMOVE_HEAD(&chip->nandsim_events, links);
NANDSIM_CHIP_UNLOCK(chip);
if (ev->type == NANDSIM_EV_EXIT) {
NANDSIM_CHIP_LOCK(chip);
destroy_event(ev);
wakeup(ev);
while (!STAILQ_EMPTY(&chip->nandsim_events)) {
ev = STAILQ_FIRST(&chip->nandsim_events);
STAILQ_REMOVE_HEAD(&chip->nandsim_events,
links);
destroy_event(ev);
wakeup(ev);
};
NANDSIM_CHIP_UNLOCK(chip);
nandsim_log(chip, NANDSIM_LOG_SM, "destroyed\n");
mtx_destroy(&chip->ns_lock);
nandsim_blk_state_destroy(chip);
nandsim_swap_destroy(chip->swap);
free(chip, M_NANDSIM);
nandsim_proc = NULL;
kthread_exit();
}
if (!(chip->flags & NANDSIM_CHIP_FROZEN)) {
nand_debug(NDBG_SIM,"Chip [%x] get event [%x]",
chip->chip_num, ev->type);
chip->ev_handler(chip, ev->type, ev->data);
}
wakeup(ev);
destroy_event(ev);
}
}
struct nandsim_ev *
create_event(struct nandsim_chip *chip, uint8_t type, uint8_t data_size)
{
struct nandsim_ev *ev;
ev = malloc(sizeof(*ev), M_NANDSIM, M_NOWAIT | M_ZERO);
if (!ev) {
nand_debug(NDBG_SIM,"Cannot create event");
return (NULL);
}
if (data_size > 0)
ev->data = malloc(sizeof(*ev), M_NANDSIM, M_NOWAIT | M_ZERO);
ev->type = type;
ev->chip = chip;
return (ev);
}
void
destroy_event(struct nandsim_ev *ev)
{
if (ev->data)
free(ev->data, M_NANDSIM);
free(ev, M_NANDSIM);
}
int
send_event(struct nandsim_ev *ev)
{
struct nandsim_chip *chip = ev->chip;
if (!(chip->flags & NANDSIM_CHIP_FROZEN)) {
nand_debug(NDBG_SIM,"Chip%d [%p] send event %x",
chip->chip_num, chip, ev->type);
NANDSIM_CHIP_LOCK(chip);
STAILQ_INSERT_TAIL(&chip->nandsim_events, ev, links);
NANDSIM_CHIP_UNLOCK(chip);
wakeup(chip);
if ((ev->type != NANDSIM_EV_TIMEOUT) && chip->nandsim_td &&
(curthread != chip->nandsim_td))
tsleep(ev, PWAIT, "ns_ev", 5 * hz);
}
return (0);
}
static void
nandsim_callout_eh(void *arg)
{
struct nandsim_ev *ev = (struct nandsim_ev *)arg;
send_event(ev);
}
static int
nandsim_delay(struct nandsim_chip *chip, int timeout)
{
struct nandsim_ev *ev;
struct timeval delay;
int tm;
nand_debug(NDBG_SIM,"Chip[%d] Set delay: %d", chip->chip_num, timeout);
ev = create_event(chip, NANDSIM_EV_TIMEOUT, 0);
if (!ev)
return (-1);
chip->sm_state = NANDSIM_STATE_TIMEOUT;
tm = (timeout/10000) * (hz / 100);
if (callout_reset(&chip->ns_callout, tm, nandsim_callout_eh, ev))
return (-1);
delay.tv_sec = chip->read_delay / 1000000;
delay.tv_usec = chip->read_delay % 1000000;
timevaladd(&chip->delay_tv, &delay);
return (0);
}
static void
nandsim_start_handler(struct nandsim_chip *chip, nandsim_evh_t evh)
{
struct nandsim_ev *ev;
chip->ev_handler = evh;
nand_debug(NDBG_SIM,"Start handler %p for chip%d [%p]", evh,
chip->chip_num, chip);
ev = create_event(chip, NANDSIM_EV_START, 0);
if (!ev)
nandsim_sm_error(chip);
send_event(ev);
}
static void
nandchip_set_data(struct nandsim_chip *chip, uint8_t *data, uint32_t len,
uint32_t idx)
{
nand_debug(NDBG_SIM,"Chip [%x] data %p [%x] at %x", chip->chip_num,
data, len, idx);
chip->data.data_ptr = data;
chip->data.size = len;
chip->data.index = idx;
}
static int
nandchip_chip_space(struct nandsim_chip *chip, int32_t row, int32_t column,
size_t size, uint8_t writing)
{
struct block_space *blk_space;
uint32_t lun, block, page, offset, block_size;
int err;
block_size = chip->cg.block_size +
(chip->cg.oob_size * chip->cg.pgs_per_blk);
err = nand_row_to_blkpg(&chip->cg, row, &lun, &block, &page);
if (err) {
nand_debug(NDBG_SIM,"cannot get address\n");
return (-1);
}
if (!nandchip_is_block_valid(chip, block)) {
nandchip_set_data(chip, NULL, 0, 0);
return (-1);
}
blk_space = get_bs(chip->swap, block, writing);
if (!blk_space) {
nandchip_set_data(chip, NULL, 0, 0);
return (-1);
}
if (size > block_size)
size = block_size;
if (size == block_size) {
offset = 0;
column = 0;
} else
offset = page * (chip->cg.page_size + chip->cg.oob_size);
nandchip_set_data(chip, &blk_space->blk_ptr[offset], size, column);
return (0);
}
static int
nandchip_get_addr_byte(struct nandsim_chip *chip, void *data, uint32_t *value)
{
int ncycles = 0;
uint8_t byte;
uint8_t *buffer;
buffer = (uint8_t *)value;
byte = *((uint8_t *)data);
KASSERT((chip->sm_state == NANDSIM_STATE_WAIT_ADDR_ROW ||
chip->sm_state == NANDSIM_STATE_WAIT_ADDR_COL),
("unexpected state"));
if (chip->sm_state == NANDSIM_STATE_WAIT_ADDR_ROW) {
ncycles = chip->params.address_cycles & 0xf;
buffer[chip->sm_addr_cycle++] = byte;
} else if (chip->sm_state == NANDSIM_STATE_WAIT_ADDR_COL) {
ncycles = (chip->params.address_cycles >> 4) & 0xf;
buffer[chip->sm_addr_cycle++] = byte;
}
nand_debug(NDBG_SIM, "Chip [%x] read addr byte: %02x (%d of %d)\n",
chip->chip_num, byte, chip->sm_addr_cycle, ncycles);
if (chip->sm_addr_cycle == ncycles) {
chip->sm_addr_cycle = 0;
return (0);
}
return (1);
}
static int
nandchip_is_block_valid(struct nandsim_chip *chip, int block_num)
{
if (!chip || !chip->blk_state)
return (0);
if (chip->blk_state[block_num].wear_lev == 0 ||
chip->blk_state[block_num].is_bad)
return (0);
return (1);
}
static void
nandchip_set_status(struct nandsim_chip *chip, uint8_t flags)
{
chip->chip_status |= flags;
}
static void
nandchip_clear_status(struct nandsim_chip *chip, uint8_t flags)
{
chip->chip_status &= ~flags;
}
uint8_t
nandchip_get_status(struct nandsim_chip *chip)
{
return (chip->chip_status);
}
void
nandsim_chip_timeout(struct nandsim_chip *chip)
{
struct timeval tv;
getmicrotime(&tv);
if (chip->sm_state == NANDSIM_STATE_TIMEOUT &&
timevalcmp(&tv, &chip->delay_tv, >=)) {
nandchip_set_status(chip, NAND_STATUS_RDY);
}
}
void
poweron_evh(struct nandsim_chip *chip, uint32_t type, void *data)
{
uint8_t cmd;
if (type == NANDSIM_EV_START)
chip->sm_state = NANDSIM_STATE_IDLE;
else if (type == NANDSIM_EV_CMD) {
cmd = *(uint8_t *)data;
switch(cmd) {
case NAND_CMD_RESET:
nandsim_log(chip, NANDSIM_LOG_SM, "in RESET state\n");
nandsim_start_handler(chip, reset_evh);
break;
default:
nandsim_undefined(chip, type);
break;
}
} else
nandsim_undefined(chip, type);
}
void
idle_evh(struct nandsim_chip *chip, uint32_t type, void *data)
{
uint8_t cmd;
if (type == NANDSIM_EV_START) {
nandsim_log(chip, NANDSIM_LOG_SM, "in IDLE state\n");
chip->sm_state = NANDSIM_STATE_WAIT_CMD;
} else if (type == NANDSIM_EV_CMD) {
nandchip_clear_status(chip, NAND_STATUS_FAIL);
getmicrotime(&chip->delay_tv);
cmd = *(uint8_t *)data;
switch(cmd) {
case NAND_CMD_READ_ID:
nandsim_start_handler(chip, readid_evh);
break;
case NAND_CMD_READ_PARAMETER:
nandsim_start_handler(chip, readparam_evh);
break;
case NAND_CMD_READ:
nandsim_start_handler(chip, read_evh);
break;
case NAND_CMD_PROG:
nandsim_start_handler(chip, write_evh);
break;
case NAND_CMD_ERASE:
nandsim_start_handler(chip, erase_evh);
break;
default:
nandsim_undefined(chip, type);
break;
}
} else
nandsim_undefined(chip, type);
}
void
readid_evh(struct nandsim_chip *chip, uint32_t type, void *data)
{
struct onfi_params *params;
uint8_t addr;
params = &chip->params;
if (type == NANDSIM_EV_START) {
nandsim_log(chip, NANDSIM_LOG_SM, "in READID state\n");
chip->sm_state = NANDSIM_STATE_WAIT_ADDR_BYTE;
} else if (type == NANDSIM_EV_ADDR) {
addr = *((uint8_t *)data);
if (addr == 0x0)
nandchip_set_data(chip, (uint8_t *)&chip->id, 2, 0);
else if (addr == ONFI_SIG_ADDR)
nandchip_set_data(chip, (uint8_t *)&params->signature,
4, 0);
else
nandsim_bad_address(chip, &addr);
nandsim_start_handler(chip, idle_evh);
} else
nandsim_undefined(chip, type);
}
void
readparam_evh(struct nandsim_chip *chip, uint32_t type, void *data)
{
struct onfi_params *params;
uint8_t addr;
params = &chip->params;
if (type == NANDSIM_EV_START) {
nandsim_log(chip, NANDSIM_LOG_SM, "in READPARAM state\n");
chip->sm_state = NANDSIM_STATE_WAIT_ADDR_BYTE;
} else if (type == NANDSIM_EV_ADDR) {
addr = *((uint8_t *)data);
if (addr == 0) {
nandchip_set_data(chip, (uint8_t *)params,
sizeof(*params), 0);
} else
nandsim_bad_address(chip, &addr);
nandsim_start_handler(chip, idle_evh);
} else
nandsim_undefined(chip, type);
}
void
read_evh(struct nandsim_chip *chip, uint32_t type, void *data)
{
static uint32_t column = 0, row = 0;
uint32_t size;
uint8_t cmd;
size = chip->cg.page_size + chip->cg.oob_size;
switch (type) {
case NANDSIM_EV_START:
nandsim_log(chip, NANDSIM_LOG_SM, "in READ state\n");
chip->sm_state = NANDSIM_STATE_WAIT_ADDR_COL;
break;
case NANDSIM_EV_ADDR:
if (chip->sm_state == NANDSIM_STATE_WAIT_ADDR_COL) {
if (nandchip_get_addr_byte(chip, data, &column))
break;
chip->sm_state = NANDSIM_STATE_WAIT_ADDR_ROW;
} else if (chip->sm_state == NANDSIM_STATE_WAIT_ADDR_ROW) {
if (nandchip_get_addr_byte(chip, data, &row))
break;
chip->sm_state = NANDSIM_STATE_WAIT_CMD;
} else
nandsim_ignore_address(chip, *((uint8_t *)data));
break;
case NANDSIM_EV_CMD:
cmd = *(uint8_t *)data;
if (chip->sm_state == NANDSIM_STATE_WAIT_CMD &&
cmd == NAND_CMD_READ_END) {
if (chip->read_delay != 0 &&
nandsim_delay(chip, chip->read_delay) == 0)
nandchip_clear_status(chip, NAND_STATUS_RDY);
else {
nandchip_chip_space(chip, row, column, size, 0);
nandchip_set_status(chip, NAND_STATUS_RDY);
nandsim_start_handler(chip, idle_evh);
}
} else
nandsim_undefined(chip, type);
break;
case NANDSIM_EV_TIMEOUT:
if (chip->sm_state == NANDSIM_STATE_TIMEOUT) {
nandchip_chip_space(chip, row, column, size, 0);
nandchip_set_status(chip, NAND_STATUS_RDY);
nandsim_start_handler(chip, idle_evh);
} else
nandsim_undefined(chip, type);
break;
}
}
void
write_evh(struct nandsim_chip *chip, uint32_t type, void *data)
{
static uint32_t column, row;
uint32_t size;
uint8_t cmd;
int err;
size = chip->cg.page_size + chip->cg.oob_size;
switch(type) {
case NANDSIM_EV_START:
nandsim_log(chip, NANDSIM_LOG_SM, "in WRITE state\n");
chip->sm_state = NANDSIM_STATE_WAIT_ADDR_COL;
break;
case NANDSIM_EV_ADDR:
if (chip->sm_state == NANDSIM_STATE_WAIT_ADDR_COL) {
if (nandchip_get_addr_byte(chip, data, &column))
break;
chip->sm_state = NANDSIM_STATE_WAIT_ADDR_ROW;
} else if (chip->sm_state == NANDSIM_STATE_WAIT_ADDR_ROW) {
if (nandchip_get_addr_byte(chip, data, &row))
break;
err = nandchip_chip_space(chip, row, column, size, 1);
if (err == -1)
nandchip_set_status(chip, NAND_STATUS_FAIL);
chip->sm_state = NANDSIM_STATE_WAIT_CMD;
} else
nandsim_ignore_address(chip, *((uint8_t *)data));
break;
case NANDSIM_EV_CMD:
cmd = *(uint8_t *)data;
if (chip->sm_state == NANDSIM_STATE_WAIT_CMD &&
cmd == NAND_CMD_PROG_END) {
if (chip->prog_delay != 0 &&
nandsim_delay(chip, chip->prog_delay) == 0)
nandchip_clear_status(chip, NAND_STATUS_RDY);
else {
nandchip_set_status(chip, NAND_STATUS_RDY);
nandsim_start_handler(chip, idle_evh);
}
} else
nandsim_undefined(chip, type);
break;
case NANDSIM_EV_TIMEOUT:
if (chip->sm_state == NANDSIM_STATE_TIMEOUT) {
nandsim_start_handler(chip, idle_evh);
nandchip_set_status(chip, NAND_STATUS_RDY);
} else
nandsim_undefined(chip, type);
break;
}
}
void
erase_evh(struct nandsim_chip *chip, uint32_t type, void *data)
{
static uint32_t row, block_size;
uint32_t lun, block, page;
int err;
uint8_t cmd;
block_size = chip->cg.block_size +
(chip->cg.oob_size * chip->cg.pgs_per_blk);
switch (type) {
case NANDSIM_EV_START:
nandsim_log(chip, NANDSIM_LOG_SM, "in ERASE state\n");
chip->sm_state = NANDSIM_STATE_WAIT_ADDR_ROW;
break;
case NANDSIM_EV_CMD:
cmd = *(uint8_t *)data;
if (chip->sm_state == NANDSIM_STATE_WAIT_CMD &&
cmd == NAND_CMD_ERASE_END) {
if (chip->data.data_ptr != NULL &&
chip->data.size == block_size)
memset(chip->data.data_ptr, 0xff, block_size);
else
nand_debug(NDBG_SIM,"Bad block erase data\n");
err = nand_row_to_blkpg(&chip->cg, row, &lun,
&block, &page);
if (!err) {
if (chip->blk_state[block].wear_lev > 0)
chip->blk_state[block].wear_lev--;
}
if (chip->erase_delay != 0 &&
nandsim_delay(chip, chip->erase_delay) == 0)
nandchip_clear_status(chip, NAND_STATUS_RDY);
else {
nandchip_set_status(chip, NAND_STATUS_RDY);
nandsim_start_handler(chip, idle_evh);
}
} else
nandsim_undefined(chip, type);
break;
case NANDSIM_EV_ADDR:
if (chip->sm_state == NANDSIM_STATE_WAIT_ADDR_ROW) {
if (nandchip_get_addr_byte(chip, data, &row))
break;
err = nandchip_chip_space(chip, row, 0, block_size, 1);
if (err == -1) {
nandchip_set_status(chip, NAND_STATUS_FAIL);
}
chip->sm_state = NANDSIM_STATE_WAIT_CMD;
} else
nandsim_ignore_address(chip, *((uint8_t *)data));
break;
case NANDSIM_EV_TIMEOUT:
if (chip->sm_state == NANDSIM_STATE_TIMEOUT) {
nandchip_set_status(chip, NAND_STATUS_RDY);
nandsim_start_handler(chip, idle_evh);
} else
nandsim_undefined(chip, type);
break;
}
}
void
reset_evh(struct nandsim_chip *chip, uint32_t type, void *data)
{
if (type == NANDSIM_EV_START) {
nandsim_log(chip, NANDSIM_LOG_SM, "in RESET state\n");
chip->sm_state = NANDSIM_STATE_TIMEOUT;
nandchip_set_data(chip, NULL, 0, 0);
DELAY(500);
nandsim_start_handler(chip, idle_evh);
} else
nandsim_undefined(chip, type);
}
static void
nandsim_undefined(struct nandsim_chip *chip, uint8_t type)
{
nandsim_log(chip, NANDSIM_LOG_ERR,
"ERR: Chip received ev %x in state %x\n",
type, chip->sm_state);
nandsim_start_handler(chip, idle_evh);
}
static void
nandsim_bad_address(struct nandsim_chip *chip, uint8_t *addr)
{
nandsim_log(chip, NANDSIM_LOG_ERR,
"ERR: Chip received out of range address"
"%02x%02x - %02x%02x%02x\n", addr[0], addr[1], addr[2],
addr[3], addr[4]);
}
static void
nandsim_ignore_address(struct nandsim_chip *chip, uint8_t byte)
{
nandsim_log(chip, NANDSIM_LOG_SM, "ignored address byte: %d\n", byte);
}
static void
nandsim_sm_error(struct nandsim_chip *chip)
{
nandsim_log(chip, NANDSIM_LOG_ERR, "ERR: State machine error."
"Restart required.\n");
}
Index: head/sys/dev/ntb/if_ntb/if_ntb.c
===================================================================
--- head/sys/dev/ntb/if_ntb/if_ntb.c (revision 283290)
+++ head/sys/dev/ntb/if_ntb/if_ntb.c (revision 283291)
@@ -1,1384 +1,1384 @@
/*-
* Copyright (C) 2013 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/queue.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/taskqueue.h>
#include <net/if.h>
#include <net/if_media.h>
#include <net/if_types.h>
#include <net/if_var.h>
#include <net/bpf.h>
#include <net/ethernet.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/bus.h>
#include <machine/cpufunc.h>
#include <machine/pmap.h>
#include "../ntb_hw/ntb_hw.h"
/*
* The Non-Transparent Bridge (NTB) is a device on some Intel processors that
* allows you to connect two systems using a PCI-e link.
*
* This module contains a protocol for sending and receiving messages, and
* exposes that protocol through a simulated ethernet device called ntb.
*
* NOTE: Much of the code in this module is shared with Linux. Any patches may
* be picked up and redistributed in Linux with a dual GPL/BSD license.
*/
/* TODO: These functions should really be part of the kernel */
#define test_bit(pos, bitmap_addr) (*(bitmap_addr) & 1UL << (pos))
#define set_bit(pos, bitmap_addr) *(bitmap_addr) |= 1UL << (pos)
#define clear_bit(pos, bitmap_addr) *(bitmap_addr) &= ~(1UL << (pos))
#define KTR_NTB KTR_SPARE3
#define NTB_TRANSPORT_VERSION 3
#define NTB_RX_MAX_PKTS 64
#define NTB_RXQ_SIZE 300
static unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN;
static unsigned int max_num_clients = 1;
STAILQ_HEAD(ntb_queue_list, ntb_queue_entry);
struct ntb_queue_entry {
/* ntb_queue list reference */
STAILQ_ENTRY(ntb_queue_entry) entry;
/* info on data to be transfered */
void *cb_data;
void *buf;
uint64_t len;
uint64_t flags;
};
struct ntb_rx_info {
unsigned int entry;
};
struct ntb_transport_qp {
struct ntb_netdev *transport;
struct ntb_softc *ntb;
void *cb_data;
bool client_ready;
bool qp_link;
uint8_t qp_num; /* Only 64 QPs are allowed. 0-63 */
struct ntb_rx_info *rx_info;
struct ntb_rx_info *remote_rx_info;
void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
void *data, int len);
struct ntb_queue_list tx_free_q;
struct mtx ntb_tx_free_q_lock;
void *tx_mw;
uint64_t tx_index;
uint64_t tx_max_entry;
uint64_t tx_max_frame;
void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
void *data, int len);
struct ntb_queue_list rx_pend_q;
struct ntb_queue_list rx_free_q;
struct mtx ntb_rx_pend_q_lock;
struct mtx ntb_rx_free_q_lock;
struct task rx_completion_task;
void *rx_buff;
uint64_t rx_index;
uint64_t rx_max_entry;
uint64_t rx_max_frame;
void (*event_handler) (void *data, int status);
struct callout link_work;
struct callout queue_full;
struct callout rx_full;
uint64_t last_rx_no_buf;
/* Stats */
uint64_t rx_bytes;
uint64_t rx_pkts;
uint64_t rx_ring_empty;
uint64_t rx_err_no_buf;
uint64_t rx_err_oflow;
uint64_t rx_err_ver;
uint64_t tx_bytes;
uint64_t tx_pkts;
uint64_t tx_ring_full;
};
struct ntb_queue_handlers {
void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
void *data, int len);
void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data,
void *data, int len);
void (*event_handler) (void *data, int status);
};
struct ntb_transport_mw {
size_t size;
void *virt_addr;
vm_paddr_t dma_addr;
};
struct ntb_netdev {
struct ntb_softc *ntb;
struct ifnet *ifp;
struct ntb_transport_mw mw[NTB_NUM_MW];
struct ntb_transport_qp *qps;
uint64_t max_qps;
uint64_t qp_bitmap;
bool transport_link;
struct callout link_work;
struct ntb_transport_qp *qp;
uint64_t bufsize;
u_char eaddr[ETHER_ADDR_LEN];
struct mtx tx_lock;
struct mtx rx_lock;
};
static struct ntb_netdev net_softc;
enum {
IF_NTB_DESC_DONE_FLAG = 1 << 0,
IF_NTB_LINK_DOWN_FLAG = 1 << 1,
};
struct ntb_payload_header {
uint64_t ver;
uint64_t len;
uint64_t flags;
};
enum {
IF_NTB_VERSION = 0,
IF_NTB_MW0_SZ,
IF_NTB_MW1_SZ,
IF_NTB_NUM_QPS,
IF_NTB_QP_LINKS,
IF_NTB_MAX_SPAD,
};
#define QP_TO_MW(qp) ((qp) % NTB_NUM_MW)
#define NTB_QP_DEF_NUM_ENTRIES 100
#define NTB_LINK_DOWN_TIMEOUT 10
static int ntb_handle_module_events(struct module *m, int what, void *arg);
static int ntb_setup_interface(void);
static int ntb_teardown_interface(void);
static void ntb_net_init(void *arg);
static int ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
static void ntb_start(struct ifnet *ifp);
static void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data,
void *data, int len);
static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
void *data, int len);
static void ntb_net_event_handler(void *data, int status);
static int ntb_transport_init(struct ntb_softc *ntb);
static void ntb_transport_free(void *transport);
static void ntb_transport_init_queue(struct ntb_netdev *nt,
unsigned int qp_num);
static void ntb_transport_free_queue(struct ntb_transport_qp *qp);
static struct ntb_transport_qp * ntb_transport_create_queue(void *data,
struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers);
static void ntb_transport_link_up(struct ntb_transport_qp *qp);
static int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb,
void *data, unsigned int len);
static int ntb_process_tx(struct ntb_transport_qp *qp,
struct ntb_queue_entry *entry);
static void ntb_tx_copy_task(struct ntb_transport_qp *qp,
struct ntb_queue_entry *entry, void *offset);
static void ntb_qp_full(void *arg);
static void ntb_transport_rxc_db(void *data, int db_num);
static void ntb_rx_pendq_full(void *arg);
static void ntb_transport_rx(struct ntb_transport_qp *qp);
static int ntb_process_rxc(struct ntb_transport_qp *qp);
static void ntb_rx_copy_task(struct ntb_transport_qp *qp,
struct ntb_queue_entry *entry, void *offset);
static void ntb_rx_completion_task(void *arg, int pending);
static void ntb_transport_event_callback(void *data, enum ntb_hw_event event);
static void ntb_transport_link_work(void *arg);
static int ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size);
static void ntb_transport_setup_qp_mw(struct ntb_netdev *nt,
unsigned int qp_num);
static void ntb_qp_link_work(void *arg);
static void ntb_transport_link_cleanup(struct ntb_netdev *nt);
static void ntb_qp_link_down(struct ntb_transport_qp *qp);
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp);
static void ntb_transport_link_down(struct ntb_transport_qp *qp);
static void ntb_send_link_down(struct ntb_transport_qp *qp);
static void ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry,
struct ntb_queue_list *list);
static struct ntb_queue_entry *ntb_list_rm(struct mtx *lock,
struct ntb_queue_list *list);
static void create_random_local_eui48(u_char *eaddr);
static unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp);
MALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver");
/* Module setup and teardown */
static int
ntb_handle_module_events(struct module *m, int what, void *arg)
{
int err = 0;
switch (what) {
case MOD_LOAD:
err = ntb_setup_interface();
break;
case MOD_UNLOAD:
err = ntb_teardown_interface();
break;
default:
err = EOPNOTSUPP;
break;
}
return (err);
}
static moduledata_t if_ntb_mod = {
"if_ntb",
ntb_handle_module_events,
NULL
};
DECLARE_MODULE(if_ntb, if_ntb_mod, SI_SUB_KLD, SI_ORDER_ANY);
MODULE_DEPEND(if_ntb, ntb_hw, 1, 1, 1);
static int
ntb_setup_interface()
{
struct ifnet *ifp;
struct ntb_queue_handlers handlers = { ntb_net_rx_handler,
ntb_net_tx_handler, ntb_net_event_handler };
net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0);
if (net_softc.ntb == NULL) {
printf("ntb: Cannot find devclass\n");
return (ENXIO);
}
ntb_transport_init(net_softc.ntb);
ifp = net_softc.ifp = if_alloc(IFT_ETHER);
if (ifp == NULL) {
printf("ntb: cannot allocate ifnet structure\n");
return (ENOMEM);
}
net_softc.qp = ntb_transport_create_queue(ifp, net_softc.ntb,
&handlers);
if_initname(ifp, "ntb", 0);
ifp->if_init = ntb_net_init;
ifp->if_softc = &net_softc;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX;
ifp->if_ioctl = ntb_ioctl;
ifp->if_start = ntb_start;
IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN);
ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN;
IFQ_SET_READY(&ifp->if_snd);
create_random_local_eui48(net_softc.eaddr);
ether_ifattach(ifp, net_softc.eaddr);
ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU;
ifp->if_capenable = ifp->if_capabilities;
ntb_transport_link_up(net_softc.qp);
net_softc.bufsize = ntb_transport_max_size(net_softc.qp) +
sizeof(struct ether_header);
return (0);
}
static int
ntb_teardown_interface()
{
if (net_softc.qp != NULL)
ntb_transport_link_down(net_softc.qp);
if (net_softc.ifp != NULL) {
ether_ifdetach(net_softc.ifp);
if_free(net_softc.ifp);
}
if (net_softc.qp != NULL) {
ntb_transport_free_queue(net_softc.qp);
ntb_transport_free(&net_softc);
}
return (0);
}
/* Network device interface */
static void
ntb_net_init(void *arg)
{
struct ntb_netdev *ntb_softc = arg;
struct ifnet *ifp = ntb_softc->ifp;
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
ifp->if_flags |= IFF_UP;
if_link_state_change(ifp, LINK_STATE_UP);
}
static int
ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
{
struct ntb_netdev *nt = ifp->if_softc;
struct ifreq *ifr = (struct ifreq *)data;
int error = 0;
switch (command) {
case SIOCSIFMTU:
{
if (ifr->ifr_mtu > ntb_transport_max_size(nt->qp) -
ETHER_HDR_LEN - ETHER_CRC_LEN) {
error = EINVAL;
break;
}
ifp->if_mtu = ifr->ifr_mtu;
break;
}
default:
error = ether_ioctl(ifp, command, data);
break;
}
return (error);
}
static void
ntb_start(struct ifnet *ifp)
{
struct mbuf *m_head;
struct ntb_netdev *nt = ifp->if_softc;
int rc;
mtx_lock(&nt->tx_lock);
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
CTR0(KTR_NTB, "TX: ntb_start");
while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
CTR1(KTR_NTB, "TX: start mbuf %p", m_head);
rc = ntb_transport_tx_enqueue(nt->qp, m_head, m_head,
m_length(m_head, NULL));
if (rc != 0) {
CTR1(KTR_NTB,
"TX: could not tx mbuf %p. Returning to snd q",
m_head);
if (rc == EAGAIN) {
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
callout_reset(&nt->qp->queue_full, hz / 1000,
ntb_qp_full, ifp);
}
break;
}
}
mtx_unlock(&nt->tx_lock);
}
/* Network Device Callbacks */
static void
ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data,
int len)
{
m_freem(data);
CTR1(KTR_NTB, "TX: tx_handler freeing mbuf %p", data);
}
static void
ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data,
int len)
{
struct mbuf *m = data;
struct ifnet *ifp = qp_data;
CTR0(KTR_NTB, "RX: rx handler");
(*ifp->if_input)(ifp, m);
}
static void
ntb_net_event_handler(void *data, int status)
{
}
/* Transport Init and teardown */
static int
ntb_transport_init(struct ntb_softc *ntb)
{
struct ntb_netdev *nt = &net_softc;
int rc, i;
nt->max_qps = max_num_clients;
ntb_register_transport(ntb, nt);
mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF);
mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF);
nt->qps = malloc(nt->max_qps * sizeof(struct ntb_transport_qp),
M_NTB_IF, M_WAITOK|M_ZERO);
nt->qp_bitmap = ((uint64_t) 1 << nt->max_qps) - 1;
for (i = 0; i < nt->max_qps; i++)
ntb_transport_init_queue(nt, i);
callout_init(&nt->link_work, 0);
rc = ntb_register_event_callback(ntb,
ntb_transport_event_callback);
if (rc != 0)
goto err;
if (ntb_query_link_status(ntb)) {
if (bootverbose)
device_printf(ntb_get_device(ntb), "link up\n");
callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
}
return (0);
err:
free(nt->qps, M_NTB_IF);
ntb_unregister_transport(ntb);
return (rc);
}
static void
ntb_transport_free(void *transport)
{
struct ntb_netdev *nt = transport;
struct ntb_softc *ntb = nt->ntb;
int i;
nt->transport_link = NTB_LINK_DOWN;
callout_drain(&nt->link_work);
/* verify that all the qps are freed */
for (i = 0; i < nt->max_qps; i++)
if (!test_bit(i, &nt->qp_bitmap))
ntb_transport_free_queue(&nt->qps[i]);
ntb_unregister_event_callback(ntb);
for (i = 0; i < NTB_NUM_MW; i++)
if (nt->mw[i].virt_addr != NULL)
contigfree(nt->mw[i].virt_addr, nt->mw[i].size,
M_NTB_IF);
free(nt->qps, M_NTB_IF);
ntb_unregister_transport(ntb);
}
static void
ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num)
{
struct ntb_transport_qp *qp;
unsigned int num_qps_mw, tx_size;
uint8_t mw_num = QP_TO_MW(qp_num);
qp = &nt->qps[qp_num];
qp->qp_num = qp_num;
qp->transport = nt;
qp->ntb = nt->ntb;
qp->qp_link = NTB_LINK_DOWN;
qp->client_ready = NTB_LINK_DOWN;
qp->event_handler = NULL;
if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
else
num_qps_mw = nt->max_qps / NTB_NUM_MW;
tx_size = (unsigned int) ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw;
qp->rx_info = (struct ntb_rx_info *)
((char *)ntb_get_mw_vbase(qp->ntb, mw_num) +
(qp_num / NTB_NUM_MW * tx_size));
tx_size -= sizeof(struct ntb_rx_info);
qp->tx_mw = qp->rx_info + sizeof(struct ntb_rx_info);
qp->tx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
tx_size);
qp->tx_max_entry = tx_size / qp->tx_max_frame;
qp->tx_index = 0;
callout_init(&qp->link_work, 0);
- callout_init(&qp->queue_full, CALLOUT_MPSAFE);
- callout_init(&qp->rx_full, CALLOUT_MPSAFE);
+ callout_init(&qp->queue_full, 1);
+ callout_init(&qp->rx_full, 1);
mtx_init(&qp->ntb_rx_pend_q_lock, "ntb rx pend q", NULL, MTX_SPIN);
mtx_init(&qp->ntb_rx_free_q_lock, "ntb rx free q", NULL, MTX_SPIN);
mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN);
TASK_INIT(&qp->rx_completion_task, 0, ntb_rx_completion_task, qp);
STAILQ_INIT(&qp->rx_pend_q);
STAILQ_INIT(&qp->rx_free_q);
STAILQ_INIT(&qp->tx_free_q);
}
static void
ntb_transport_free_queue(struct ntb_transport_qp *qp)
{
struct ntb_queue_entry *entry;
if (qp == NULL)
return;
callout_drain(&qp->link_work);
ntb_unregister_db_callback(qp->ntb, qp->qp_num);
while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
free(entry, M_NTB_IF);
while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q)))
free(entry, M_NTB_IF);
while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
free(entry, M_NTB_IF);
set_bit(qp->qp_num, &qp->transport->qp_bitmap);
}
/**
* ntb_transport_create_queue - Create a new NTB transport layer queue
* @rx_handler: receive callback function
* @tx_handler: transmit callback function
* @event_handler: event callback function
*
* Create a new NTB transport layer queue and provide the queue with a callback
* routine for both transmit and receive. The receive callback routine will be
* used to pass up data when the transport has received it on the queue. The
* transmit callback routine will be called when the transport has completed the
* transmission of the data on the queue and the data is ready to be freed.
*
* RETURNS: pointer to newly created ntb_queue, NULL on error.
*/
static struct ntb_transport_qp *
ntb_transport_create_queue(void *data, struct ntb_softc *pdev,
const struct ntb_queue_handlers *handlers)
{
struct ntb_queue_entry *entry;
struct ntb_transport_qp *qp;
struct ntb_netdev *nt;
unsigned int free_queue;
int rc, i;
nt = ntb_find_transport(pdev);
if (nt == NULL)
goto err;
free_queue = ffs(nt->qp_bitmap);
if (free_queue == 0)
goto err;
/* decrement free_queue to make it zero based */
free_queue--;
clear_bit(free_queue, &nt->qp_bitmap);
qp = &nt->qps[free_queue];
qp->cb_data = data;
qp->rx_handler = handlers->rx_handler;
qp->tx_handler = handlers->tx_handler;
qp->event_handler = handlers->event_handler;
for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
M_WAITOK|M_ZERO);
entry->cb_data = nt->ifp;
entry->buf = NULL;
entry->len = transport_mtu;
ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
}
for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
M_WAITOK|M_ZERO);
ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
}
rc = ntb_register_db_callback(qp->ntb, free_queue, qp,
ntb_transport_rxc_db);
if (rc != 0)
goto err1;
return (qp);
err1:
while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
free(entry, M_NTB_IF);
while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
free(entry, M_NTB_IF);
set_bit(free_queue, &nt->qp_bitmap);
err:
return (NULL);
}
/**
* ntb_transport_link_up - Notify NTB transport of client readiness to use queue
* @qp: NTB transport layer queue to be enabled
*
* Notify NTB transport layer of client readiness to use queue
*/
static void
ntb_transport_link_up(struct ntb_transport_qp *qp)
{
if (qp == NULL)
return;
qp->client_ready = NTB_LINK_UP;
if (bootverbose)
device_printf(ntb_get_device(qp->ntb), "qp client ready\n");
if (qp->transport->transport_link == NTB_LINK_UP)
callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);
}
/* Transport Tx */
/**
* ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
* @qp: NTB transport layer queue the entry is to be enqueued on
* @cb: per buffer pointer for callback function to use
* @data: pointer to data buffer that will be sent
* @len: length of the data buffer
*
* Enqueue a new transmit buffer onto the transport queue from which a NTB
* payload will be transmitted. This assumes that a lock is behing held to
* serialize access to the qp.
*
* RETURNS: An appropriate ERRNO error value on error, or zero for success.
*/
static int
ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
unsigned int len)
{
struct ntb_queue_entry *entry;
int rc;
if (qp == NULL || qp->qp_link != NTB_LINK_UP || len == 0) {
CTR0(KTR_NTB, "TX: link not up");
return (EINVAL);
}
entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
if (entry == NULL) {
CTR0(KTR_NTB, "TX: could not get entry from tx_free_q");
return (ENOMEM);
}
CTR1(KTR_NTB, "TX: got entry %p from tx_free_q", entry);
entry->cb_data = cb;
entry->buf = data;
entry->len = len;
entry->flags = 0;
rc = ntb_process_tx(qp, entry);
if (rc != 0) {
ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
CTR1(KTR_NTB,
"TX: process_tx failed. Returning entry %p to tx_free_q",
entry);
}
return (rc);
}
static int
ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry)
{
void *offset;
offset = (char *)qp->tx_mw + qp->tx_max_frame * qp->tx_index;
CTR3(KTR_NTB,
"TX: process_tx: tx_pkts=%u, tx_index=%u, remote entry=%u",
qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry);
if (qp->tx_index == qp->remote_rx_info->entry) {
CTR0(KTR_NTB, "TX: ring full");
qp->tx_ring_full++;
return (EAGAIN);
}
if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {
if (qp->tx_handler != NULL)
qp->tx_handler(qp, qp->cb_data, entry->buf,
EIO);
ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
CTR1(KTR_NTB,
"TX: frame too big. returning entry %p to tx_free_q",
entry);
return (0);
}
CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset);
ntb_tx_copy_task(qp, entry, offset);
qp->tx_index++;
qp->tx_index %= qp->tx_max_entry;
qp->tx_pkts++;
return (0);
}
static void
ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
void *offset)
{
struct ntb_payload_header *hdr;
CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset);
if (entry->buf != NULL)
m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset);
hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame -
sizeof(struct ntb_payload_header));
hdr->len = entry->len; /* TODO: replace with bus_space_write */
hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */
wmb();
/* TODO: replace with bus_space_write */
hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG;
ntb_ring_sdb(qp->ntb, qp->qp_num);
/*
* The entry length can only be zero if the packet is intended to be a
* "link down" or similar. Since no payload is being sent in these
* cases, there is nothing to add to the completion queue.
*/
if (entry->len > 0) {
qp->tx_bytes += entry->len;
if (qp->tx_handler)
qp->tx_handler(qp, qp->cb_data, entry->cb_data,
entry->len);
}
CTR2(KTR_NTB,
"TX: entry %p sent. hdr->ver = %d, Returning to tx_free_q", entry,
hdr->ver);
ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
}
static void
ntb_qp_full(void *arg)
{
CTR0(KTR_NTB, "TX: qp_full callout");
ntb_start(arg);
}
/* Transport Rx */
static void
ntb_transport_rxc_db(void *data, int db_num)
{
struct ntb_transport_qp *qp = data;
ntb_transport_rx(qp);
}
static void
ntb_rx_pendq_full(void *arg)
{
CTR0(KTR_NTB, "RX: ntb_rx_pendq_full callout");
ntb_transport_rx(arg);
}
static void
ntb_transport_rx(struct ntb_transport_qp *qp)
{
int rc, i;
/*
* Limit the number of packets processed in a single interrupt to
* provide fairness to others
*/
mtx_lock(&qp->transport->rx_lock);
CTR0(KTR_NTB, "RX: transport_rx");
for (i = 0; i < NTB_RX_MAX_PKTS; i++) {
rc = ntb_process_rxc(qp);
if (rc != 0) {
CTR0(KTR_NTB, "RX: process_rxc failed");
break;
}
}
mtx_unlock(&qp->transport->rx_lock);
}
static int
ntb_process_rxc(struct ntb_transport_qp *qp)
{
struct ntb_payload_header *hdr;
struct ntb_queue_entry *entry;
void *offset;
offset = (void *)
((char *)qp->rx_buff + qp->rx_max_frame * qp->rx_index);
hdr = (void *)
((char *)offset + qp->rx_max_frame -
sizeof(struct ntb_payload_header));
CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index);
entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
if (entry == NULL) {
qp->rx_err_no_buf++;
CTR0(KTR_NTB, "RX: No entries in rx_pend_q");
return (ENOMEM);
}
callout_stop(&qp->rx_full);
CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry);
if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) {
CTR1(KTR_NTB,
"RX: hdr not done. Returning entry %p to rx_pend_q", entry);
ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
qp->rx_ring_empty++;
return (EAGAIN);
}
if (hdr->ver != (uint32_t) qp->rx_pkts) {
CTR3(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). "
"Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts,
entry);
ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
qp->rx_err_ver++;
return (EIO);
}
if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) {
ntb_qp_link_down(qp);
CTR1(KTR_NTB,
"RX: link down. adding entry %p back to rx_pend_q", entry);
ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
goto out;
}
if (hdr->len <= entry->len) {
entry->len = hdr->len;
ntb_rx_copy_task(qp, entry, offset);
} else {
CTR1(KTR_NTB,
"RX: len too long. Returning entry %p to rx_pend_q", entry);
ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
qp->rx_err_oflow++;
}
qp->rx_bytes += hdr->len;
qp->rx_pkts++;
CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts);
out:
/* Ensure that the data is globally visible before clearing the flag */
wmb();
hdr->flags = 0;
/* TODO: replace with bus_space_write */
qp->rx_info->entry = qp->rx_index;
qp->rx_index++;
qp->rx_index %= qp->rx_max_entry;
return (0);
}
static void
ntb_rx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
void *offset)
{
struct ifnet *ifp = entry->cb_data;
unsigned int len = entry->len;
struct mbuf *m;
CTR2(KTR_NTB, "RX: copying %d bytes from offset %p", len, offset);
m = m_devget(offset, len, 0, ifp, NULL);
m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID;
entry->buf = (void *)m;
CTR2(KTR_NTB,
"RX: copied entry %p to mbuf %p. Adding entry to rx_free_q", entry,
m);
ntb_list_add(&qp->ntb_rx_free_q_lock, entry, &qp->rx_free_q);
taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task);
}
static void
ntb_rx_completion_task(void *arg, int pending)
{
struct ntb_transport_qp *qp = arg;
struct mbuf *m;
struct ntb_queue_entry *entry;
CTR0(KTR_NTB, "RX: rx_completion_task");
while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) {
m = entry->buf;
CTR2(KTR_NTB, "RX: completing entry %p, mbuf %p", entry, m);
if (qp->rx_handler && qp->client_ready == NTB_LINK_UP)
qp->rx_handler(qp, qp->cb_data, m, entry->len);
entry->buf = NULL;
entry->len = qp->transport->bufsize;
CTR1(KTR_NTB,"RX: entry %p removed from rx_free_q "
"and added to rx_pend_q", entry);
ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
if (qp->rx_err_no_buf > qp->last_rx_no_buf) {
qp->last_rx_no_buf = qp->rx_err_no_buf;
CTR0(KTR_NTB, "RX: could spawn rx task");
callout_reset(&qp->rx_full, hz / 1000, ntb_rx_pendq_full,
qp);
}
}
}
/* Link Event handler */
static void
ntb_transport_event_callback(void *data, enum ntb_hw_event event)
{
struct ntb_netdev *nt = data;
switch (event) {
case NTB_EVENT_HW_LINK_UP:
if (bootverbose)
device_printf(ntb_get_device(nt->ntb), "HW link up\n");
callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
break;
case NTB_EVENT_HW_LINK_DOWN:
if (bootverbose)
device_printf(ntb_get_device(nt->ntb), "HW link down\n");
ntb_transport_link_cleanup(nt);
break;
default:
panic("ntb: Unknown NTB event");
}
}
/* Link bring up */
static void
ntb_transport_link_work(void *arg)
{
struct ntb_netdev *nt = arg;
struct ntb_softc *ntb = nt->ntb;
struct ntb_transport_qp *qp;
uint32_t val;
int rc, i;
/* send the local info */
rc = ntb_write_remote_spad(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION);
if (rc != 0)
goto out;
rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ, ntb_get_mw_size(ntb, 0));
if (rc != 0)
goto out;
rc = ntb_write_remote_spad(ntb, IF_NTB_MW1_SZ, ntb_get_mw_size(ntb, 1));
if (rc != 0)
goto out;
rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_QPS, nt->max_qps);
if (rc != 0)
goto out;
rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val);
if (rc != 0)
goto out;
rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val);
if (rc != 0)
goto out;
/* Query the remote side for its info */
rc = ntb_read_local_spad(ntb, IF_NTB_VERSION, &val);
if (rc != 0)
goto out;
if (val != NTB_TRANSPORT_VERSION)
goto out;
rc = ntb_read_local_spad(ntb, IF_NTB_NUM_QPS, &val);
if (rc != 0)
goto out;
if (val != nt->max_qps)
goto out;
rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ, &val);
if (rc != 0)
goto out;
if (val == 0)
goto out;
rc = ntb_set_mw(nt, 0, val);
if (rc != 0)
return;
rc = ntb_read_local_spad(ntb, IF_NTB_MW1_SZ, &val);
if (rc != 0)
goto out;
if (val == 0)
goto out;
rc = ntb_set_mw(nt, 1, val);
if (rc != 0)
return;
nt->transport_link = NTB_LINK_UP;
if (bootverbose)
device_printf(ntb_get_device(ntb), "transport link up\n");
for (i = 0; i < nt->max_qps; i++) {
qp = &nt->qps[i];
ntb_transport_setup_qp_mw(nt, i);
if (qp->client_ready == NTB_LINK_UP)
callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);
}
return;
out:
if (ntb_query_link_status(ntb))
callout_reset(&nt->link_work,
NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt);
}
static int
ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size)
{
struct ntb_transport_mw *mw = &nt->mw[num_mw];
/* Alloc memory for receiving data. Must be 4k aligned */
mw->size = size;
mw->virt_addr = contigmalloc(mw->size, M_NTB_IF, M_ZERO, 0,
BUS_SPACE_MAXADDR, mw->size, 0);
if (mw->virt_addr == NULL) {
printf("ntb: Unable to allocate MW buffer of size %d\n",
(int)mw->size);
return (ENOMEM);
}
/* TODO: replace with bus_space_* functions */
mw->dma_addr = vtophys(mw->virt_addr);
/* Notify HW the memory location of the receive buffer */
ntb_set_mw_addr(nt->ntb, num_mw, mw->dma_addr);
return (0);
}
static void
ntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num)
{
struct ntb_transport_qp *qp = &nt->qps[qp_num];
void *offset;
unsigned int rx_size, num_qps_mw;
uint8_t mw_num = QP_TO_MW(qp_num);
unsigned int i;
if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW)
num_qps_mw = nt->max_qps / NTB_NUM_MW + 1;
else
num_qps_mw = nt->max_qps / NTB_NUM_MW;
rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw;
qp->remote_rx_info = (void *)((uint8_t *)nt->mw[mw_num].virt_addr +
(qp_num / NTB_NUM_MW * rx_size));
rx_size -= sizeof(struct ntb_rx_info);
qp->rx_buff = qp->remote_rx_info + sizeof(struct ntb_rx_info);
qp->rx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
rx_size);
qp->rx_max_entry = rx_size / qp->rx_max_frame;
qp->rx_index = 0;
qp->tx_index = 0;
qp->remote_rx_info->entry = qp->rx_max_entry;
/* setup the hdr offsets with 0's */
for (i = 0; i < qp->rx_max_entry; i++) {
offset = (void *)((uint8_t *)qp->rx_buff +
qp->rx_max_frame * (i + 1) -
sizeof(struct ntb_payload_header));
memset(offset, 0, sizeof(struct ntb_payload_header));
}
qp->rx_pkts = 0;
qp->tx_pkts = 0;
}
static void
ntb_qp_link_work(void *arg)
{
struct ntb_transport_qp *qp = arg;
struct ntb_softc *ntb = qp->ntb;
struct ntb_netdev *nt = qp->transport;
int rc, val;
rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val);
if (rc != 0)
return;
rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num);
/* query remote spad for qp ready bits */
rc = ntb_read_local_spad(ntb, IF_NTB_QP_LINKS, &val);
/* See if the remote side is up */
if ((1 << qp->qp_num & val) != 0) {
qp->qp_link = NTB_LINK_UP;
if (qp->event_handler != NULL)
qp->event_handler(qp->cb_data, NTB_LINK_UP);
if (bootverbose)
device_printf(ntb_get_device(ntb), "qp link up\n");
} else if (nt->transport_link == NTB_LINK_UP) {
callout_reset(&qp->link_work,
NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
}
}
/* Link down event*/
static void
ntb_transport_link_cleanup(struct ntb_netdev *nt)
{
int i;
if (nt->transport_link == NTB_LINK_DOWN)
callout_drain(&nt->link_work);
else
nt->transport_link = NTB_LINK_DOWN;
/* Pass along the info to any clients */
for (i = 0; i < nt->max_qps; i++)
if (!test_bit(i, &nt->qp_bitmap))
ntb_qp_link_down(&nt->qps[i]);
/*
* The scratchpad registers keep the values if the remote side
* goes down, blast them now to give them a sane value the next
* time they are accessed
*/
for (i = 0; i < IF_NTB_MAX_SPAD; i++)
ntb_write_local_spad(nt->ntb, i, 0);
}
static void
ntb_qp_link_down(struct ntb_transport_qp *qp)
{
ntb_qp_link_cleanup(qp);
}
static void
ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
{
struct ntb_netdev *nt = qp->transport;
if (qp->qp_link == NTB_LINK_DOWN) {
callout_drain(&qp->link_work);
return;
}
if (qp->event_handler != NULL)
qp->event_handler(qp->cb_data, NTB_LINK_DOWN);
qp->qp_link = NTB_LINK_DOWN;
if (nt->transport_link == NTB_LINK_UP)
callout_reset(&qp->link_work,
NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);
}
/* Link commanded down */
/**
* ntb_transport_link_down - Notify NTB transport to no longer enqueue data
* @qp: NTB transport layer queue to be disabled
*
* Notify NTB transport layer of client's desire to no longer receive data on
* transport queue specified. It is the client's responsibility to ensure all
* entries on queue are purged or otherwise handled appropraitely.
*/
static void
ntb_transport_link_down(struct ntb_transport_qp *qp)
{
int rc, val;
if (qp == NULL)
return;
qp->client_ready = NTB_LINK_DOWN;
rc = ntb_read_remote_spad(qp->ntb, IF_NTB_QP_LINKS, &val);
if (rc != 0)
return;
rc = ntb_write_remote_spad(qp->ntb, IF_NTB_QP_LINKS,
val & ~(1 << qp->qp_num));
if (qp->qp_link == NTB_LINK_UP)
ntb_send_link_down(qp);
else
callout_drain(&qp->link_work);
}
static void
ntb_send_link_down(struct ntb_transport_qp *qp)
{
struct ntb_queue_entry *entry;
int i, rc;
if (qp->qp_link == NTB_LINK_DOWN)
return;
qp->qp_link = NTB_LINK_DOWN;
for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) {
entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
if (entry != NULL)
break;
pause("NTB Wait for link down", hz / 10);
}
if (entry == NULL)
return;
entry->cb_data = NULL;
entry->buf = NULL;
entry->len = 0;
entry->flags = IF_NTB_LINK_DOWN_FLAG;
mtx_lock(&qp->transport->tx_lock);
rc = ntb_process_tx(qp, entry);
if (rc != 0)
printf("ntb: Failed to send link down\n");
mtx_unlock(&qp->transport->tx_lock);
}
/* List Management */
static void
ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry,
struct ntb_queue_list *list)
{
mtx_lock_spin(lock);
STAILQ_INSERT_TAIL(list, entry, entry);
mtx_unlock_spin(lock);
}
static struct ntb_queue_entry *
ntb_list_rm(struct mtx *lock, struct ntb_queue_list *list)
{
struct ntb_queue_entry *entry;
mtx_lock_spin(lock);
if (STAILQ_EMPTY(list)) {
entry = NULL;
goto out;
}
entry = STAILQ_FIRST(list);
STAILQ_REMOVE_HEAD(list, entry);
out:
mtx_unlock_spin(lock);
return (entry);
}
/* Helper functions */
/* TODO: This too should really be part of the kernel */
#define EUI48_MULTICAST 1 << 0
#define EUI48_LOCALLY_ADMINISTERED 1 << 1
static void
create_random_local_eui48(u_char *eaddr)
{
static uint8_t counter = 0;
uint32_t seed = ticks;
eaddr[0] = EUI48_LOCALLY_ADMINISTERED;
memcpy(&eaddr[1], &seed, sizeof(uint32_t));
eaddr[5] = counter++;
}
/**
* ntb_transport_max_size - Query the max payload size of a qp
* @qp: NTB transport layer queue to be queried
*
* Query the maximum payload size permissible on the given qp
*
* RETURNS: the max payload size of a qp
*/
static unsigned int
ntb_transport_max_size(struct ntb_transport_qp *qp)
{
if (qp == NULL)
return (0);
return (qp->tx_max_frame - sizeof(struct ntb_payload_header));
}
Index: head/sys/dev/ntb/ntb_hw/ntb_hw.c
===================================================================
--- head/sys/dev/ntb/ntb_hw/ntb_hw.c (revision 283290)
+++ head/sys/dev/ntb/ntb_hw/ntb_hw.c (revision 283291)
@@ -1,1414 +1,1414 @@
/*-
* Copyright (C) 2013 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/queue.h>
#include <sys/rman.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/bus.h>
#include <machine/pmap.h>
#include <machine/resource.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include "ntb_regs.h"
#include "ntb_hw.h"
/*
* The Non-Transparent Bridge (NTB) is a device on some Intel processors that
* allows you to connect two systems using a PCI-e link.
*
* This module contains the hardware abstraction layer for the NTB. It allows
* you to send and recieve interrupts, map the memory windows and send and
* receive messages in the scratch-pad registers.
*
* NOTE: Much of the code in this module is shared with Linux. Any patches may
* be picked up and redistributed in Linux with a dual GPL/BSD license.
*/
#define NTB_CONFIG_BAR 0
#define NTB_B2B_BAR_1 1
#define NTB_B2B_BAR_2 2
#define NTB_MAX_BARS 3
#define NTB_MW_TO_BAR(mw) ((mw) + 1)
#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS)
#define NTB_HB_TIMEOUT 1 /* second */
#define SOC_LINK_RECOVERY_TIME 500
#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev))
enum ntb_device_type {
NTB_XEON,
NTB_SOC
};
/* Device features and workarounds */
#define HAS_FEATURE(feature) \
((ntb->features & (feature)) != 0)
#define NTB_BAR_SIZE_4K (1 << 0)
#define NTB_REGS_THRU_MW (1 << 1)
struct ntb_hw_info {
uint32_t device_id;
const char *desc;
enum ntb_device_type type;
uint64_t features;
};
struct ntb_pci_bar_info {
bus_space_tag_t pci_bus_tag;
bus_space_handle_t pci_bus_handle;
int pci_resource_id;
struct resource *pci_resource;
vm_paddr_t pbase;
void *vbase;
u_long size;
};
struct ntb_int_info {
struct resource *res;
int rid;
void *tag;
};
struct ntb_db_cb {
ntb_db_callback callback;
unsigned int db_num;
void *data;
struct ntb_softc *ntb;
};
struct ntb_softc {
device_t device;
enum ntb_device_type type;
uint64_t features;
struct ntb_pci_bar_info bar_info[NTB_MAX_BARS];
struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS];
uint32_t allocated_interrupts;
struct callout heartbeat_timer;
struct callout lr_timer;
void *ntb_transport;
ntb_event_callback event_cb;
struct ntb_db_cb *db_cb;
struct {
uint32_t max_spads;
uint32_t max_db_bits;
uint32_t msix_cnt;
} limits;
struct {
uint32_t pdb;
uint32_t pdb_mask;
uint32_t sdb;
uint32_t sbar2_xlat;
uint32_t sbar4_xlat;
uint32_t spad_remote;
uint32_t spad_local;
uint32_t lnk_cntl;
uint32_t lnk_stat;
uint32_t spci_cmd;
} reg_ofs;
uint8_t conn_type;
uint8_t dev_type;
uint8_t bits_per_vector;
uint8_t link_status;
uint8_t link_width;
uint8_t link_speed;
};
#define ntb_bar_read(SIZE, bar, offset) \
bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
ntb->bar_info[(bar)].pci_bus_handle, (offset))
#define ntb_bar_write(SIZE, bar, offset, val) \
bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
ntb->bar_info[(bar)].pci_bus_handle, (offset), (val))
#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset)
#define ntb_reg_write(SIZE, offset, val) \
ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset)
#define ntb_mw_write(SIZE, offset, val) \
ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val)
typedef int (*bar_map_strategy)(struct ntb_softc *ntb,
struct ntb_pci_bar_info *bar);
static int ntb_probe(device_t device);
static int ntb_attach(device_t device);
static int ntb_detach(device_t device);
static int ntb_map_pci_bars(struct ntb_softc *ntb);
static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
struct ntb_pci_bar_info *bar);
static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
static int map_memory_window_bar(struct ntb_softc *ntb,
struct ntb_pci_bar_info *bar);
static void ntb_unmap_pci_bar(struct ntb_softc *ntb);
static int ntb_setup_interrupts(struct ntb_softc *ntb);
static void ntb_teardown_interrupts(struct ntb_softc *ntb);
static void handle_soc_irq(void *arg);
static void handle_xeon_irq(void *arg);
static void handle_xeon_event_irq(void *arg);
static void ntb_handle_legacy_interrupt(void *arg);
static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors);
static void ntb_free_callbacks(struct ntb_softc *ntb);
static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
static int ntb_initialize_hw(struct ntb_softc *ntb);
static int ntb_setup_xeon(struct ntb_softc *ntb);
static int ntb_setup_soc(struct ntb_softc *ntb);
static void configure_soc_secondary_side_bars(struct ntb_softc *ntb);
static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb);
static void ntb_handle_heartbeat(void *arg);
static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state);
static void recover_soc_link(void *arg);
static int ntb_check_link_status(struct ntb_softc *ntb);
static void save_bar_parameters(struct ntb_pci_bar_info *bar);
static struct ntb_hw_info pci_ids[] = {
{ 0x3C0D8086, "Xeon E5/Core i7 Non-Transparent Bridge B2B", NTB_XEON,
NTB_REGS_THRU_MW },
{ 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 },
{ 0x0E0D8086, "Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
NTB_REGS_THRU_MW | NTB_BAR_SIZE_4K },
{ 0x00000000, NULL, NTB_SOC, 0 }
};
/*
* OS <-> Driver interface structures
*/
MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations");
static device_method_t ntb_pci_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, ntb_probe),
DEVMETHOD(device_attach, ntb_attach),
DEVMETHOD(device_detach, ntb_detach),
DEVMETHOD_END
};
static driver_t ntb_pci_driver = {
"ntb_hw",
ntb_pci_methods,
sizeof(struct ntb_softc),
};
static devclass_t ntb_devclass;
DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL);
MODULE_VERSION(ntb_hw, 1);
/*
* OS <-> Driver linkage functions
*/
static int
ntb_probe(device_t device)
{
struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device));
if (p != NULL) {
device_set_desc(device, p->desc);
return (0);
} else
return (ENXIO);
}
#define DETACH_ON_ERROR(func) \
error = func; \
if (error < 0) { \
ntb_detach(device); \
return (error); \
}
static int
ntb_attach(device_t device)
{
struct ntb_softc *ntb = DEVICE2SOFTC(device);
struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device));
int error;
ntb->device = device;
ntb->type = p->type;
ntb->features = p->features;
/* Heartbeat timer for NTB_SOC since there is no link interrupt */
- callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE);
- callout_init(&ntb->lr_timer, CALLOUT_MPSAFE);
+ callout_init(&ntb->heartbeat_timer, 1);
+ callout_init(&ntb->lr_timer, 1);
DETACH_ON_ERROR(ntb_map_pci_bars(ntb));
DETACH_ON_ERROR(ntb_initialize_hw(ntb));
DETACH_ON_ERROR(ntb_setup_interrupts(ntb));
pci_enable_busmaster(ntb->device);
return (error);
}
static int
ntb_detach(device_t device)
{
struct ntb_softc *ntb = DEVICE2SOFTC(device);
callout_drain(&ntb->heartbeat_timer);
callout_drain(&ntb->lr_timer);
ntb_teardown_interrupts(ntb);
ntb_unmap_pci_bar(ntb);
return (0);
}
static int
ntb_map_pci_bars(struct ntb_softc *ntb)
{
int rc;
ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0);
rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]);
if (rc != 0)
return rc;
ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2);
rc = map_pci_bar(ntb, map_memory_window_bar,
&ntb->bar_info[NTB_B2B_BAR_1]);
if (rc != 0)
return rc;
ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
if (HAS_FEATURE(NTB_REGS_THRU_MW))
rc = map_pci_bar(ntb, map_mmr_bar,
&ntb->bar_info[NTB_B2B_BAR_2]);
else
rc = map_pci_bar(ntb, map_memory_window_bar,
&ntb->bar_info[NTB_B2B_BAR_2]);
if (rc != 0)
return rc;
return (0);
}
static int
map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
struct ntb_pci_bar_info *bar)
{
int rc;
rc = strategy(ntb, bar);
if (rc != 0) {
device_printf(ntb->device,
"unable to allocate pci resource\n");
} else {
device_printf(ntb->device,
"Bar size = %lx, v %p, p %p\n",
bar->size, bar->vbase,
(void *)(bar->pbase));
}
return (rc);
}
static int
map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
{
bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
&bar->pci_resource_id, RF_ACTIVE);
if (bar->pci_resource == NULL)
return (ENXIO);
else {
save_bar_parameters(bar);
return (0);
}
}
static int
map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
{
int rc;
uint8_t bar_size_bits = 0;
bar->pci_resource = bus_alloc_resource_any(ntb->device,
SYS_RES_MEMORY, &bar->pci_resource_id, RF_ACTIVE);
if (bar->pci_resource == NULL)
return (ENXIO);
else {
save_bar_parameters(bar);
/*
* Ivytown NTB BAR sizes are misreported by the hardware due to
* a hardware issue. To work around this, query the size it
* should be configured to by the device and modify the resource
* to correspond to this new size. The BIOS on systems with this
* problem is required to provide enough address space to allow
* the driver to make this change safely.
*
* Ideally I could have just specified the size when I allocated
* the resource like:
* bus_alloc_resource(ntb->device,
* SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul,
* 1ul << bar_size_bits, RF_ACTIVE);
* but the PCI driver does not honor the size in this call, so
* we have to modify it after the fact.
*/
if (HAS_FEATURE(NTB_BAR_SIZE_4K)) {
if (bar->pci_resource_id == PCIR_BAR(2))
bar_size_bits = pci_read_config(ntb->device,
XEON_PBAR23SZ_OFFSET, 1);
else
bar_size_bits = pci_read_config(ntb->device,
XEON_PBAR45SZ_OFFSET, 1);
rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY,
bar->pci_resource, bar->pbase,
bar->pbase + (1ul << bar_size_bits) - 1);
if (rc != 0 ) {
device_printf(ntb->device,
"unable to resize bar\n");
return (rc);
} else
save_bar_parameters(bar);
}
/* Mark bar region as write combining to improve performance. */
rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size,
VM_MEMATTR_WRITE_COMBINING);
if (rc != 0) {
device_printf(ntb->device, "unable to mark bar as"
" WRITE_COMBINING\n");
return (rc);
}
}
return (0);
}
static void
ntb_unmap_pci_bar(struct ntb_softc *ntb)
{
struct ntb_pci_bar_info *current_bar;
int i;
for (i = 0; i< NTB_MAX_BARS; i++) {
current_bar = &ntb->bar_info[i];
if (current_bar->pci_resource != NULL)
bus_release_resource(ntb->device, SYS_RES_MEMORY,
current_bar->pci_resource_id,
current_bar->pci_resource);
}
}
static int
ntb_setup_interrupts(struct ntb_softc *ntb)
{
void (*interrupt_handler)(void *);
void *int_arg;
bool use_msix = 0;
uint32_t num_vectors;
int i;
ntb->allocated_interrupts = 0;
/*
* On SOC, disable all interrupts. On XEON, disable all but Link
* Interrupt. The rest will be unmasked as callbacks are registered.
*/
if (ntb->type == NTB_SOC)
ntb_reg_write(8, ntb->reg_ofs.pdb_mask, ~0);
else
ntb_reg_write(2, ntb->reg_ofs.pdb_mask,
~(1 << ntb->limits.max_db_bits));
num_vectors = MIN(pci_msix_count(ntb->device),
ntb->limits.max_db_bits);
if (num_vectors >= 1) {
pci_alloc_msix(ntb->device, &num_vectors);
if (num_vectors >= 4)
use_msix = TRUE;
}
ntb_create_callbacks(ntb, num_vectors);
if (use_msix == TRUE) {
for (i = 0; i < num_vectors; i++) {
ntb->int_info[i].rid = i + 1;
ntb->int_info[i].res = bus_alloc_resource_any(
ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid,
RF_ACTIVE);
if (ntb->int_info[i].res == NULL) {
device_printf(ntb->device,
"bus_alloc_resource failed\n");
return (-1);
}
ntb->int_info[i].tag = NULL;
ntb->allocated_interrupts++;
if (ntb->type == NTB_SOC) {
interrupt_handler = handle_soc_irq;
int_arg = &ntb->db_cb[i];
} else {
if (i == num_vectors - 1) {
interrupt_handler =
handle_xeon_event_irq;
int_arg = ntb;
} else {
interrupt_handler =
handle_xeon_irq;
int_arg = &ntb->db_cb[i];
}
}
if (bus_setup_intr(ntb->device, ntb->int_info[i].res,
INTR_MPSAFE | INTR_TYPE_MISC, NULL,
interrupt_handler, int_arg,
&ntb->int_info[i].tag) != 0) {
device_printf(ntb->device,
"bus_setup_intr failed\n");
return (ENXIO);
}
}
}
else {
ntb->int_info[0].rid = 0;
ntb->int_info[0].res = bus_alloc_resource_any(ntb->device,
SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
interrupt_handler = ntb_handle_legacy_interrupt;
if (ntb->int_info[0].res == NULL) {
device_printf(ntb->device,
"bus_alloc_resource failed\n");
return (-1);
}
ntb->int_info[0].tag = NULL;
ntb->allocated_interrupts = 1;
if (bus_setup_intr(ntb->device, ntb->int_info[0].res,
INTR_MPSAFE | INTR_TYPE_MISC, NULL,
interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) {
device_printf(ntb->device, "bus_setup_intr failed\n");
return (ENXIO);
}
}
return (0);
}
static void
ntb_teardown_interrupts(struct ntb_softc *ntb)
{
struct ntb_int_info *current_int;
int i;
for (i=0; i<ntb->allocated_interrupts; i++) {
current_int = &ntb->int_info[i];
if (current_int->tag != NULL)
bus_teardown_intr(ntb->device, current_int->res,
current_int->tag);
if (current_int->res != NULL)
bus_release_resource(ntb->device, SYS_RES_IRQ,
rman_get_rid(current_int->res), current_int->res);
}
ntb_free_callbacks(ntb);
pci_release_msi(ntb->device);
}
static void
handle_soc_irq(void *arg)
{
struct ntb_db_cb *db_cb = arg;
struct ntb_softc *ntb = db_cb->ntb;
ntb_reg_write(8, ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num);
if (db_cb->callback != NULL)
db_cb->callback(db_cb->data, db_cb->db_num);
}
static void
handle_xeon_irq(void *arg)
{
struct ntb_db_cb *db_cb = arg;
struct ntb_softc *ntb = db_cb->ntb;
/*
* On Xeon, there are 16 bits in the interrupt register
* but only 4 vectors. So, 5 bits are assigned to the first 3
* vectors, with the 4th having a single bit for link
* interrupts.
*/
ntb_reg_write(2, ntb->reg_ofs.pdb,
((1 << ntb->bits_per_vector) - 1) <<
(db_cb->db_num * ntb->bits_per_vector));
if (db_cb->callback != NULL)
db_cb->callback(db_cb->data, db_cb->db_num);
}
/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */
static void
handle_xeon_event_irq(void *arg)
{
struct ntb_softc *ntb = arg;
int rc;
rc = ntb_check_link_status(ntb);
if (rc != 0)
device_printf(ntb->device, "Error determining link status\n");
/* bit 15 is always the link bit */
ntb_reg_write(2, ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits);
}
static void
ntb_handle_legacy_interrupt(void *arg)
{
struct ntb_softc *ntb = arg;
unsigned int i = 0;
uint64_t pdb64;
uint16_t pdb16;
if (ntb->type == NTB_SOC) {
pdb64 = ntb_reg_read(8, ntb->reg_ofs.pdb);
while (pdb64) {
i = ffs(pdb64);
pdb64 &= pdb64 - 1;
handle_soc_irq(&ntb->db_cb[i]);
}
} else {
pdb16 = ntb_reg_read(2, ntb->reg_ofs.pdb);
if ((pdb16 & XEON_DB_HW_LINK) != 0) {
handle_xeon_event_irq(ntb);
pdb16 &= ~XEON_DB_HW_LINK;
}
while (pdb16 != 0) {
i = ffs(pdb16);
pdb16 &= pdb16 - 1;
handle_xeon_irq(&ntb->db_cb[i]);
}
}
}
static int
ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors)
{
int i;
ntb->db_cb = malloc(num_vectors * sizeof(struct ntb_db_cb), M_NTB,
M_ZERO | M_WAITOK);
for (i = 0; i < num_vectors; i++) {
ntb->db_cb[i].db_num = i;
ntb->db_cb[i].ntb = ntb;
}
return (0);
}
static void
ntb_free_callbacks(struct ntb_softc *ntb)
{
int i;
for (i = 0; i < ntb->limits.max_db_bits; i++)
ntb_unregister_db_callback(ntb, i);
free(ntb->db_cb, M_NTB);
}
static struct ntb_hw_info *
ntb_get_device_info(uint32_t device_id)
{
struct ntb_hw_info *ep = pci_ids;
while (ep->device_id) {
if (ep->device_id == device_id)
return (ep);
++ep;
}
return (NULL);
}
static int
ntb_initialize_hw(struct ntb_softc *ntb)
{
if (ntb->type == NTB_SOC)
return (ntb_setup_soc(ntb));
else
return (ntb_setup_xeon(ntb));
}
static int
ntb_setup_xeon(struct ntb_softc *ntb)
{
uint8_t val, connection_type;
val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
connection_type = val & XEON_PPD_CONN_TYPE;
switch (connection_type) {
case NTB_CONN_B2B:
ntb->conn_type = NTB_CONN_B2B;
break;
case NTB_CONN_CLASSIC:
case NTB_CONN_RP:
default:
device_printf(ntb->device, "Connection type %d not supported\n",
connection_type);
return (ENXIO);
}
if ((val & XEON_PPD_DEV_TYPE) != 0)
ntb->dev_type = NTB_DEV_DSD;
else
ntb->dev_type = NTB_DEV_USD;
ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET;
ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET;
ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET;
ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET;
ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET;
ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET;
ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET;
ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET;
if (ntb->conn_type == NTB_CONN_B2B) {
ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
ntb->limits.max_spads = XEON_MAX_SPADS;
} else {
ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET;
ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS;
}
ntb->limits.max_db_bits = XEON_MAX_DB_BITS;
ntb->limits.msix_cnt = XEON_MSIX_CNT;
ntb->bits_per_vector = XEON_DB_BITS_PER_VEC;
configure_xeon_secondary_side_bars(ntb);
/* Enable Bus Master and Memory Space on the secondary side */
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
/* Enable link training */
ntb_reg_write(4, ntb->reg_ofs.lnk_cntl,
NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);
return (0);
}
static int
ntb_setup_soc(struct ntb_softc *ntb)
{
uint32_t val, connection_type;
val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
connection_type = (val & SOC_PPD_CONN_TYPE) >> 8;
switch (connection_type) {
case NTB_CONN_B2B:
ntb->conn_type = NTB_CONN_B2B;
break;
case NTB_CONN_RP:
default:
device_printf(ntb->device, "Connection type %d not supported\n",
connection_type);
return (ENXIO);
}
if ((val & SOC_PPD_DEV_TYPE) != 0)
ntb->dev_type = NTB_DEV_DSD;
else
ntb->dev_type = NTB_DEV_USD;
/* Initiate PCI-E link training */
pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK,
4);
ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET;
ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET;
ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET;
ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET;
ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET;
ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET;
ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET;
ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET;
if (ntb->conn_type == NTB_CONN_B2B) {
ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET;
ntb->limits.max_spads = SOC_MAX_SPADS;
} else {
ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET;
ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS;
}
ntb->limits.max_db_bits = SOC_MAX_DB_BITS;
ntb->limits.msix_cnt = SOC_MSIX_CNT;
ntb->bits_per_vector = SOC_DB_BITS_PER_VEC;
/*
* FIXME - MSI-X bug on early SOC HW, remove once internal issue is
* resolved. Mask transaction layer internal parity errors.
*/
pci_write_config(ntb->device, 0xFC, 0x4, 4);
configure_soc_secondary_side_bars(ntb);
/* Enable Bus Master and Memory Space on the secondary side */
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb);
return (0);
}
static void
configure_soc_secondary_side_bars(struct ntb_softc *ntb)
{
if (ntb->dev_type == NTB_DEV_USD) {
ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR);
ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR);
ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR);
} else {
ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR);
ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR);
ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR);
ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR);
}
}
static void
configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
{
if (ntb->dev_type == NTB_DEV_USD) {
ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
if (HAS_FEATURE(NTB_REGS_THRU_MW))
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
MBAR01_DSD_ADDR);
else
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
PBAR4XLAT_USD_ADDR);
ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR);
ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR);
ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR);
} else {
ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR);
if (HAS_FEATURE(NTB_REGS_THRU_MW))
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
MBAR01_USD_ADDR);
else
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
PBAR4XLAT_DSD_ADDR);
ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR);
ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR);
ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR);
}
}
/* SOC does not have link status interrupt, poll on that platform */
static void
ntb_handle_heartbeat(void *arg)
{
struct ntb_softc *ntb = arg;
uint32_t status32;
int rc = ntb_check_link_status(ntb);
if (rc != 0)
device_printf(ntb->device,
"Error determining link status\n");
/* Check to see if a link error is the cause of the link down */
if (ntb->link_status == NTB_LINK_DOWN) {
status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) {
callout_reset(&ntb->lr_timer, 0, recover_soc_link,
ntb);
return;
}
}
callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
ntb_handle_heartbeat, ntb);
}
static void
soc_perform_link_restart(struct ntb_softc *ntb)
{
uint32_t status;
/* Driver resets the NTB ModPhy lanes - magic! */
ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0);
ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40);
ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60);
ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60);
/* Driver waits 100ms to allow the NTB ModPhy to settle */
pause("ModPhy", hz / 10);
/* Clear AER Errors, write to clear */
status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET);
status &= PCIM_AER_COR_REPLAY_ROLLOVER;
ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status);
/* Clear unexpected electrical idle event in LTSSM, write to clear */
status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET);
status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI;
ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status);
/* Clear DeSkew Buffer error, write to clear */
status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET);
status |= SOC_DESKEWSTS_DBERR;
ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status);
status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
status &= SOC_IBIST_ERR_OFLOW;
ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status);
/* Releases the NTB state machine to allow the link to retrain */
status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT;
ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status);
}
static void
ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
{
enum ntb_hw_event event;
uint16_t status;
if (ntb->link_status == link_state)
return;
if (link_state == NTB_LINK_UP) {
device_printf(ntb->device, "Link Up\n");
ntb->link_status = NTB_LINK_UP;
event = NTB_EVENT_HW_LINK_UP;
if (ntb->type == NTB_SOC)
status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat);
else
status = pci_read_config(ntb->device,
XEON_LINK_STATUS_OFFSET, 2);
ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
ntb->link_speed = (status & NTB_LINK_SPEED_MASK);
device_printf(ntb->device, "Link Width %d, Link Speed %d\n",
ntb->link_width, ntb->link_speed);
callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
ntb_handle_heartbeat, ntb);
} else {
device_printf(ntb->device, "Link Down\n");
ntb->link_status = NTB_LINK_DOWN;
event = NTB_EVENT_HW_LINK_DOWN;
/* Do not modify link width/speed, we need it in link recovery */
}
/* notify the upper layer if we have an event change */
if (ntb->event_cb != NULL)
ntb->event_cb(ntb->ntb_transport, event);
}
static void
recover_soc_link(void *arg)
{
struct ntb_softc *ntb = arg;
uint8_t speed, width;
uint32_t status32;
uint16_t status16;
soc_perform_link_restart(ntb);
pause("Link", SOC_LINK_RECOVERY_TIME * hz / 1000);
status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0)
goto retry;
status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
if ((status32 & SOC_IBIST_ERR_OFLOW) != 0)
goto retry;
status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat);
width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
speed = (status16 & NTB_LINK_SPEED_MASK);
if (ntb->link_width != width || ntb->link_speed != speed)
goto retry;
callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
ntb_handle_heartbeat, ntb);
return;
retry:
callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link,
ntb);
}
static int
ntb_check_link_status(struct ntb_softc *ntb)
{
int link_state;
uint32_t ntb_cntl;
uint16_t status;
if (ntb->type == NTB_SOC) {
ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0)
link_state = NTB_LINK_DOWN;
else
link_state = NTB_LINK_UP;
} else {
status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET,
2);
if ((status & NTB_LINK_STATUS_ACTIVE) != 0)
link_state = NTB_LINK_UP;
else
link_state = NTB_LINK_DOWN;
}
ntb_handle_link_event(ntb, link_state);
return (0);
}
/**
* ntb_register_event_callback() - register event callback
* @ntb: pointer to ntb_softc instance
* @func: callback function to register
*
* This function registers a callback for any HW driver events such as link
* up/down, power management notices and etc.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int
ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func)
{
if (ntb->event_cb != NULL)
return (EINVAL);
ntb->event_cb = func;
return (0);
}
/**
* ntb_unregister_event_callback() - unregisters the event callback
* @ntb: pointer to ntb_softc instance
*
* This function unregisters the existing callback from transport
*/
void
ntb_unregister_event_callback(struct ntb_softc *ntb)
{
ntb->event_cb = NULL;
}
/**
* ntb_register_db_callback() - register a callback for doorbell interrupt
* @ntb: pointer to ntb_softc instance
* @idx: doorbell index to register callback, zero based
* @func: callback function to register
*
* This function registers a callback function for the doorbell interrupt
* on the primary side. The function will unmask the doorbell as well to
* allow interrupt.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int
ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
ntb_db_callback func)
{
uint16_t mask;
if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) {
device_printf(ntb->device, "Invalid Index.\n");
return (EINVAL);
}
ntb->db_cb[idx].callback = func;
ntb->db_cb[idx].data = data;
/* unmask interrupt */
mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask);
mask &= ~(1 << (idx * ntb->bits_per_vector));
ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask);
return (0);
}
/**
* ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
* @ntb: pointer to ntb_softc instance
* @idx: doorbell index to register callback, zero based
*
* This function unregisters a callback function for the doorbell interrupt
* on the primary side. The function will also mask the said doorbell.
*/
void
ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx)
{
unsigned long mask;
if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback)
return;
mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask);
mask |= 1 << (idx * ntb->bits_per_vector);
ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask);
ntb->db_cb[idx].callback = NULL;
}
/**
* ntb_find_transport() - find the transport pointer
* @transport: pointer to pci device
*
* Given the pci device pointer, return the transport pointer passed in when
* the transport attached when it was inited.
*
* RETURNS: pointer to transport.
*/
void *
ntb_find_transport(struct ntb_softc *ntb)
{
return (ntb->ntb_transport);
}
/**
* ntb_register_transport() - Register NTB transport with NTB HW driver
* @transport: transport identifier
*
* This function allows a transport to reserve the hardware driver for
* NTB usage.
*
* RETURNS: pointer to ntb_softc, NULL on error.
*/
struct ntb_softc *
ntb_register_transport(struct ntb_softc *ntb, void *transport)
{
/*
* TODO: when we have more than one transport, we will need to rewrite
* this to prevent race conditions
*/
if (ntb->ntb_transport != NULL)
return (NULL);
ntb->ntb_transport = transport;
return (ntb);
}
/**
* ntb_unregister_transport() - Unregister the transport with the NTB HW driver
* @ntb - ntb_softc of the transport to be freed
*
* This function unregisters the transport from the HW driver and performs any
* necessary cleanups.
*/
void
ntb_unregister_transport(struct ntb_softc *ntb)
{
int i;
if (ntb->ntb_transport == NULL)
return;
for (i = 0; i < ntb->allocated_interrupts; i++)
ntb_unregister_db_callback(ntb, i);
ntb_unregister_event_callback(ntb);
ntb->ntb_transport = NULL;
}
/**
* ntb_get_max_spads() - get the total scratch regs usable
* @ntb: pointer to ntb_softc instance
*
* This function returns the max 32bit scratchpad registers usable by the
* upper layer.
*
* RETURNS: total number of scratch pad registers available
*/
int
ntb_get_max_spads(struct ntb_softc *ntb)
{
return (ntb->limits.max_spads);
}
/**
* ntb_write_local_spad() - write to the secondary scratchpad register
* @ntb: pointer to ntb_softc instance
* @idx: index to the scratchpad register, 0 based
* @val: the data value to put into the register
*
* This function allows writing of a 32bit value to the indexed scratchpad
* register. The register resides on the secondary (external) side.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int
ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
{
if (idx >= ntb->limits.max_spads)
return (EINVAL);
ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val);
return (0);
}
/**
* ntb_read_local_spad() - read from the primary scratchpad register
* @ntb: pointer to ntb_softc instance
* @idx: index to scratchpad register, 0 based
* @val: pointer to 32bit integer for storing the register value
*
* This function allows reading of the 32bit scratchpad register on
* the primary (internal) side.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int
ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
{
if (idx >= ntb->limits.max_spads)
return (EINVAL);
*val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4);
return (0);
}
/**
* ntb_write_remote_spad() - write to the secondary scratchpad register
* @ntb: pointer to ntb_softc instance
* @idx: index to the scratchpad register, 0 based
* @val: the data value to put into the register
*
* This function allows writing of a 32bit value to the indexed scratchpad
* register. The register resides on the secondary (external) side.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int
ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
{
if (idx >= ntb->limits.max_spads)
return (EINVAL);
if (HAS_FEATURE(NTB_REGS_THRU_MW))
ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val);
else
ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val);
return (0);
}
/**
* ntb_read_remote_spad() - read from the primary scratchpad register
* @ntb: pointer to ntb_softc instance
* @idx: index to scratchpad register, 0 based
* @val: pointer to 32bit integer for storing the register value
*
* This function allows reading of the 32bit scratchpad register on
* the primary (internal) side.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int
ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
{
if (idx >= ntb->limits.max_spads)
return (EINVAL);
if (HAS_FEATURE(NTB_REGS_THRU_MW))
*val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4);
else
*val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4);
return (0);
}
/**
* ntb_get_mw_vbase() - get virtual addr for the NTB memory window
* @ntb: pointer to ntb_softc instance
* @mw: memory window number
*
* This function provides the base virtual address of the memory window
* specified.
*
* RETURNS: pointer to virtual address, or NULL on error.
*/
void *
ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw)
{
if (mw >= NTB_NUM_MW)
return (NULL);
return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase);
}
vm_paddr_t
ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw)
{
if (mw >= NTB_NUM_MW)
return (0);
return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase);
}
/**
* ntb_get_mw_size() - return size of NTB memory window
* @ntb: pointer to ntb_softc instance
* @mw: memory window number
*
* This function provides the physical size of the memory window specified
*
* RETURNS: the size of the memory window or zero on error
*/
u_long
ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw)
{
if (mw >= NTB_NUM_MW)
return (0);
return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size);
}
/**
* ntb_set_mw_addr - set the memory window address
* @ntb: pointer to ntb_softc instance
* @mw: memory window number
* @addr: base address for data
*
* This function sets the base physical address of the memory window. This
* memory address is where data from the remote system will be transfered into
* or out of depending on how the transport is configured.
*/
void
ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
{
if (mw >= NTB_NUM_MW)
return;
switch (NTB_MW_TO_BAR(mw)) {
case NTB_B2B_BAR_1:
ntb_reg_write(8, ntb->reg_ofs.sbar2_xlat, addr);
break;
case NTB_B2B_BAR_2:
ntb_reg_write(8, ntb->reg_ofs.sbar4_xlat, addr);
break;
}
}
/**
* ntb_ring_sdb() - Set the doorbell on the secondary/external side
* @ntb: pointer to ntb_softc instance
* @db: doorbell to ring
*
* This function allows triggering of a doorbell on the secondary/external
* side that will initiate an interrupt on the remote host
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
void
ntb_ring_sdb(struct ntb_softc *ntb, unsigned int db)
{
if (ntb->type == NTB_SOC)
ntb_reg_write(8, ntb->reg_ofs.sdb, (uint64_t) 1 << db);
else
if (HAS_FEATURE(NTB_REGS_THRU_MW))
ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET,
((1 << ntb->bits_per_vector) - 1) <<
(db * ntb->bits_per_vector));
else
ntb_reg_write(2, ntb->reg_ofs.sdb,
((1 << ntb->bits_per_vector) - 1) <<
(db * ntb->bits_per_vector));
}
/**
* ntb_query_link_status() - return the hardware link status
* @ndev: pointer to ntb_device instance
*
* Returns true if the hardware is connected to the remote system
*
* RETURNS: true or false based on the hardware link state
*/
bool
ntb_query_link_status(struct ntb_softc *ntb)
{
return (ntb->link_status == NTB_LINK_UP);
}
static void
save_bar_parameters(struct ntb_pci_bar_info *bar)
{
bar->pci_bus_tag =
rman_get_bustag(bar->pci_resource);
bar->pci_bus_handle =
rman_get_bushandle(bar->pci_resource);
bar->pbase =
rman_get_start(bar->pci_resource);
bar->size =
rman_get_size(bar->pci_resource);
bar->vbase =
rman_get_virtual(bar->pci_resource);
}
device_t ntb_get_device(struct ntb_softc *ntb)
{
return (ntb->device);
}
Index: head/sys/dev/nxge/if_nxge.c
===================================================================
--- head/sys/dev/nxge/if_nxge.c (revision 283290)
+++ head/sys/dev/nxge/if_nxge.c (revision 283291)
@@ -1,3522 +1,3522 @@
/*-
* Copyright (c) 2002-2007 Neterion, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include <dev/nxge/if_nxge.h>
#include <dev/nxge/xge-osdep.h>
#include <net/if_arp.h>
#include <sys/types.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_vlan_var.h>
int copyright_print = 0;
int hal_driver_init_count = 0;
size_t size = sizeof(int);
static void inline xge_flush_txds(xge_hal_channel_h);
/**
* xge_probe
* Probes for Xframe devices
*
* @dev Device handle
*
* Returns
* BUS_PROBE_DEFAULT if device is supported
* ENXIO if device is not supported
*/
int
xge_probe(device_t dev)
{
int devid = pci_get_device(dev);
int vendorid = pci_get_vendor(dev);
int retValue = ENXIO;
if(vendorid == XGE_PCI_VENDOR_ID) {
if((devid == XGE_PCI_DEVICE_ID_XENA_2) ||
(devid == XGE_PCI_DEVICE_ID_HERC_2)) {
if(!copyright_print) {
xge_os_printf(XGE_COPYRIGHT);
copyright_print = 1;
}
device_set_desc_copy(dev,
"Neterion Xframe 10 Gigabit Ethernet Adapter");
retValue = BUS_PROBE_DEFAULT;
}
}
return retValue;
}
/**
* xge_init_params
* Sets HAL parameter values (from kenv).
*
* @dconfig Device Configuration
* @dev Device Handle
*/
void
xge_init_params(xge_hal_device_config_t *dconfig, device_t dev)
{
int qindex, tindex, revision;
device_t checkdev;
xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
dconfig->mtu = XGE_DEFAULT_INITIAL_MTU;
dconfig->pci_freq_mherz = XGE_DEFAULT_USER_HARDCODED;
dconfig->device_poll_millis = XGE_HAL_DEFAULT_DEVICE_POLL_MILLIS;
dconfig->link_stability_period = XGE_HAL_DEFAULT_LINK_STABILITY_PERIOD;
dconfig->mac.rmac_bcast_en = XGE_DEFAULT_MAC_RMAC_BCAST_EN;
dconfig->fifo.alignment_size = XGE_DEFAULT_FIFO_ALIGNMENT_SIZE;
XGE_GET_PARAM("hw.xge.enable_tso", (*lldev), enabled_tso,
XGE_DEFAULT_ENABLED_TSO);
XGE_GET_PARAM("hw.xge.enable_lro", (*lldev), enabled_lro,
XGE_DEFAULT_ENABLED_LRO);
XGE_GET_PARAM("hw.xge.enable_msi", (*lldev), enabled_msi,
XGE_DEFAULT_ENABLED_MSI);
XGE_GET_PARAM("hw.xge.latency_timer", (*dconfig), latency_timer,
XGE_DEFAULT_LATENCY_TIMER);
XGE_GET_PARAM("hw.xge.max_splits_trans", (*dconfig), max_splits_trans,
XGE_DEFAULT_MAX_SPLITS_TRANS);
XGE_GET_PARAM("hw.xge.mmrb_count", (*dconfig), mmrb_count,
XGE_DEFAULT_MMRB_COUNT);
XGE_GET_PARAM("hw.xge.shared_splits", (*dconfig), shared_splits,
XGE_DEFAULT_SHARED_SPLITS);
XGE_GET_PARAM("hw.xge.isr_polling_cnt", (*dconfig), isr_polling_cnt,
XGE_DEFAULT_ISR_POLLING_CNT);
XGE_GET_PARAM("hw.xge.stats_refresh_time_sec", (*dconfig),
stats_refresh_time_sec, XGE_DEFAULT_STATS_REFRESH_TIME_SEC);
XGE_GET_PARAM_MAC("hw.xge.mac_tmac_util_period", tmac_util_period,
XGE_DEFAULT_MAC_TMAC_UTIL_PERIOD);
XGE_GET_PARAM_MAC("hw.xge.mac_rmac_util_period", rmac_util_period,
XGE_DEFAULT_MAC_RMAC_UTIL_PERIOD);
XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_gen_en", rmac_pause_gen_en,
XGE_DEFAULT_MAC_RMAC_PAUSE_GEN_EN);
XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_rcv_en", rmac_pause_rcv_en,
XGE_DEFAULT_MAC_RMAC_PAUSE_RCV_EN);
XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_time", rmac_pause_time,
XGE_DEFAULT_MAC_RMAC_PAUSE_TIME);
XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q0q3",
mc_pause_threshold_q0q3, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q0Q3);
XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q4q7",
mc_pause_threshold_q4q7, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q4Q7);
XGE_GET_PARAM_FIFO("hw.xge.fifo_memblock_size", memblock_size,
XGE_DEFAULT_FIFO_MEMBLOCK_SIZE);
XGE_GET_PARAM_FIFO("hw.xge.fifo_reserve_threshold", reserve_threshold,
XGE_DEFAULT_FIFO_RESERVE_THRESHOLD);
XGE_GET_PARAM_FIFO("hw.xge.fifo_max_frags", max_frags,
XGE_DEFAULT_FIFO_MAX_FRAGS);
for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_intr", intr, qindex,
XGE_DEFAULT_FIFO_QUEUE_INTR);
XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_max", max, qindex,
XGE_DEFAULT_FIFO_QUEUE_MAX);
XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_initial", initial,
qindex, XGE_DEFAULT_FIFO_QUEUE_INITIAL);
for (tindex = 0; tindex < XGE_HAL_MAX_FIFO_TTI_NUM; tindex++) {
dconfig->fifo.queue[qindex].tti[tindex].enabled = 1;
dconfig->fifo.queue[qindex].configured = 1;
XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_a",
urange_a, qindex, tindex,
XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_A);
XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_b",
urange_b, qindex, tindex,
XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_B);
XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_c",
urange_c, qindex, tindex,
XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_C);
XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_a",
ufc_a, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_A);
XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_b",
ufc_b, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_B);
XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_c",
ufc_c, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_C);
XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_d",
ufc_d, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_D);
XGE_GET_PARAM_FIFO_QUEUE_TTI(
"hw.xge.fifo_queue_tti_timer_ci_en", timer_ci_en, qindex,
tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_CI_EN);
XGE_GET_PARAM_FIFO_QUEUE_TTI(
"hw.xge.fifo_queue_tti_timer_ac_en", timer_ac_en, qindex,
tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_AC_EN);
XGE_GET_PARAM_FIFO_QUEUE_TTI(
"hw.xge.fifo_queue_tti_timer_val_us", timer_val_us, qindex,
tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_VAL_US);
}
}
XGE_GET_PARAM_RING("hw.xge.ring_memblock_size", memblock_size,
XGE_DEFAULT_RING_MEMBLOCK_SIZE);
XGE_GET_PARAM_RING("hw.xge.ring_strip_vlan_tag", strip_vlan_tag,
XGE_DEFAULT_RING_STRIP_VLAN_TAG);
XGE_GET_PARAM("hw.xge.buffer_mode", (*lldev), buffer_mode,
XGE_DEFAULT_BUFFER_MODE);
if((lldev->buffer_mode < XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ||
(lldev->buffer_mode > XGE_HAL_RING_QUEUE_BUFFER_MODE_2)) {
xge_trace(XGE_ERR, "Supported buffer modes are 1 and 2");
lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_1;
}
for (qindex = 0; qindex < XGE_RING_COUNT; qindex++) {
dconfig->ring.queue[qindex].max_frm_len = XGE_HAL_RING_USE_MTU;
dconfig->ring.queue[qindex].priority = 0;
dconfig->ring.queue[qindex].configured = 1;
dconfig->ring.queue[qindex].buffer_mode =
(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) ?
XGE_HAL_RING_QUEUE_BUFFER_MODE_3 : lldev->buffer_mode;
XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_max", max, qindex,
XGE_DEFAULT_RING_QUEUE_MAX);
XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_initial", initial,
qindex, XGE_DEFAULT_RING_QUEUE_INITIAL);
XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_dram_size_mb",
dram_size_mb, qindex, XGE_DEFAULT_RING_QUEUE_DRAM_SIZE_MB);
XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_indicate_max_pkts",
indicate_max_pkts, qindex,
XGE_DEFAULT_RING_QUEUE_INDICATE_MAX_PKTS);
XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_backoff_interval_us",
backoff_interval_us, qindex,
XGE_DEFAULT_RING_QUEUE_BACKOFF_INTERVAL_US);
XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_a", ufc_a,
qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_A);
XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_b", ufc_b,
qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_B);
XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_c", ufc_c,
qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_C);
XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_d", ufc_d,
qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_D);
XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_ac_en",
timer_ac_en, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_AC_EN);
XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_val_us",
timer_val_us, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_VAL_US);
XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_a",
urange_a, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_A);
XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_b",
urange_b, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_B);
XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_c",
urange_c, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_C);
}
if(dconfig->fifo.max_frags > (PAGE_SIZE/32)) {
xge_os_printf("fifo_max_frags = %d", dconfig->fifo.max_frags)
xge_os_printf("fifo_max_frags should be <= (PAGE_SIZE / 32) = %d",
(int)(PAGE_SIZE / 32))
xge_os_printf("Using fifo_max_frags = %d", (int)(PAGE_SIZE / 32))
dconfig->fifo.max_frags = (PAGE_SIZE / 32);
}
checkdev = pci_find_device(VENDOR_ID_AMD, DEVICE_ID_8131_PCI_BRIDGE);
if(checkdev != NULL) {
/* Check Revision for 0x12 */
revision = pci_read_config(checkdev,
xge_offsetof(xge_hal_pci_config_t, revision), 1);
if(revision <= 0x12) {
/* Set mmrb_count to 1k and max splits = 2 */
dconfig->mmrb_count = 1;
dconfig->max_splits_trans = XGE_HAL_THREE_SPLIT_TRANSACTION;
}
}
}
/**
* xge_buffer_sizes_set
* Set buffer sizes based on Rx buffer mode
*
* @lldev Per-adapter Data
* @buffer_mode Rx Buffer Mode
*/
void
xge_rx_buffer_sizes_set(xge_lldev_t *lldev, int buffer_mode, int mtu)
{
int index = 0;
int frame_header = XGE_HAL_MAC_HEADER_MAX_SIZE;
int buffer_size = mtu + frame_header;
xge_os_memzero(lldev->rxd_mbuf_len, sizeof(lldev->rxd_mbuf_len));
if(buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
lldev->rxd_mbuf_len[buffer_mode - 1] = mtu;
lldev->rxd_mbuf_len[0] = (buffer_mode == 1) ? buffer_size:frame_header;
if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
lldev->rxd_mbuf_len[1] = XGE_HAL_TCPIP_HEADER_MAX_SIZE;
if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
index = 2;
buffer_size -= XGE_HAL_TCPIP_HEADER_MAX_SIZE;
while(buffer_size > MJUMPAGESIZE) {
lldev->rxd_mbuf_len[index++] = MJUMPAGESIZE;
buffer_size -= MJUMPAGESIZE;
}
XGE_ALIGN_TO(buffer_size, 128);
lldev->rxd_mbuf_len[index] = buffer_size;
lldev->rxd_mbuf_cnt = index + 1;
}
for(index = 0; index < buffer_mode; index++)
xge_trace(XGE_TRACE, "Buffer[%d] %d\n", index,
lldev->rxd_mbuf_len[index]);
}
/**
* xge_buffer_mode_init
* Init Rx buffer mode
*
* @lldev Per-adapter Data
* @mtu Interface MTU
*/
void
xge_buffer_mode_init(xge_lldev_t *lldev, int mtu)
{
int index = 0, buffer_size = 0;
xge_hal_ring_config_t *ring_config = &((lldev->devh)->config.ring);
buffer_size = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE;
if(lldev->enabled_lro)
(lldev->ifnetp)->if_capenable |= IFCAP_LRO;
else
(lldev->ifnetp)->if_capenable &= ~IFCAP_LRO;
lldev->rxd_mbuf_cnt = lldev->buffer_mode;
if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
XGE_SET_BUFFER_MODE_IN_RINGS(XGE_HAL_RING_QUEUE_BUFFER_MODE_3);
ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_B;
}
else {
XGE_SET_BUFFER_MODE_IN_RINGS(lldev->buffer_mode);
ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_A;
}
xge_rx_buffer_sizes_set(lldev, lldev->buffer_mode, mtu);
xge_os_printf("%s: TSO %s", device_get_nameunit(lldev->device),
((lldev->enabled_tso) ? "Enabled":"Disabled"));
xge_os_printf("%s: LRO %s", device_get_nameunit(lldev->device),
((lldev->ifnetp)->if_capenable & IFCAP_LRO) ? "Enabled":"Disabled");
xge_os_printf("%s: Rx %d Buffer Mode Enabled",
device_get_nameunit(lldev->device), lldev->buffer_mode);
}
/**
* xge_driver_initialize
* Initializes HAL driver (common for all devices)
*
* Returns
* XGE_HAL_OK if success
* XGE_HAL_ERR_BAD_DRIVER_CONFIG if driver configuration parameters are invalid
*/
int
xge_driver_initialize(void)
{
xge_hal_uld_cbs_t uld_callbacks;
xge_hal_driver_config_t driver_config;
xge_hal_status_e status = XGE_HAL_OK;
/* Initialize HAL driver */
if(!hal_driver_init_count) {
xge_os_memzero(&uld_callbacks, sizeof(xge_hal_uld_cbs_t));
xge_os_memzero(&driver_config, sizeof(xge_hal_driver_config_t));
/*
* Initial and maximum size of the queue used to store the events
* like Link up/down (xge_hal_event_e)
*/
driver_config.queue_size_initial = XGE_HAL_MIN_QUEUE_SIZE_INITIAL;
driver_config.queue_size_max = XGE_HAL_MAX_QUEUE_SIZE_MAX;
uld_callbacks.link_up = xge_callback_link_up;
uld_callbacks.link_down = xge_callback_link_down;
uld_callbacks.crit_err = xge_callback_crit_err;
uld_callbacks.event = xge_callback_event;
status = xge_hal_driver_initialize(&driver_config, &uld_callbacks);
if(status != XGE_HAL_OK) {
XGE_EXIT_ON_ERR("xgeX: Initialization of HAL driver failed",
xdi_out, status);
}
}
hal_driver_init_count = hal_driver_init_count + 1;
xge_hal_driver_debug_module_mask_set(0xffffffff);
xge_hal_driver_debug_level_set(XGE_TRACE);
xdi_out:
return status;
}
/**
* xge_media_init
* Initializes, adds and sets media
*
* @devc Device Handle
*/
void
xge_media_init(device_t devc)
{
xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(devc);
/* Initialize Media */
ifmedia_init(&lldev->media, IFM_IMASK, xge_ifmedia_change,
xge_ifmedia_status);
/* Add supported media */
ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 0, NULL);
ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
ifmedia_add(&lldev->media, IFM_ETHER | IFM_AUTO, 0, NULL);
ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
/* Set media */
ifmedia_set(&lldev->media, IFM_ETHER | IFM_AUTO);
}
/**
* xge_pci_space_save
* Save PCI configuration space
*
* @dev Device Handle
*/
void
xge_pci_space_save(device_t dev)
{
struct pci_devinfo *dinfo = NULL;
dinfo = device_get_ivars(dev);
xge_trace(XGE_TRACE, "Saving PCI configuration space");
pci_cfg_save(dev, dinfo, 0);
}
/**
* xge_pci_space_restore
* Restore saved PCI configuration space
*
* @dev Device Handle
*/
void
xge_pci_space_restore(device_t dev)
{
struct pci_devinfo *dinfo = NULL;
dinfo = device_get_ivars(dev);
xge_trace(XGE_TRACE, "Restoring PCI configuration space");
pci_cfg_restore(dev, dinfo);
}
/**
* xge_msi_info_save
* Save MSI info
*
* @lldev Per-adapter Data
*/
void
xge_msi_info_save(xge_lldev_t * lldev)
{
xge_os_pci_read16(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_control),
&lldev->msi_info.msi_control);
xge_os_pci_read32(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address),
&lldev->msi_info.msi_lower_address);
xge_os_pci_read32(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address),
&lldev->msi_info.msi_higher_address);
xge_os_pci_read16(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_data),
&lldev->msi_info.msi_data);
}
/**
* xge_msi_info_restore
* Restore saved MSI info
*
* @dev Device Handle
*/
void
xge_msi_info_restore(xge_lldev_t *lldev)
{
/*
* If interface is made down and up, traffic fails. It was observed that
* MSI information were getting reset on down. Restoring them.
*/
xge_os_pci_write16(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_control),
lldev->msi_info.msi_control);
xge_os_pci_write32(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address),
lldev->msi_info.msi_lower_address);
xge_os_pci_write32(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address),
lldev->msi_info.msi_higher_address);
xge_os_pci_write16(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_data),
lldev->msi_info.msi_data);
}
/**
* xge_init_mutex
* Initializes mutexes used in driver
*
* @lldev Per-adapter Data
*/
void
xge_mutex_init(xge_lldev_t *lldev)
{
int qindex;
sprintf(lldev->mtx_name_drv, "%s_drv",
device_get_nameunit(lldev->device));
mtx_init(&lldev->mtx_drv, lldev->mtx_name_drv, MTX_NETWORK_LOCK,
MTX_DEF);
for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
sprintf(lldev->mtx_name_tx[qindex], "%s_tx_%d",
device_get_nameunit(lldev->device), qindex);
mtx_init(&lldev->mtx_tx[qindex], lldev->mtx_name_tx[qindex], NULL,
MTX_DEF);
}
}
/**
* xge_mutex_destroy
* Destroys mutexes used in driver
*
* @lldev Per-adapter Data
*/
void
xge_mutex_destroy(xge_lldev_t *lldev)
{
int qindex;
for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++)
mtx_destroy(&lldev->mtx_tx[qindex]);
mtx_destroy(&lldev->mtx_drv);
}
/**
* xge_print_info
* Print device and driver information
*
* @lldev Per-adapter Data
*/
void
xge_print_info(xge_lldev_t *lldev)
{
device_t dev = lldev->device;
xge_hal_device_t *hldev = lldev->devh;
xge_hal_status_e status = XGE_HAL_OK;
u64 val64 = 0;
const char *xge_pci_bus_speeds[17] = {
"PCI 33MHz Bus",
"PCI 66MHz Bus",
"PCIX(M1) 66MHz Bus",
"PCIX(M1) 100MHz Bus",
"PCIX(M1) 133MHz Bus",
"PCIX(M2) 133MHz Bus",
"PCIX(M2) 200MHz Bus",
"PCIX(M2) 266MHz Bus",
"PCIX(M1) Reserved",
"PCIX(M1) 66MHz Bus (Not Supported)",
"PCIX(M1) 100MHz Bus (Not Supported)",
"PCIX(M1) 133MHz Bus (Not Supported)",
"PCIX(M2) Reserved",
"PCIX 533 Reserved",
"PCI Basic Mode",
"PCIX Basic Mode",
"PCI Invalid Mode"
};
xge_os_printf("%s: Xframe%s %s Revision %d Driver v%s",
device_get_nameunit(dev),
((hldev->device_id == XGE_PCI_DEVICE_ID_XENA_2) ? "I" : "II"),
hldev->vpd_data.product_name, hldev->revision, XGE_DRIVER_VERSION);
xge_os_printf("%s: Serial Number %s",
device_get_nameunit(dev), hldev->vpd_data.serial_num);
if(pci_get_device(dev) == XGE_PCI_DEVICE_ID_HERC_2) {
status = xge_hal_mgmt_reg_read(hldev, 0,
xge_offsetof(xge_hal_pci_bar0_t, pci_info), &val64);
if(status != XGE_HAL_OK)
xge_trace(XGE_ERR, "Error for getting bus speed");
xge_os_printf("%s: Adapter is on %s bit %s",
device_get_nameunit(dev), ((val64 & BIT(8)) ? "32":"64"),
(xge_pci_bus_speeds[((val64 & XGE_HAL_PCI_INFO) >> 60)]));
}
xge_os_printf("%s: Using %s Interrupts",
device_get_nameunit(dev),
(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) ? "MSI":"Line");
}
/**
* xge_create_dma_tags
* Creates DMA tags for both Tx and Rx
*
* @dev Device Handle
*
* Returns XGE_HAL_OK or XGE_HAL_FAIL (if errors)
*/
xge_hal_status_e
xge_create_dma_tags(device_t dev)
{
xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
xge_hal_status_e status = XGE_HAL_FAIL;
int mtu = (lldev->ifnetp)->if_mtu, maxsize;
/* DMA tag for Tx */
status = bus_dma_tag_create(
bus_get_dma_tag(dev), /* Parent */
PAGE_SIZE, /* Alignment */
0, /* Bounds */
BUS_SPACE_MAXADDR, /* Low Address */
BUS_SPACE_MAXADDR, /* High Address */
NULL, /* Filter Function */
NULL, /* Filter Function Arguments */
MCLBYTES * XGE_MAX_SEGS, /* Maximum Size */
XGE_MAX_SEGS, /* Number of Segments */
MCLBYTES, /* Maximum Segment Size */
BUS_DMA_ALLOCNOW, /* Flags */
NULL, /* Lock Function */
NULL, /* Lock Function Arguments */
(&lldev->dma_tag_tx)); /* DMA Tag */
if(status != 0)
goto _exit;
maxsize = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE;
if(maxsize <= MCLBYTES) {
maxsize = MCLBYTES;
}
else {
if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
maxsize = MJUMPAGESIZE;
else
maxsize = (maxsize <= MJUMPAGESIZE) ? MJUMPAGESIZE : MJUM9BYTES;
}
/* DMA tag for Rx */
status = bus_dma_tag_create(
bus_get_dma_tag(dev), /* Parent */
PAGE_SIZE, /* Alignment */
0, /* Bounds */
BUS_SPACE_MAXADDR, /* Low Address */
BUS_SPACE_MAXADDR, /* High Address */
NULL, /* Filter Function */
NULL, /* Filter Function Arguments */
maxsize, /* Maximum Size */
1, /* Number of Segments */
maxsize, /* Maximum Segment Size */
BUS_DMA_ALLOCNOW, /* Flags */
NULL, /* Lock Function */
NULL, /* Lock Function Arguments */
(&lldev->dma_tag_rx)); /* DMA Tag */
if(status != 0)
goto _exit1;
status = bus_dmamap_create(lldev->dma_tag_rx, BUS_DMA_NOWAIT,
&lldev->extra_dma_map);
if(status != 0)
goto _exit2;
status = XGE_HAL_OK;
goto _exit;
_exit2:
status = bus_dma_tag_destroy(lldev->dma_tag_rx);
if(status != 0)
xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
_exit1:
status = bus_dma_tag_destroy(lldev->dma_tag_tx);
if(status != 0)
xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
status = XGE_HAL_FAIL;
_exit:
return status;
}
/**
* xge_confirm_changes
* Disables and Enables interface to apply requested change
*
* @lldev Per-adapter Data
* @mtu_set Is it called for changing MTU? (Yes: 1, No: 0)
*
* Returns 0 or Error Number
*/
void
xge_confirm_changes(xge_lldev_t *lldev, xge_option_e option)
{
if(lldev->initialized == 0) goto _exit1;
mtx_lock(&lldev->mtx_drv);
if_down(lldev->ifnetp);
xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
if(option == XGE_SET_MTU)
(lldev->ifnetp)->if_mtu = lldev->mtu;
else
xge_buffer_mode_init(lldev, lldev->mtu);
xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
if_up(lldev->ifnetp);
mtx_unlock(&lldev->mtx_drv);
goto _exit;
_exit1:
/* Request was to change MTU and device not initialized */
if(option == XGE_SET_MTU) {
(lldev->ifnetp)->if_mtu = lldev->mtu;
xge_buffer_mode_init(lldev, lldev->mtu);
}
_exit:
return;
}
/**
* xge_change_lro_status
* Enable/Disable LRO feature
*
* @SYSCTL_HANDLER_ARGS sysctl_oid structure with arguments
*
* Returns 0 or error number.
*/
static int
xge_change_lro_status(SYSCTL_HANDLER_ARGS)
{
xge_lldev_t *lldev = (xge_lldev_t *)arg1;
int request = lldev->enabled_lro, status = XGE_HAL_OK;
status = sysctl_handle_int(oidp, &request, arg2, req);
if((status != XGE_HAL_OK) || (!req->newptr))
goto _exit;
if((request < 0) || (request > 1)) {
status = EINVAL;
goto _exit;
}
/* Return if current and requested states are same */
if(request == lldev->enabled_lro){
xge_trace(XGE_ERR, "LRO is already %s",
((request) ? "enabled" : "disabled"));
goto _exit;
}
lldev->enabled_lro = request;
xge_confirm_changes(lldev, XGE_CHANGE_LRO);
arg2 = lldev->enabled_lro;
_exit:
return status;
}
/**
* xge_add_sysctl_handlers
* Registers sysctl parameter value update handlers
*
* @lldev Per-adapter data
*/
void
xge_add_sysctl_handlers(xge_lldev_t *lldev)
{
struct sysctl_ctx_list *context_list =
device_get_sysctl_ctx(lldev->device);
struct sysctl_oid *oid = device_get_sysctl_tree(lldev->device);
SYSCTL_ADD_PROC(context_list, SYSCTL_CHILDREN(oid), OID_AUTO,
"enable_lro", CTLTYPE_INT | CTLFLAG_RW, lldev, 0,
xge_change_lro_status, "I", "Enable or disable LRO feature");
}
/**
* xge_attach
* Connects driver to the system if probe was success
*
* @dev Device Handle
*/
int
xge_attach(device_t dev)
{
xge_hal_device_config_t *device_config;
xge_hal_device_attr_t attr;
xge_lldev_t *lldev;
xge_hal_device_t *hldev;
xge_pci_info_t *pci_info;
struct ifnet *ifnetp;
int rid, rid0, rid1, error;
int msi_count = 0, status = XGE_HAL_OK;
int enable_msi = XGE_HAL_INTR_MODE_IRQLINE;
device_config = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t));
if(!device_config) {
XGE_EXIT_ON_ERR("Memory allocation for device configuration failed",
attach_out_config, ENOMEM);
}
lldev = (xge_lldev_t *) device_get_softc(dev);
if(!lldev) {
XGE_EXIT_ON_ERR("Adapter softc is NULL", attach_out, ENOMEM);
}
lldev->device = dev;
xge_mutex_init(lldev);
error = xge_driver_initialize();
if(error != XGE_HAL_OK) {
xge_resources_free(dev, xge_free_mutex);
XGE_EXIT_ON_ERR("Initializing driver failed", attach_out, ENXIO);
}
/* HAL device */
hldev =
(xge_hal_device_t *)xge_os_malloc(NULL, sizeof(xge_hal_device_t));
if(!hldev) {
xge_resources_free(dev, xge_free_terminate_hal_driver);
XGE_EXIT_ON_ERR("Memory allocation for HAL device failed",
attach_out, ENOMEM);
}
lldev->devh = hldev;
/* Our private structure */
pci_info =
(xge_pci_info_t*) xge_os_malloc(NULL, sizeof(xge_pci_info_t));
if(!pci_info) {
xge_resources_free(dev, xge_free_hal_device);
XGE_EXIT_ON_ERR("Memory allocation for PCI info. failed",
attach_out, ENOMEM);
}
lldev->pdev = pci_info;
pci_info->device = dev;
/* Set bus master */
pci_enable_busmaster(dev);
/* Get virtual address for BAR0 */
rid0 = PCIR_BAR(0);
pci_info->regmap0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid0,
RF_ACTIVE);
if(pci_info->regmap0 == NULL) {
xge_resources_free(dev, xge_free_pci_info);
XGE_EXIT_ON_ERR("Bus resource allocation for BAR0 failed",
attach_out, ENOMEM);
}
attr.bar0 = (char *)pci_info->regmap0;
pci_info->bar0resource = (xge_bus_resource_t*)
xge_os_malloc(NULL, sizeof(xge_bus_resource_t));
if(pci_info->bar0resource == NULL) {
xge_resources_free(dev, xge_free_bar0);
XGE_EXIT_ON_ERR("Memory allocation for BAR0 Resources failed",
attach_out, ENOMEM);
}
((xge_bus_resource_t *)(pci_info->bar0resource))->bus_tag =
rman_get_bustag(pci_info->regmap0);
((xge_bus_resource_t *)(pci_info->bar0resource))->bus_handle =
rman_get_bushandle(pci_info->regmap0);
((xge_bus_resource_t *)(pci_info->bar0resource))->bar_start_addr =
pci_info->regmap0;
/* Get virtual address for BAR1 */
rid1 = PCIR_BAR(2);
pci_info->regmap1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid1,
RF_ACTIVE);
if(pci_info->regmap1 == NULL) {
xge_resources_free(dev, xge_free_bar0_resource);
XGE_EXIT_ON_ERR("Bus resource allocation for BAR1 failed",
attach_out, ENOMEM);
}
attr.bar1 = (char *)pci_info->regmap1;
pci_info->bar1resource = (xge_bus_resource_t*)
xge_os_malloc(NULL, sizeof(xge_bus_resource_t));
if(pci_info->bar1resource == NULL) {
xge_resources_free(dev, xge_free_bar1);
XGE_EXIT_ON_ERR("Memory allocation for BAR1 Resources failed",
attach_out, ENOMEM);
}
((xge_bus_resource_t *)(pci_info->bar1resource))->bus_tag =
rman_get_bustag(pci_info->regmap1);
((xge_bus_resource_t *)(pci_info->bar1resource))->bus_handle =
rman_get_bushandle(pci_info->regmap1);
((xge_bus_resource_t *)(pci_info->bar1resource))->bar_start_addr =
pci_info->regmap1;
/* Save PCI config space */
xge_pci_space_save(dev);
attr.regh0 = (xge_bus_resource_t *) pci_info->bar0resource;
attr.regh1 = (xge_bus_resource_t *) pci_info->bar1resource;
attr.irqh = lldev->irqhandle;
attr.cfgh = pci_info;
attr.pdev = pci_info;
/* Initialize device configuration parameters */
xge_init_params(device_config, dev);
rid = 0;
if(lldev->enabled_msi) {
/* Number of MSI messages supported by device */
msi_count = pci_msi_count(dev);
if(msi_count > 1) {
/* Device supports MSI */
if(bootverbose) {
xge_trace(XGE_ERR, "MSI count: %d", msi_count);
xge_trace(XGE_ERR, "Now, driver supporting 1 message");
}
msi_count = 1;
error = pci_alloc_msi(dev, &msi_count);
if(error == 0) {
if(bootverbose)
xge_trace(XGE_ERR, "Allocated messages: %d", msi_count);
enable_msi = XGE_HAL_INTR_MODE_MSI;
rid = 1;
}
else {
if(bootverbose)
xge_trace(XGE_ERR, "pci_alloc_msi failed, %d", error);
}
}
}
lldev->enabled_msi = enable_msi;
/* Allocate resource for irq */
lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
(RF_SHAREABLE | RF_ACTIVE));
if(lldev->irq == NULL) {
xge_trace(XGE_ERR, "Allocating irq resource for %s failed",
((rid == 0) ? "line interrupt" : "MSI"));
if(rid == 1) {
error = pci_release_msi(dev);
if(error != 0) {
xge_trace(XGE_ERR, "Releasing MSI resources failed %d",
error);
xge_trace(XGE_ERR, "Requires reboot to use MSI again");
}
xge_trace(XGE_ERR, "Trying line interrupts");
rid = 0;
lldev->enabled_msi = XGE_HAL_INTR_MODE_IRQLINE;
lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
(RF_SHAREABLE | RF_ACTIVE));
}
if(lldev->irq == NULL) {
xge_trace(XGE_ERR, "Allocating irq resource failed");
xge_resources_free(dev, xge_free_bar1_resource);
status = ENOMEM;
goto attach_out;
}
}
device_config->intr_mode = lldev->enabled_msi;
if(bootverbose) {
xge_trace(XGE_TRACE, "rid: %d, Mode: %d, MSI count: %d", rid,
lldev->enabled_msi, msi_count);
}
/* Initialize HAL device */
error = xge_hal_device_initialize(hldev, &attr, device_config);
if(error != XGE_HAL_OK) {
xge_resources_free(dev, xge_free_irq_resource);
XGE_EXIT_ON_ERR("Initializing HAL device failed", attach_out,
ENXIO);
}
xge_hal_device_private_set(hldev, lldev);
error = xge_interface_setup(dev);
if(error != 0) {
status = error;
goto attach_out;
}
ifnetp = lldev->ifnetp;
ifnetp->if_mtu = device_config->mtu;
xge_media_init(dev);
/* Associate interrupt handler with the device */
if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
error = bus_setup_intr(dev, lldev->irq,
(INTR_TYPE_NET | INTR_MPSAFE),
#if __FreeBSD_version > 700030
NULL,
#endif
xge_isr_msi, lldev, &lldev->irqhandle);
xge_msi_info_save(lldev);
}
else {
error = bus_setup_intr(dev, lldev->irq,
(INTR_TYPE_NET | INTR_MPSAFE),
#if __FreeBSD_version > 700030
xge_isr_filter,
#endif
xge_isr_line, lldev, &lldev->irqhandle);
}
if(error != 0) {
xge_resources_free(dev, xge_free_media_interface);
XGE_EXIT_ON_ERR("Associating interrupt handler with device failed",
attach_out, ENXIO);
}
xge_print_info(lldev);
xge_add_sysctl_handlers(lldev);
xge_buffer_mode_init(lldev, device_config->mtu);
attach_out:
xge_os_free(NULL, device_config, sizeof(xge_hal_device_config_t));
attach_out_config:
return status;
}
/**
* xge_resources_free
* Undo what-all we did during load/attach
*
* @dev Device Handle
* @error Identifies what-all to undo
*/
void
xge_resources_free(device_t dev, xge_lables_e error)
{
xge_lldev_t *lldev;
xge_pci_info_t *pci_info;
xge_hal_device_t *hldev;
int rid, status;
/* LL Device */
lldev = (xge_lldev_t *) device_get_softc(dev);
pci_info = lldev->pdev;
/* HAL Device */
hldev = lldev->devh;
switch(error) {
case xge_free_all:
/* Teardown interrupt handler - device association */
bus_teardown_intr(dev, lldev->irq, lldev->irqhandle);
case xge_free_media_interface:
/* Media */
ifmedia_removeall(&lldev->media);
/* Detach Ether */
ether_ifdetach(lldev->ifnetp);
if_free(lldev->ifnetp);
xge_hal_device_private_set(hldev, NULL);
xge_hal_device_disable(hldev);
case xge_free_terminate_hal_device:
/* HAL Device */
xge_hal_device_terminate(hldev);
case xge_free_irq_resource:
/* Release IRQ resource */
bus_release_resource(dev, SYS_RES_IRQ,
((lldev->enabled_msi == XGE_HAL_INTR_MODE_IRQLINE) ? 0:1),
lldev->irq);
if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
status = pci_release_msi(dev);
if(status != 0) {
if(bootverbose) {
xge_trace(XGE_ERR,
"pci_release_msi returned %d", status);
}
}
}
case xge_free_bar1_resource:
/* Restore PCI configuration space */
xge_pci_space_restore(dev);
/* Free bar1resource */
xge_os_free(NULL, pci_info->bar1resource,
sizeof(xge_bus_resource_t));
case xge_free_bar1:
/* Release BAR1 */
rid = PCIR_BAR(2);
bus_release_resource(dev, SYS_RES_MEMORY, rid,
pci_info->regmap1);
case xge_free_bar0_resource:
/* Free bar0resource */
xge_os_free(NULL, pci_info->bar0resource,
sizeof(xge_bus_resource_t));
case xge_free_bar0:
/* Release BAR0 */
rid = PCIR_BAR(0);
bus_release_resource(dev, SYS_RES_MEMORY, rid,
pci_info->regmap0);
case xge_free_pci_info:
/* Disable Bus Master */
pci_disable_busmaster(dev);
/* Free pci_info_t */
lldev->pdev = NULL;
xge_os_free(NULL, pci_info, sizeof(xge_pci_info_t));
case xge_free_hal_device:
/* Free device configuration struct and HAL device */
xge_os_free(NULL, hldev, sizeof(xge_hal_device_t));
case xge_free_terminate_hal_driver:
/* Terminate HAL driver */
hal_driver_init_count = hal_driver_init_count - 1;
if(!hal_driver_init_count) {
xge_hal_driver_terminate();
}
case xge_free_mutex:
xge_mutex_destroy(lldev);
}
}
/**
* xge_detach
* Detaches driver from the Kernel subsystem
*
* @dev Device Handle
*/
int
xge_detach(device_t dev)
{
xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
if(lldev->in_detach == 0) {
lldev->in_detach = 1;
xge_stop(lldev);
xge_resources_free(dev, xge_free_all);
}
return 0;
}
/**
* xge_shutdown
* To shutdown device before system shutdown
*
* @dev Device Handle
*/
int
xge_shutdown(device_t dev)
{
xge_lldev_t *lldev = (xge_lldev_t *) device_get_softc(dev);
xge_stop(lldev);
return 0;
}
/**
* xge_interface_setup
* Setup interface
*
* @dev Device Handle
*
* Returns 0 on success, ENXIO/ENOMEM on failure
*/
int
xge_interface_setup(device_t dev)
{
u8 mcaddr[ETHER_ADDR_LEN];
xge_hal_status_e status;
xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
struct ifnet *ifnetp;
xge_hal_device_t *hldev = lldev->devh;
/* Get the MAC address of the device */
status = xge_hal_device_macaddr_get(hldev, 0, &mcaddr);
if(status != XGE_HAL_OK) {
xge_resources_free(dev, xge_free_terminate_hal_device);
XGE_EXIT_ON_ERR("Getting MAC address failed", ifsetup_out, ENXIO);
}
/* Get interface ifnet structure for this Ether device */
ifnetp = lldev->ifnetp = if_alloc(IFT_ETHER);
if(ifnetp == NULL) {
xge_resources_free(dev, xge_free_terminate_hal_device);
XGE_EXIT_ON_ERR("Allocation ifnet failed", ifsetup_out, ENOMEM);
}
/* Initialize interface ifnet structure */
if_initname(ifnetp, device_get_name(dev), device_get_unit(dev));
ifnetp->if_mtu = XGE_HAL_DEFAULT_MTU;
ifnetp->if_baudrate = XGE_BAUDRATE;
ifnetp->if_init = xge_init;
ifnetp->if_softc = lldev;
ifnetp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifnetp->if_ioctl = xge_ioctl;
ifnetp->if_start = xge_send;
/* TODO: Check and assign optimal value */
ifnetp->if_snd.ifq_maxlen = ifqmaxlen;
ifnetp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU |
IFCAP_HWCSUM;
if(lldev->enabled_tso)
ifnetp->if_capabilities |= IFCAP_TSO4;
if(lldev->enabled_lro)
ifnetp->if_capabilities |= IFCAP_LRO;
ifnetp->if_capenable = ifnetp->if_capabilities;
/* Attach the interface */
ether_ifattach(ifnetp, mcaddr);
ifsetup_out:
return status;
}
/**
* xge_callback_link_up
* Callback for Link-up indication from HAL
*
* @userdata Per-adapter data
*/
void
xge_callback_link_up(void *userdata)
{
xge_lldev_t *lldev = (xge_lldev_t *)userdata;
struct ifnet *ifnetp = lldev->ifnetp;
ifnetp->if_flags &= ~IFF_DRV_OACTIVE;
if_link_state_change(ifnetp, LINK_STATE_UP);
}
/**
* xge_callback_link_down
* Callback for Link-down indication from HAL
*
* @userdata Per-adapter data
*/
void
xge_callback_link_down(void *userdata)
{
xge_lldev_t *lldev = (xge_lldev_t *)userdata;
struct ifnet *ifnetp = lldev->ifnetp;
ifnetp->if_flags |= IFF_DRV_OACTIVE;
if_link_state_change(ifnetp, LINK_STATE_DOWN);
}
/**
* xge_callback_crit_err
* Callback for Critical error indication from HAL
*
* @userdata Per-adapter data
* @type Event type (Enumerated hardware error)
* @serr_data Hardware status
*/
void
xge_callback_crit_err(void *userdata, xge_hal_event_e type, u64 serr_data)
{
xge_trace(XGE_ERR, "Critical Error");
xge_reset(userdata);
}
/**
* xge_callback_event
* Callback from HAL indicating that some event has been queued
*
* @item Queued event item
*/
void
xge_callback_event(xge_queue_item_t *item)
{
xge_lldev_t *lldev = NULL;
xge_hal_device_t *hldev = NULL;
struct ifnet *ifnetp = NULL;
hldev = item->context;
lldev = xge_hal_device_private(hldev);
ifnetp = lldev->ifnetp;
switch((int)item->event_type) {
case XGE_LL_EVENT_TRY_XMIT_AGAIN:
if(lldev->initialized) {
if(xge_hal_channel_dtr_count(lldev->fifo_channel[0]) > 0) {
ifnetp->if_flags &= ~IFF_DRV_OACTIVE;
}
else {
xge_queue_produce_context(
xge_hal_device_queue(lldev->devh),
XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh);
}
}
break;
case XGE_LL_EVENT_DEVICE_RESETTING:
xge_reset(item->context);
break;
default:
break;
}
}
/**
* xge_ifmedia_change
* Media change driver callback
*
* @ifnetp Interface Handle
*
* Returns 0 if media is Ether else EINVAL
*/
int
xge_ifmedia_change(struct ifnet *ifnetp)
{
xge_lldev_t *lldev = ifnetp->if_softc;
struct ifmedia *ifmediap = &lldev->media;
return (IFM_TYPE(ifmediap->ifm_media) != IFM_ETHER) ? EINVAL:0;
}
/**
* xge_ifmedia_status
* Media status driver callback
*
* @ifnetp Interface Handle
* @ifmr Interface Media Settings
*/
void
xge_ifmedia_status(struct ifnet *ifnetp, struct ifmediareq *ifmr)
{
xge_hal_status_e status;
u64 regvalue;
xge_lldev_t *lldev = ifnetp->if_softc;
xge_hal_device_t *hldev = lldev->devh;
ifmr->ifm_status = IFM_AVALID;
ifmr->ifm_active = IFM_ETHER;
status = xge_hal_mgmt_reg_read(hldev, 0,
xge_offsetof(xge_hal_pci_bar0_t, adapter_status), &regvalue);
if(status != XGE_HAL_OK) {
xge_trace(XGE_TRACE, "Getting adapter status failed");
goto _exit;
}
if((regvalue & (XGE_HAL_ADAPTER_STATUS_RMAC_REMOTE_FAULT |
XGE_HAL_ADAPTER_STATUS_RMAC_LOCAL_FAULT)) == 0) {
ifmr->ifm_status |= IFM_ACTIVE;
ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
if_link_state_change(ifnetp, LINK_STATE_UP);
}
else {
if_link_state_change(ifnetp, LINK_STATE_DOWN);
}
_exit:
return;
}
/**
* xge_ioctl_stats
* IOCTL to get statistics
*
* @lldev Per-adapter data
* @ifreqp Interface request
*/
int
xge_ioctl_stats(xge_lldev_t *lldev, struct ifreq *ifreqp)
{
xge_hal_status_e status = XGE_HAL_OK;
char *data = (char *)ifreqp->ifr_data;
void *info = NULL;
int retValue = EINVAL;
switch(*data) {
case XGE_QUERY_STATS:
mtx_lock(&lldev->mtx_drv);
status = xge_hal_stats_hw(lldev->devh,
(xge_hal_stats_hw_info_t **)&info);
mtx_unlock(&lldev->mtx_drv);
if(status == XGE_HAL_OK) {
if(copyout(info, ifreqp->ifr_data,
sizeof(xge_hal_stats_hw_info_t)) == 0)
retValue = 0;
}
else {
xge_trace(XGE_ERR, "Getting statistics failed (Status: %d)",
status);
}
break;
case XGE_QUERY_PCICONF:
info = xge_os_malloc(NULL, sizeof(xge_hal_pci_config_t));
if(info != NULL) {
mtx_lock(&lldev->mtx_drv);
status = xge_hal_mgmt_pci_config(lldev->devh, info,
sizeof(xge_hal_pci_config_t));
mtx_unlock(&lldev->mtx_drv);
if(status == XGE_HAL_OK) {
if(copyout(info, ifreqp->ifr_data,
sizeof(xge_hal_pci_config_t)) == 0)
retValue = 0;
}
else {
xge_trace(XGE_ERR,
"Getting PCI configuration failed (%d)", status);
}
xge_os_free(NULL, info, sizeof(xge_hal_pci_config_t));
}
break;
case XGE_QUERY_DEVSTATS:
info = xge_os_malloc(NULL, sizeof(xge_hal_stats_device_info_t));
if(info != NULL) {
mtx_lock(&lldev->mtx_drv);
status =xge_hal_mgmt_device_stats(lldev->devh, info,
sizeof(xge_hal_stats_device_info_t));
mtx_unlock(&lldev->mtx_drv);
if(status == XGE_HAL_OK) {
if(copyout(info, ifreqp->ifr_data,
sizeof(xge_hal_stats_device_info_t)) == 0)
retValue = 0;
}
else {
xge_trace(XGE_ERR, "Getting device info failed (%d)",
status);
}
xge_os_free(NULL, info,
sizeof(xge_hal_stats_device_info_t));
}
break;
case XGE_QUERY_SWSTATS:
info = xge_os_malloc(NULL, sizeof(xge_hal_stats_sw_err_t));
if(info != NULL) {
mtx_lock(&lldev->mtx_drv);
status =xge_hal_mgmt_sw_stats(lldev->devh, info,
sizeof(xge_hal_stats_sw_err_t));
mtx_unlock(&lldev->mtx_drv);
if(status == XGE_HAL_OK) {
if(copyout(info, ifreqp->ifr_data,
sizeof(xge_hal_stats_sw_err_t)) == 0)
retValue = 0;
}
else {
xge_trace(XGE_ERR,
"Getting tcode statistics failed (%d)", status);
}
xge_os_free(NULL, info, sizeof(xge_hal_stats_sw_err_t));
}
break;
case XGE_QUERY_DRIVERSTATS:
if(copyout(&lldev->driver_stats, ifreqp->ifr_data,
sizeof(xge_driver_stats_t)) == 0) {
retValue = 0;
}
else {
xge_trace(XGE_ERR,
"Copyout of driver statistics failed (%d)", status);
}
break;
case XGE_READ_VERSION:
info = xge_os_malloc(NULL, XGE_BUFFER_SIZE);
if(version != NULL) {
strcpy(info, XGE_DRIVER_VERSION);
if(copyout(info, ifreqp->ifr_data, XGE_BUFFER_SIZE) == 0)
retValue = 0;
xge_os_free(NULL, info, XGE_BUFFER_SIZE);
}
break;
case XGE_QUERY_DEVCONF:
info = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t));
if(info != NULL) {
mtx_lock(&lldev->mtx_drv);
status = xge_hal_mgmt_device_config(lldev->devh, info,
sizeof(xge_hal_device_config_t));
mtx_unlock(&lldev->mtx_drv);
if(status == XGE_HAL_OK) {
if(copyout(info, ifreqp->ifr_data,
sizeof(xge_hal_device_config_t)) == 0)
retValue = 0;
}
else {
xge_trace(XGE_ERR, "Getting devconfig failed (%d)",
status);
}
xge_os_free(NULL, info, sizeof(xge_hal_device_config_t));
}
break;
case XGE_QUERY_BUFFER_MODE:
if(copyout(&lldev->buffer_mode, ifreqp->ifr_data,
sizeof(int)) == 0)
retValue = 0;
break;
case XGE_SET_BUFFER_MODE_1:
case XGE_SET_BUFFER_MODE_2:
case XGE_SET_BUFFER_MODE_5:
*data = (*data == XGE_SET_BUFFER_MODE_1) ? 'Y':'N';
if(copyout(data, ifreqp->ifr_data, sizeof(data)) == 0)
retValue = 0;
break;
default:
xge_trace(XGE_TRACE, "Nothing is matching");
retValue = ENOTTY;
break;
}
return retValue;
}
/**
* xge_ioctl_registers
* IOCTL to get registers
*
* @lldev Per-adapter data
* @ifreqp Interface request
*/
int
xge_ioctl_registers(xge_lldev_t *lldev, struct ifreq *ifreqp)
{
xge_register_t *data = (xge_register_t *)ifreqp->ifr_data;
xge_hal_status_e status = XGE_HAL_OK;
int retValue = EINVAL, offset = 0, index = 0;
u64 val64 = 0;
/* Reading a register */
if(strcmp(data->option, "-r") == 0) {
data->value = 0x0000;
mtx_lock(&lldev->mtx_drv);
status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset,
&data->value);
mtx_unlock(&lldev->mtx_drv);
if(status == XGE_HAL_OK) {
if(copyout(data, ifreqp->ifr_data, sizeof(xge_register_t)) == 0)
retValue = 0;
}
}
/* Writing to a register */
else if(strcmp(data->option, "-w") == 0) {
mtx_lock(&lldev->mtx_drv);
status = xge_hal_mgmt_reg_write(lldev->devh, 0, data->offset,
data->value);
if(status == XGE_HAL_OK) {
val64 = 0x0000;
status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset,
&val64);
if(status != XGE_HAL_OK) {
xge_trace(XGE_ERR, "Reading back updated register failed");
}
else {
if(val64 != data->value) {
xge_trace(XGE_ERR,
"Read and written register values mismatched");
}
else retValue = 0;
}
}
else {
xge_trace(XGE_ERR, "Getting register value failed");
}
mtx_unlock(&lldev->mtx_drv);
}
else {
mtx_lock(&lldev->mtx_drv);
for(index = 0, offset = 0; offset <= XGE_OFFSET_OF_LAST_REG;
index++, offset += 0x0008) {
val64 = 0;
status = xge_hal_mgmt_reg_read(lldev->devh, 0, offset, &val64);
if(status != XGE_HAL_OK) {
xge_trace(XGE_ERR, "Getting register value failed");
break;
}
*((u64 *)((u64 *)data + index)) = val64;
retValue = 0;
}
mtx_unlock(&lldev->mtx_drv);
if(retValue == 0) {
if(copyout(data, ifreqp->ifr_data,
sizeof(xge_hal_pci_bar0_t)) != 0) {
xge_trace(XGE_ERR, "Copyout of register values failed");
retValue = EINVAL;
}
}
else {
xge_trace(XGE_ERR, "Getting register values failed");
}
}
return retValue;
}
/**
* xge_ioctl
* Callback to control the device - Interface configuration
*
* @ifnetp Interface Handle
* @command Device control command
* @data Parameters associated with command (if any)
*/
int
xge_ioctl(struct ifnet *ifnetp, unsigned long command, caddr_t data)
{
struct ifreq *ifreqp = (struct ifreq *)data;
xge_lldev_t *lldev = ifnetp->if_softc;
struct ifmedia *ifmediap = &lldev->media;
int retValue = 0, mask = 0;
if(lldev->in_detach) {
return retValue;
}
switch(command) {
/* Set/Get ifnet address */
case SIOCSIFADDR:
case SIOCGIFADDR:
ether_ioctl(ifnetp, command, data);
break;
/* Set ifnet MTU */
case SIOCSIFMTU:
retValue = xge_change_mtu(lldev, ifreqp->ifr_mtu);
break;
/* Set ifnet flags */
case SIOCSIFFLAGS:
if(ifnetp->if_flags & IFF_UP) {
/* Link status is UP */
if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) {
xge_init(lldev);
}
xge_disable_promisc(lldev);
xge_enable_promisc(lldev);
}
else {
/* Link status is DOWN */
/* If device is in running, make it down */
if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
xge_stop(lldev);
}
}
break;
/* Add/delete multicast address */
case SIOCADDMULTI:
case SIOCDELMULTI:
if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
xge_setmulti(lldev);
}
break;
/* Set/Get net media */
case SIOCSIFMEDIA:
case SIOCGIFMEDIA:
retValue = ifmedia_ioctl(ifnetp, ifreqp, ifmediap, command);
break;
/* Set capabilities */
case SIOCSIFCAP:
mtx_lock(&lldev->mtx_drv);
mask = ifreqp->ifr_reqcap ^ ifnetp->if_capenable;
if(mask & IFCAP_TXCSUM) {
if(ifnetp->if_capenable & IFCAP_TXCSUM) {
ifnetp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TXCSUM);
ifnetp->if_hwassist &=
~(CSUM_TCP | CSUM_UDP | CSUM_TSO);
}
else {
ifnetp->if_capenable |= IFCAP_TXCSUM;
ifnetp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
}
}
if(mask & IFCAP_TSO4) {
if(ifnetp->if_capenable & IFCAP_TSO4) {
ifnetp->if_capenable &= ~IFCAP_TSO4;
ifnetp->if_hwassist &= ~CSUM_TSO;
xge_os_printf("%s: TSO Disabled",
device_get_nameunit(lldev->device));
}
else if(ifnetp->if_capenable & IFCAP_TXCSUM) {
ifnetp->if_capenable |= IFCAP_TSO4;
ifnetp->if_hwassist |= CSUM_TSO;
xge_os_printf("%s: TSO Enabled",
device_get_nameunit(lldev->device));
}
}
mtx_unlock(&lldev->mtx_drv);
break;
/* Custom IOCTL 0 */
case SIOCGPRIVATE_0:
retValue = xge_ioctl_stats(lldev, ifreqp);
break;
/* Custom IOCTL 1 */
case SIOCGPRIVATE_1:
retValue = xge_ioctl_registers(lldev, ifreqp);
break;
default:
retValue = EINVAL;
break;
}
return retValue;
}
/**
* xge_init
* Initialize the interface
*
* @plldev Per-adapter Data
*/
void
xge_init(void *plldev)
{
xge_lldev_t *lldev = (xge_lldev_t *)plldev;
mtx_lock(&lldev->mtx_drv);
xge_os_memzero(&lldev->driver_stats, sizeof(xge_driver_stats_t));
xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
mtx_unlock(&lldev->mtx_drv);
}
/**
* xge_device_init
* Initialize the interface (called by holding lock)
*
* @pdevin Per-adapter Data
*/
void
xge_device_init(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
{
struct ifnet *ifnetp = lldev->ifnetp;
xge_hal_device_t *hldev = lldev->devh;
struct ifaddr *ifaddrp;
unsigned char *macaddr;
struct sockaddr_dl *sockaddrp;
int status = XGE_HAL_OK;
mtx_assert((&lldev->mtx_drv), MA_OWNED);
/* If device is in running state, initializing is not required */
if(ifnetp->if_drv_flags & IFF_DRV_RUNNING)
return;
/* Initializing timer */
- callout_init(&lldev->timer, CALLOUT_MPSAFE);
+ callout_init(&lldev->timer, 1);
xge_trace(XGE_TRACE, "Set MTU size");
status = xge_hal_device_mtu_set(hldev, ifnetp->if_mtu);
if(status != XGE_HAL_OK) {
xge_trace(XGE_ERR, "Setting MTU in HAL device failed");
goto _exit;
}
/* Enable HAL device */
xge_hal_device_enable(hldev);
/* Get MAC address and update in HAL */
ifaddrp = ifnetp->if_addr;
sockaddrp = (struct sockaddr_dl *)ifaddrp->ifa_addr;
sockaddrp->sdl_type = IFT_ETHER;
sockaddrp->sdl_alen = ifnetp->if_addrlen;
macaddr = LLADDR(sockaddrp);
xge_trace(XGE_TRACE,
"Setting MAC address: %02x:%02x:%02x:%02x:%02x:%02x\n",
*macaddr, *(macaddr + 1), *(macaddr + 2), *(macaddr + 3),
*(macaddr + 4), *(macaddr + 5));
status = xge_hal_device_macaddr_set(hldev, 0, macaddr);
if(status != XGE_HAL_OK)
xge_trace(XGE_ERR, "Setting MAC address failed (%d)", status);
/* Opening channels */
mtx_unlock(&lldev->mtx_drv);
status = xge_channel_open(lldev, option);
mtx_lock(&lldev->mtx_drv);
if(status != XGE_HAL_OK)
goto _exit;
/* Set appropriate flags */
ifnetp->if_drv_flags |= IFF_DRV_RUNNING;
ifnetp->if_flags &= ~IFF_DRV_OACTIVE;
/* Checksum capability */
ifnetp->if_hwassist = (ifnetp->if_capenable & IFCAP_TXCSUM) ?
(CSUM_TCP | CSUM_UDP) : 0;
if((lldev->enabled_tso) && (ifnetp->if_capenable & IFCAP_TSO4))
ifnetp->if_hwassist |= CSUM_TSO;
/* Enable interrupts */
xge_hal_device_intr_enable(hldev);
callout_reset(&lldev->timer, 10*hz, xge_timer, lldev);
/* Disable promiscuous mode */
xge_trace(XGE_TRACE, "If opted, enable promiscuous mode");
xge_enable_promisc(lldev);
/* Device is initialized */
lldev->initialized = 1;
xge_os_mdelay(1000);
_exit:
return;
}
/**
* xge_timer
* Timer timeout function to handle link status
*
* @devp Per-adapter Data
*/
void
xge_timer(void *devp)
{
xge_lldev_t *lldev = (xge_lldev_t *)devp;
xge_hal_device_t *hldev = lldev->devh;
/* Poll for changes */
xge_hal_device_poll(hldev);
/* Reset timer */
callout_reset(&lldev->timer, hz, xge_timer, lldev);
return;
}
/**
* xge_stop
* De-activate the interface
*
* @lldev Per-adater Data
*/
void
xge_stop(xge_lldev_t *lldev)
{
mtx_lock(&lldev->mtx_drv);
xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
mtx_unlock(&lldev->mtx_drv);
}
/**
* xge_isr_filter
* ISR filter function - to filter interrupts from other devices (shared)
*
* @handle Per-adapter Data
*
* Returns
* FILTER_STRAY if interrupt is from other device
* FILTER_SCHEDULE_THREAD if interrupt is from Xframe device
*/
int
xge_isr_filter(void *handle)
{
xge_lldev_t *lldev = (xge_lldev_t *)handle;
xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)((lldev->devh)->bar0);
u16 retValue = FILTER_STRAY;
u64 val64 = 0;
XGE_DRV_STATS(isr_filter);
val64 = xge_os_pio_mem_read64(lldev->pdev, (lldev->devh)->regh0,
&bar0->general_int_status);
retValue = (!val64) ? FILTER_STRAY : FILTER_SCHEDULE_THREAD;
return retValue;
}
/**
* xge_isr_line
* Interrupt service routine for Line interrupts
*
* @plldev Per-adapter Data
*/
void
xge_isr_line(void *plldev)
{
xge_hal_status_e status;
xge_lldev_t *lldev = (xge_lldev_t *)plldev;
xge_hal_device_t *hldev = (xge_hal_device_t *)lldev->devh;
struct ifnet *ifnetp = lldev->ifnetp;
XGE_DRV_STATS(isr_line);
if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
status = xge_hal_device_handle_irq(hldev);
if(!(IFQ_DRV_IS_EMPTY(&ifnetp->if_snd)))
xge_send(ifnetp);
}
}
/*
* xge_isr_msi
* ISR for Message signaled interrupts
*/
void
xge_isr_msi(void *plldev)
{
xge_lldev_t *lldev = (xge_lldev_t *)plldev;
XGE_DRV_STATS(isr_msi);
xge_hal_device_continue_irq(lldev->devh);
}
/**
* xge_rx_open
* Initiate and open all Rx channels
*
* @qid Ring Index
* @lldev Per-adapter Data
* @rflag Channel open/close/reopen flag
*
* Returns 0 or Error Number
*/
int
xge_rx_open(int qid, xge_lldev_t *lldev, xge_hal_channel_reopen_e rflag)
{
u64 adapter_status = 0x0;
xge_hal_status_e status = XGE_HAL_FAIL;
xge_hal_channel_attr_t attr = {
.post_qid = qid,
.compl_qid = 0,
.callback = xge_rx_compl,
.per_dtr_space = sizeof(xge_rx_priv_t),
.flags = 0,
.type = XGE_HAL_CHANNEL_TYPE_RING,
.userdata = lldev,
.dtr_init = xge_rx_initial_replenish,
.dtr_term = xge_rx_term
};
/* If device is not ready, return */
status = xge_hal_device_status(lldev->devh, &adapter_status);
if(status != XGE_HAL_OK) {
xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status);
XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL);
}
else {
status = xge_hal_channel_open(lldev->devh, &attr,
&lldev->ring_channel[qid], rflag);
}
_exit:
return status;
}
/**
* xge_tx_open
* Initialize and open all Tx channels
*
* @lldev Per-adapter Data
* @tflag Channel open/close/reopen flag
*
* Returns 0 or Error Number
*/
int
xge_tx_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e tflag)
{
xge_hal_status_e status = XGE_HAL_FAIL;
u64 adapter_status = 0x0;
int qindex, index;
xge_hal_channel_attr_t attr = {
.compl_qid = 0,
.callback = xge_tx_compl,
.per_dtr_space = sizeof(xge_tx_priv_t),
.flags = 0,
.type = XGE_HAL_CHANNEL_TYPE_FIFO,
.userdata = lldev,
.dtr_init = xge_tx_initial_replenish,
.dtr_term = xge_tx_term
};
/* If device is not ready, return */
status = xge_hal_device_status(lldev->devh, &adapter_status);
if(status != XGE_HAL_OK) {
xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status);
XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL);
}
for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
attr.post_qid = qindex,
status = xge_hal_channel_open(lldev->devh, &attr,
&lldev->fifo_channel[qindex], tflag);
if(status != XGE_HAL_OK) {
for(index = 0; index < qindex; index++)
xge_hal_channel_close(lldev->fifo_channel[index], tflag);
}
}
_exit:
return status;
}
/**
* xge_enable_msi
* Enables MSI
*
* @lldev Per-adapter Data
*/
void
xge_enable_msi(xge_lldev_t *lldev)
{
xge_list_t *item = NULL;
xge_hal_device_t *hldev = lldev->devh;
xge_hal_channel_t *channel = NULL;
u16 offset = 0, val16 = 0;
xge_os_pci_read16(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16);
/* Update msi_data */
offset = (val16 & 0x80) ? 0x4c : 0x48;
xge_os_pci_read16(lldev->pdev, NULL, offset, &val16);
if(val16 & 0x1)
val16 &= 0xfffe;
else
val16 |= 0x1;
xge_os_pci_write16(lldev->pdev, NULL, offset, val16);
/* Update msi_control */
xge_os_pci_read16(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16);
val16 |= 0x10;
xge_os_pci_write16(lldev->pdev, NULL,
xge_offsetof(xge_hal_pci_config_le_t, msi_control), val16);
/* Set TxMAT and RxMAT registers with MSI */
xge_list_for_each(item, &hldev->free_channels) {
channel = xge_container_of(item, xge_hal_channel_t, item);
xge_hal_channel_msi_set(channel, 1, (u32)val16);
}
}
/**
* xge_channel_open
* Open both Tx and Rx channels
*
* @lldev Per-adapter Data
* @option Channel reopen option
*/
int
xge_channel_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
{
xge_lro_entry_t *lro_session = NULL;
xge_hal_status_e status = XGE_HAL_OK;
int index = 0, index2 = 0;
if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
xge_msi_info_restore(lldev);
xge_enable_msi(lldev);
}
_exit2:
status = xge_create_dma_tags(lldev->device);
if(status != XGE_HAL_OK)
XGE_EXIT_ON_ERR("DMA tag creation failed", _exit, status);
/* Open ring (Rx) channel */
for(index = 0; index < XGE_RING_COUNT; index++) {
status = xge_rx_open(index, lldev, option);
if(status != XGE_HAL_OK) {
/*
* DMA mapping fails in the unpatched Kernel which can't
* allocate contiguous memory for Jumbo frames.
* Try using 5 buffer mode.
*/
if((lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) &&
(((lldev->ifnetp)->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE) >
MJUMPAGESIZE)) {
/* Close so far opened channels */
for(index2 = 0; index2 < index; index2++) {
xge_hal_channel_close(lldev->ring_channel[index2],
option);
}
/* Destroy DMA tags intended to use for 1 buffer mode */
if(bus_dmamap_destroy(lldev->dma_tag_rx,
lldev->extra_dma_map)) {
xge_trace(XGE_ERR, "Rx extra DMA map destroy failed");
}
if(bus_dma_tag_destroy(lldev->dma_tag_rx))
xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
if(bus_dma_tag_destroy(lldev->dma_tag_tx))
xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
/* Switch to 5 buffer mode */
lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_5;
xge_buffer_mode_init(lldev, (lldev->ifnetp)->if_mtu);
/* Restart init */
goto _exit2;
}
else {
XGE_EXIT_ON_ERR("Opening Rx channel failed", _exit1,
status);
}
}
}
if(lldev->enabled_lro) {
SLIST_INIT(&lldev->lro_free);
SLIST_INIT(&lldev->lro_active);
lldev->lro_num = XGE_LRO_DEFAULT_ENTRIES;
for(index = 0; index < lldev->lro_num; index++) {
lro_session = (xge_lro_entry_t *)
xge_os_malloc(NULL, sizeof(xge_lro_entry_t));
if(lro_session == NULL) {
lldev->lro_num = index;
break;
}
SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next);
}
}
/* Open FIFO (Tx) channel */
status = xge_tx_open(lldev, option);
if(status != XGE_HAL_OK)
XGE_EXIT_ON_ERR("Opening Tx channel failed", _exit1, status);
goto _exit;
_exit1:
/*
* Opening Rx channel(s) failed (index is <last ring index - 1>) or
* Initialization of LRO failed (index is XGE_RING_COUNT)
* Opening Tx channel failed (index is XGE_RING_COUNT)
*/
for(index2 = 0; index2 < index; index2++)
xge_hal_channel_close(lldev->ring_channel[index2], option);
_exit:
return status;
}
/**
* xge_channel_close
* Close both Tx and Rx channels
*
* @lldev Per-adapter Data
* @option Channel reopen option
*
*/
void
xge_channel_close(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
{
int qindex = 0;
DELAY(1000 * 1000);
/* Close FIFO (Tx) channel */
for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++)
xge_hal_channel_close(lldev->fifo_channel[qindex], option);
/* Close Ring (Rx) channels */
for(qindex = 0; qindex < XGE_RING_COUNT; qindex++)
xge_hal_channel_close(lldev->ring_channel[qindex], option);
if(bus_dmamap_destroy(lldev->dma_tag_rx, lldev->extra_dma_map))
xge_trace(XGE_ERR, "Rx extra map destroy failed");
if(bus_dma_tag_destroy(lldev->dma_tag_rx))
xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
if(bus_dma_tag_destroy(lldev->dma_tag_tx))
xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
}
/**
* dmamap_cb
* DMA map callback
*
* @arg Parameter passed from dmamap
* @segs Segments
* @nseg Number of segments
* @error Error
*/
void
dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
if(!error) {
*(bus_addr_t *) arg = segs->ds_addr;
}
}
/**
* xge_reset
* Device Reset
*
* @lldev Per-adapter Data
*/
void
xge_reset(xge_lldev_t *lldev)
{
xge_trace(XGE_TRACE, "Reseting the chip");
/* If the device is not initialized, return */
if(lldev->initialized) {
mtx_lock(&lldev->mtx_drv);
xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
mtx_unlock(&lldev->mtx_drv);
}
return;
}
/**
* xge_setmulti
* Set an address as a multicast address
*
* @lldev Per-adapter Data
*/
void
xge_setmulti(xge_lldev_t *lldev)
{
struct ifmultiaddr *ifma;
u8 *lladdr;
xge_hal_device_t *hldev = (xge_hal_device_t *)lldev->devh;
struct ifnet *ifnetp = lldev->ifnetp;
int index = 0;
int offset = 1;
int table_size = 47;
xge_hal_status_e status = XGE_HAL_OK;
u8 initial_addr[]= {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
if((ifnetp->if_flags & IFF_MULTICAST) && (!lldev->all_multicast)) {
status = xge_hal_device_mcast_enable(hldev);
lldev->all_multicast = 1;
}
else if((ifnetp->if_flags & IFF_MULTICAST) && (lldev->all_multicast)) {
status = xge_hal_device_mcast_disable(hldev);
lldev->all_multicast = 0;
}
if(status != XGE_HAL_OK) {
xge_trace(XGE_ERR, "Enabling/disabling multicast failed");
goto _exit;
}
/* Updating address list */
if_maddr_rlock(ifnetp);
index = 0;
TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) {
if(ifma->ifma_addr->sa_family != AF_LINK) {
continue;
}
lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
index += 1;
}
if_maddr_runlock(ifnetp);
if((!lldev->all_multicast) && (index)) {
lldev->macaddr_count = (index + 1);
if(lldev->macaddr_count > table_size) {
goto _exit;
}
/* Clear old addresses */
for(index = 0; index < 48; index++) {
xge_hal_device_macaddr_set(hldev, (offset + index),
initial_addr);
}
}
/* Add new addresses */
if_maddr_rlock(ifnetp);
index = 0;
TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) {
if(ifma->ifma_addr->sa_family != AF_LINK) {
continue;
}
lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
xge_hal_device_macaddr_set(hldev, (offset + index), lladdr);
index += 1;
}
if_maddr_runlock(ifnetp);
_exit:
return;
}
/**
* xge_enable_promisc
* Enable Promiscuous Mode
*
* @lldev Per-adapter Data
*/
void
xge_enable_promisc(xge_lldev_t *lldev)
{
struct ifnet *ifnetp = lldev->ifnetp;
xge_hal_device_t *hldev = lldev->devh;
xge_hal_pci_bar0_t *bar0 = NULL;
u64 val64 = 0;
bar0 = (xge_hal_pci_bar0_t *) hldev->bar0;
if(ifnetp->if_flags & IFF_PROMISC) {
xge_hal_device_promisc_enable(lldev->devh);
/*
* When operating in promiscuous mode, don't strip the VLAN tag
*/
val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0,
&bar0->rx_pa_cfg);
val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(0);
xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64,
&bar0->rx_pa_cfg);
xge_trace(XGE_TRACE, "Promiscuous mode ON");
}
}
/**
* xge_disable_promisc
* Disable Promiscuous Mode
*
* @lldev Per-adapter Data
*/
void
xge_disable_promisc(xge_lldev_t *lldev)
{
xge_hal_device_t *hldev = lldev->devh;
xge_hal_pci_bar0_t *bar0 = NULL;
u64 val64 = 0;
bar0 = (xge_hal_pci_bar0_t *) hldev->bar0;
xge_hal_device_promisc_disable(lldev->devh);
/*
* Strip VLAN tag when operating in non-promiscuous mode
*/
val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0,
&bar0->rx_pa_cfg);
val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64,
&bar0->rx_pa_cfg);
xge_trace(XGE_TRACE, "Promiscuous mode OFF");
}
/**
* xge_change_mtu
* Change interface MTU to a requested valid size
*
* @lldev Per-adapter Data
* @NewMtu Requested MTU
*
* Returns 0 or Error Number
*/
int
xge_change_mtu(xge_lldev_t *lldev, int new_mtu)
{
int status = XGE_HAL_OK;
/* Check requested MTU size for boundary */
if(xge_hal_device_mtu_check(lldev->devh, new_mtu) != XGE_HAL_OK) {
XGE_EXIT_ON_ERR("Invalid MTU", _exit, EINVAL);
}
lldev->mtu = new_mtu;
xge_confirm_changes(lldev, XGE_SET_MTU);
_exit:
return status;
}
/**
* xge_device_stop
*
* Common code for both stop and part of reset. Disables device, interrupts and
* closes channels
*
* @dev Device Handle
* @option Channel normal/reset option
*/
void
xge_device_stop(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
{
xge_hal_device_t *hldev = lldev->devh;
struct ifnet *ifnetp = lldev->ifnetp;
u64 val64 = 0;
mtx_assert((&lldev->mtx_drv), MA_OWNED);
/* If device is not in "Running" state, return */
if (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING))
goto _exit;
/* Set appropriate flags */
ifnetp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
/* Stop timer */
callout_stop(&lldev->timer);
/* Disable interrupts */
xge_hal_device_intr_disable(hldev);
mtx_unlock(&lldev->mtx_drv);
xge_queue_flush(xge_hal_device_queue(lldev->devh));
mtx_lock(&lldev->mtx_drv);
/* Disable HAL device */
if(xge_hal_device_disable(hldev) != XGE_HAL_OK) {
xge_trace(XGE_ERR, "Disabling HAL device failed");
xge_hal_device_status(hldev, &val64);
xge_trace(XGE_ERR, "Adapter Status: 0x%llx", (long long)val64);
}
/* Close Tx and Rx channels */
xge_channel_close(lldev, option);
/* Reset HAL device */
xge_hal_device_reset(hldev);
xge_os_mdelay(1000);
lldev->initialized = 0;
if_link_state_change(ifnetp, LINK_STATE_DOWN);
_exit:
return;
}
/**
* xge_set_mbuf_cflags
* set checksum flag for the mbuf
*
* @pkt Packet
*/
void
xge_set_mbuf_cflags(mbuf_t pkt)
{
pkt->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
pkt->m_pkthdr.csum_flags |= CSUM_IP_VALID;
pkt->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
pkt->m_pkthdr.csum_data = htons(0xffff);
}
/**
* xge_lro_flush_sessions
* Flush LRO session and send accumulated LRO packet to upper layer
*
* @lldev Per-adapter Data
*/
void
xge_lro_flush_sessions(xge_lldev_t *lldev)
{
xge_lro_entry_t *lro_session = NULL;
while(!SLIST_EMPTY(&lldev->lro_active)) {
lro_session = SLIST_FIRST(&lldev->lro_active);
SLIST_REMOVE_HEAD(&lldev->lro_active, next);
xge_lro_flush(lldev, lro_session);
}
}
/**
* xge_lro_flush
* Flush LRO session. Send accumulated LRO packet to upper layer
*
* @lldev Per-adapter Data
* @lro LRO session to be flushed
*/
static void
xge_lro_flush(xge_lldev_t *lldev, xge_lro_entry_t *lro_session)
{
struct ip *header_ip;
struct tcphdr *header_tcp;
u32 *ptr;
if(lro_session->append_cnt) {
header_ip = lro_session->lro_header_ip;
header_ip->ip_len = htons(lro_session->len - ETHER_HDR_LEN);
lro_session->m_head->m_pkthdr.len = lro_session->len;
header_tcp = (struct tcphdr *)(header_ip + 1);
header_tcp->th_ack = lro_session->ack_seq;
header_tcp->th_win = lro_session->window;
if(lro_session->timestamp) {
ptr = (u32 *)(header_tcp + 1);
ptr[1] = htonl(lro_session->tsval);
ptr[2] = lro_session->tsecr;
}
}
(*lldev->ifnetp->if_input)(lldev->ifnetp, lro_session->m_head);
lro_session->m_head = NULL;
lro_session->timestamp = 0;
lro_session->append_cnt = 0;
SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next);
}
/**
* xge_lro_accumulate
* Accumulate packets to form a large LRO packet based on various conditions
*
* @lldev Per-adapter Data
* @m_head Current Packet
*
* Returns XGE_HAL_OK or XGE_HAL_FAIL (failure)
*/
static int
xge_lro_accumulate(xge_lldev_t *lldev, struct mbuf *m_head)
{
struct ether_header *header_ethernet;
struct ip *header_ip;
struct tcphdr *header_tcp;
u32 seq, *ptr;
struct mbuf *buffer_next, *buffer_tail;
xge_lro_entry_t *lro_session;
xge_hal_status_e status = XGE_HAL_FAIL;
int hlen, ip_len, tcp_hdr_len, tcp_data_len, tot_len, tcp_options;
int trim;
/* Get Ethernet header */
header_ethernet = mtod(m_head, struct ether_header *);
/* Return if it is not IP packet */
if(header_ethernet->ether_type != htons(ETHERTYPE_IP))
goto _exit;
/* Get IP header */
header_ip = lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1 ?
(struct ip *)(header_ethernet + 1) :
mtod(m_head->m_next, struct ip *);
/* Return if it is not TCP packet */
if(header_ip->ip_p != IPPROTO_TCP)
goto _exit;
/* Return if packet has options */
if((header_ip->ip_hl << 2) != sizeof(*header_ip))
goto _exit;
/* Return if packet is fragmented */
if(header_ip->ip_off & htons(IP_MF | IP_OFFMASK))
goto _exit;
/* Get TCP header */
header_tcp = (struct tcphdr *)(header_ip + 1);
/* Return if not ACK or PUSH */
if((header_tcp->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
goto _exit;
/* Only timestamp option is handled */
tcp_options = (header_tcp->th_off << 2) - sizeof(*header_tcp);
tcp_hdr_len = sizeof(*header_tcp) + tcp_options;
ptr = (u32 *)(header_tcp + 1);
if(tcp_options != 0) {
if(__predict_false(tcp_options != TCPOLEN_TSTAMP_APPA) ||
(*ptr != ntohl(TCPOPT_NOP << 24 | TCPOPT_NOP << 16 |
TCPOPT_TIMESTAMP << 8 | TCPOLEN_TIMESTAMP))) {
goto _exit;
}
}
/* Total length of packet (IP) */
ip_len = ntohs(header_ip->ip_len);
/* TCP data size */
tcp_data_len = ip_len - (header_tcp->th_off << 2) - sizeof(*header_ip);
/* If the frame is padded, trim it */
tot_len = m_head->m_pkthdr.len;
trim = tot_len - (ip_len + ETHER_HDR_LEN);
if(trim != 0) {
if(trim < 0)
goto _exit;
m_adj(m_head, -trim);
tot_len = m_head->m_pkthdr.len;
}
buffer_next = m_head;
buffer_tail = NULL;
while(buffer_next != NULL) {
buffer_tail = buffer_next;
buffer_next = buffer_tail->m_next;
}
/* Total size of only headers */
hlen = ip_len + ETHER_HDR_LEN - tcp_data_len;
/* Get sequence number */
seq = ntohl(header_tcp->th_seq);
SLIST_FOREACH(lro_session, &lldev->lro_active, next) {
if(lro_session->source_port == header_tcp->th_sport &&
lro_session->dest_port == header_tcp->th_dport &&
lro_session->source_ip == header_ip->ip_src.s_addr &&
lro_session->dest_ip == header_ip->ip_dst.s_addr) {
/* Unmatched sequence number, flush LRO session */
if(__predict_false(seq != lro_session->next_seq)) {
SLIST_REMOVE(&lldev->lro_active, lro_session,
xge_lro_entry_t, next);
xge_lro_flush(lldev, lro_session);
goto _exit;
}
/* Handle timestamp option */
if(tcp_options) {
u32 tsval = ntohl(*(ptr + 1));
if(__predict_false(lro_session->tsval > tsval ||
*(ptr + 2) == 0)) {
goto _exit;
}
lro_session->tsval = tsval;
lro_session->tsecr = *(ptr + 2);
}
lro_session->next_seq += tcp_data_len;
lro_session->ack_seq = header_tcp->th_ack;
lro_session->window = header_tcp->th_win;
/* If TCP data/payload is of 0 size, free mbuf */
if(tcp_data_len == 0) {
m_freem(m_head);
status = XGE_HAL_OK;
goto _exit;
}
lro_session->append_cnt++;
lro_session->len += tcp_data_len;
/* Adjust mbuf so that m_data points to payload than headers */
m_adj(m_head, hlen);
/* Append this packet to LRO accumulated packet */
lro_session->m_tail->m_next = m_head;
lro_session->m_tail = buffer_tail;
/* Flush if LRO packet is exceeding maximum size */
if(lro_session->len >
(XGE_HAL_LRO_DEFAULT_FRM_LEN - lldev->ifnetp->if_mtu)) {
SLIST_REMOVE(&lldev->lro_active, lro_session,
xge_lro_entry_t, next);
xge_lro_flush(lldev, lro_session);
}
status = XGE_HAL_OK;
goto _exit;
}
}
if(SLIST_EMPTY(&lldev->lro_free))
goto _exit;
/* Start a new LRO session */
lro_session = SLIST_FIRST(&lldev->lro_free);
SLIST_REMOVE_HEAD(&lldev->lro_free, next);
SLIST_INSERT_HEAD(&lldev->lro_active, lro_session, next);
lro_session->source_port = header_tcp->th_sport;
lro_session->dest_port = header_tcp->th_dport;
lro_session->source_ip = header_ip->ip_src.s_addr;
lro_session->dest_ip = header_ip->ip_dst.s_addr;
lro_session->next_seq = seq + tcp_data_len;
lro_session->mss = tcp_data_len;
lro_session->ack_seq = header_tcp->th_ack;
lro_session->window = header_tcp->th_win;
lro_session->lro_header_ip = header_ip;
/* Handle timestamp option */
if(tcp_options) {
lro_session->timestamp = 1;
lro_session->tsval = ntohl(*(ptr + 1));
lro_session->tsecr = *(ptr + 2);
}
lro_session->len = tot_len;
lro_session->m_head = m_head;
lro_session->m_tail = buffer_tail;
status = XGE_HAL_OK;
_exit:
return status;
}
/**
* xge_accumulate_large_rx
* Accumulate packets to form a large LRO packet based on various conditions
*
* @lldev Per-adapter Data
* @pkt Current packet
* @pkt_length Packet Length
* @rxd_priv Rx Descriptor Private Data
*/
void
xge_accumulate_large_rx(xge_lldev_t *lldev, struct mbuf *pkt, int pkt_length,
xge_rx_priv_t *rxd_priv)
{
if(xge_lro_accumulate(lldev, pkt) != XGE_HAL_OK) {
bus_dmamap_sync(lldev->dma_tag_rx, rxd_priv->dmainfo[0].dma_map,
BUS_DMASYNC_POSTREAD);
(*lldev->ifnetp->if_input)(lldev->ifnetp, pkt);
}
}
/**
* xge_rx_compl
* If the interrupt is due to received frame (Rx completion), send it up
*
* @channelh Ring Channel Handle
* @dtr Current Descriptor
* @t_code Transfer Code indicating success or error
* @userdata Per-adapter Data
*
* Returns XGE_HAL_OK or HAL error enums
*/
xge_hal_status_e
xge_rx_compl(xge_hal_channel_h channelh, xge_hal_dtr_h dtr, u8 t_code,
void *userdata)
{
struct ifnet *ifnetp;
xge_rx_priv_t *rxd_priv = NULL;
mbuf_t mbuf_up = NULL;
xge_hal_status_e status = XGE_HAL_OK;
xge_hal_dtr_info_t ext_info;
int index;
u16 vlan_tag;
/*get the user data portion*/
xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
if(!lldev) {
XGE_EXIT_ON_ERR("Failed to get user data", _exit, XGE_HAL_FAIL);
}
XGE_DRV_STATS(rx_completions);
/* get the interface pointer */
ifnetp = lldev->ifnetp;
do {
XGE_DRV_STATS(rx_desc_compl);
if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) {
status = XGE_HAL_FAIL;
goto _exit;
}
if(t_code) {
xge_trace(XGE_TRACE, "Packet dropped because of %d", t_code);
XGE_DRV_STATS(rx_tcode);
xge_hal_device_handle_tcode(channelh, dtr, t_code);
xge_hal_ring_dtr_post(channelh,dtr);
continue;
}
/* Get the private data for this descriptor*/
rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh,
dtr);
if(!rxd_priv) {
XGE_EXIT_ON_ERR("Failed to get descriptor private data", _exit,
XGE_HAL_FAIL);
}
/*
* Prepare one buffer to send it to upper layer -- since the upper
* layer frees the buffer do not use rxd_priv->buffer. Meanwhile
* prepare a new buffer, do mapping, use it in the current
* descriptor and post descriptor back to ring channel
*/
mbuf_up = rxd_priv->bufferArray[0];
/* Gets details of mbuf i.e., packet length */
xge_ring_dtr_get(mbuf_up, channelh, dtr, lldev, rxd_priv);
status =
(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ?
xge_get_buf(dtr, rxd_priv, lldev, 0) :
xge_get_buf_3b_5b(dtr, rxd_priv, lldev);
if(status != XGE_HAL_OK) {
xge_trace(XGE_ERR, "No memory");
XGE_DRV_STATS(rx_no_buf);
/*
* Unable to allocate buffer. Instead of discarding, post
* descriptor back to channel for future processing of same
* packet.
*/
xge_hal_ring_dtr_post(channelh, dtr);
continue;
}
/* Get the extended information */
xge_hal_ring_dtr_info_get(channelh, dtr, &ext_info);
/*
* As we have allocated a new mbuf for this descriptor, post this
* descriptor with new mbuf back to ring channel
*/
vlan_tag = ext_info.vlan;
xge_hal_ring_dtr_post(channelh, dtr);
if ((!(ext_info.proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) &&
(ext_info.proto & XGE_HAL_FRAME_PROTO_TCP_OR_UDP) &&
(ext_info.l3_cksum == XGE_HAL_L3_CKSUM_OK) &&
(ext_info.l4_cksum == XGE_HAL_L4_CKSUM_OK))) {
/* set Checksum Flag */
xge_set_mbuf_cflags(mbuf_up);
if(lldev->enabled_lro) {
xge_accumulate_large_rx(lldev, mbuf_up, mbuf_up->m_len,
rxd_priv);
}
else {
/* Post-Read sync for buffers*/
for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
bus_dmamap_sync(lldev->dma_tag_rx,
rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD);
}
(*ifnetp->if_input)(ifnetp, mbuf_up);
}
}
else {
/*
* Packet with erroneous checksum , let the upper layer deal
* with it
*/
/* Post-Read sync for buffers*/
for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
bus_dmamap_sync(lldev->dma_tag_rx,
rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD);
}
if(vlan_tag) {
mbuf_up->m_pkthdr.ether_vtag = vlan_tag;
mbuf_up->m_flags |= M_VLANTAG;
}
if(lldev->enabled_lro)
xge_lro_flush_sessions(lldev);
(*ifnetp->if_input)(ifnetp, mbuf_up);
}
} while(xge_hal_ring_dtr_next_completed(channelh, &dtr, &t_code)
== XGE_HAL_OK);
if(lldev->enabled_lro)
xge_lro_flush_sessions(lldev);
_exit:
return status;
}
/**
* xge_ring_dtr_get
* Get descriptors
*
* @mbuf_up Packet to send up
* @channelh Ring Channel Handle
* @dtr Descriptor
* @lldev Per-adapter Data
* @rxd_priv Rx Descriptor Private Data
*
* Returns XGE_HAL_OK or HAL error enums
*/
int
xge_ring_dtr_get(mbuf_t mbuf_up, xge_hal_channel_h channelh, xge_hal_dtr_h dtr,
xge_lldev_t *lldev, xge_rx_priv_t *rxd_priv)
{
mbuf_t m;
int pkt_length[5]={0,0}, pkt_len=0;
dma_addr_t dma_data[5];
int index;
m = mbuf_up;
pkt_len = 0;
if(lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
xge_os_memzero(pkt_length, sizeof(pkt_length));
/*
* Retrieve data of interest from the completed descriptor -- This
* returns the packet length
*/
if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
xge_hal_ring_dtr_5b_get(channelh, dtr, dma_data, pkt_length);
}
else {
xge_hal_ring_dtr_3b_get(channelh, dtr, dma_data, pkt_length);
}
for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
m->m_len = pkt_length[index];
if(index < (lldev->rxd_mbuf_cnt-1)) {
m->m_next = rxd_priv->bufferArray[index + 1];
m = m->m_next;
}
else {
m->m_next = NULL;
}
pkt_len+=pkt_length[index];
}
/*
* Since 2 buffer mode is an exceptional case where data is in 3rd
* buffer but not in 2nd buffer
*/
if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
m->m_len = pkt_length[2];
pkt_len+=pkt_length[2];
}
/*
* Update length of newly created buffer to be sent up with packet
* length
*/
mbuf_up->m_pkthdr.len = pkt_len;
}
else {
/*
* Retrieve data of interest from the completed descriptor -- This
* returns the packet length
*/
xge_hal_ring_dtr_1b_get(channelh, dtr,&dma_data[0], &pkt_length[0]);
/*
* Update length of newly created buffer to be sent up with packet
* length
*/
mbuf_up->m_len = mbuf_up->m_pkthdr.len = pkt_length[0];
}
return XGE_HAL_OK;
}
/**
* xge_flush_txds
* Flush Tx descriptors
*
* @channelh Channel handle
*/
static void inline
xge_flush_txds(xge_hal_channel_h channelh)
{
xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
xge_hal_dtr_h tx_dtr;
xge_tx_priv_t *tx_priv;
u8 t_code;
while(xge_hal_fifo_dtr_next_completed(channelh, &tx_dtr, &t_code)
== XGE_HAL_OK) {
XGE_DRV_STATS(tx_desc_compl);
if(t_code) {
xge_trace(XGE_TRACE, "Tx descriptor with t_code %d", t_code);
XGE_DRV_STATS(tx_tcode);
xge_hal_device_handle_tcode(channelh, tx_dtr, t_code);
}
tx_priv = xge_hal_fifo_dtr_private(tx_dtr);
bus_dmamap_unload(lldev->dma_tag_tx, tx_priv->dma_map);
m_freem(tx_priv->buffer);
tx_priv->buffer = NULL;
xge_hal_fifo_dtr_free(channelh, tx_dtr);
}
}
/**
* xge_send
* Transmit function
*
* @ifnetp Interface Handle
*/
void
xge_send(struct ifnet *ifnetp)
{
int qindex = 0;
xge_lldev_t *lldev = ifnetp->if_softc;
for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
if(mtx_trylock(&lldev->mtx_tx[qindex]) == 0) {
XGE_DRV_STATS(tx_lock_fail);
break;
}
xge_send_locked(ifnetp, qindex);
mtx_unlock(&lldev->mtx_tx[qindex]);
}
}
static void inline
xge_send_locked(struct ifnet *ifnetp, int qindex)
{
xge_hal_dtr_h dtr;
static bus_dma_segment_t segs[XGE_MAX_SEGS];
xge_hal_status_e status;
unsigned int max_fragments;
xge_lldev_t *lldev = ifnetp->if_softc;
xge_hal_channel_h channelh = lldev->fifo_channel[qindex];
mbuf_t m_head = NULL;
mbuf_t m_buf = NULL;
xge_tx_priv_t *ll_tx_priv = NULL;
register unsigned int count = 0;
unsigned int nsegs = 0;
u16 vlan_tag;
max_fragments = ((xge_hal_fifo_t *)channelh)->config->max_frags;
/* If device is not initialized, return */
if((!lldev->initialized) || (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)))
return;
XGE_DRV_STATS(tx_calls);
/*
* This loop will be executed for each packet in the kernel maintained
* queue -- each packet can be with fragments as an mbuf chain
*/
for(;;) {
IF_DEQUEUE(&ifnetp->if_snd, m_head);
if (m_head == NULL) {
ifnetp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
return;
}
for(m_buf = m_head; m_buf != NULL; m_buf = m_buf->m_next) {
if(m_buf->m_len) count += 1;
}
if(count >= max_fragments) {
m_buf = m_defrag(m_head, M_NOWAIT);
if(m_buf != NULL) m_head = m_buf;
XGE_DRV_STATS(tx_defrag);
}
/* Reserve descriptors */
status = xge_hal_fifo_dtr_reserve(channelh, &dtr);
if(status != XGE_HAL_OK) {
XGE_DRV_STATS(tx_no_txd);
xge_flush_txds(channelh);
break;
}
vlan_tag =
(m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;
xge_hal_fifo_dtr_vlan_set(dtr, vlan_tag);
/* Update Tx private structure for this descriptor */
ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
ll_tx_priv->buffer = m_head;
/*
* Do mapping -- Required DMA tag has been created in xge_init
* function and DMA maps have already been created in the
* xgell_tx_replenish function.
* Returns number of segments through nsegs
*/
if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_tx,
ll_tx_priv->dma_map, m_head, segs, &nsegs, BUS_DMA_NOWAIT)) {
xge_trace(XGE_TRACE, "DMA map load failed");
XGE_DRV_STATS(tx_map_fail);
break;
}
if(lldev->driver_stats.tx_max_frags < nsegs)
lldev->driver_stats.tx_max_frags = nsegs;
/* Set descriptor buffer for header and each fragment/segment */
count = 0;
do {
xge_hal_fifo_dtr_buffer_set(channelh, dtr, count,
(dma_addr_t)htole64(segs[count].ds_addr),
segs[count].ds_len);
count++;
} while(count < nsegs);
/* Pre-write Sync of mapping */
bus_dmamap_sync(lldev->dma_tag_tx, ll_tx_priv->dma_map,
BUS_DMASYNC_PREWRITE);
if((lldev->enabled_tso) &&
(m_head->m_pkthdr.csum_flags & CSUM_TSO)) {
XGE_DRV_STATS(tx_tso);
xge_hal_fifo_dtr_mss_set(dtr, m_head->m_pkthdr.tso_segsz);
}
/* Checksum */
if(ifnetp->if_hwassist > 0) {
xge_hal_fifo_dtr_cksum_set_bits(dtr, XGE_HAL_TXD_TX_CKO_IPV4_EN
| XGE_HAL_TXD_TX_CKO_TCP_EN | XGE_HAL_TXD_TX_CKO_UDP_EN);
}
/* Post descriptor to FIFO channel */
xge_hal_fifo_dtr_post(channelh, dtr);
XGE_DRV_STATS(tx_posted);
/* Send the same copy of mbuf packet to BPF (Berkely Packet Filter)
* listener so that we can use tools like tcpdump */
ETHER_BPF_MTAP(ifnetp, m_head);
}
/* Prepend the packet back to queue */
IF_PREPEND(&ifnetp->if_snd, m_head);
ifnetp->if_drv_flags |= IFF_DRV_OACTIVE;
xge_queue_produce_context(xge_hal_device_queue(lldev->devh),
XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh);
XGE_DRV_STATS(tx_again);
}
/**
* xge_get_buf
* Allocates new mbufs to be placed into descriptors
*
* @dtrh Descriptor Handle
* @rxd_priv Rx Descriptor Private Data
* @lldev Per-adapter Data
* @index Buffer Index (if multi-buffer mode)
*
* Returns XGE_HAL_OK or HAL error enums
*/
int
xge_get_buf(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv,
xge_lldev_t *lldev, int index)
{
register mbuf_t mp = NULL;
struct ifnet *ifnetp = lldev->ifnetp;
int status = XGE_HAL_OK;
int buffer_size = 0, cluster_size = 0, count;
bus_dmamap_t map = rxd_priv->dmainfo[index].dma_map;
bus_dma_segment_t segs[3];
buffer_size = (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ?
ifnetp->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE :
lldev->rxd_mbuf_len[index];
if(buffer_size <= MCLBYTES) {
cluster_size = MCLBYTES;
mp = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
}
else {
cluster_size = MJUMPAGESIZE;
if((lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5) &&
(buffer_size > MJUMPAGESIZE)) {
cluster_size = MJUM9BYTES;
}
mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, cluster_size);
}
if(!mp) {
xge_trace(XGE_ERR, "Out of memory to allocate mbuf");
status = XGE_HAL_FAIL;
goto getbuf_out;
}
/* Update mbuf's length, packet length and receive interface */
mp->m_len = mp->m_pkthdr.len = buffer_size;
mp->m_pkthdr.rcvif = ifnetp;
/* Load DMA map */
if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_rx, lldev->extra_dma_map,
mp, segs, &count, BUS_DMA_NOWAIT)) {
XGE_DRV_STATS(rx_map_fail);
m_freem(mp);
XGE_EXIT_ON_ERR("DMA map load failed", getbuf_out, XGE_HAL_FAIL);
}
/* Update descriptor private data */
rxd_priv->bufferArray[index] = mp;
rxd_priv->dmainfo[index].dma_phyaddr = htole64(segs->ds_addr);
rxd_priv->dmainfo[index].dma_map = lldev->extra_dma_map;
lldev->extra_dma_map = map;
/* Pre-Read/Write sync */
bus_dmamap_sync(lldev->dma_tag_rx, map, BUS_DMASYNC_POSTREAD);
/* Unload DMA map of mbuf in current descriptor */
bus_dmamap_unload(lldev->dma_tag_rx, map);
/* Set descriptor buffer */
if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
xge_hal_ring_dtr_1b_set(dtrh, rxd_priv->dmainfo[0].dma_phyaddr,
cluster_size);
}
getbuf_out:
return status;
}
/**
* xge_get_buf_3b_5b
* Allocates new mbufs to be placed into descriptors (in multi-buffer modes)
*
* @dtrh Descriptor Handle
* @rxd_priv Rx Descriptor Private Data
* @lldev Per-adapter Data
*
* Returns XGE_HAL_OK or HAL error enums
*/
int
xge_get_buf_3b_5b(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv,
xge_lldev_t *lldev)
{
bus_addr_t dma_pointers[5];
int dma_sizes[5];
int status = XGE_HAL_OK, index;
int newindex = 0;
for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
status = xge_get_buf(dtrh, rxd_priv, lldev, index);
if(status != XGE_HAL_OK) {
for(newindex = 0; newindex < index; newindex++) {
m_freem(rxd_priv->bufferArray[newindex]);
}
XGE_EXIT_ON_ERR("mbuf allocation failed", _exit, status);
}
}
for(index = 0; index < lldev->buffer_mode; index++) {
if(lldev->rxd_mbuf_len[index] != 0) {
dma_pointers[index] = rxd_priv->dmainfo[index].dma_phyaddr;
dma_sizes[index] = lldev->rxd_mbuf_len[index];
}
else {
dma_pointers[index] = rxd_priv->dmainfo[index-1].dma_phyaddr;
dma_sizes[index] = 1;
}
}
/* Assigning second buffer to third pointer in 2 buffer mode */
if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
dma_pointers[2] = dma_pointers[1];
dma_sizes[2] = dma_sizes[1];
dma_sizes[1] = 1;
}
if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
xge_hal_ring_dtr_5b_set(dtrh, dma_pointers, dma_sizes);
}
else {
xge_hal_ring_dtr_3b_set(dtrh, dma_pointers, dma_sizes);
}
_exit:
return status;
}
/**
* xge_tx_compl
* If the interrupt is due to Tx completion, free the sent buffer
*
* @channelh Channel Handle
* @dtr Descriptor
* @t_code Transfer Code indicating success or error
* @userdata Per-adapter Data
*
* Returns XGE_HAL_OK or HAL error enum
*/
xge_hal_status_e
xge_tx_compl(xge_hal_channel_h channelh,
xge_hal_dtr_h dtr, u8 t_code, void *userdata)
{
xge_tx_priv_t *ll_tx_priv = NULL;
xge_lldev_t *lldev = (xge_lldev_t *)userdata;
struct ifnet *ifnetp = lldev->ifnetp;
mbuf_t m_buffer = NULL;
int qindex = xge_hal_channel_id(channelh);
mtx_lock(&lldev->mtx_tx[qindex]);
XGE_DRV_STATS(tx_completions);
/*
* For each completed descriptor: Get private structure, free buffer,
* do unmapping, and free descriptor
*/
do {
XGE_DRV_STATS(tx_desc_compl);
if(t_code) {
XGE_DRV_STATS(tx_tcode);
xge_trace(XGE_TRACE, "t_code %d", t_code);
xge_hal_device_handle_tcode(channelh, dtr, t_code);
}
ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
m_buffer = ll_tx_priv->buffer;
bus_dmamap_unload(lldev->dma_tag_tx, ll_tx_priv->dma_map);
m_freem(m_buffer);
ll_tx_priv->buffer = NULL;
xge_hal_fifo_dtr_free(channelh, dtr);
} while(xge_hal_fifo_dtr_next_completed(channelh, &dtr, &t_code)
== XGE_HAL_OK);
xge_send_locked(ifnetp, qindex);
ifnetp->if_drv_flags &= ~IFF_DRV_OACTIVE;
mtx_unlock(&lldev->mtx_tx[qindex]);
return XGE_HAL_OK;
}
/**
* xge_tx_initial_replenish
* Initially allocate buffers and set them into descriptors for later use
*
* @channelh Tx Channel Handle
* @dtrh Descriptor Handle
* @index
* @userdata Per-adapter Data
* @reopen Channel open/reopen option
*
* Returns XGE_HAL_OK or HAL error enums
*/
xge_hal_status_e
xge_tx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
int index, void *userdata, xge_hal_channel_reopen_e reopen)
{
xge_tx_priv_t *txd_priv = NULL;
int status = XGE_HAL_OK;
/* Get the user data portion from channel handle */
xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
if(lldev == NULL) {
XGE_EXIT_ON_ERR("Failed to get user data from channel", txinit_out,
XGE_HAL_FAIL);
}
/* Get the private data */
txd_priv = (xge_tx_priv_t *) xge_hal_fifo_dtr_private(dtrh);
if(txd_priv == NULL) {
XGE_EXIT_ON_ERR("Failed to get descriptor private data", txinit_out,
XGE_HAL_FAIL);
}
/* Create DMA map for this descriptor */
if(bus_dmamap_create(lldev->dma_tag_tx, BUS_DMA_NOWAIT,
&txd_priv->dma_map)) {
XGE_EXIT_ON_ERR("DMA map creation for Tx descriptor failed",
txinit_out, XGE_HAL_FAIL);
}
txinit_out:
return status;
}
/**
* xge_rx_initial_replenish
* Initially allocate buffers and set them into descriptors for later use
*
* @channelh Tx Channel Handle
* @dtrh Descriptor Handle
* @index Ring Index
* @userdata Per-adapter Data
* @reopen Channel open/reopen option
*
* Returns XGE_HAL_OK or HAL error enums
*/
xge_hal_status_e
xge_rx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
int index, void *userdata, xge_hal_channel_reopen_e reopen)
{
xge_rx_priv_t *rxd_priv = NULL;
int status = XGE_HAL_OK;
int index1 = 0, index2 = 0;
/* Get the user data portion from channel handle */
xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
if(lldev == NULL) {
XGE_EXIT_ON_ERR("Failed to get user data from channel", rxinit_out,
XGE_HAL_FAIL);
}
/* Get the private data */
rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh);
if(rxd_priv == NULL) {
XGE_EXIT_ON_ERR("Failed to get descriptor private data", rxinit_out,
XGE_HAL_FAIL);
}
rxd_priv->bufferArray = xge_os_malloc(NULL,
(sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
if(rxd_priv->bufferArray == NULL) {
XGE_EXIT_ON_ERR("Failed to allocate Rxd private", rxinit_out,
XGE_HAL_FAIL);
}
if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
/* Create DMA map for these descriptors*/
if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT,
&rxd_priv->dmainfo[0].dma_map)) {
XGE_EXIT_ON_ERR("DMA map creation for Rx descriptor failed",
rxinit_err_out, XGE_HAL_FAIL);
}
/* Get a buffer, attach it to this descriptor */
status = xge_get_buf(dtrh, rxd_priv, lldev, 0);
}
else {
for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) {
/* Create DMA map for this descriptor */
if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT ,
&rxd_priv->dmainfo[index1].dma_map)) {
for(index2 = index1 - 1; index2 >= 0; index2--) {
bus_dmamap_destroy(lldev->dma_tag_rx,
rxd_priv->dmainfo[index2].dma_map);
}
XGE_EXIT_ON_ERR(
"Jumbo DMA map creation for Rx descriptor failed",
rxinit_err_out, XGE_HAL_FAIL);
}
}
status = xge_get_buf_3b_5b(dtrh, rxd_priv, lldev);
}
if(status != XGE_HAL_OK) {
for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) {
bus_dmamap_destroy(lldev->dma_tag_rx,
rxd_priv->dmainfo[index1].dma_map);
}
goto rxinit_err_out;
}
else {
goto rxinit_out;
}
rxinit_err_out:
xge_os_free(NULL, rxd_priv->bufferArray,
(sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
rxinit_out:
return status;
}
/**
* xge_rx_term
* During unload terminate and free all descriptors
*
* @channelh Rx Channel Handle
* @dtrh Rx Descriptor Handle
* @state Descriptor State
* @userdata Per-adapter Data
* @reopen Channel open/reopen option
*/
void
xge_rx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
xge_hal_dtr_state_e state, void *userdata,
xge_hal_channel_reopen_e reopen)
{
xge_rx_priv_t *rxd_priv = NULL;
xge_lldev_t *lldev = NULL;
int index = 0;
/* Descriptor state is not "Posted" */
if(state != XGE_HAL_DTR_STATE_POSTED) goto rxterm_out;
/* Get the user data portion */
lldev = xge_hal_channel_userdata(channelh);
/* Get the private data */
rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh);
for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
if(rxd_priv->dmainfo[index].dma_map != NULL) {
bus_dmamap_sync(lldev->dma_tag_rx,
rxd_priv->dmainfo[index].dma_map, BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(lldev->dma_tag_rx,
rxd_priv->dmainfo[index].dma_map);
if(rxd_priv->bufferArray[index] != NULL)
m_free(rxd_priv->bufferArray[index]);
bus_dmamap_destroy(lldev->dma_tag_rx,
rxd_priv->dmainfo[index].dma_map);
}
}
xge_os_free(NULL, rxd_priv->bufferArray,
(sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
/* Free the descriptor */
xge_hal_ring_dtr_free(channelh, dtrh);
rxterm_out:
return;
}
/**
* xge_tx_term
* During unload terminate and free all descriptors
*
* @channelh Rx Channel Handle
* @dtrh Rx Descriptor Handle
* @state Descriptor State
* @userdata Per-adapter Data
* @reopen Channel open/reopen option
*/
void
xge_tx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtr,
xge_hal_dtr_state_e state, void *userdata,
xge_hal_channel_reopen_e reopen)
{
xge_tx_priv_t *ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
xge_lldev_t *lldev = (xge_lldev_t *)userdata;
/* Destroy DMA map */
bus_dmamap_destroy(lldev->dma_tag_tx, ll_tx_priv->dma_map);
}
/**
* xge_methods
*
* FreeBSD device interface entry points
*/
static device_method_t xge_methods[] = {
DEVMETHOD(device_probe, xge_probe),
DEVMETHOD(device_attach, xge_attach),
DEVMETHOD(device_detach, xge_detach),
DEVMETHOD(device_shutdown, xge_shutdown),
DEVMETHOD_END
};
static driver_t xge_driver = {
"nxge",
xge_methods,
sizeof(xge_lldev_t),
};
static devclass_t xge_devclass;
DRIVER_MODULE(nxge, pci, xge_driver, xge_devclass, 0, 0);
Index: head/sys/dev/oce/oce_if.c
===================================================================
--- head/sys/dev/oce/oce_if.c (revision 283290)
+++ head/sys/dev/oce/oce_if.c (revision 283291)
@@ -1,2359 +1,2359 @@
/*-
* Copyright (C) 2013 Emulex
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. Neither the name of the Emulex Corporation nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* Contact Information:
* freebsd-drivers@emulex.com
*
* Emulex
* 3333 Susan Street
* Costa Mesa, CA 92626
*/
/* $FreeBSD$ */
#include "opt_inet6.h"
#include "opt_inet.h"
#include "oce_if.h"
/* UE Status Low CSR */
static char *ue_status_low_desc[] = {
"CEV",
"CTX",
"DBUF",
"ERX",
"Host",
"MPU",
"NDMA",
"PTC ",
"RDMA ",
"RXF ",
"RXIPS ",
"RXULP0 ",
"RXULP1 ",
"RXULP2 ",
"TIM ",
"TPOST ",
"TPRE ",
"TXIPS ",
"TXULP0 ",
"TXULP1 ",
"UC ",
"WDMA ",
"TXULP2 ",
"HOST1 ",
"P0_OB_LINK ",
"P1_OB_LINK ",
"HOST_GPIO ",
"MBOX ",
"AXGMAC0",
"AXGMAC1",
"JTAG",
"MPU_INTPEND"
};
/* UE Status High CSR */
static char *ue_status_hi_desc[] = {
"LPCMEMHOST",
"MGMT_MAC",
"PCS0ONLINE",
"MPU_IRAM",
"PCS1ONLINE",
"PCTL0",
"PCTL1",
"PMEM",
"RR",
"TXPB",
"RXPP",
"XAUI",
"TXP",
"ARM",
"IPC",
"HOST2",
"HOST3",
"HOST4",
"HOST5",
"HOST6",
"HOST7",
"HOST8",
"HOST9",
"NETC",
"Unknown",
"Unknown",
"Unknown",
"Unknown",
"Unknown",
"Unknown",
"Unknown",
"Unknown"
};
/* Driver entry points prototypes */
static int oce_probe(device_t dev);
static int oce_attach(device_t dev);
static int oce_detach(device_t dev);
static int oce_shutdown(device_t dev);
static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
static void oce_init(void *xsc);
static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m);
static void oce_multiq_flush(struct ifnet *ifp);
/* Driver interrupt routines protypes */
static void oce_intr(void *arg, int pending);
static int oce_setup_intr(POCE_SOFTC sc);
static int oce_fast_isr(void *arg);
static int oce_alloc_intr(POCE_SOFTC sc, int vector,
void (*isr) (void *arg, int pending));
/* Media callbacks prototypes */
static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req);
static int oce_media_change(struct ifnet *ifp);
/* Transmit routines prototypes */
static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
static void oce_tx_complete(struct oce_wq *wq, uint32_t wqe_idx,
uint32_t status);
static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m,
struct oce_wq *wq);
/* Receive routines prototypes */
static void oce_discard_rx_comp(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
static void oce_rx(struct oce_rq *rq, uint32_t rqe_idx,
struct oce_nic_rx_cqe *cqe);
/* Helper function prototypes in this file */
static int oce_attach_ifp(POCE_SOFTC sc);
static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
static int oce_vid_config(POCE_SOFTC sc);
static void oce_mac_addr_set(POCE_SOFTC sc);
static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data);
static void oce_local_timer(void *arg);
static void oce_if_deactivate(POCE_SOFTC sc);
static void oce_if_activate(POCE_SOFTC sc);
static void setup_max_queues_want(POCE_SOFTC sc);
static void update_queues_got(POCE_SOFTC sc);
static void process_link_state(POCE_SOFTC sc,
struct oce_async_cqe_link_state *acqe);
static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m);
static void oce_get_config(POCE_SOFTC sc);
static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete);
/* IP specific */
#if defined(INET6) || defined(INET)
static int oce_init_lro(POCE_SOFTC sc);
static void oce_rx_flush_lro(struct oce_rq *rq);
static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
#endif
static device_method_t oce_dispatch[] = {
DEVMETHOD(device_probe, oce_probe),
DEVMETHOD(device_attach, oce_attach),
DEVMETHOD(device_detach, oce_detach),
DEVMETHOD(device_shutdown, oce_shutdown),
DEVMETHOD_END
};
static driver_t oce_driver = {
"oce",
oce_dispatch,
sizeof(OCE_SOFTC)
};
static devclass_t oce_devclass;
DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0);
MODULE_DEPEND(oce, pci, 1, 1, 1);
MODULE_DEPEND(oce, ether, 1, 1, 1);
MODULE_VERSION(oce, 1);
/* global vars */
const char component_revision[32] = {"///" COMPONENT_REVISION "///"};
/* Module capabilites and parameters */
uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED;
uint32_t oce_enable_rss = OCE_MODCAP_RSS;
TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled);
TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss);
/* Supported devices table */
static uint32_t supportedDevices[] = {
(PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2,
(PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3,
(PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3,
(PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201,
(PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF,
(PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH
};
/*****************************************************************************
* Driver entry points functions *
*****************************************************************************/
static int
oce_probe(device_t dev)
{
uint16_t vendor = 0;
uint16_t device = 0;
int i = 0;
char str[256] = {0};
POCE_SOFTC sc;
sc = device_get_softc(dev);
bzero(sc, sizeof(OCE_SOFTC));
sc->dev = dev;
vendor = pci_get_vendor(dev);
device = pci_get_device(dev);
for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
if (device == (supportedDevices[i] & 0xffff)) {
sprintf(str, "%s:%s", "Emulex CNA NIC function",
component_revision);
device_set_desc_copy(dev, str);
switch (device) {
case PCI_PRODUCT_BE2:
sc->flags |= OCE_FLAGS_BE2;
break;
case PCI_PRODUCT_BE3:
sc->flags |= OCE_FLAGS_BE3;
break;
case PCI_PRODUCT_XE201:
case PCI_PRODUCT_XE201_VF:
sc->flags |= OCE_FLAGS_XE201;
break;
case PCI_PRODUCT_SH:
sc->flags |= OCE_FLAGS_SH;
break;
default:
return ENXIO;
}
return BUS_PROBE_DEFAULT;
}
}
}
return ENXIO;
}
static int
oce_attach(device_t dev)
{
POCE_SOFTC sc;
int rc = 0;
sc = device_get_softc(dev);
rc = oce_hw_pci_alloc(sc);
if (rc)
return rc;
sc->tx_ring_size = OCE_TX_RING_SIZE;
sc->rx_ring_size = OCE_RX_RING_SIZE;
sc->rq_frag_size = OCE_RQ_BUF_SIZE;
sc->flow_control = OCE_DEFAULT_FLOW_CONTROL;
sc->promisc = OCE_DEFAULT_PROMISCUOUS;
LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock");
LOCK_CREATE(&sc->dev_lock, "Device_lock");
/* initialise the hardware */
rc = oce_hw_init(sc);
if (rc)
goto pci_res_free;
oce_get_config(sc);
setup_max_queues_want(sc);
rc = oce_setup_intr(sc);
if (rc)
goto mbox_free;
rc = oce_queue_init_all(sc);
if (rc)
goto intr_free;
rc = oce_attach_ifp(sc);
if (rc)
goto queues_free;
#if defined(INET6) || defined(INET)
rc = oce_init_lro(sc);
if (rc)
goto ifp_free;
#endif
rc = oce_hw_start(sc);
if (rc)
goto lro_free;
sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST);
rc = oce_stats_init(sc);
if (rc)
goto vlan_free;
oce_add_sysctls(sc);
- callout_init(&sc->timer, CALLOUT_MPSAFE);
+ callout_init(&sc->timer, 1);
rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
if (rc)
goto stats_free;
return 0;
stats_free:
callout_drain(&sc->timer);
oce_stats_free(sc);
vlan_free:
if (sc->vlan_attach)
EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
if (sc->vlan_detach)
EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
oce_hw_intr_disable(sc);
lro_free:
#if defined(INET6) || defined(INET)
oce_free_lro(sc);
ifp_free:
#endif
ether_ifdetach(sc->ifp);
if_free(sc->ifp);
queues_free:
oce_queue_release_all(sc);
intr_free:
oce_intr_free(sc);
mbox_free:
oce_dma_free(sc, &sc->bsmbx);
pci_res_free:
oce_hw_pci_free(sc);
LOCK_DESTROY(&sc->dev_lock);
LOCK_DESTROY(&sc->bmbx_lock);
return rc;
}
static int
oce_detach(device_t dev)
{
POCE_SOFTC sc = device_get_softc(dev);
LOCK(&sc->dev_lock);
oce_if_deactivate(sc);
UNLOCK(&sc->dev_lock);
callout_drain(&sc->timer);
if (sc->vlan_attach != NULL)
EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
if (sc->vlan_detach != NULL)
EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
ether_ifdetach(sc->ifp);
if_free(sc->ifp);
oce_hw_shutdown(sc);
bus_generic_detach(dev);
return 0;
}
static int
oce_shutdown(device_t dev)
{
int rc;
rc = oce_detach(dev);
return rc;
}
static int
oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
{
struct ifreq *ifr = (struct ifreq *)data;
POCE_SOFTC sc = ifp->if_softc;
int rc = 0;
uint32_t u;
switch (command) {
case SIOCGIFMEDIA:
rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
break;
case SIOCSIFMTU:
if (ifr->ifr_mtu > OCE_MAX_MTU)
rc = EINVAL;
else
ifp->if_mtu = ifr->ifr_mtu;
break;
case SIOCSIFFLAGS:
if (ifp->if_flags & IFF_UP) {
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
oce_init(sc);
}
device_printf(sc->dev, "Interface Up\n");
} else {
LOCK(&sc->dev_lock);
sc->ifp->if_drv_flags &=
~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
oce_if_deactivate(sc);
UNLOCK(&sc->dev_lock);
device_printf(sc->dev, "Interface Down\n");
}
if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) {
if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1))))
sc->promisc = TRUE;
} else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) {
if (!oce_rxf_set_promiscuous(sc, 0))
sc->promisc = FALSE;
}
break;
case SIOCADDMULTI:
case SIOCDELMULTI:
rc = oce_hw_update_multicast(sc);
if (rc)
device_printf(sc->dev,
"Update multicast address failed\n");
break;
case SIOCSIFCAP:
u = ifr->ifr_reqcap ^ ifp->if_capenable;
if (u & IFCAP_TXCSUM) {
ifp->if_capenable ^= IFCAP_TXCSUM;
ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
if (IFCAP_TSO & ifp->if_capenable &&
!(IFCAP_TXCSUM & ifp->if_capenable)) {
ifp->if_capenable &= ~IFCAP_TSO;
ifp->if_hwassist &= ~CSUM_TSO;
if_printf(ifp,
"TSO disabled due to -txcsum.\n");
}
}
if (u & IFCAP_RXCSUM)
ifp->if_capenable ^= IFCAP_RXCSUM;
if (u & IFCAP_TSO4) {
ifp->if_capenable ^= IFCAP_TSO4;
if (IFCAP_TSO & ifp->if_capenable) {
if (IFCAP_TXCSUM & ifp->if_capenable)
ifp->if_hwassist |= CSUM_TSO;
else {
ifp->if_capenable &= ~IFCAP_TSO;
ifp->if_hwassist &= ~CSUM_TSO;
if_printf(ifp,
"Enable txcsum first.\n");
rc = EAGAIN;
}
} else
ifp->if_hwassist &= ~CSUM_TSO;
}
if (u & IFCAP_VLAN_HWTAGGING)
ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
if (u & IFCAP_VLAN_HWFILTER) {
ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
oce_vid_config(sc);
}
#if defined(INET6) || defined(INET)
if (u & IFCAP_LRO)
ifp->if_capenable ^= IFCAP_LRO;
#endif
break;
case SIOCGPRIVATE_0:
rc = oce_handle_passthrough(ifp, data);
break;
default:
rc = ether_ioctl(ifp, command, data);
break;
}
return rc;
}
static void
oce_init(void *arg)
{
POCE_SOFTC sc = arg;
LOCK(&sc->dev_lock);
if (sc->ifp->if_flags & IFF_UP) {
oce_if_deactivate(sc);
oce_if_activate(sc);
}
UNLOCK(&sc->dev_lock);
}
static int
oce_multiq_start(struct ifnet *ifp, struct mbuf *m)
{
POCE_SOFTC sc = ifp->if_softc;
struct oce_wq *wq = NULL;
int queue_index = 0;
int status = 0;
if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
queue_index = m->m_pkthdr.flowid % sc->nwqs;
wq = sc->wq[queue_index];
LOCK(&wq->tx_lock);
status = oce_multiq_transmit(ifp, m, wq);
UNLOCK(&wq->tx_lock);
return status;
}
static void
oce_multiq_flush(struct ifnet *ifp)
{
POCE_SOFTC sc = ifp->if_softc;
struct mbuf *m;
int i = 0;
for (i = 0; i < sc->nwqs; i++) {
while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)
m_freem(m);
}
if_qflush(ifp);
}
/*****************************************************************************
* Driver interrupt routines functions *
*****************************************************************************/
static void
oce_intr(void *arg, int pending)
{
POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
POCE_SOFTC sc = ii->sc;
struct oce_eq *eq = ii->eq;
struct oce_eqe *eqe;
struct oce_cq *cq = NULL;
int i, num_eqes = 0;
bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
BUS_DMASYNC_POSTWRITE);
do {
eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
if (eqe->evnt == 0)
break;
eqe->evnt = 0;
bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
BUS_DMASYNC_POSTWRITE);
RING_GET(eq->ring, 1);
num_eqes++;
} while (TRUE);
if (!num_eqes)
goto eq_arm; /* Spurious */
/* Clear EQ entries, but dont arm */
oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE);
/* Process TX, RX and MCC. But dont arm CQ*/
for (i = 0; i < eq->cq_valid; i++) {
cq = eq->cq[i];
(*cq->cq_handler)(cq->cb_arg);
}
/* Arm all cqs connected to this EQ */
for (i = 0; i < eq->cq_valid; i++) {
cq = eq->cq[i];
oce_arm_cq(sc, cq->cq_id, 0, TRUE);
}
eq_arm:
oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
return;
}
static int
oce_setup_intr(POCE_SOFTC sc)
{
int rc = 0, use_intx = 0;
int vector = 0, req_vectors = 0;
if (is_rss_enabled(sc))
req_vectors = MAX((sc->nrqs - 1), sc->nwqs);
else
req_vectors = 1;
if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) {
sc->intr_count = req_vectors;
rc = pci_alloc_msix(sc->dev, &sc->intr_count);
if (rc != 0) {
use_intx = 1;
pci_release_msi(sc->dev);
} else
sc->flags |= OCE_FLAGS_USING_MSIX;
} else
use_intx = 1;
if (use_intx)
sc->intr_count = 1;
/* Scale number of queues based on intr we got */
update_queues_got(sc);
if (use_intx) {
device_printf(sc->dev, "Using legacy interrupt\n");
rc = oce_alloc_intr(sc, vector, oce_intr);
if (rc)
goto error;
} else {
for (; vector < sc->intr_count; vector++) {
rc = oce_alloc_intr(sc, vector, oce_intr);
if (rc)
goto error;
}
}
return 0;
error:
oce_intr_free(sc);
return rc;
}
static int
oce_fast_isr(void *arg)
{
POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
POCE_SOFTC sc = ii->sc;
if (ii->eq == NULL)
return FILTER_STRAY;
oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE);
taskqueue_enqueue_fast(ii->tq, &ii->task);
ii->eq->intr++;
return FILTER_HANDLED;
}
static int
oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending))
{
POCE_INTR_INFO ii = &sc->intrs[vector];
int rc = 0, rr;
if (vector >= OCE_MAX_EQ)
return (EINVAL);
/* Set the resource id for the interrupt.
* MSIx is vector + 1 for the resource id,
* INTx is 0 for the resource id.
*/
if (sc->flags & OCE_FLAGS_USING_MSIX)
rr = vector + 1;
else
rr = 0;
ii->intr_res = bus_alloc_resource_any(sc->dev,
SYS_RES_IRQ,
&rr, RF_ACTIVE|RF_SHAREABLE);
ii->irq_rr = rr;
if (ii->intr_res == NULL) {
device_printf(sc->dev,
"Could not allocate interrupt\n");
rc = ENXIO;
return rc;
}
TASK_INIT(&ii->task, 0, isr, ii);
ii->vector = vector;
sprintf(ii->task_name, "oce_task[%d]", ii->vector);
ii->tq = taskqueue_create_fast(ii->task_name,
M_NOWAIT,
taskqueue_thread_enqueue,
&ii->tq);
taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq",
device_get_nameunit(sc->dev));
ii->sc = sc;
rc = bus_setup_intr(sc->dev,
ii->intr_res,
INTR_TYPE_NET,
oce_fast_isr, NULL, ii, &ii->tag);
return rc;
}
void
oce_intr_free(POCE_SOFTC sc)
{
int i = 0;
for (i = 0; i < sc->intr_count; i++) {
if (sc->intrs[i].tag != NULL)
bus_teardown_intr(sc->dev, sc->intrs[i].intr_res,
sc->intrs[i].tag);
if (sc->intrs[i].tq != NULL)
taskqueue_free(sc->intrs[i].tq);
if (sc->intrs[i].intr_res != NULL)
bus_release_resource(sc->dev, SYS_RES_IRQ,
sc->intrs[i].irq_rr,
sc->intrs[i].intr_res);
sc->intrs[i].tag = NULL;
sc->intrs[i].intr_res = NULL;
}
if (sc->flags & OCE_FLAGS_USING_MSIX)
pci_release_msi(sc->dev);
}
/******************************************************************************
* Media callbacks functions *
******************************************************************************/
static void
oce_media_status(struct ifnet *ifp, struct ifmediareq *req)
{
POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc;
req->ifm_status = IFM_AVALID;
req->ifm_active = IFM_ETHER;
if (sc->link_status == 1)
req->ifm_status |= IFM_ACTIVE;
else
return;
switch (sc->link_speed) {
case 1: /* 10 Mbps */
req->ifm_active |= IFM_10_T | IFM_FDX;
sc->speed = 10;
break;
case 2: /* 100 Mbps */
req->ifm_active |= IFM_100_TX | IFM_FDX;
sc->speed = 100;
break;
case 3: /* 1 Gbps */
req->ifm_active |= IFM_1000_T | IFM_FDX;
sc->speed = 1000;
break;
case 4: /* 10 Gbps */
req->ifm_active |= IFM_10G_SR | IFM_FDX;
sc->speed = 10000;
break;
case 5: /* 20 Gbps */
req->ifm_active |= IFM_10G_SR | IFM_FDX;
sc->speed = 20000;
break;
case 6: /* 25 Gbps */
req->ifm_active |= IFM_10G_SR | IFM_FDX;
sc->speed = 25000;
break;
case 7: /* 40 Gbps */
req->ifm_active |= IFM_40G_SR4 | IFM_FDX;
sc->speed = 40000;
break;
default:
sc->speed = 0;
break;
}
return;
}
int
oce_media_change(struct ifnet *ifp)
{
return 0;
}
/*****************************************************************************
* Transmit routines functions *
*****************************************************************************/
static int
oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
{
int rc = 0, i, retry_cnt = 0;
bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS];
struct mbuf *m, *m_temp;
struct oce_wq *wq = sc->wq[wq_index];
struct oce_packet_desc *pd;
struct oce_nic_hdr_wqe *nichdr;
struct oce_nic_frag_wqe *nicfrag;
int num_wqes;
uint32_t reg_value;
boolean_t complete = TRUE;
m = *mpp;
if (!m)
return EINVAL;
if (!(m->m_flags & M_PKTHDR)) {
rc = ENXIO;
goto free_ret;
}
if(oce_tx_asic_stall_verify(sc, m)) {
m = oce_insert_vlan_tag(sc, m, &complete);
if(!m) {
device_printf(sc->dev, "Insertion unsuccessful\n");
return 0;
}
}
if (m->m_pkthdr.csum_flags & CSUM_TSO) {
/* consolidate packet buffers for TSO/LSO segment offload */
#if defined(INET6) || defined(INET)
m = oce_tso_setup(sc, mpp);
#else
m = NULL;
#endif
if (m == NULL) {
rc = ENXIO;
goto free_ret;
}
}
pd = &wq->pckts[wq->pkt_desc_head];
retry:
rc = bus_dmamap_load_mbuf_sg(wq->tag,
pd->map,
m, segs, &pd->nsegs, BUS_DMA_NOWAIT);
if (rc == 0) {
num_wqes = pd->nsegs + 1;
if (IS_BE(sc) || IS_SH(sc)) {
/*Dummy required only for BE3.*/
if (num_wqes & 1)
num_wqes++;
}
if (num_wqes >= RING_NUM_FREE(wq->ring)) {
bus_dmamap_unload(wq->tag, pd->map);
return EBUSY;
}
atomic_store_rel_int(&wq->pkt_desc_head,
(wq->pkt_desc_head + 1) % \
OCE_WQ_PACKET_ARRAY_SIZE);
bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);
pd->mbuf = m;
nichdr =
RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);
nichdr->u0.dw[0] = 0;
nichdr->u0.dw[1] = 0;
nichdr->u0.dw[2] = 0;
nichdr->u0.dw[3] = 0;
nichdr->u0.s.complete = complete;
nichdr->u0.s.event = 1;
nichdr->u0.s.crc = 1;
nichdr->u0.s.forward = 0;
nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0;
nichdr->u0.s.udpcs =
(m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0;
nichdr->u0.s.tcpcs =
(m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0;
nichdr->u0.s.num_wqe = num_wqes;
nichdr->u0.s.total_length = m->m_pkthdr.len;
if (m->m_flags & M_VLANTAG) {
nichdr->u0.s.vlan = 1; /*Vlan present*/
nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag;
}
if (m->m_pkthdr.csum_flags & CSUM_TSO) {
if (m->m_pkthdr.tso_segsz) {
nichdr->u0.s.lso = 1;
nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz;
}
if (!IS_BE(sc) || !IS_SH(sc))
nichdr->u0.s.ipcs = 1;
}
RING_PUT(wq->ring, 1);
atomic_add_int(&wq->ring->num_used, 1);
for (i = 0; i < pd->nsegs; i++) {
nicfrag =
RING_GET_PRODUCER_ITEM_VA(wq->ring,
struct oce_nic_frag_wqe);
nicfrag->u0.s.rsvd0 = 0;
nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr);
nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr);
nicfrag->u0.s.frag_len = segs[i].ds_len;
pd->wqe_idx = wq->ring->pidx;
RING_PUT(wq->ring, 1);
atomic_add_int(&wq->ring->num_used, 1);
}
if (num_wqes > (pd->nsegs + 1)) {
nicfrag =
RING_GET_PRODUCER_ITEM_VA(wq->ring,
struct oce_nic_frag_wqe);
nicfrag->u0.dw[0] = 0;
nicfrag->u0.dw[1] = 0;
nicfrag->u0.dw[2] = 0;
nicfrag->u0.dw[3] = 0;
pd->wqe_idx = wq->ring->pidx;
RING_PUT(wq->ring, 1);
atomic_add_int(&wq->ring->num_used, 1);
pd->nsegs++;
}
if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
wq->tx_stats.tx_reqs++;
wq->tx_stats.tx_wrbs += num_wqes;
wq->tx_stats.tx_bytes += m->m_pkthdr.len;
wq->tx_stats.tx_pkts++;
bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
reg_value = (num_wqes << 16) | wq->wq_id;
OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);
} else if (rc == EFBIG) {
if (retry_cnt == 0) {
m_temp = m_defrag(m, M_NOWAIT);
if (m_temp == NULL)
goto free_ret;
m = m_temp;
*mpp = m_temp;
retry_cnt = retry_cnt + 1;
goto retry;
} else
goto free_ret;
} else if (rc == ENOMEM)
return rc;
else
goto free_ret;
return 0;
free_ret:
m_freem(*mpp);
*mpp = NULL;
return rc;
}
static void
oce_tx_complete(struct oce_wq *wq, uint32_t wqe_idx, uint32_t status)
{
struct oce_packet_desc *pd;
POCE_SOFTC sc = (POCE_SOFTC) wq->parent;
struct mbuf *m;
pd = &wq->pckts[wq->pkt_desc_tail];
atomic_store_rel_int(&wq->pkt_desc_tail,
(wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);
atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);
bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(wq->tag, pd->map);
m = pd->mbuf;
m_freem(m);
pd->mbuf = NULL;
if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) {
if (wq->ring->num_used < (wq->ring->num_items / 2)) {
sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
oce_tx_restart(sc, wq);
}
}
}
static void
oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
{
if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
return;
#if __FreeBSD_version >= 800000
if (!drbr_empty(sc->ifp, wq->br))
#else
if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd))
#endif
taskqueue_enqueue_fast(taskqueue_swi, &wq->txtask);
}
#if defined(INET6) || defined(INET)
static struct mbuf *
oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
{
struct mbuf *m;
#ifdef INET
struct ip *ip;
#endif
#ifdef INET6
struct ip6_hdr *ip6;
#endif
struct ether_vlan_header *eh;
struct tcphdr *th;
uint16_t etype;
int total_len = 0, ehdrlen = 0;
m = *mpp;
if (M_WRITABLE(m) == 0) {
m = m_dup(*mpp, M_NOWAIT);
if (!m)
return NULL;
m_freem(*mpp);
*mpp = m;
}
eh = mtod(m, struct ether_vlan_header *);
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
etype = ntohs(eh->evl_proto);
ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
} else {
etype = ntohs(eh->evl_encap_proto);
ehdrlen = ETHER_HDR_LEN;
}
switch (etype) {
#ifdef INET
case ETHERTYPE_IP:
ip = (struct ip *)(m->m_data + ehdrlen);
if (ip->ip_p != IPPROTO_TCP)
return NULL;
th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2);
break;
#endif
#ifdef INET6
case ETHERTYPE_IPV6:
ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen);
if (ip6->ip6_nxt != IPPROTO_TCP)
return NULL;
th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2);
break;
#endif
default:
return NULL;
}
m = m_pullup(m, total_len);
if (!m)
return NULL;
*mpp = m;
return m;
}
#endif /* INET6 || INET */
void
oce_tx_task(void *arg, int npending)
{
struct oce_wq *wq = arg;
POCE_SOFTC sc = wq->parent;
struct ifnet *ifp = sc->ifp;
int rc = 0;
#if __FreeBSD_version >= 800000
LOCK(&wq->tx_lock);
rc = oce_multiq_transmit(ifp, NULL, wq);
if (rc) {
device_printf(sc->dev,
"TX[%d] restart failed\n", wq->queue_index);
}
UNLOCK(&wq->tx_lock);
#else
oce_start(ifp);
#endif
}
void
oce_start(struct ifnet *ifp)
{
POCE_SOFTC sc = ifp->if_softc;
struct mbuf *m;
int rc = 0;
int def_q = 0; /* Defualt tx queue is 0*/
if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
IFF_DRV_RUNNING)
return;
if (!sc->link_status)
return;
do {
IF_DEQUEUE(&sc->ifp->if_snd, m);
if (m == NULL)
break;
LOCK(&sc->wq[def_q]->tx_lock);
rc = oce_tx(sc, &m, def_q);
UNLOCK(&sc->wq[def_q]->tx_lock);
if (rc) {
if (m != NULL) {
sc->wq[def_q]->tx_stats.tx_stops ++;
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IFQ_DRV_PREPEND(&ifp->if_snd, m);
m = NULL;
}
break;
}
if (m != NULL)
ETHER_BPF_MTAP(ifp, m);
} while (TRUE);
return;
}
/* Handle the Completion Queue for transmit */
uint16_t
oce_wq_handler(void *arg)
{
struct oce_wq *wq = (struct oce_wq *)arg;
POCE_SOFTC sc = wq->parent;
struct oce_cq *cq = wq->cq;
struct oce_nic_tx_cqe *cqe;
int num_cqes = 0;
bus_dmamap_sync(cq->ring->dma.tag,
cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
while (cqe->u0.dw[3]) {
DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe));
wq->ring->cidx = cqe->u0.s.wqe_index + 1;
if (wq->ring->cidx >= wq->ring->num_items)
wq->ring->cidx -= wq->ring->num_items;
oce_tx_complete(wq, cqe->u0.s.wqe_index, cqe->u0.s.status);
wq->tx_stats.tx_compl++;
cqe->u0.dw[3] = 0;
RING_GET(cq->ring, 1);
bus_dmamap_sync(cq->ring->dma.tag,
cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
cqe =
RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
num_cqes++;
}
if (num_cqes)
oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
return 0;
}
static int
oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq)
{
POCE_SOFTC sc = ifp->if_softc;
int status = 0, queue_index = 0;
struct mbuf *next = NULL;
struct buf_ring *br = NULL;
br = wq->br;
queue_index = wq->queue_index;
if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
IFF_DRV_RUNNING) {
if (m != NULL)
status = drbr_enqueue(ifp, br, m);
return status;
}
if (m != NULL) {
if ((status = drbr_enqueue(ifp, br, m)) != 0)
return status;
}
while ((next = drbr_peek(ifp, br)) != NULL) {
if (oce_tx(sc, &next, queue_index)) {
if (next == NULL) {
drbr_advance(ifp, br);
} else {
drbr_putback(ifp, br, next);
wq->tx_stats.tx_stops ++;
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
}
break;
}
drbr_advance(ifp, br);
if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len);
if (next->m_flags & M_MCAST)
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
ETHER_BPF_MTAP(ifp, next);
}
return 0;
}
/*****************************************************************************
* Receive routines functions *
*****************************************************************************/
static void
oce_rx(struct oce_rq *rq, uint32_t rqe_idx, struct oce_nic_rx_cqe *cqe)
{
uint32_t out;
struct oce_packet_desc *pd;
POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
int i, len, frag_len;
struct mbuf *m = NULL, *tail = NULL;
uint16_t vtag;
len = cqe->u0.s.pkt_size;
if (!len) {
/*partial DMA workaround for Lancer*/
oce_discard_rx_comp(rq, cqe);
goto exit;
}
/* Get vlan_tag value */
if(IS_BE(sc) || IS_SH(sc))
vtag = BSWAP_16(cqe->u0.s.vlan_tag);
else
vtag = cqe->u0.s.vlan_tag;
for (i = 0; i < cqe->u0.s.num_fragments; i++) {
if (rq->packets_out == rq->packets_in) {
device_printf(sc->dev,
"RQ transmit descriptor missing\n");
}
out = rq->packets_out + 1;
if (out == OCE_RQ_PACKET_ARRAY_SIZE)
out = 0;
pd = &rq->pckts[rq->packets_out];
rq->packets_out = out;
bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(rq->tag, pd->map);
rq->pending--;
frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len;
pd->mbuf->m_len = frag_len;
if (tail != NULL) {
/* additional fragments */
pd->mbuf->m_flags &= ~M_PKTHDR;
tail->m_next = pd->mbuf;
tail = pd->mbuf;
} else {
/* first fragment, fill out much of the packet header */
pd->mbuf->m_pkthdr.len = len;
pd->mbuf->m_pkthdr.csum_flags = 0;
if (IF_CSUM_ENABLED(sc)) {
if (cqe->u0.s.l4_cksum_pass) {
pd->mbuf->m_pkthdr.csum_flags |=
(CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
pd->mbuf->m_pkthdr.csum_data = 0xffff;
}
if (cqe->u0.s.ip_cksum_pass) {
if (!cqe->u0.s.ip_ver) { /* IPV4 */
pd->mbuf->m_pkthdr.csum_flags |=
(CSUM_IP_CHECKED|CSUM_IP_VALID);
}
}
}
m = tail = pd->mbuf;
}
pd->mbuf = NULL;
len -= frag_len;
}
if (m) {
if (!oce_cqe_portid_valid(sc, cqe)) {
m_freem(m);
goto exit;
}
m->m_pkthdr.rcvif = sc->ifp;
#if __FreeBSD_version >= 800000
if (rq->queue_index)
m->m_pkthdr.flowid = (rq->queue_index - 1);
else
m->m_pkthdr.flowid = rq->queue_index;
M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
#endif
/* This deternies if vlan tag is Valid */
if (oce_cqe_vtp_valid(sc, cqe)) {
if (sc->function_mode & FNM_FLEX10_MODE) {
/* FLEX10. If QnQ is not set, neglect VLAN */
if (cqe->u0.s.qnq) {
m->m_pkthdr.ether_vtag = vtag;
m->m_flags |= M_VLANTAG;
}
} else if (sc->pvid != (vtag & VLAN_VID_MASK)) {
/* In UMC mode generally pvid will be striped by
hw. But in some cases we have seen it comes
with pvid. So if pvid == vlan, neglect vlan.
*/
m->m_pkthdr.ether_vtag = vtag;
m->m_flags |= M_VLANTAG;
}
}
if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
#if defined(INET6) || defined(INET)
/* Try to queue to LRO */
if (IF_LRO_ENABLED(sc) &&
(cqe->u0.s.ip_cksum_pass) &&
(cqe->u0.s.l4_cksum_pass) &&
(!cqe->u0.s.ip_ver) &&
(rq->lro.lro_cnt != 0)) {
if (tcp_lro_rx(&rq->lro, m, 0) == 0) {
rq->lro_pkts_queued ++;
goto post_done;
}
/* If LRO posting fails then try to post to STACK */
}
#endif
(*sc->ifp->if_input) (sc->ifp, m);
#if defined(INET6) || defined(INET)
post_done:
#endif
/* Update rx stats per queue */
rq->rx_stats.rx_pkts++;
rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size;
rq->rx_stats.rx_frags += cqe->u0.s.num_fragments;
if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET)
rq->rx_stats.rx_mcast_pkts++;
if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET)
rq->rx_stats.rx_ucast_pkts++;
}
exit:
return;
}
static void
oce_discard_rx_comp(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe)
{
uint32_t out, i = 0;
struct oce_packet_desc *pd;
POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
int num_frags = cqe->u0.s.num_fragments;
for (i = 0; i < num_frags; i++) {
if (rq->packets_out == rq->packets_in) {
device_printf(sc->dev,
"RQ transmit descriptor missing\n");
}
out = rq->packets_out + 1;
if (out == OCE_RQ_PACKET_ARRAY_SIZE)
out = 0;
pd = &rq->pckts[rq->packets_out];
rq->packets_out = out;
bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(rq->tag, pd->map);
rq->pending--;
m_freem(pd->mbuf);
}
}
static int
oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
{
struct oce_nic_rx_cqe_v1 *cqe_v1;
int vtp = 0;
if (sc->be3_native) {
cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
vtp = cqe_v1->u0.s.vlan_tag_present;
} else
vtp = cqe->u0.s.vlan_tag_present;
return vtp;
}
static int
oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
{
struct oce_nic_rx_cqe_v1 *cqe_v1;
int port_id = 0;
if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) {
cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
port_id = cqe_v1->u0.s.port;
if (sc->port_id != port_id)
return 0;
} else
;/* For BE3 legacy and Lancer this is dummy */
return 1;
}
#if defined(INET6) || defined(INET)
static void
oce_rx_flush_lro(struct oce_rq *rq)
{
struct lro_ctrl *lro = &rq->lro;
struct lro_entry *queued;
POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
if (!IF_LRO_ENABLED(sc))
return;
while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
SLIST_REMOVE_HEAD(&lro->lro_active, next);
tcp_lro_flush(lro, queued);
}
rq->lro_pkts_queued = 0;
return;
}
static int
oce_init_lro(POCE_SOFTC sc)
{
struct lro_ctrl *lro = NULL;
int i = 0, rc = 0;
for (i = 0; i < sc->nrqs; i++) {
lro = &sc->rq[i]->lro;
rc = tcp_lro_init(lro);
if (rc != 0) {
device_printf(sc->dev, "LRO init failed\n");
return rc;
}
lro->ifp = sc->ifp;
}
return rc;
}
void
oce_free_lro(POCE_SOFTC sc)
{
struct lro_ctrl *lro = NULL;
int i = 0;
for (i = 0; i < sc->nrqs; i++) {
lro = &sc->rq[i]->lro;
if (lro)
tcp_lro_free(lro);
}
}
#endif
int
oce_alloc_rx_bufs(struct oce_rq *rq, int count)
{
POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
int i, in, rc;
struct oce_packet_desc *pd;
bus_dma_segment_t segs[6];
int nsegs, added = 0;
struct oce_nic_rqe *rqe;
pd_rxulp_db_t rxdb_reg;
bzero(&rxdb_reg, sizeof(pd_rxulp_db_t));
for (i = 0; i < count; i++) {
in = rq->packets_in + 1;
if (in == OCE_RQ_PACKET_ARRAY_SIZE)
in = 0;
if (in == rq->packets_out)
break; /* no more room */
pd = &rq->pckts[rq->packets_in];
pd->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
if (pd->mbuf == NULL)
break;
pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = MCLBYTES;
rc = bus_dmamap_load_mbuf_sg(rq->tag,
pd->map,
pd->mbuf,
segs, &nsegs, BUS_DMA_NOWAIT);
if (rc) {
m_free(pd->mbuf);
break;
}
if (nsegs != 1) {
i--;
continue;
}
rq->packets_in = in;
bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD);
rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe);
rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr);
rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr);
DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe));
RING_PUT(rq->ring, 1);
added++;
rq->pending++;
}
if (added != 0) {
for (i = added / OCE_MAX_RQ_POSTS; i > 0; i--) {
rxdb_reg.bits.num_posted = OCE_MAX_RQ_POSTS;
rxdb_reg.bits.qid = rq->rq_id;
OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
added -= OCE_MAX_RQ_POSTS;
}
if (added > 0) {
rxdb_reg.bits.qid = rq->rq_id;
rxdb_reg.bits.num_posted = added;
OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
}
}
return 0;
}
/* Handle the Completion Queue for receive */
uint16_t
oce_rq_handler(void *arg)
{
struct oce_rq *rq = (struct oce_rq *)arg;
struct oce_cq *cq = rq->cq;
POCE_SOFTC sc = rq->parent;
struct oce_nic_rx_cqe *cqe;
int num_cqes = 0, rq_buffers_used = 0;
bus_dmamap_sync(cq->ring->dma.tag,
cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
while (cqe->u0.dw[2]) {
DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe));
RING_GET(rq->ring, 1);
if (cqe->u0.s.error == 0) {
oce_rx(rq, cqe->u0.s.frag_index, cqe);
} else {
rq->rx_stats.rxcp_err++;
if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
/* Post L3/L4 errors to stack.*/
oce_rx(rq, cqe->u0.s.frag_index, cqe);
}
rq->rx_stats.rx_compl++;
cqe->u0.dw[2] = 0;
#if defined(INET6) || defined(INET)
if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) {
oce_rx_flush_lro(rq);
}
#endif
RING_GET(cq->ring, 1);
bus_dmamap_sync(cq->ring->dma.tag,
cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
cqe =
RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
num_cqes++;
if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
break;
}
#if defined(INET6) || defined(INET)
if (IF_LRO_ENABLED(sc))
oce_rx_flush_lro(rq);
#endif
if (num_cqes) {
oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
rq_buffers_used = OCE_RQ_PACKET_ARRAY_SIZE - rq->pending;
if (rq_buffers_used > 1)
oce_alloc_rx_bufs(rq, (rq_buffers_used - 1));
}
return 0;
}
/*****************************************************************************
* Helper function prototypes in this file *
*****************************************************************************/
static int
oce_attach_ifp(POCE_SOFTC sc)
{
sc->ifp = if_alloc(IFT_ETHER);
if (!sc->ifp)
return ENOMEM;
ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status);
ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST;
sc->ifp->if_ioctl = oce_ioctl;
sc->ifp->if_start = oce_start;
sc->ifp->if_init = oce_init;
sc->ifp->if_mtu = ETHERMTU;
sc->ifp->if_softc = sc;
#if __FreeBSD_version >= 800000
sc->ifp->if_transmit = oce_multiq_start;
sc->ifp->if_qflush = oce_multiq_flush;
#endif
if_initname(sc->ifp,
device_get_name(sc->dev), device_get_unit(sc->dev));
sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1;
IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen);
IFQ_SET_READY(&sc->ifp->if_snd);
sc->ifp->if_hwassist = OCE_IF_HWASSIST;
sc->ifp->if_hwassist |= CSUM_TSO;
sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP);
sc->ifp->if_capabilities = OCE_IF_CAPABILITIES;
sc->ifp->if_capabilities |= IFCAP_HWCSUM;
sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
#if defined(INET6) || defined(INET)
sc->ifp->if_capabilities |= IFCAP_TSO;
sc->ifp->if_capabilities |= IFCAP_LRO;
sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
#endif
sc->ifp->if_capenable = sc->ifp->if_capabilities;
sc->ifp->if_baudrate = IF_Gbps(10);
#if __FreeBSD_version >= 1000000
sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS;
sc->ifp->if_hw_tsomaxsegsize = 4096;
#endif
ether_ifattach(sc->ifp, sc->macaddr.mac_addr);
return 0;
}
static void
oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
{
POCE_SOFTC sc = ifp->if_softc;
if (ifp->if_softc != arg)
return;
if ((vtag == 0) || (vtag > 4095))
return;
sc->vlan_tag[vtag] = 1;
sc->vlans_added++;
if (sc->vlans_added <= (sc->max_vlans + 1))
oce_vid_config(sc);
}
static void
oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
{
POCE_SOFTC sc = ifp->if_softc;
if (ifp->if_softc != arg)
return;
if ((vtag == 0) || (vtag > 4095))
return;
sc->vlan_tag[vtag] = 0;
sc->vlans_added--;
oce_vid_config(sc);
}
/*
* A max of 64 vlans can be configured in BE. If the user configures
* more, place the card in vlan promiscuous mode.
*/
static int
oce_vid_config(POCE_SOFTC sc)
{
struct normal_vlan vtags[MAX_VLANFILTER_SIZE];
uint16_t ntags = 0, i;
int status = 0;
if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) &&
(sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) {
for (i = 0; i < MAX_VLANS; i++) {
if (sc->vlan_tag[i]) {
vtags[ntags].vtag = i;
ntags++;
}
}
if (ntags)
status = oce_config_vlan(sc, (uint8_t) sc->if_id,
vtags, ntags, 1, 0);
} else
status = oce_config_vlan(sc, (uint8_t) sc->if_id,
NULL, 0, 1, 1);
return status;
}
static void
oce_mac_addr_set(POCE_SOFTC sc)
{
uint32_t old_pmac_id = sc->pmac_id;
int status = 0;
status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
sc->macaddr.size_of_struct);
if (!status)
return;
status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)),
sc->if_id, &sc->pmac_id);
if (!status) {
status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id);
bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
sc->macaddr.size_of_struct);
}
if (status)
device_printf(sc->dev, "Failed update macaddress\n");
}
static int
oce_handle_passthrough(struct ifnet *ifp, caddr_t data)
{
POCE_SOFTC sc = ifp->if_softc;
struct ifreq *ifr = (struct ifreq *)data;
int rc = ENXIO;
char cookie[32] = {0};
void *priv_data = (void *)ifr->ifr_data;
void *ioctl_ptr;
uint32_t req_size;
struct mbx_hdr req;
OCE_DMA_MEM dma_mem;
struct mbx_common_get_cntl_attr *fw_cmd;
if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE)))
return EFAULT;
if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE)))
return EINVAL;
ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE);
if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr)))
return EFAULT;
req_size = le32toh(req.u0.req.request_length);
if (req_size > 65536)
return EINVAL;
req_size += sizeof(struct mbx_hdr);
rc = oce_dma_alloc(sc, req_size, &dma_mem, 0);
if (rc)
return ENOMEM;
if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) {
rc = EFAULT;
goto dma_free;
}
rc = oce_pass_through_mbox(sc, &dma_mem, req_size);
if (rc) {
rc = EIO;
goto dma_free;
}
if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size))
rc = EFAULT;
/*
firmware is filling all the attributes for this ioctl except
the driver version..so fill it
*/
if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) {
fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr;
strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str,
COMPONENT_REVISION, strlen(COMPONENT_REVISION));
}
dma_free:
oce_dma_free(sc, &dma_mem);
return rc;
}
static void
oce_eqd_set_periodic(POCE_SOFTC sc)
{
struct oce_set_eqd set_eqd[OCE_MAX_EQ];
struct oce_aic_obj *aic;
struct oce_eq *eqo;
uint64_t now = 0, delta;
int eqd, i, num = 0;
uint32_t ips = 0;
int tps;
for (i = 0 ; i < sc->neqs; i++) {
eqo = sc->eq[i];
aic = &sc->aic_obj[i];
/* When setting the static eq delay from the user space */
if (!aic->enable) {
eqd = aic->et_eqd;
goto modify_eqd;
}
now = ticks;
/* Over flow check */
if ((now < aic->ticks) || (eqo->intr < aic->intr_prev))
goto done;
delta = now - aic->ticks;
tps = delta/hz;
/* Interrupt rate based on elapsed ticks */
if(tps)
ips = (uint32_t)(eqo->intr - aic->intr_prev) / tps;
if (ips > INTR_RATE_HWM)
eqd = aic->cur_eqd + 20;
else if (ips < INTR_RATE_LWM)
eqd = aic->cur_eqd / 2;
else
goto done;
if (eqd < 10)
eqd = 0;
/* Make sure that the eq delay is in the known range */
eqd = min(eqd, aic->max_eqd);
eqd = max(eqd, aic->min_eqd);
modify_eqd:
if (eqd != aic->cur_eqd) {
set_eqd[num].delay_multiplier = (eqd * 65)/100;
set_eqd[num].eq_id = eqo->eq_id;
aic->cur_eqd = eqd;
num++;
}
done:
aic->intr_prev = eqo->intr;
aic->ticks = now;
}
/* Is there atleast one eq that needs to be modified? */
if(num)
oce_mbox_eqd_modify_periodic(sc, set_eqd, num);
}
static void oce_detect_hw_error(POCE_SOFTC sc)
{
uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0;
uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
uint32_t i;
if (sc->hw_error)
return;
if (IS_XE201(sc)) {
sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET);
if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET);
sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET);
}
} else {
ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW);
ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH);
ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK);
ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK);
ue_low = (ue_low & ~ue_low_mask);
ue_high = (ue_high & ~ue_high_mask);
}
/* On certain platforms BE hardware can indicate spurious UEs.
* Allow the h/w to stop working completely in case of a real UE.
* Hence not setting the hw_error for UE detection.
*/
if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
sc->hw_error = TRUE;
device_printf(sc->dev, "Error detected in the card\n");
}
if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
device_printf(sc->dev,
"ERR: sliport status 0x%x\n", sliport_status);
device_printf(sc->dev,
"ERR: sliport error1 0x%x\n", sliport_err1);
device_printf(sc->dev,
"ERR: sliport error2 0x%x\n", sliport_err2);
}
if (ue_low) {
for (i = 0; ue_low; ue_low >>= 1, i++) {
if (ue_low & 1)
device_printf(sc->dev, "UE: %s bit set\n",
ue_status_low_desc[i]);
}
}
if (ue_high) {
for (i = 0; ue_high; ue_high >>= 1, i++) {
if (ue_high & 1)
device_printf(sc->dev, "UE: %s bit set\n",
ue_status_hi_desc[i]);
}
}
}
static void
oce_local_timer(void *arg)
{
POCE_SOFTC sc = arg;
int i = 0;
oce_detect_hw_error(sc);
oce_refresh_nic_stats(sc);
oce_refresh_queue_stats(sc);
oce_mac_addr_set(sc);
/* TX Watch Dog*/
for (i = 0; i < sc->nwqs; i++)
oce_tx_restart(sc, sc->wq[i]);
/* calculate and set the eq delay for optimal interrupt rate */
if (IS_BE(sc) || IS_SH(sc))
oce_eqd_set_periodic(sc);
callout_reset(&sc->timer, hz, oce_local_timer, sc);
}
/* NOTE : This should only be called holding
* DEVICE_LOCK.
*/
static void
oce_if_deactivate(POCE_SOFTC sc)
{
int i, mtime = 0;
int wait_req = 0;
struct oce_rq *rq;
struct oce_wq *wq;
struct oce_eq *eq;
sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
/*Wait for max of 400ms for TX completions to be done */
while (mtime < 400) {
wait_req = 0;
for_all_wq_queues(sc, wq, i) {
if (wq->ring->num_used) {
wait_req = 1;
DELAY(1);
break;
}
}
mtime += 1;
if (!wait_req)
break;
}
/* Stop intrs and finish any bottom halves pending */
oce_hw_intr_disable(sc);
/* Since taskqueue_drain takes a Gaint Lock, We should not acquire
any other lock. So unlock device lock and require after
completing taskqueue_drain.
*/
UNLOCK(&sc->dev_lock);
for (i = 0; i < sc->intr_count; i++) {
if (sc->intrs[i].tq != NULL) {
taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task);
}
}
LOCK(&sc->dev_lock);
/* Delete RX queue in card with flush param */
oce_stop_rx(sc);
/* Invalidate any pending cq and eq entries*/
for_all_evnt_queues(sc, eq, i)
oce_drain_eq(eq);
for_all_rq_queues(sc, rq, i)
oce_drain_rq_cq(rq);
for_all_wq_queues(sc, wq, i)
oce_drain_wq_cq(wq);
/* But still we need to get MCC aync events.
So enable intrs and also arm first EQ
*/
oce_hw_intr_enable(sc);
oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
DELAY(10);
}
static void
oce_if_activate(POCE_SOFTC sc)
{
struct oce_eq *eq;
struct oce_rq *rq;
struct oce_wq *wq;
int i, rc = 0;
sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
oce_hw_intr_disable(sc);
oce_start_rx(sc);
for_all_rq_queues(sc, rq, i) {
rc = oce_start_rq(rq);
if (rc)
device_printf(sc->dev, "Unable to start RX\n");
}
for_all_wq_queues(sc, wq, i) {
rc = oce_start_wq(wq);
if (rc)
device_printf(sc->dev, "Unable to start TX\n");
}
for_all_evnt_queues(sc, eq, i)
oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
oce_hw_intr_enable(sc);
}
static void
process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
{
/* Update Link status */
if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
ASYNC_EVENT_LINK_UP) {
sc->link_status = ASYNC_EVENT_LINK_UP;
if_link_state_change(sc->ifp, LINK_STATE_UP);
} else {
sc->link_status = ASYNC_EVENT_LINK_DOWN;
if_link_state_change(sc->ifp, LINK_STATE_DOWN);
}
}
/* Handle the Completion Queue for the Mailbox/Async notifications */
uint16_t
oce_mq_handler(void *arg)
{
struct oce_mq *mq = (struct oce_mq *)arg;
POCE_SOFTC sc = mq->parent;
struct oce_cq *cq = mq->cq;
int num_cqes = 0, evt_type = 0, optype = 0;
struct oce_mq_cqe *cqe;
struct oce_async_cqe_link_state *acqe;
struct oce_async_event_grp5_pvid_state *gcqe;
struct oce_async_event_qnq *dbgcqe;
bus_dmamap_sync(cq->ring->dma.tag,
cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
while (cqe->u0.dw[3]) {
DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
if (cqe->u0.s.async_event) {
evt_type = cqe->u0.s.event_type;
optype = cqe->u0.s.async_type;
if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) {
/* Link status evt */
acqe = (struct oce_async_cqe_link_state *)cqe;
process_link_state(sc, acqe);
} else if ((evt_type == ASYNC_EVENT_GRP5) &&
(optype == ASYNC_EVENT_PVID_STATE)) {
/* GRP5 PVID */
gcqe =
(struct oce_async_event_grp5_pvid_state *)cqe;
if (gcqe->enabled)
sc->pvid = gcqe->tag & VLAN_VID_MASK;
else
sc->pvid = 0;
}
else if(evt_type == ASYNC_EVENT_CODE_DEBUG &&
optype == ASYNC_EVENT_DEBUG_QNQ) {
dbgcqe =
(struct oce_async_event_qnq *)cqe;
if(dbgcqe->valid)
sc->qnqid = dbgcqe->vlan_tag;
sc->qnq_debug_event = TRUE;
}
}
cqe->u0.dw[3] = 0;
RING_GET(cq->ring, 1);
bus_dmamap_sync(cq->ring->dma.tag,
cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
num_cqes++;
}
if (num_cqes)
oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
return 0;
}
static void
setup_max_queues_want(POCE_SOFTC sc)
{
/* Check if it is FLEX machine. Is so dont use RSS */
if ((sc->function_mode & FNM_FLEX10_MODE) ||
(sc->function_mode & FNM_UMC_MODE) ||
(sc->function_mode & FNM_VNIC_MODE) ||
(!is_rss_enabled(sc)) ||
IS_BE2(sc)) {
sc->nrqs = 1;
sc->nwqs = 1;
} else {
sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs);
}
if (IS_BE2(sc) && is_rss_enabled(sc))
sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
}
static void
update_queues_got(POCE_SOFTC sc)
{
if (is_rss_enabled(sc)) {
sc->nrqs = sc->intr_count + 1;
sc->nwqs = sc->intr_count;
} else {
sc->nrqs = 1;
sc->nwqs = 1;
}
if (IS_BE2(sc))
sc->nwqs = 1;
}
static int
oce_check_ipv6_ext_hdr(struct mbuf *m)
{
struct ether_header *eh = mtod(m, struct ether_header *);
caddr_t m_datatemp = m->m_data;
if (eh->ether_type == htons(ETHERTYPE_IPV6)) {
m->m_data += sizeof(struct ether_header);
struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
if((ip6->ip6_nxt != IPPROTO_TCP) && \
(ip6->ip6_nxt != IPPROTO_UDP)){
struct ip6_ext *ip6e = NULL;
m->m_data += sizeof(struct ip6_hdr);
ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *);
if(ip6e->ip6e_len == 0xff) {
m->m_data = m_datatemp;
return TRUE;
}
}
m->m_data = m_datatemp;
}
return FALSE;
}
static int
is_be3_a1(POCE_SOFTC sc)
{
if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) {
return TRUE;
}
return FALSE;
}
static struct mbuf *
oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete)
{
uint16_t vlan_tag = 0;
if(!M_WRITABLE(m))
return NULL;
/* Embed vlan tag in the packet if it is not part of it */
if(m->m_flags & M_VLANTAG) {
vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag);
m->m_flags &= ~M_VLANTAG;
}
/* if UMC, ignore vlan tag insertion and instead insert pvid */
if(sc->pvid) {
if(!vlan_tag)
vlan_tag = sc->pvid;
*complete = FALSE;
}
if(vlan_tag) {
m = ether_vlanencap(m, vlan_tag);
}
if(sc->qnqid) {
m = ether_vlanencap(m, sc->qnqid);
*complete = FALSE;
}
return m;
}
static int
oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m)
{
if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \
oce_check_ipv6_ext_hdr(m)) {
return TRUE;
}
return FALSE;
}
static void
oce_get_config(POCE_SOFTC sc)
{
int rc = 0;
uint32_t max_rss = 0;
if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native))
max_rss = OCE_LEGACY_MODE_RSS;
else
max_rss = OCE_MAX_RSS;
if (!IS_BE(sc)) {
rc = oce_get_profile_config(sc, max_rss);
if (rc) {
sc->nwqs = OCE_MAX_WQ;
sc->nrssqs = max_rss;
sc->nrqs = sc->nrssqs + 1;
}
}
else { /* For BE3 don't rely on fw for determining the resources */
sc->nrssqs = max_rss;
sc->nrqs = sc->nrssqs + 1;
sc->nwqs = OCE_MAX_WQ;
sc->max_vlans = MAX_VLANFILTER_SIZE;
}
}
Index: head/sys/dev/patm/if_patm_attach.c
===================================================================
--- head/sys/dev/patm/if_patm_attach.c (revision 283290)
+++ head/sys/dev/patm/if_patm_attach.c (revision 283291)
@@ -1,1076 +1,1076 @@
/*-
* Copyright (c) 2003
* Fraunhofer Institute for Open Communication Systems (FhG Fokus).
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* Author: Hartmut Brandt <harti@freebsd.org>
*
* Driver for IDT77252 based cards like ProSum's.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_natm.h"
#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/bus.h>
#include <sys/errno.h>
#include <sys/conf.h>
#include <sys/module.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <sys/queue.h>
#include <sys/condvar.h>
#include <vm/uma.h>
#include <sys/sockio.h>
#include <sys/mbuf.h>
#include <sys/socket.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_media.h>
#include <net/if_types.h>
#include <net/if_atm.h>
#include <net/route.h>
#ifdef ENABLE_BPF
#include <net/bpf.h>
#endif
#include <netinet/in.h>
#include <netinet/if_atm.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <sys/mbpool.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/utopia/utopia.h>
#include <dev/patm/idt77252reg.h>
#include <dev/patm/if_patmvar.h>
MODULE_DEPEND(patm, utopia, 1, 1, 1);
MODULE_DEPEND(patm, pci, 1, 1, 1);
MODULE_DEPEND(patm, atm, 1, 1, 1);
MODULE_DEPEND(patm, libmbpool, 1, 1, 1);
devclass_t patm_devclass;
static int patm_probe(device_t dev);
static int patm_attach(device_t dev);
static int patm_detach(device_t dev);
static device_method_t patm_methods[] = {
DEVMETHOD(device_probe, patm_probe),
DEVMETHOD(device_attach, patm_attach),
DEVMETHOD(device_detach, patm_detach),
{0,0}
};
static driver_t patm_driver = {
"patm",
patm_methods,
sizeof(struct patm_softc),
};
DRIVER_MODULE(patm, pci, patm_driver, patm_devclass, NULL, 0);
static const struct {
u_int devid;
const char *desc;
} devs[] = {
{ PCI_DEVICE_IDT77252, "NICStAR (77222/77252) ATM adapter" },
{ PCI_DEVICE_IDT77v252, "NICStAR (77v252) ATM adapter" },
{ PCI_DEVICE_IDT77v222, "NICStAR (77v222) ATM adapter" },
{ 0, NULL }
};
SYSCTL_DECL(_hw_atm);
static int patm_phy_readregs(struct ifatm *, u_int, uint8_t *, u_int *);
static int patm_phy_writereg(struct ifatm *, u_int, u_int, u_int);
static const struct utopia_methods patm_utopia_methods = {
patm_phy_readregs,
patm_phy_writereg
};
static void patm_destroy(struct patm_softc *sc);
static int patm_sysctl_istats(SYSCTL_HANDLER_ARGS);
static int patm_sysctl_eeprom(SYSCTL_HANDLER_ARGS);
static void patm_read_eeprom(struct patm_softc *sc);
static int patm_sq_init(struct patm_softc *sc);
static int patm_rbuf_init(struct patm_softc *sc);
static int patm_txmap_init(struct patm_softc *sc);
static void patm_env_getuint(struct patm_softc *, u_int *, const char *);
#ifdef PATM_DEBUG
static int patm_sysctl_regs(SYSCTL_HANDLER_ARGS);
static int patm_sysctl_tsq(SYSCTL_HANDLER_ARGS);
int patm_dump_vc(u_int unit, u_int vc) __unused;
int patm_dump_regs(u_int unit) __unused;
int patm_dump_sram(u_int unit, u_int from, u_int words) __unused;
#endif
/*
* Probe for a IDT77252 controller
*/
static int
patm_probe(device_t dev)
{
u_int i;
if (pci_get_vendor(dev) == PCI_VENDOR_IDT) {
for (i = 0; devs[i].desc != NULL; i++)
if (pci_get_device(dev) == devs[i].devid) {
device_set_desc(dev, devs[i].desc);
return (BUS_PROBE_DEFAULT);
}
}
return (ENXIO);
}
/*
* Attach
*/
static int
patm_attach(device_t dev)
{
struct patm_softc *sc;
int error;
struct ifnet *ifp;
int rid;
u_int a;
static const struct idt_mmap idt_mmap[4] = IDT_MMAP;
sc = device_get_softc(dev);
sc->dev = dev;
#ifdef IATM_DEBUG
sc->debug = IATM_DEBUG;
#endif
ifp = sc->ifp = if_alloc(IFT_ATM);
if (ifp == NULL) {
return (ENOSPC);
}
IFP2IFATM(sc->ifp)->mib.device = ATM_DEVICE_IDTABR25;
IFP2IFATM(sc->ifp)->mib.serial = 0;
IFP2IFATM(sc->ifp)->mib.hw_version = 0;
IFP2IFATM(sc->ifp)->mib.sw_version = 0;
IFP2IFATM(sc->ifp)->mib.vpi_bits = PATM_VPI_BITS;
IFP2IFATM(sc->ifp)->mib.vci_bits = 0; /* set below */;
IFP2IFATM(sc->ifp)->mib.max_vpcs = 0;
IFP2IFATM(sc->ifp)->mib.max_vccs = 0; /* set below */
IFP2IFATM(sc->ifp)->mib.media = IFM_ATM_UNKNOWN;
IFP2IFATM(sc->ifp)->phy = &sc->utopia;
ifp->if_softc = sc;
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_flags = IFF_SIMPLEX;
ifp->if_init = patm_init;
ifp->if_ioctl = patm_ioctl;
ifp->if_start = patm_start;
/* do this early so we can destroy unconditionally */
mtx_init(&sc->mtx, device_get_nameunit(dev),
MTX_NETWORK_LOCK, MTX_DEF);
mtx_init(&sc->tst_lock, "tst lock", NULL, MTX_DEF);
cv_init(&sc->vcc_cv, "vcc_close");
- callout_init(&sc->tst_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->tst_callout, 1);
sysctl_ctx_init(&sc->sysctl_ctx);
/*
* Get revision
*/
sc->revision = pci_read_config(dev, PCIR_REVID, 4) & 0xf;
/*
* Enable PCI bus master and memory
*/
pci_enable_busmaster(dev);
rid = IDT_PCI_REG_MEMBASE;
sc->memres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (sc->memres == NULL) {
patm_printf(sc, "could not map memory\n");
error = ENXIO;
goto fail;
}
sc->memh = rman_get_bushandle(sc->memres);
sc->memt = rman_get_bustag(sc->memres);
/*
* Allocate the interrupt (enable it later)
*/
sc->irqid = 0;
sc->irqres = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irqid,
RF_SHAREABLE | RF_ACTIVE);
if (sc->irqres == 0) {
patm_printf(sc, "could not allocate irq\n");
error = ENXIO;
goto fail;
}
/*
* Construct the sysctl tree
*/
error = ENOMEM;
if ((sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx,
SYSCTL_STATIC_CHILDREN(_hw_atm), OID_AUTO,
device_get_nameunit(dev), CTLFLAG_RD, 0, "")) == NULL)
goto fail;
if (SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
OID_AUTO, "istats", CTLTYPE_OPAQUE | CTLFLAG_RD, sc, 0,
patm_sysctl_istats, "S", "internal statistics") == NULL)
goto fail;
if (SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
OID_AUTO, "eeprom", CTLTYPE_OPAQUE | CTLFLAG_RD, sc, 0,
patm_sysctl_eeprom, "S", "EEPROM contents") == NULL)
goto fail;
if (SYSCTL_ADD_UINT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
OID_AUTO, "lbuf_max", CTLFLAG_RD, &sc->lbuf_max,
0, "maximum number of large receive buffers") == NULL)
goto fail;
patm_env_getuint(sc, &sc->lbuf_max, "lbuf_max");
if (SYSCTL_ADD_UINT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
OID_AUTO, "max_txmaps", CTLFLAG_RW, &sc->tx_maxmaps,
0, "maximum number of TX DMA maps") == NULL)
goto fail;
patm_env_getuint(sc, &sc->tx_maxmaps, "tx_maxmaps");
#ifdef PATM_DEBUG
if (SYSCTL_ADD_UINT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
OID_AUTO, "debug", CTLFLAG_RW, &sc->debug,
0, "debug flags") == NULL)
goto fail;
sc->debug = PATM_DEBUG;
patm_env_getuint(sc, &sc->debug, "debug");
if (SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
OID_AUTO, "regs", CTLTYPE_OPAQUE | CTLFLAG_RD, sc, 0,
patm_sysctl_regs, "S", "registers") == NULL)
goto fail;
if (SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
OID_AUTO, "tsq", CTLTYPE_OPAQUE | CTLFLAG_RD, sc, 0,
patm_sysctl_tsq, "S", "TSQ") == NULL)
goto fail;
#endif
patm_reset(sc);
/*
* Detect and attach the phy.
*/
patm_debug(sc, ATTACH, "attaching utopia");
IFP2IFATM(sc->ifp)->phy = &sc->utopia;
utopia_attach(&sc->utopia, IFP2IFATM(sc->ifp), &sc->media, &sc->mtx,
&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
&patm_utopia_methods);
/*
* Start the PHY because we need the autodetection
*/
patm_debug(sc, ATTACH, "starting utopia");
mtx_lock(&sc->mtx);
utopia_start(&sc->utopia);
utopia_reset(&sc->utopia);
mtx_unlock(&sc->mtx);
/* Read EEPROM */
patm_read_eeprom(sc);
/* analyze it */
if (strncmp(sc->eeprom + PATM_PROATM_NAME_OFFSET, PATM_PROATM_NAME,
strlen(PATM_PROATM_NAME)) == 0) {
if (sc->utopia.chip->type == UTP_TYPE_IDT77105) {
IFP2IFATM(sc->ifp)->mib.device = ATM_DEVICE_PROATM25;
IFP2IFATM(sc->ifp)->mib.pcr = ATM_RATE_25_6M;
IFP2IFATM(sc->ifp)->mib.media = IFM_ATM_UTP_25;
sc->flags |= PATM_25M;
patm_printf(sc, "ProATM 25 interface; ");
} else {
/* cannot really know which media */
IFP2IFATM(sc->ifp)->mib.device = ATM_DEVICE_PROATM155;
IFP2IFATM(sc->ifp)->mib.pcr = ATM_RATE_155M;
IFP2IFATM(sc->ifp)->mib.media = IFM_ATM_MM_155;
patm_printf(sc, "ProATM 155 interface; ");
}
bcopy(sc->eeprom + PATM_PROATM_MAC_OFFSET, IFP2IFATM(sc->ifp)->mib.esi,
sizeof(IFP2IFATM(sc->ifp)->mib.esi));
} else {
if (sc->utopia.chip->type == UTP_TYPE_IDT77105) {
IFP2IFATM(sc->ifp)->mib.device = ATM_DEVICE_IDTABR25;
IFP2IFATM(sc->ifp)->mib.pcr = ATM_RATE_25_6M;
IFP2IFATM(sc->ifp)->mib.media = IFM_ATM_UTP_25;
sc->flags |= PATM_25M;
patm_printf(sc, "IDT77252 25MBit interface; ");
} else {
/* cannot really know which media */
IFP2IFATM(sc->ifp)->mib.device = ATM_DEVICE_IDTABR155;
IFP2IFATM(sc->ifp)->mib.pcr = ATM_RATE_155M;
IFP2IFATM(sc->ifp)->mib.media = IFM_ATM_MM_155;
patm_printf(sc, "IDT77252 155MBit interface; ");
}
bcopy(sc->eeprom + PATM_IDT_MAC_OFFSET, IFP2IFATM(sc->ifp)->mib.esi,
sizeof(IFP2IFATM(sc->ifp)->mib.esi));
}
printf("idt77252 Rev. %c; %s PHY\n", 'A' + sc->revision,
sc->utopia.chip->name);
utopia_reset_media(&sc->utopia);
utopia_init_media(&sc->utopia);
/*
* Determine RAM size
*/
for (a = 0; a < 0x20000; a++)
patm_sram_write(sc, a, 0);
patm_sram_write(sc, 0, 0xdeadbeef);
if (patm_sram_read(sc, 0x4004) == 0xdeadbeef)
sc->mmap = &idt_mmap[0];
else if (patm_sram_read(sc, 0x8000) == 0xdeadbeef)
sc->mmap = &idt_mmap[1];
else if (patm_sram_read(sc, 0x20000) == 0xdeadbeef)
sc->mmap = &idt_mmap[2];
else
sc->mmap = &idt_mmap[3];
IFP2IFATM(sc->ifp)->mib.vci_bits = sc->mmap->vcbits - IFP2IFATM(sc->ifp)->mib.vpi_bits;
IFP2IFATM(sc->ifp)->mib.max_vccs = sc->mmap->max_conn;
patm_sram_write(sc, 0, 0);
patm_printf(sc, "%uK x 32 SRAM; %u connections\n", sc->mmap->sram,
sc->mmap->max_conn);
/* initialize status queues */
error = patm_sq_init(sc);
if (error != 0)
goto fail;
/* get TST */
sc->tst_soft = malloc(sizeof(uint32_t) * sc->mmap->tst_size,
M_DEVBUF, M_WAITOK);
/* allocate all the receive buffer stuff */
error = patm_rbuf_init(sc);
if (error != 0)
goto fail;
/*
* Allocate SCD tag
*
* Don't use BUS_DMA_ALLOCNOW, because we never need bouncing with
* bus_dmamem_alloc()
*/
error = bus_dma_tag_create(bus_get_dma_tag(dev), PAGE_SIZE, 0,
BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
NULL, NULL, sizeof(struct patm_scd), 1,
sizeof(struct patm_scd), 0, NULL, NULL, &sc->scd_tag);
if (error) {
patm_printf(sc, "SCD DMA tag create %d\n", error);
goto fail;
}
LIST_INIT(&sc->scd_list);
/* allocate VCC zone and pointers */
if ((sc->vcc_zone = uma_zcreate("PATM vccs", sizeof(struct patm_vcc),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0)) == NULL) {
patm_printf(sc, "cannot allocate zone for vccs\n");
goto fail;
}
sc->vccs = malloc(sizeof(sc->vccs[0]) * sc->mmap->max_conn,
M_DEVBUF, M_WAITOK | M_ZERO);
/* allocate transmission resources */
error = patm_txmap_init(sc);
if (error != 0)
goto fail;
/* poll while we are not running */
sc->utopia.flags |= UTP_FL_POLL_CARRIER;
patm_debug(sc, ATTACH, "attaching interface");
atm_ifattach(ifp);
#ifdef ENABLE_BPF
bpfattach(ifp, DLT_ATM_RFC1483, sizeof(struct atmllc));
#endif
patm_debug(sc, ATTACH, "attaching interrupt handler");
error = bus_setup_intr(dev, sc->irqres, INTR_TYPE_NET | INTR_MPSAFE,
NULL, patm_intr, sc, &sc->ih);
if (error != 0) {
patm_printf(sc, "could not setup interrupt\n");
atm_ifdetach(sc->ifp);
if_free(sc->ifp);
goto fail;
}
return (0);
fail:
patm_destroy(sc);
return (error);
}
/*
* Detach
*/
static int
patm_detach(device_t dev)
{
struct patm_softc *sc;
sc = device_get_softc(dev);
mtx_lock(&sc->mtx);
patm_stop(sc);
if (sc->utopia.state & UTP_ST_ATTACHED) {
patm_debug(sc, ATTACH, "detaching utopia");
utopia_stop(&sc->utopia);
utopia_detach(&sc->utopia);
}
mtx_unlock(&sc->mtx);
atm_ifdetach(sc->ifp);
patm_destroy(sc);
return (0);
}
/*
* Destroy everything. Assume we are stopped.
*/
static void
patm_destroy(struct patm_softc *sc)
{
u_int i;
struct patm_txmap *map;
if (sc->ih != NULL)
bus_teardown_intr(sc->dev, sc->irqres, sc->ih);
if (sc->tx_mapzone != NULL) {
/* all maps must be free */
while ((map = SLIST_FIRST(&sc->tx_maps_free)) != NULL) {
bus_dmamap_destroy(sc->tx_tag, map->map);
SLIST_REMOVE_HEAD(&sc->tx_maps_free, link);
uma_zfree(sc->tx_mapzone, map);
}
uma_zdestroy(sc->tx_mapzone);
}
if (sc->scd_tag != NULL)
bus_dma_tag_destroy(sc->scd_tag);
if (sc->tx_tag != NULL)
bus_dma_tag_destroy(sc->scd_tag);
if (sc->vccs != NULL) {
for (i = 0; i < sc->mmap->max_conn; i++)
if (sc->vccs[i] != NULL)
uma_zfree(sc->vcc_zone, sc->vccs[i]);
free(sc->vccs, M_DEVBUF);
}
if (sc->vcc_zone != NULL)
uma_zdestroy(sc->vcc_zone);
if (sc->lbufs != NULL) {
for (i = 0; i < sc->lbuf_max; i++)
bus_dmamap_destroy(sc->lbuf_tag, sc->lbufs[i].map);
free(sc->lbufs, M_DEVBUF);
}
if (sc->lbuf_tag != NULL)
bus_dma_tag_destroy(sc->lbuf_tag);
if (sc->sbuf_pool != NULL)
mbp_destroy(sc->sbuf_pool);
if (sc->vbuf_pool != NULL)
mbp_destroy(sc->vbuf_pool);
if (sc->sbuf_tag != NULL)
bus_dma_tag_destroy(sc->sbuf_tag);
if (sc->tst_soft != NULL)
free(sc->tst_soft, M_DEVBUF);
/*
* Free all status queue memory resources
*/
if (sc->tsq != NULL) {
bus_dmamap_unload(sc->sq_tag, sc->sq_map);
bus_dmamem_free(sc->sq_tag, sc->tsq, sc->sq_map);
bus_dma_tag_destroy(sc->sq_tag);
}
if (sc->irqres != NULL)
bus_release_resource(sc->dev, SYS_RES_IRQ,
sc->irqid, sc->irqres);
if (sc->memres != NULL)
bus_release_resource(sc->dev, SYS_RES_MEMORY,
IDT_PCI_REG_MEMBASE, sc->memres);
/* this was initialize unconditionally */
sysctl_ctx_free(&sc->sysctl_ctx);
cv_destroy(&sc->vcc_cv);
mtx_destroy(&sc->tst_lock);
mtx_destroy(&sc->mtx);
if (sc->ifp != NULL)
if_free(sc->ifp);
}
/*
* Try to find a variable in the environment and parse it as an unsigned
* integer.
*/
static void
patm_env_getuint(struct patm_softc *sc, u_int *var, const char *name)
{
char full[IFNAMSIZ + 3 + 20];
char *val, *end;
u_long u;
snprintf(full, sizeof(full), "hw.%s.%s",
device_get_nameunit(sc->dev), name);
if ((val = kern_getenv(full)) != NULL) {
u = strtoul(val, &end, 0);
if (end > val && *end == '\0') {
if (bootverbose)
patm_printf(sc, "%s=%lu\n", full, u);
*var = u;
}
freeenv(val);
}
}
/*
* Sysctl handler for internal statistics
*
* LOCK: unlocked, needed
*/
static int
patm_sysctl_istats(SYSCTL_HANDLER_ARGS)
{
struct patm_softc *sc = arg1;
uint32_t *ret;
int error;
ret = malloc(sizeof(sc->stats), M_TEMP, M_WAITOK);
mtx_lock(&sc->mtx);
bcopy(&sc->stats, ret, sizeof(sc->stats));
mtx_unlock(&sc->mtx);
error = SYSCTL_OUT(req, ret, sizeof(sc->stats));
free(ret, M_TEMP);
return (error);
}
/*
* Sysctl handler for EEPROM
*
* LOCK: unlocked, needed
*/
static int
patm_sysctl_eeprom(SYSCTL_HANDLER_ARGS)
{
struct patm_softc *sc = arg1;
void *ret;
int error;
ret = malloc(sizeof(sc->eeprom), M_TEMP, M_WAITOK);
mtx_lock(&sc->mtx);
bcopy(sc->eeprom, ret, sizeof(sc->eeprom));
mtx_unlock(&sc->mtx);
error = SYSCTL_OUT(req, ret, sizeof(sc->eeprom));
free(ret, M_TEMP);
return (error);
}
/*
* Read the EEPROM. We assume that this is a XIRCOM 25020
*/
static void
patm_read_eeprom(struct patm_softc *sc)
{
u_int gp;
uint8_t byte;
int i, addr;
static const uint32_t tab[] = {
/* CS transition to reset the chip */
IDT_GP_EECS | IDT_GP_EESCLK, 0,
/* read command 0x03 */
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, IDT_GP_EEDO,
IDT_GP_EESCLK | IDT_GP_EEDO, IDT_GP_EEDO,
IDT_GP_EESCLK | IDT_GP_EEDO, 0,
/* address 0x00 */
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
IDT_GP_EESCLK, 0,
};
/* go to a known state (chip enabled) */
gp = patm_nor_read(sc, IDT_NOR_GP);
gp &= ~(IDT_GP_EESCLK | IDT_GP_EECS | IDT_GP_EEDO);
for (i = 0; i < sizeof(tab) / sizeof(tab[0]); i++) {
patm_nor_write(sc, IDT_NOR_GP, gp | tab[i]);
DELAY(40);
}
/* read out the prom */
for (addr = 0; addr < 256; addr++) {
byte = 0;
for (i = 0; i < 8; i++) {
byte <<= 1;
if (patm_nor_read(sc, IDT_NOR_GP) & IDT_GP_EEDI)
byte |= 1;
/* rising CLK */
patm_nor_write(sc, IDT_NOR_GP, gp | IDT_GP_EESCLK);
DELAY(40);
/* falling clock */
patm_nor_write(sc, IDT_NOR_GP, gp);
DELAY(40);
}
sc->eeprom[addr] = byte;
}
}
/*
* PHY access read
*/
static int
patm_phy_readregs(struct ifatm *ifatm, u_int reg, uint8_t *val, u_int *n)
{
struct patm_softc *sc = ifatm->ifp->if_softc;
u_int cnt = *n;
if (reg >= 0x100)
return (EINVAL);
patm_cmd_wait(sc);
while (reg < 0x100 && cnt > 0) {
patm_nor_write(sc, IDT_NOR_CMD, IDT_MKCMD_RUTIL(1, 0, reg));
patm_cmd_wait(sc);
*val = patm_nor_read(sc, IDT_NOR_D0);
patm_debug(sc, PHY, "phy(%02x)=%02x", reg, *val);
val++;
reg++;
cnt--;
}
*n = *n - cnt;
return (0);
}
/*
* Write PHY reg
*/
static int
patm_phy_writereg(struct ifatm *ifatm, u_int reg, u_int mask, u_int val)
{
struct patm_softc *sc = ifatm->ifp->if_softc;
u_int old, new;
if (reg >= 0x100)
return (EINVAL);
patm_cmd_wait(sc);
patm_nor_write(sc, IDT_NOR_CMD, IDT_MKCMD_RUTIL(1, 0, reg));
patm_cmd_wait(sc);
old = patm_nor_read(sc, IDT_NOR_D0);
new = (old & ~mask) | (val & mask);
patm_debug(sc, PHY, "phy(%02x) %02x -> %02x", reg, old, new);
patm_nor_write(sc, IDT_NOR_D0, new);
patm_nor_write(sc, IDT_NOR_CMD, IDT_MKCMD_WUTIL(1, 0, reg));
patm_cmd_wait(sc);
return (0);
}
/*
* Allocate a large chunk of DMA able memory for the transmit
* and receive status queues. We align this to a page boundary
* to ensure the alignment.
*/
static int
patm_sq_init(struct patm_softc *sc)
{
int error;
void *p;
/* compute size of the two queues */
sc->sq_size = IDT_TSQ_SIZE * IDT_TSQE_SIZE +
PATM_RSQ_SIZE * IDT_RSQE_SIZE +
IDT_RAWHND_SIZE;
patm_debug(sc, ATTACH,
"allocating status queues (%zu) ...", sc->sq_size);
/*
* allocate tag
* Don't use BUS_DMA_ALLOCNOW, because we never need bouncing with
* bus_dmamem_alloc()
*/
error = bus_dma_tag_create(bus_get_dma_tag(sc->dev),
PATM_SQ_ALIGNMENT, 0,
BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
NULL, NULL, sc->sq_size, 1, sc->sq_size,
0, NULL, NULL, &sc->sq_tag);
if (error) {
patm_printf(sc, "memory DMA tag create %d\n", error);
return (error);
}
/* allocate memory */
error = bus_dmamem_alloc(sc->sq_tag, &p, 0, &sc->sq_map);
if (error) {
patm_printf(sc, "memory DMA alloc %d\n", error);
bus_dma_tag_destroy(sc->sq_tag);
return (error);
}
/* map it */
sc->tsq_phy = 0x1fff;
error = bus_dmamap_load(sc->sq_tag, sc->sq_map, p,
sc->sq_size, patm_load_callback, &sc->tsq_phy, BUS_DMA_NOWAIT);
if (error) {
patm_printf(sc, "memory DMA map load %d\n", error);
bus_dmamem_free(sc->sq_tag, p, sc->sq_map);
bus_dma_tag_destroy(sc->sq_tag);
return (error);
}
/* set queue start */
sc->tsq = p;
sc->rsq = (void *)((char *)p + IDT_TSQ_SIZE * IDT_TSQE_SIZE);
sc->rsq_phy = sc->tsq_phy + IDT_TSQ_SIZE * IDT_TSQE_SIZE;
sc->rawhnd = (void *)((char *)sc->rsq + PATM_RSQ_SIZE * IDT_RSQE_SIZE);
sc->rawhnd_phy = sc->rsq_phy + PATM_RSQ_SIZE * IDT_RSQE_SIZE;
return (0);
}
/*
* Initialize all receive buffer stuff
*/
static int
patm_rbuf_init(struct patm_softc *sc)
{
u_int i;
int error;
patm_debug(sc, ATTACH, "allocating Rx buffer resources ...");
/*
* Create a tag for small buffers. We allocate these page wise.
* Don't use BUS_DMA_ALLOCNOW, because we never need bouncing with
* bus_dmamem_alloc()
*/
if ((error = bus_dma_tag_create(bus_get_dma_tag(sc->dev), PAGE_SIZE, 0,
BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
SMBUF_PAGE_SIZE, 1, SMBUF_PAGE_SIZE, 0,
NULL, NULL, &sc->sbuf_tag)) != 0) {
patm_printf(sc, "sbuf DMA tag create %d\n", error);
return (error);
}
error = mbp_create(&sc->sbuf_pool, "patm sbufs", sc->sbuf_tag,
SMBUF_MAX_PAGES, SMBUF_PAGE_SIZE, SMBUF_CHUNK_SIZE);
if (error != 0) {
patm_printf(sc, "smbuf pool create %d\n", error);
return (error);
}
error = mbp_create(&sc->vbuf_pool, "patm vbufs", sc->sbuf_tag,
VMBUF_MAX_PAGES, SMBUF_PAGE_SIZE, VMBUF_CHUNK_SIZE);
if (error != 0) {
patm_printf(sc, "vmbuf pool create %d\n", error);
return (error);
}
/*
* Create a tag for large buffers.
* Don't use BUS_DMA_ALLOCNOW, because it makes no sense with multiple
* maps using one tag. Rather use BUS_DMA_NOWAIT when loading the map
* to prevent EINPROGRESS.
*/
if ((error = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 4, 0,
BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
MCLBYTES, 1, MCLBYTES, 0,
NULL, NULL, &sc->lbuf_tag)) != 0) {
patm_printf(sc, "lbuf DMA tag create %d\n", error);
return (error);
}
if (sc->lbuf_max < IDT_FBQ_SIZE)
sc->lbuf_max = LMBUF_MAX;
sc->lbufs = malloc(sizeof(sc->lbufs[0]) * sc->lbuf_max,
M_DEVBUF, M_ZERO | M_WAITOK);
SLIST_INIT(&sc->lbuf_free_list);
for (i = 0; i < sc->lbuf_max; i++) {
struct lmbuf *b = &sc->lbufs[i];
error = bus_dmamap_create(sc->lbuf_tag, 0, &b->map);
if (error) {
/* must deallocate here, because a test for NULL
* does not work on most archs */
while (i-- > 0)
bus_dmamap_destroy(sc->lbuf_tag,
sc->lbufs[i].map);
free(sc->lbufs, M_DEVBUF);
sc->lbufs = NULL;
return (error);
}
b->handle = i;
SLIST_INSERT_HEAD(&sc->lbuf_free_list, b, link);
}
return (0);
}
/*
* Allocate everything needed for the transmission maps.
*/
static int
patm_txmap_init(struct patm_softc *sc)
{
int error;
struct patm_txmap *map;
/* get transmission tag */
error = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
NULL, NULL, 65536, IDT_SCQ_SIZE - 1, 65536,
0, NULL, NULL, &sc->tx_tag);
if (error) {
patm_printf(sc, "cannot allocate TX tag %d\n", error);
return (error);
}
if ((sc->tx_mapzone = uma_zcreate("PATM tx maps",
sizeof(struct patm_txmap), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0)) == NULL)
return (ENOMEM);
if (sc->tx_maxmaps < PATM_CFG_TXMAPS_MAX)
sc->tx_maxmaps = PATM_CFG_TXMAPS_MAX;
sc->tx_nmaps = PATM_CFG_TXMAPS_INIT;
for (sc->tx_nmaps = 0; sc->tx_nmaps < PATM_CFG_TXMAPS_INIT;
sc->tx_nmaps++) {
map = uma_zalloc(sc->tx_mapzone, M_WAITOK);
error = bus_dmamap_create(sc->tx_tag, 0, &map->map);
if (error) {
uma_zfree(sc->tx_mapzone, map);
return (ENOMEM);
}
SLIST_INSERT_HEAD(&sc->tx_maps_free, map, link);
}
return (0);
}
#ifdef PATM_DEBUG
/*
* Sysctl handler for REGS
*
* LOCK: unlocked, needed
*/
static int
patm_sysctl_regs(SYSCTL_HANDLER_ARGS)
{
struct patm_softc *sc = arg1;
uint32_t *ret;
int error, i;
ret = malloc(IDT_NOR_END, M_TEMP, M_WAITOK);
mtx_lock(&sc->mtx);
for (i = 0; i < IDT_NOR_END; i += 4)
ret[i / 4] = patm_nor_read(sc, i);
mtx_unlock(&sc->mtx);
error = SYSCTL_OUT(req, ret, IDT_NOR_END);
free(ret, M_TEMP);
return (error);
}
/*
* Sysctl handler for TSQ
*
* LOCK: unlocked, needed
*/
static int
patm_sysctl_tsq(SYSCTL_HANDLER_ARGS)
{
struct patm_softc *sc = arg1;
void *ret;
int error;
ret = malloc(IDT_TSQ_SIZE * IDT_TSQE_SIZE, M_TEMP, M_WAITOK);
mtx_lock(&sc->mtx);
memcpy(ret, sc->tsq, IDT_TSQ_SIZE * IDT_TSQE_SIZE);
mtx_unlock(&sc->mtx);
error = SYSCTL_OUT(req, ret, IDT_TSQ_SIZE * IDT_TSQE_SIZE);
free(ret, M_TEMP);
return (error);
}
/*
* debugging
*/
static struct patm_softc *
patm_dump_unit(u_int unit)
{
devclass_t dc;
struct patm_softc *sc;
dc = devclass_find("patm");
if (dc == NULL) {
printf("%s: can't find devclass\n", __func__);
return (NULL);
}
sc = devclass_get_softc(dc, unit);
if (sc == NULL) {
printf("%s: invalid unit number: %d\n", __func__, unit);
return (NULL);
}
return (sc);
}
int
patm_dump_vc(u_int unit, u_int vc)
{
struct patm_softc *sc;
uint32_t tct[8];
uint32_t rct[4];
uint32_t scd[12];
u_int i;
if ((sc = patm_dump_unit(unit)) == NULL)
return (0);
for (i = 0; i < 8; i++)
tct[i] = patm_sram_read(sc, vc * 8 + i);
for (i = 0; i < 4; i++)
rct[i] = patm_sram_read(sc, sc->mmap->rct + vc * 4 + i);
for (i = 0; i < 12; i++)
scd[i] = patm_sram_read(sc, (tct[0] & 0x7ffff) + i);
printf("TCT%3u: %08x %08x %08x %08x %08x %08x %08x %08x\n", vc,
tct[0], tct[1], tct[2], tct[3], tct[4], tct[5], tct[6], tct[7]);
printf("RCT%3u: %08x %08x %08x %08x\n", vc,
rct[0], rct[1], rct[2], rct[3]);
printf("SCD%3u: %08x %08x %08x %08x %08x %08x %08x %08x\n", vc,
scd[0], scd[1], scd[2], scd[3], scd[4], scd[5], scd[6], scd[7]);
printf(" %08x %08x %08x %08x\n",
scd[8], scd[9], scd[10], scd[11]);
return (0);
}
int
patm_dump_regs(u_int unit)
{
struct patm_softc *sc;
u_int i;
if ((sc = patm_dump_unit(unit)) == NULL)
return (0);
for (i = 0; i <= IDT_NOR_DNOW; i += 4)
printf("%x: %08x\n", i, patm_nor_read(sc, i));
return (0);
}
int
patm_dump_sram(u_int unit, u_int from, u_int words)
{
struct patm_softc *sc;
u_int i;
if ((sc = patm_dump_unit(unit)) == NULL)
return (0);
for (i = 0; i < words; i++) {
if (i % 8 == 0)
printf("%05x:", from + i);
printf(" %08x", patm_sram_read(sc, from + i));
if (i % 8 == 7)
printf("\n");
}
if (i % 8 != 0)
printf("\n");
return (0);
}
#endif
Index: head/sys/dev/qlxgb/qla_os.c
===================================================================
--- head/sys/dev/qlxgb/qla_os.c (revision 283290)
+++ head/sys/dev/qlxgb/qla_os.c (revision 283291)
@@ -1,1487 +1,1487 @@
/*
* Copyright (c) 2011-2013 Qlogic Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* File: qla_os.c
* Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "qla_os.h"
#include "qla_reg.h"
#include "qla_hw.h"
#include "qla_def.h"
#include "qla_inline.h"
#include "qla_ver.h"
#include "qla_glbl.h"
#include "qla_dbg.h"
/*
* Some PCI Configuration Space Related Defines
*/
#ifndef PCI_VENDOR_QLOGIC
#define PCI_VENDOR_QLOGIC 0x1077
#endif
#ifndef PCI_PRODUCT_QLOGIC_ISP8020
#define PCI_PRODUCT_QLOGIC_ISP8020 0x8020
#endif
#define PCI_QLOGIC_ISP8020 \
((PCI_PRODUCT_QLOGIC_ISP8020 << 16) | PCI_VENDOR_QLOGIC)
/*
* static functions
*/
static int qla_alloc_parent_dma_tag(qla_host_t *ha);
static void qla_free_parent_dma_tag(qla_host_t *ha);
static int qla_alloc_xmt_bufs(qla_host_t *ha);
static void qla_free_xmt_bufs(qla_host_t *ha);
static int qla_alloc_rcv_bufs(qla_host_t *ha);
static void qla_free_rcv_bufs(qla_host_t *ha);
static void qla_init_ifnet(device_t dev, qla_host_t *ha);
static int qla_sysctl_get_stats(SYSCTL_HANDLER_ARGS);
static void qla_release(qla_host_t *ha);
static void qla_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
int error);
static void qla_stop(qla_host_t *ha);
static int qla_send(qla_host_t *ha, struct mbuf **m_headp);
static void qla_tx_done(void *context, int pending);
/*
* Hooks to the Operating Systems
*/
static int qla_pci_probe (device_t);
static int qla_pci_attach (device_t);
static int qla_pci_detach (device_t);
static void qla_init(void *arg);
static int qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
static int qla_media_change(struct ifnet *ifp);
static void qla_media_status(struct ifnet *ifp, struct ifmediareq *ifmr);
static device_method_t qla_pci_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, qla_pci_probe),
DEVMETHOD(device_attach, qla_pci_attach),
DEVMETHOD(device_detach, qla_pci_detach),
{ 0, 0 }
};
static driver_t qla_pci_driver = {
"ql", qla_pci_methods, sizeof (qla_host_t),
};
static devclass_t qla80xx_devclass;
DRIVER_MODULE(qla80xx, pci, qla_pci_driver, qla80xx_devclass, 0, 0);
MODULE_DEPEND(qla80xx, pci, 1, 1, 1);
MODULE_DEPEND(qla80xx, ether, 1, 1, 1);
MALLOC_DEFINE(M_QLA8XXXBUF, "qla80xxbuf", "Buffers for qla80xx driver");
uint32_t std_replenish = 8;
uint32_t jumbo_replenish = 2;
uint32_t rcv_pkt_thres = 128;
uint32_t rcv_pkt_thres_d = 32;
uint32_t snd_pkt_thres = 16;
uint32_t free_pkt_thres = (NUM_TX_DESCRIPTORS / 2);
static char dev_str[64];
/*
* Name: qla_pci_probe
* Function: Validate the PCI device to be a QLA80XX device
*/
static int
qla_pci_probe(device_t dev)
{
switch ((pci_get_device(dev) << 16) | (pci_get_vendor(dev))) {
case PCI_QLOGIC_ISP8020:
snprintf(dev_str, sizeof(dev_str), "%s v%d.%d.%d",
"Qlogic ISP 80xx PCI CNA Adapter-Ethernet Function",
QLA_VERSION_MAJOR, QLA_VERSION_MINOR,
QLA_VERSION_BUILD);
device_set_desc(dev, dev_str);
break;
default:
return (ENXIO);
}
if (bootverbose)
printf("%s: %s\n ", __func__, dev_str);
return (BUS_PROBE_DEFAULT);
}
static void
qla_add_sysctls(qla_host_t *ha)
{
device_t dev = ha->pci_dev;
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RD,
(void *)ha, 0,
qla_sysctl_get_stats, "I", "Statistics");
SYSCTL_ADD_STRING(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "fw_version", CTLFLAG_RD,
ha->fw_ver_str, 0, "firmware version");
dbg_level = 0;
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "debug", CTLFLAG_RW,
&dbg_level, dbg_level, "Debug Level");
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "std_replenish", CTLFLAG_RW,
&std_replenish, std_replenish,
"Threshold for Replenishing Standard Frames");
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "jumbo_replenish", CTLFLAG_RW,
&jumbo_replenish, jumbo_replenish,
"Threshold for Replenishing Jumbo Frames");
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "rcv_pkt_thres", CTLFLAG_RW,
&rcv_pkt_thres, rcv_pkt_thres,
"Threshold for # of rcv pkts to trigger indication isr");
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "rcv_pkt_thres_d", CTLFLAG_RW,
&rcv_pkt_thres_d, rcv_pkt_thres_d,
"Threshold for # of rcv pkts to trigger indication defered");
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "snd_pkt_thres", CTLFLAG_RW,
&snd_pkt_thres, snd_pkt_thres,
"Threshold for # of snd packets");
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "free_pkt_thres", CTLFLAG_RW,
&free_pkt_thres, free_pkt_thres,
"Threshold for # of packets to free at a time");
return;
}
static void
qla_watchdog(void *arg)
{
qla_host_t *ha = arg;
qla_hw_t *hw;
struct ifnet *ifp;
hw = &ha->hw;
ifp = ha->ifp;
if (ha->flags.qla_watchdog_exit)
return;
if (!ha->flags.qla_watchdog_pause) {
if (qla_le32_to_host(*(hw->tx_cons)) != hw->txr_comp) {
taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
} else if ((ifp->if_snd.ifq_head != NULL) && QL_RUNNING(ifp)) {
taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
}
}
ha->watchdog_ticks = ha->watchdog_ticks++ % 1000;
callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
qla_watchdog, ha);
}
/*
* Name: qla_pci_attach
* Function: attaches the device to the operating system
*/
static int
qla_pci_attach(device_t dev)
{
qla_host_t *ha = NULL;
uint32_t rsrc_len, i;
QL_DPRINT2((dev, "%s: enter\n", __func__));
if ((ha = device_get_softc(dev)) == NULL) {
device_printf(dev, "cannot get softc\n");
return (ENOMEM);
}
memset(ha, 0, sizeof (qla_host_t));
if (pci_get_device(dev) != PCI_PRODUCT_QLOGIC_ISP8020) {
device_printf(dev, "device is not ISP8020\n");
return (ENXIO);
}
ha->pci_func = pci_get_function(dev);
ha->pci_dev = dev;
pci_enable_busmaster(dev);
ha->reg_rid = PCIR_BAR(0);
ha->pci_reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->reg_rid,
RF_ACTIVE);
if (ha->pci_reg == NULL) {
device_printf(dev, "unable to map any ports\n");
goto qla_pci_attach_err;
}
rsrc_len = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY,
ha->reg_rid);
mtx_init(&ha->hw_lock, "qla80xx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF);
mtx_init(&ha->tx_lock, "qla80xx_tx_lock", MTX_NETWORK_LOCK, MTX_DEF);
mtx_init(&ha->rx_lock, "qla80xx_rx_lock", MTX_NETWORK_LOCK, MTX_DEF);
mtx_init(&ha->rxj_lock, "qla80xx_rxj_lock", MTX_NETWORK_LOCK, MTX_DEF);
ha->flags.lock_init = 1;
ha->msix_count = pci_msix_count(dev);
if (ha->msix_count < qla_get_msix_count(ha)) {
device_printf(dev, "%s: msix_count[%d] not enough\n", __func__,
ha->msix_count);
goto qla_pci_attach_err;
}
QL_DPRINT2((dev, "%s: ha %p irq %p pci_func 0x%x rsrc_count 0x%08x"
" msix_count 0x%x pci_reg %p\n", __func__, ha,
ha->irq, ha->pci_func, rsrc_len, ha->msix_count, ha->pci_reg));
ha->msix_count = qla_get_msix_count(ha);
if (pci_alloc_msix(dev, &ha->msix_count)) {
device_printf(dev, "%s: pci_alloc_msi[%d] failed\n", __func__,
ha->msix_count);
ha->msix_count = 0;
goto qla_pci_attach_err;
}
TASK_INIT(&ha->tx_task, 0, qla_tx_done, ha);
ha->tx_tq = taskqueue_create_fast("qla_txq", M_NOWAIT,
taskqueue_thread_enqueue, &ha->tx_tq);
taskqueue_start_threads(&ha->tx_tq, 1, PI_NET, "%s txq",
device_get_nameunit(ha->pci_dev));
for (i = 0; i < ha->msix_count; i++) {
ha->irq_vec[i].irq_rid = i+1;
ha->irq_vec[i].ha = ha;
ha->irq_vec[i].irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
&ha->irq_vec[i].irq_rid,
(RF_ACTIVE | RF_SHAREABLE));
if (ha->irq_vec[i].irq == NULL) {
device_printf(dev, "could not allocate interrupt\n");
goto qla_pci_attach_err;
}
if (bus_setup_intr(dev, ha->irq_vec[i].irq,
(INTR_TYPE_NET | INTR_MPSAFE),
NULL, qla_isr, &ha->irq_vec[i],
&ha->irq_vec[i].handle)) {
device_printf(dev, "could not setup interrupt\n");
goto qla_pci_attach_err;
}
TASK_INIT(&ha->irq_vec[i].rcv_task, 0, qla_rcv,\
&ha->irq_vec[i]);
ha->irq_vec[i].rcv_tq = taskqueue_create_fast("qla_rcvq",
M_NOWAIT, taskqueue_thread_enqueue,
&ha->irq_vec[i].rcv_tq);
taskqueue_start_threads(&ha->irq_vec[i].rcv_tq, 1, PI_NET,
"%s rcvq",
device_get_nameunit(ha->pci_dev));
}
qla_add_sysctls(ha);
/* add hardware specific sysctls */
qla_hw_add_sysctls(ha);
/* initialize hardware */
if (qla_init_hw(ha)) {
device_printf(dev, "%s: qla_init_hw failed\n", __func__);
goto qla_pci_attach_err;
}
device_printf(dev, "%s: firmware[%d.%d.%d.%d]\n", __func__,
ha->fw_ver_major, ha->fw_ver_minor, ha->fw_ver_sub,
ha->fw_ver_build);
snprintf(ha->fw_ver_str, sizeof(ha->fw_ver_str), "%d.%d.%d.%d",
ha->fw_ver_major, ha->fw_ver_minor, ha->fw_ver_sub,
ha->fw_ver_build);
//qla_get_hw_caps(ha);
qla_read_mac_addr(ha);
/* allocate parent dma tag */
if (qla_alloc_parent_dma_tag(ha)) {
device_printf(dev, "%s: qla_alloc_parent_dma_tag failed\n",
__func__);
goto qla_pci_attach_err;
}
/* alloc all dma buffers */
if (qla_alloc_dma(ha)) {
device_printf(dev, "%s: qla_alloc_dma failed\n", __func__);
goto qla_pci_attach_err;
}
/* create the o.s ethernet interface */
qla_init_ifnet(dev, ha);
ha->flags.qla_watchdog_active = 1;
ha->flags.qla_watchdog_pause = 1;
- callout_init(&ha->tx_callout, TRUE);
+ callout_init(&ha->tx_callout, 1);
/* create ioctl device interface */
if (qla_make_cdev(ha)) {
device_printf(dev, "%s: qla_make_cdev failed\n", __func__);
goto qla_pci_attach_err;
}
callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
qla_watchdog, ha);
QL_DPRINT2((dev, "%s: exit 0\n", __func__));
return (0);
qla_pci_attach_err:
qla_release(ha);
QL_DPRINT2((dev, "%s: exit ENXIO\n", __func__));
return (ENXIO);
}
/*
* Name: qla_pci_detach
* Function: Unhooks the device from the operating system
*/
static int
qla_pci_detach(device_t dev)
{
qla_host_t *ha = NULL;
struct ifnet *ifp;
int i;
QL_DPRINT2((dev, "%s: enter\n", __func__));
if ((ha = device_get_softc(dev)) == NULL) {
device_printf(dev, "cannot get softc\n");
return (ENOMEM);
}
ifp = ha->ifp;
QLA_LOCK(ha, __func__);
qla_stop(ha);
QLA_UNLOCK(ha, __func__);
if (ha->tx_tq) {
taskqueue_drain(ha->tx_tq, &ha->tx_task);
taskqueue_free(ha->tx_tq);
}
for (i = 0; i < ha->msix_count; i++) {
taskqueue_drain(ha->irq_vec[i].rcv_tq,
&ha->irq_vec[i].rcv_task);
taskqueue_free(ha->irq_vec[i].rcv_tq);
}
qla_release(ha);
QL_DPRINT2((dev, "%s: exit\n", __func__));
return (0);
}
/*
* SYSCTL Related Callbacks
*/
static int
qla_sysctl_get_stats(SYSCTL_HANDLER_ARGS)
{
int err, ret = 0;
qla_host_t *ha;
err = sysctl_handle_int(oidp, &ret, 0, req);
if (err)
return (err);
ha = (qla_host_t *)arg1;
//qla_get_stats(ha);
QL_DPRINT2((ha->pci_dev, "%s: called ret %d\n", __func__, ret));
return (err);
}
/*
* Name: qla_release
* Function: Releases the resources allocated for the device
*/
static void
qla_release(qla_host_t *ha)
{
device_t dev;
int i;
dev = ha->pci_dev;
qla_del_cdev(ha);
if (ha->flags.qla_watchdog_active)
ha->flags.qla_watchdog_exit = 1;
callout_stop(&ha->tx_callout);
qla_mdelay(__func__, 100);
if (ha->ifp != NULL)
ether_ifdetach(ha->ifp);
qla_free_dma(ha);
qla_free_parent_dma_tag(ha);
for (i = 0; i < ha->msix_count; i++) {
if (ha->irq_vec[i].handle)
(void)bus_teardown_intr(dev, ha->irq_vec[i].irq,
ha->irq_vec[i].handle);
if (ha->irq_vec[i].irq)
(void) bus_release_resource(dev, SYS_RES_IRQ,
ha->irq_vec[i].irq_rid,
ha->irq_vec[i].irq);
}
if (ha->msix_count)
pci_release_msi(dev);
if (ha->flags.lock_init) {
mtx_destroy(&ha->tx_lock);
mtx_destroy(&ha->rx_lock);
mtx_destroy(&ha->rxj_lock);
mtx_destroy(&ha->hw_lock);
}
if (ha->pci_reg)
(void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid,
ha->pci_reg);
}
/*
* DMA Related Functions
*/
static void
qla_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
{
*((bus_addr_t *)arg) = 0;
if (error) {
printf("%s: bus_dmamap_load failed (%d)\n", __func__, error);
return;
}
QL_ASSERT((nsegs == 1), ("%s: %d segments returned!", __func__, nsegs));
*((bus_addr_t *)arg) = segs[0].ds_addr;
return;
}
int
qla_alloc_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
{
int ret = 0;
device_t dev;
bus_addr_t b_addr;
dev = ha->pci_dev;
QL_DPRINT2((dev, "%s: enter\n", __func__));
ret = bus_dma_tag_create(
ha->parent_tag,/* parent */
dma_buf->alignment,
((bus_size_t)(1ULL << 32)),/* boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
dma_buf->size, /* maxsize */
1, /* nsegments */
dma_buf->size, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&dma_buf->dma_tag);
if (ret) {
device_printf(dev, "%s: could not create dma tag\n", __func__);
goto qla_alloc_dmabuf_exit;
}
ret = bus_dmamem_alloc(dma_buf->dma_tag,
(void **)&dma_buf->dma_b,
(BUS_DMA_ZERO | BUS_DMA_COHERENT | BUS_DMA_NOWAIT),
&dma_buf->dma_map);
if (ret) {
bus_dma_tag_destroy(dma_buf->dma_tag);
device_printf(dev, "%s: bus_dmamem_alloc failed\n", __func__);
goto qla_alloc_dmabuf_exit;
}
ret = bus_dmamap_load(dma_buf->dma_tag,
dma_buf->dma_map,
dma_buf->dma_b,
dma_buf->size,
qla_dmamap_callback,
&b_addr, BUS_DMA_NOWAIT);
if (ret || !b_addr) {
bus_dma_tag_destroy(dma_buf->dma_tag);
bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b,
dma_buf->dma_map);
ret = -1;
goto qla_alloc_dmabuf_exit;
}
dma_buf->dma_addr = b_addr;
qla_alloc_dmabuf_exit:
QL_DPRINT2((dev, "%s: exit ret 0x%08x tag %p map %p b %p sz 0x%x\n",
__func__, ret, (void *)dma_buf->dma_tag,
(void *)dma_buf->dma_map, (void *)dma_buf->dma_b,
dma_buf->size));
return ret;
}
void
qla_free_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
{
bus_dmamap_unload(dma_buf->dma_tag, dma_buf->dma_map);
bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, dma_buf->dma_map);
bus_dma_tag_destroy(dma_buf->dma_tag);
}
static int
qla_alloc_parent_dma_tag(qla_host_t *ha)
{
int ret;
device_t dev;
dev = ha->pci_dev;
/*
* Allocate parent DMA Tag
*/
ret = bus_dma_tag_create(
bus_get_dma_tag(dev), /* parent */
1,((bus_size_t)(1ULL << 32)),/* alignment, boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
0, /* nsegments */
BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&ha->parent_tag);
if (ret) {
device_printf(dev, "%s: could not create parent dma tag\n",
__func__);
return (-1);
}
ha->flags.parent_tag = 1;
return (0);
}
static void
qla_free_parent_dma_tag(qla_host_t *ha)
{
if (ha->flags.parent_tag) {
bus_dma_tag_destroy(ha->parent_tag);
ha->flags.parent_tag = 0;
}
}
/*
* Name: qla_init_ifnet
* Function: Creates the Network Device Interface and Registers it with the O.S
*/
static void
qla_init_ifnet(device_t dev, qla_host_t *ha)
{
struct ifnet *ifp;
QL_DPRINT2((dev, "%s: enter\n", __func__));
ifp = ha->ifp = if_alloc(IFT_ETHER);
if (ifp == NULL)
panic("%s: cannot if_alloc()\n", device_get_nameunit(dev));
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_baudrate = IF_Gbps(10);
ifp->if_init = qla_init;
ifp->if_softc = ha;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = qla_ioctl;
ifp->if_start = qla_start;
IFQ_SET_MAXLEN(&ifp->if_snd, qla_get_ifq_snd_maxlen(ha));
ifp->if_snd.ifq_drv_maxlen = qla_get_ifq_snd_maxlen(ha);
IFQ_SET_READY(&ifp->if_snd);
ha->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
ether_ifattach(ifp, qla_get_mac_addr(ha));
ifp->if_capabilities = IFCAP_HWCSUM |
IFCAP_TSO4 |
IFCAP_JUMBO_MTU;
ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
ifp->if_capabilities |= IFCAP_LINKSTATE;
#if defined(__FreeBSD_version) && (__FreeBSD_version < 900002)
ifp->if_timer = 0;
ifp->if_watchdog = NULL;
#endif /* #if defined(__FreeBSD_version) && (__FreeBSD_version < 900002) */
ifp->if_capenable = ifp->if_capabilities;
ifp->if_hdrlen = sizeof(struct ether_vlan_header);
ifmedia_init(&ha->media, IFM_IMASK, qla_media_change, qla_media_status);
ifmedia_add(&ha->media, (IFM_ETHER | qla_get_optics(ha) | IFM_FDX), 0,
NULL);
ifmedia_add(&ha->media, (IFM_ETHER | IFM_AUTO), 0, NULL);
ifmedia_set(&ha->media, (IFM_ETHER | IFM_AUTO));
QL_DPRINT2((dev, "%s: exit\n", __func__));
return;
}
static void
qla_init_locked(qla_host_t *ha)
{
struct ifnet *ifp = ha->ifp;
qla_stop(ha);
if (qla_alloc_xmt_bufs(ha) != 0)
return;
if (qla_alloc_rcv_bufs(ha) != 0)
return;
if (qla_config_lro(ha))
return;
bcopy(IF_LLADDR(ha->ifp), ha->hw.mac_addr, ETHER_ADDR_LEN);
ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO;
ha->flags.stop_rcv = 0;
if (qla_init_hw_if(ha) == 0) {
ifp = ha->ifp;
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
ha->flags.qla_watchdog_pause = 0;
}
return;
}
static void
qla_init(void *arg)
{
qla_host_t *ha;
ha = (qla_host_t *)arg;
QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
QLA_LOCK(ha, __func__);
qla_init_locked(ha);
QLA_UNLOCK(ha, __func__);
QL_DPRINT2((ha->pci_dev, "%s: exit\n", __func__));
}
static void
qla_set_multi(qla_host_t *ha, uint32_t add_multi)
{
uint8_t mta[Q8_MAX_NUM_MULTICAST_ADDRS * Q8_MAC_ADDR_LEN];
struct ifmultiaddr *ifma;
int mcnt = 0;
struct ifnet *ifp = ha->ifp;
if_maddr_rlock(ifp);
TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
if (ifma->ifma_addr->sa_family != AF_LINK)
continue;
if (mcnt == Q8_MAX_NUM_MULTICAST_ADDRS)
break;
bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
&mta[mcnt * Q8_MAC_ADDR_LEN], Q8_MAC_ADDR_LEN);
mcnt++;
}
if_maddr_runlock(ifp);
qla_hw_set_multi(ha, mta, mcnt, add_multi);
return;
}
static int
qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
int ret = 0;
struct ifreq *ifr = (struct ifreq *)data;
struct ifaddr *ifa = (struct ifaddr *)data;
qla_host_t *ha;
ha = (qla_host_t *)ifp->if_softc;
switch (cmd) {
case SIOCSIFADDR:
QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFADDR (0x%lx)\n",
__func__, cmd));
if (ifa->ifa_addr->sa_family == AF_INET) {
ifp->if_flags |= IFF_UP;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
QLA_LOCK(ha, __func__);
qla_init_locked(ha);
QLA_UNLOCK(ha, __func__);
}
QL_DPRINT4((ha->pci_dev,
"%s: SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n",
__func__, cmd, ntohl(IA_SIN(ifa)->sin_addr.s_addr)));
arp_ifinit(ifp, ifa);
if (ntohl(IA_SIN(ifa)->sin_addr.s_addr) != INADDR_ANY) {
qla_config_ipv4_addr(ha,
(IA_SIN(ifa)->sin_addr.s_addr));
}
} else {
ether_ioctl(ifp, cmd, data);
}
break;
case SIOCSIFMTU:
QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFMTU (0x%lx)\n",
__func__, cmd));
if (ifr->ifr_mtu > QLA_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
ret = EINVAL;
} else {
QLA_LOCK(ha, __func__);
ifp->if_mtu = ifr->ifr_mtu;
ha->max_frame_size =
ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
ret = qla_set_max_mtu(ha, ha->max_frame_size,
(ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id);
}
QLA_UNLOCK(ha, __func__);
if (ret)
ret = EINVAL;
}
break;
case SIOCSIFFLAGS:
QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFFLAGS (0x%lx)\n",
__func__, cmd));
if (ifp->if_flags & IFF_UP) {
if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
if ((ifp->if_flags ^ ha->if_flags) &
IFF_PROMISC) {
qla_set_promisc(ha);
} else if ((ifp->if_flags ^ ha->if_flags) &
IFF_ALLMULTI) {
qla_set_allmulti(ha);
}
} else {
QLA_LOCK(ha, __func__);
qla_init_locked(ha);
ha->max_frame_size = ifp->if_mtu +
ETHER_HDR_LEN + ETHER_CRC_LEN;
ret = qla_set_max_mtu(ha, ha->max_frame_size,
(ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id);
QLA_UNLOCK(ha, __func__);
}
} else {
QLA_LOCK(ha, __func__);
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
qla_stop(ha);
ha->if_flags = ifp->if_flags;
QLA_UNLOCK(ha, __func__);
}
break;
case SIOCADDMULTI:
QL_DPRINT4((ha->pci_dev,
"%s: %s (0x%lx)\n", __func__, "SIOCADDMULTI", cmd));
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
qla_set_multi(ha, 1);
}
break;
case SIOCDELMULTI:
QL_DPRINT4((ha->pci_dev,
"%s: %s (0x%lx)\n", __func__, "SIOCDELMULTI", cmd));
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
qla_set_multi(ha, 0);
}
break;
case SIOCSIFMEDIA:
case SIOCGIFMEDIA:
QL_DPRINT4((ha->pci_dev,
"%s: SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n",
__func__, cmd));
ret = ifmedia_ioctl(ifp, ifr, &ha->media, cmd);
break;
case SIOCSIFCAP:
{
int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFCAP (0x%lx)\n",
__func__, cmd));
if (mask & IFCAP_HWCSUM)
ifp->if_capenable ^= IFCAP_HWCSUM;
if (mask & IFCAP_TSO4)
ifp->if_capenable ^= IFCAP_TSO4;
if (mask & IFCAP_TSO6)
ifp->if_capenable ^= IFCAP_TSO6;
if (mask & IFCAP_VLAN_HWTAGGING)
ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
qla_init(ha);
VLAN_CAPABILITIES(ifp);
break;
}
default:
QL_DPRINT4((ha->pci_dev, "%s: default (0x%lx)\n",
__func__, cmd));
ret = ether_ioctl(ifp, cmd, data);
break;
}
return (ret);
}
static int
qla_media_change(struct ifnet *ifp)
{
qla_host_t *ha;
struct ifmedia *ifm;
int ret = 0;
ha = (qla_host_t *)ifp->if_softc;
QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
ifm = &ha->media;
if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
ret = EINVAL;
QL_DPRINT2((ha->pci_dev, "%s: exit\n", __func__));
return (ret);
}
static void
qla_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
qla_host_t *ha;
ha = (qla_host_t *)ifp->if_softc;
QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
ifmr->ifm_status = IFM_AVALID;
ifmr->ifm_active = IFM_ETHER;
qla_update_link_state(ha);
if (ha->hw.flags.link_up) {
ifmr->ifm_status |= IFM_ACTIVE;
ifmr->ifm_active |= (IFM_FDX | qla_get_optics(ha));
}
QL_DPRINT2((ha->pci_dev, "%s: exit (%s)\n", __func__,\
(ha->hw.flags.link_up ? "link_up" : "link_down")));
return;
}
void
qla_start(struct ifnet *ifp)
{
struct mbuf *m_head;
qla_host_t *ha = (qla_host_t *)ifp->if_softc;
QL_DPRINT8((ha->pci_dev, "%s: enter\n", __func__));
if (!mtx_trylock(&ha->tx_lock)) {
QL_DPRINT8((ha->pci_dev,
"%s: mtx_trylock(&ha->tx_lock) failed\n", __func__));
return;
}
if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
IFF_DRV_RUNNING) {
QL_DPRINT8((ha->pci_dev, "%s: !IFF_DRV_RUNNING\n", __func__));
QLA_TX_UNLOCK(ha);
return;
}
if (!ha->watchdog_ticks)
qla_update_link_state(ha);
if (!ha->hw.flags.link_up) {
QL_DPRINT8((ha->pci_dev, "%s: link down\n", __func__));
QLA_TX_UNLOCK(ha);
return;
}
while (ifp->if_snd.ifq_head != NULL) {
IF_DEQUEUE(&ifp->if_snd, m_head);
if (m_head == NULL) {
QL_DPRINT8((ha->pci_dev, "%s: m_head == NULL\n",
__func__));
break;
}
if (qla_send(ha, &m_head)) {
if (m_head == NULL)
break;
QL_DPRINT8((ha->pci_dev, "%s: PREPEND\n", __func__));
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IF_PREPEND(&ifp->if_snd, m_head);
break;
}
/* Send a copy of the frame to the BPF listener */
ETHER_BPF_MTAP(ifp, m_head);
}
QLA_TX_UNLOCK(ha);
QL_DPRINT8((ha->pci_dev, "%s: exit\n", __func__));
return;
}
static int
qla_send(qla_host_t *ha, struct mbuf **m_headp)
{
bus_dma_segment_t segs[QLA_MAX_SEGMENTS];
bus_dmamap_t map;
int nsegs;
int ret = -1;
uint32_t tx_idx;
struct mbuf *m_head = *m_headp;
QL_DPRINT8((ha->pci_dev, "%s: enter\n", __func__));
if ((ret = bus_dmamap_create(ha->tx_tag, BUS_DMA_NOWAIT, &map))) {
ha->err_tx_dmamap_create++;
device_printf(ha->pci_dev,
"%s: bus_dmamap_create failed[%d, %d]\n",
__func__, ret, m_head->m_pkthdr.len);
return (ret);
}
ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs,
BUS_DMA_NOWAIT);
if (ret == EFBIG) {
struct mbuf *m;
QL_DPRINT8((ha->pci_dev, "%s: EFBIG [%d]\n", __func__,
m_head->m_pkthdr.len));
m = m_defrag(m_head, M_NOWAIT);
if (m == NULL) {
ha->err_tx_defrag++;
m_freem(m_head);
*m_headp = NULL;
device_printf(ha->pci_dev,
"%s: m_defrag() = NULL [%d]\n",
__func__, ret);
return (ENOBUFS);
}
m_head = m;
if ((ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head,
segs, &nsegs, BUS_DMA_NOWAIT))) {
ha->err_tx_dmamap_load++;
device_printf(ha->pci_dev,
"%s: bus_dmamap_load_mbuf_sg failed0[%d, %d]\n",
__func__, ret, m_head->m_pkthdr.len);
bus_dmamap_destroy(ha->tx_tag, map);
if (ret != ENOMEM) {
m_freem(m_head);
*m_headp = NULL;
}
return (ret);
}
} else if (ret) {
ha->err_tx_dmamap_load++;
device_printf(ha->pci_dev,
"%s: bus_dmamap_load_mbuf_sg failed1[%d, %d]\n",
__func__, ret, m_head->m_pkthdr.len);
bus_dmamap_destroy(ha->tx_tag, map);
if (ret != ENOMEM) {
m_freem(m_head);
*m_headp = NULL;
}
return (ret);
}
QL_ASSERT((nsegs != 0), ("qla_send: empty packet"));
bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_PREWRITE);
if (!(ret = qla_hw_send(ha, segs, nsegs, &tx_idx, m_head))) {
ha->tx_buf[tx_idx].m_head = m_head;
ha->tx_buf[tx_idx].map = map;
} else {
if (ret == EINVAL) {
m_freem(m_head);
*m_headp = NULL;
}
}
QL_DPRINT8((ha->pci_dev, "%s: exit\n", __func__));
return (ret);
}
static void
qla_stop(qla_host_t *ha)
{
struct ifnet *ifp = ha->ifp;
device_t dev;
dev = ha->pci_dev;
ha->flags.qla_watchdog_pause = 1;
qla_mdelay(__func__, 100);
ha->flags.stop_rcv = 1;
qla_hw_stop_rcv(ha);
qla_del_hw_if(ha);
qla_free_lro(ha);
qla_free_xmt_bufs(ha);
qla_free_rcv_bufs(ha);
ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
return;
}
/*
* Buffer Management Functions for Transmit and Receive Rings
*/
static int
qla_alloc_xmt_bufs(qla_host_t *ha)
{
if (bus_dma_tag_create(NULL, /* parent */
1, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
QLA_MAX_TSO_FRAME_SIZE, /* maxsize */
QLA_MAX_SEGMENTS, /* nsegments */
PAGE_SIZE, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, /* lockfunc */
NULL, /* lockfuncarg */
&ha->tx_tag)) {
device_printf(ha->pci_dev, "%s: tx_tag alloc failed\n",
__func__);
return (ENOMEM);
}
bzero((void *)ha->tx_buf, (sizeof(qla_tx_buf_t) * NUM_TX_DESCRIPTORS));
return 0;
}
/*
* Release mbuf after it sent on the wire
*/
static void
qla_clear_tx_buf(qla_host_t *ha, qla_tx_buf_t *txb)
{
QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
if (txb->m_head) {
bus_dmamap_unload(ha->tx_tag, txb->map);
bus_dmamap_destroy(ha->tx_tag, txb->map);
m_freem(txb->m_head);
txb->m_head = NULL;
}
QL_DPRINT2((ha->pci_dev, "%s: exit\n", __func__));
}
static void
qla_free_xmt_bufs(qla_host_t *ha)
{
int i;
for (i = 0; i < NUM_TX_DESCRIPTORS; i++)
qla_clear_tx_buf(ha, &ha->tx_buf[i]);
if (ha->tx_tag != NULL) {
bus_dma_tag_destroy(ha->tx_tag);
ha->tx_tag = NULL;
}
bzero((void *)ha->tx_buf, (sizeof(qla_tx_buf_t) * NUM_TX_DESCRIPTORS));
return;
}
static int
qla_alloc_rcv_bufs(qla_host_t *ha)
{
int i, j, ret = 0;
qla_rx_buf_t *rxb;
if (bus_dma_tag_create(NULL, /* parent */
1, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
MJUM9BYTES, /* maxsize */
1, /* nsegments */
MJUM9BYTES, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, /* lockfunc */
NULL, /* lockfuncarg */
&ha->rx_tag)) {
device_printf(ha->pci_dev, "%s: rx_tag alloc failed\n",
__func__);
return (ENOMEM);
}
bzero((void *)ha->rx_buf, (sizeof(qla_rx_buf_t) * NUM_RX_DESCRIPTORS));
bzero((void *)ha->rx_jbuf,
(sizeof(qla_rx_buf_t) * NUM_RX_JUMBO_DESCRIPTORS));
for (i = 0; i < MAX_SDS_RINGS; i++) {
ha->hw.sds[i].sdsr_next = 0;
ha->hw.sds[i].rxb_free = NULL;
ha->hw.sds[i].rx_free = 0;
ha->hw.sds[i].rxjb_free = NULL;
ha->hw.sds[i].rxj_free = 0;
}
for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
rxb = &ha->rx_buf[i];
ret = bus_dmamap_create(ha->rx_tag, BUS_DMA_NOWAIT, &rxb->map);
if (ret) {
device_printf(ha->pci_dev,
"%s: dmamap[%d] failed\n", __func__, i);
for (j = 0; j < i; j++) {
bus_dmamap_destroy(ha->rx_tag,
ha->rx_buf[j].map);
}
goto qla_alloc_rcv_bufs_failed;
}
}
qla_init_hw_rcv_descriptors(ha, RDS_RING_INDEX_NORMAL);
for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
rxb = &ha->rx_buf[i];
rxb->handle = i;
if (!(ret = qla_get_mbuf(ha, rxb, NULL, 0))) {
/*
* set the physical address in the corresponding
* descriptor entry in the receive ring/queue for the
* hba
*/
qla_set_hw_rcv_desc(ha, RDS_RING_INDEX_NORMAL, i,
rxb->handle, rxb->paddr,
(rxb->m_head)->m_pkthdr.len);
} else {
device_printf(ha->pci_dev,
"%s: qla_get_mbuf [standard(%d)] failed\n",
__func__, i);
bus_dmamap_destroy(ha->rx_tag, rxb->map);
goto qla_alloc_rcv_bufs_failed;
}
}
for (i = 0; i < NUM_RX_JUMBO_DESCRIPTORS; i++) {
rxb = &ha->rx_jbuf[i];
ret = bus_dmamap_create(ha->rx_tag, BUS_DMA_NOWAIT, &rxb->map);
if (ret) {
device_printf(ha->pci_dev,
"%s: dmamap[%d] failed\n", __func__, i);
for (j = 0; j < i; j++) {
bus_dmamap_destroy(ha->rx_tag,
ha->rx_jbuf[j].map);
}
goto qla_alloc_rcv_bufs_failed;
}
}
qla_init_hw_rcv_descriptors(ha, RDS_RING_INDEX_JUMBO);
for (i = 0; i < NUM_RX_JUMBO_DESCRIPTORS; i++) {
rxb = &ha->rx_jbuf[i];
rxb->handle = i;
if (!(ret = qla_get_mbuf(ha, rxb, NULL, 1))) {
/*
* set the physical address in the corresponding
* descriptor entry in the receive ring/queue for the
* hba
*/
qla_set_hw_rcv_desc(ha, RDS_RING_INDEX_JUMBO, i,
rxb->handle, rxb->paddr,
(rxb->m_head)->m_pkthdr.len);
} else {
device_printf(ha->pci_dev,
"%s: qla_get_mbuf [jumbo(%d)] failed\n",
__func__, i);
bus_dmamap_destroy(ha->rx_tag, rxb->map);
goto qla_alloc_rcv_bufs_failed;
}
}
return (0);
qla_alloc_rcv_bufs_failed:
qla_free_rcv_bufs(ha);
return (ret);
}
static void
qla_free_rcv_bufs(qla_host_t *ha)
{
int i;
qla_rx_buf_t *rxb;
for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
rxb = &ha->rx_buf[i];
if (rxb->m_head != NULL) {
bus_dmamap_unload(ha->rx_tag, rxb->map);
bus_dmamap_destroy(ha->rx_tag, rxb->map);
m_freem(rxb->m_head);
rxb->m_head = NULL;
}
}
for (i = 0; i < NUM_RX_JUMBO_DESCRIPTORS; i++) {
rxb = &ha->rx_jbuf[i];
if (rxb->m_head != NULL) {
bus_dmamap_unload(ha->rx_tag, rxb->map);
bus_dmamap_destroy(ha->rx_tag, rxb->map);
m_freem(rxb->m_head);
rxb->m_head = NULL;
}
}
if (ha->rx_tag != NULL) {
bus_dma_tag_destroy(ha->rx_tag);
ha->rx_tag = NULL;
}
bzero((void *)ha->rx_buf, (sizeof(qla_rx_buf_t) * NUM_RX_DESCRIPTORS));
bzero((void *)ha->rx_jbuf,
(sizeof(qla_rx_buf_t) * NUM_RX_JUMBO_DESCRIPTORS));
for (i = 0; i < MAX_SDS_RINGS; i++) {
ha->hw.sds[i].sdsr_next = 0;
ha->hw.sds[i].rxb_free = NULL;
ha->hw.sds[i].rx_free = 0;
ha->hw.sds[i].rxjb_free = NULL;
ha->hw.sds[i].rxj_free = 0;
}
return;
}
int
qla_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp,
uint32_t jumbo)
{
register struct mbuf *mp = nmp;
struct ifnet *ifp;
int ret = 0;
uint32_t offset;
QL_DPRINT2((ha->pci_dev, "%s: jumbo(0x%x) enter\n", __func__, jumbo));
ifp = ha->ifp;
if (mp == NULL) {
if (!jumbo) {
mp = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
if (mp == NULL) {
ha->err_m_getcl++;
ret = ENOBUFS;
device_printf(ha->pci_dev,
"%s: m_getcl failed\n", __func__);
goto exit_qla_get_mbuf;
}
mp->m_len = mp->m_pkthdr.len = MCLBYTES;
} else {
mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR,
MJUM9BYTES);
if (mp == NULL) {
ha->err_m_getjcl++;
ret = ENOBUFS;
device_printf(ha->pci_dev,
"%s: m_getjcl failed\n", __func__);
goto exit_qla_get_mbuf;
}
mp->m_len = mp->m_pkthdr.len = MJUM9BYTES;
}
} else {
if (!jumbo)
mp->m_len = mp->m_pkthdr.len = MCLBYTES;
else
mp->m_len = mp->m_pkthdr.len = MJUM9BYTES;
mp->m_data = mp->m_ext.ext_buf;
mp->m_next = NULL;
}
offset = (uint32_t)((unsigned long long)mp->m_data & 0x7ULL);
if (offset) {
offset = 8 - offset;
m_adj(mp, offset);
}
/*
* Using memory from the mbuf cluster pool, invoke the bus_dma
* machinery to arrange the memory mapping.
*/
ret = bus_dmamap_load(ha->rx_tag, rxb->map,
mtod(mp, void *), mp->m_len,
qla_dmamap_callback, &rxb->paddr,
BUS_DMA_NOWAIT);
if (ret || !rxb->paddr) {
m_free(mp);
rxb->m_head = NULL;
device_printf(ha->pci_dev,
"%s: bus_dmamap_load failed\n", __func__);
ret = -1;
goto exit_qla_get_mbuf;
}
rxb->m_head = mp;
bus_dmamap_sync(ha->rx_tag, rxb->map, BUS_DMASYNC_PREREAD);
exit_qla_get_mbuf:
QL_DPRINT2((ha->pci_dev, "%s: exit ret = 0x%08x\n", __func__, ret));
return (ret);
}
static void
qla_tx_done(void *context, int pending)
{
qla_host_t *ha = context;
qla_hw_tx_done(ha);
qla_start(ha->ifp);
}
Index: head/sys/dev/qlxgbe/ql_os.c
===================================================================
--- head/sys/dev/qlxgbe/ql_os.c (revision 283290)
+++ head/sys/dev/qlxgbe/ql_os.c (revision 283291)
@@ -1,1701 +1,1701 @@
/*
* Copyright (c) 2013-2014 Qlogic Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* and ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* File: ql_os.c
* Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "ql_os.h"
#include "ql_hw.h"
#include "ql_def.h"
#include "ql_inline.h"
#include "ql_ver.h"
#include "ql_glbl.h"
#include "ql_dbg.h"
#include <sys/smp.h>
/*
* Some PCI Configuration Space Related Defines
*/
#ifndef PCI_VENDOR_QLOGIC
#define PCI_VENDOR_QLOGIC 0x1077
#endif
#ifndef PCI_PRODUCT_QLOGIC_ISP8030
#define PCI_PRODUCT_QLOGIC_ISP8030 0x8030
#endif
#define PCI_QLOGIC_ISP8030 \
((PCI_PRODUCT_QLOGIC_ISP8030 << 16) | PCI_VENDOR_QLOGIC)
/*
* static functions
*/
static int qla_alloc_parent_dma_tag(qla_host_t *ha);
static void qla_free_parent_dma_tag(qla_host_t *ha);
static int qla_alloc_xmt_bufs(qla_host_t *ha);
static void qla_free_xmt_bufs(qla_host_t *ha);
static int qla_alloc_rcv_bufs(qla_host_t *ha);
static void qla_free_rcv_bufs(qla_host_t *ha);
static void qla_clear_tx_buf(qla_host_t *ha, qla_tx_buf_t *txb);
static void qla_init_ifnet(device_t dev, qla_host_t *ha);
static int qla_sysctl_get_stats(SYSCTL_HANDLER_ARGS);
static int qla_sysctl_get_link_status(SYSCTL_HANDLER_ARGS);
static void qla_release(qla_host_t *ha);
static void qla_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
int error);
static void qla_stop(qla_host_t *ha);
static int qla_send(qla_host_t *ha, struct mbuf **m_headp);
static void qla_tx_done(void *context, int pending);
static void qla_get_peer(qla_host_t *ha);
static void qla_error_recovery(void *context, int pending);
/*
* Hooks to the Operating Systems
*/
static int qla_pci_probe (device_t);
static int qla_pci_attach (device_t);
static int qla_pci_detach (device_t);
static void qla_init(void *arg);
static int qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
static int qla_media_change(struct ifnet *ifp);
static void qla_media_status(struct ifnet *ifp, struct ifmediareq *ifmr);
static void qla_start(struct ifnet *ifp);
static device_method_t qla_pci_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, qla_pci_probe),
DEVMETHOD(device_attach, qla_pci_attach),
DEVMETHOD(device_detach, qla_pci_detach),
{ 0, 0 }
};
static driver_t qla_pci_driver = {
"ql", qla_pci_methods, sizeof (qla_host_t),
};
static devclass_t qla83xx_devclass;
DRIVER_MODULE(qla83xx, pci, qla_pci_driver, qla83xx_devclass, 0, 0);
MODULE_DEPEND(qla83xx, pci, 1, 1, 1);
MODULE_DEPEND(qla83xx, ether, 1, 1, 1);
MALLOC_DEFINE(M_QLA83XXBUF, "qla83xxbuf", "Buffers for qla83xx driver");
#define QL_STD_REPLENISH_THRES 0
#define QL_JUMBO_REPLENISH_THRES 32
static char dev_str[64];
/*
* Name: qla_pci_probe
* Function: Validate the PCI device to be a QLA80XX device
*/
static int
qla_pci_probe(device_t dev)
{
switch ((pci_get_device(dev) << 16) | (pci_get_vendor(dev))) {
case PCI_QLOGIC_ISP8030:
snprintf(dev_str, sizeof(dev_str), "%s v%d.%d.%d",
"Qlogic ISP 83xx PCI CNA Adapter-Ethernet Function",
QLA_VERSION_MAJOR, QLA_VERSION_MINOR,
QLA_VERSION_BUILD);
device_set_desc(dev, dev_str);
break;
default:
return (ENXIO);
}
if (bootverbose)
printf("%s: %s\n ", __func__, dev_str);
return (BUS_PROBE_DEFAULT);
}
static void
qla_add_sysctls(qla_host_t *ha)
{
device_t dev = ha->pci_dev;
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW,
(void *)ha, 0,
qla_sysctl_get_stats, "I", "Statistics");
SYSCTL_ADD_STRING(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "fw_version", CTLFLAG_RD,
ha->fw_ver_str, 0, "firmware version");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "link_status", CTLTYPE_INT | CTLFLAG_RW,
(void *)ha, 0,
qla_sysctl_get_link_status, "I", "Link Status");
ha->dbg_level = 0;
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "debug", CTLFLAG_RW,
&ha->dbg_level, ha->dbg_level, "Debug Level");
ha->std_replenish = QL_STD_REPLENISH_THRES;
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "std_replenish", CTLFLAG_RW,
&ha->std_replenish, ha->std_replenish,
"Threshold for Replenishing Standard Frames");
SYSCTL_ADD_QUAD(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "ipv4_lro",
CTLFLAG_RD, &ha->ipv4_lro,
"number of ipv4 lro completions");
SYSCTL_ADD_QUAD(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "ipv6_lro",
CTLFLAG_RD, &ha->ipv6_lro,
"number of ipv6 lro completions");
SYSCTL_ADD_QUAD(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "tx_tso_frames",
CTLFLAG_RD, &ha->tx_tso_frames,
"number of Tx TSO Frames");
SYSCTL_ADD_QUAD(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "hw_vlan_tx_frames",
CTLFLAG_RD, &ha->hw_vlan_tx_frames,
"number of Tx VLAN Frames");
return;
}
static void
qla_watchdog(void *arg)
{
qla_host_t *ha = arg;
qla_hw_t *hw;
struct ifnet *ifp;
uint32_t i;
qla_hw_tx_cntxt_t *hw_tx_cntxt;
hw = &ha->hw;
ifp = ha->ifp;
if (ha->flags.qla_watchdog_exit) {
ha->qla_watchdog_exited = 1;
return;
}
ha->qla_watchdog_exited = 0;
if (!ha->flags.qla_watchdog_pause) {
if (ql_hw_check_health(ha) || ha->qla_initiate_recovery ||
(ha->msg_from_peer == QL_PEER_MSG_RESET)) {
ha->qla_watchdog_paused = 1;
ha->flags.qla_watchdog_pause = 1;
ha->qla_initiate_recovery = 0;
ha->err_inject = 0;
taskqueue_enqueue(ha->err_tq, &ha->err_task);
} else {
for (i = 0; i < ha->hw.num_tx_rings; i++) {
hw_tx_cntxt = &hw->tx_cntxt[i];
if (qla_le32_to_host(*(hw_tx_cntxt->tx_cons)) !=
hw_tx_cntxt->txr_comp) {
taskqueue_enqueue(ha->tx_tq,
&ha->tx_task);
break;
}
}
if ((ifp->if_snd.ifq_head != NULL) && QL_RUNNING(ifp)) {
taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
}
ha->qla_watchdog_paused = 0;
}
} else {
ha->qla_watchdog_paused = 1;
}
ha->watchdog_ticks = ha->watchdog_ticks++ % 1000;
callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
qla_watchdog, ha);
}
/*
* Name: qla_pci_attach
* Function: attaches the device to the operating system
*/
static int
qla_pci_attach(device_t dev)
{
qla_host_t *ha = NULL;
uint32_t rsrc_len;
int i;
QL_DPRINT2(ha, (dev, "%s: enter\n", __func__));
if ((ha = device_get_softc(dev)) == NULL) {
device_printf(dev, "cannot get softc\n");
return (ENOMEM);
}
memset(ha, 0, sizeof (qla_host_t));
if (pci_get_device(dev) != PCI_PRODUCT_QLOGIC_ISP8030) {
device_printf(dev, "device is not ISP8030\n");
return (ENXIO);
}
ha->pci_func = pci_get_function(dev);
ha->pci_dev = dev;
pci_enable_busmaster(dev);
ha->reg_rid = PCIR_BAR(0);
ha->pci_reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->reg_rid,
RF_ACTIVE);
if (ha->pci_reg == NULL) {
device_printf(dev, "unable to map any ports\n");
goto qla_pci_attach_err;
}
rsrc_len = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY,
ha->reg_rid);
mtx_init(&ha->hw_lock, "qla83xx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF);
mtx_init(&ha->tx_lock, "qla83xx_tx_lock", MTX_NETWORK_LOCK, MTX_DEF);
qla_add_sysctls(ha);
ql_hw_add_sysctls(ha);
ha->flags.lock_init = 1;
ha->reg_rid1 = PCIR_BAR(2);
ha->pci_reg1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
&ha->reg_rid1, RF_ACTIVE);
ha->msix_count = pci_msix_count(dev);
if (ha->msix_count < (ha->hw.num_sds_rings + 1)) {
device_printf(dev, "%s: msix_count[%d] not enough\n", __func__,
ha->msix_count);
goto qla_pci_attach_err;
}
QL_DPRINT2(ha, (dev, "%s: ha %p pci_func 0x%x rsrc_count 0x%08x"
" msix_count 0x%x pci_reg %p\n", __func__, ha,
ha->pci_func, rsrc_len, ha->msix_count, ha->pci_reg));
ha->msix_count = ha->hw.num_sds_rings + 1;
if (pci_alloc_msix(dev, &ha->msix_count)) {
device_printf(dev, "%s: pci_alloc_msi[%d] failed\n", __func__,
ha->msix_count);
ha->msix_count = 0;
goto qla_pci_attach_err;
}
ha->mbx_irq_rid = 1;
ha->mbx_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
&ha->mbx_irq_rid,
(RF_ACTIVE | RF_SHAREABLE));
if (ha->mbx_irq == NULL) {
device_printf(dev, "could not allocate mbx interrupt\n");
goto qla_pci_attach_err;
}
if (bus_setup_intr(dev, ha->mbx_irq, (INTR_TYPE_NET | INTR_MPSAFE),
NULL, ql_mbx_isr, ha, &ha->mbx_handle)) {
device_printf(dev, "could not setup mbx interrupt\n");
goto qla_pci_attach_err;
}
for (i = 0; i < ha->hw.num_sds_rings; i++) {
ha->irq_vec[i].sds_idx = i;
ha->irq_vec[i].ha = ha;
ha->irq_vec[i].irq_rid = 2 + i;
ha->irq_vec[i].irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
&ha->irq_vec[i].irq_rid,
(RF_ACTIVE | RF_SHAREABLE));
if (ha->irq_vec[i].irq == NULL) {
device_printf(dev, "could not allocate interrupt\n");
goto qla_pci_attach_err;
}
if (bus_setup_intr(dev, ha->irq_vec[i].irq,
(INTR_TYPE_NET | INTR_MPSAFE),
NULL, ql_isr, &ha->irq_vec[i],
&ha->irq_vec[i].handle)) {
device_printf(dev, "could not setup interrupt\n");
goto qla_pci_attach_err;
}
}
printf("%s: mp__ncpus %d sds %d rds %d msi-x %d\n", __func__, mp_ncpus,
ha->hw.num_sds_rings, ha->hw.num_rds_rings, ha->msix_count);
/* initialize hardware */
if (ql_init_hw(ha)) {
device_printf(dev, "%s: ql_init_hw failed\n", __func__);
goto qla_pci_attach_err;
}
device_printf(dev, "%s: firmware[%d.%d.%d.%d]\n", __func__,
ha->fw_ver_major, ha->fw_ver_minor, ha->fw_ver_sub,
ha->fw_ver_build);
snprintf(ha->fw_ver_str, sizeof(ha->fw_ver_str), "%d.%d.%d.%d",
ha->fw_ver_major, ha->fw_ver_minor, ha->fw_ver_sub,
ha->fw_ver_build);
ql_read_mac_addr(ha);
/* allocate parent dma tag */
if (qla_alloc_parent_dma_tag(ha)) {
device_printf(dev, "%s: qla_alloc_parent_dma_tag failed\n",
__func__);
goto qla_pci_attach_err;
}
/* alloc all dma buffers */
if (ql_alloc_dma(ha)) {
device_printf(dev, "%s: ql_alloc_dma failed\n", __func__);
goto qla_pci_attach_err;
}
qla_get_peer(ha);
/* create the o.s ethernet interface */
qla_init_ifnet(dev, ha);
ha->flags.qla_watchdog_active = 1;
ha->flags.qla_watchdog_pause = 1;
TASK_INIT(&ha->tx_task, 0, qla_tx_done, ha);
ha->tx_tq = taskqueue_create_fast("qla_txq", M_NOWAIT,
taskqueue_thread_enqueue, &ha->tx_tq);
taskqueue_start_threads(&ha->tx_tq, 1, PI_NET, "%s txq",
device_get_nameunit(ha->pci_dev));
- callout_init(&ha->tx_callout, TRUE);
+ callout_init(&ha->tx_callout, 1);
ha->flags.qla_callout_init = 1;
/* create ioctl device interface */
if (ql_make_cdev(ha)) {
device_printf(dev, "%s: ql_make_cdev failed\n", __func__);
goto qla_pci_attach_err;
}
callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
qla_watchdog, ha);
TASK_INIT(&ha->err_task, 0, qla_error_recovery, ha);
ha->err_tq = taskqueue_create_fast("qla_errq", M_NOWAIT,
taskqueue_thread_enqueue, &ha->err_tq);
taskqueue_start_threads(&ha->err_tq, 1, PI_NET, "%s errq",
device_get_nameunit(ha->pci_dev));
QL_DPRINT2(ha, (dev, "%s: exit 0\n", __func__));
return (0);
qla_pci_attach_err:
qla_release(ha);
QL_DPRINT2(ha, (dev, "%s: exit ENXIO\n", __func__));
return (ENXIO);
}
/*
* Name: qla_pci_detach
* Function: Unhooks the device from the operating system
*/
static int
qla_pci_detach(device_t dev)
{
qla_host_t *ha = NULL;
struct ifnet *ifp;
QL_DPRINT2(ha, (dev, "%s: enter\n", __func__));
if ((ha = device_get_softc(dev)) == NULL) {
device_printf(dev, "cannot get softc\n");
return (ENOMEM);
}
ifp = ha->ifp;
(void)QLA_LOCK(ha, __func__, 0);
qla_stop(ha);
QLA_UNLOCK(ha, __func__);
qla_release(ha);
QL_DPRINT2(ha, (dev, "%s: exit\n", __func__));
return (0);
}
/*
* SYSCTL Related Callbacks
*/
static int
qla_sysctl_get_stats(SYSCTL_HANDLER_ARGS)
{
int err, ret = 0;
qla_host_t *ha;
err = sysctl_handle_int(oidp, &ret, 0, req);
if (err || !req->newptr)
return (err);
if (ret == 1) {
ha = (qla_host_t *)arg1;
ql_get_stats(ha);
}
return (err);
}
static int
qla_sysctl_get_link_status(SYSCTL_HANDLER_ARGS)
{
int err, ret = 0;
qla_host_t *ha;
err = sysctl_handle_int(oidp, &ret, 0, req);
if (err || !req->newptr)
return (err);
if (ret == 1) {
ha = (qla_host_t *)arg1;
ql_hw_link_status(ha);
}
return (err);
}
/*
* Name: qla_release
* Function: Releases the resources allocated for the device
*/
static void
qla_release(qla_host_t *ha)
{
device_t dev;
int i;
dev = ha->pci_dev;
if (ha->err_tq) {
taskqueue_drain(ha->err_tq, &ha->err_task);
taskqueue_free(ha->err_tq);
}
if (ha->tx_tq) {
taskqueue_drain(ha->tx_tq, &ha->tx_task);
taskqueue_free(ha->tx_tq);
}
ql_del_cdev(ha);
if (ha->flags.qla_watchdog_active) {
ha->flags.qla_watchdog_exit = 1;
while (ha->qla_watchdog_exited == 0)
qla_mdelay(__func__, 1);
}
if (ha->flags.qla_callout_init)
callout_stop(&ha->tx_callout);
if (ha->ifp != NULL)
ether_ifdetach(ha->ifp);
ql_free_dma(ha);
qla_free_parent_dma_tag(ha);
if (ha->mbx_handle)
(void)bus_teardown_intr(dev, ha->mbx_irq, ha->mbx_handle);
if (ha->mbx_irq)
(void) bus_release_resource(dev, SYS_RES_IRQ, ha->mbx_irq_rid,
ha->mbx_irq);
for (i = 0; i < ha->hw.num_sds_rings; i++) {
if (ha->irq_vec[i].handle) {
(void)bus_teardown_intr(dev, ha->irq_vec[i].irq,
ha->irq_vec[i].handle);
}
if (ha->irq_vec[i].irq) {
(void)bus_release_resource(dev, SYS_RES_IRQ,
ha->irq_vec[i].irq_rid,
ha->irq_vec[i].irq);
}
}
if (ha->msix_count)
pci_release_msi(dev);
if (ha->flags.lock_init) {
mtx_destroy(&ha->tx_lock);
mtx_destroy(&ha->hw_lock);
}
if (ha->pci_reg)
(void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid,
ha->pci_reg);
if (ha->pci_reg1)
(void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid1,
ha->pci_reg1);
}
/*
* DMA Related Functions
*/
static void
qla_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
{
*((bus_addr_t *)arg) = 0;
if (error) {
printf("%s: bus_dmamap_load failed (%d)\n", __func__, error);
return;
}
*((bus_addr_t *)arg) = segs[0].ds_addr;
return;
}
int
ql_alloc_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
{
int ret = 0;
device_t dev;
bus_addr_t b_addr;
dev = ha->pci_dev;
QL_DPRINT2(ha, (dev, "%s: enter\n", __func__));
ret = bus_dma_tag_create(
ha->parent_tag,/* parent */
dma_buf->alignment,
((bus_size_t)(1ULL << 32)),/* boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
dma_buf->size, /* maxsize */
1, /* nsegments */
dma_buf->size, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&dma_buf->dma_tag);
if (ret) {
device_printf(dev, "%s: could not create dma tag\n", __func__);
goto ql_alloc_dmabuf_exit;
}
ret = bus_dmamem_alloc(dma_buf->dma_tag,
(void **)&dma_buf->dma_b,
(BUS_DMA_ZERO | BUS_DMA_COHERENT | BUS_DMA_NOWAIT),
&dma_buf->dma_map);
if (ret) {
bus_dma_tag_destroy(dma_buf->dma_tag);
device_printf(dev, "%s: bus_dmamem_alloc failed\n", __func__);
goto ql_alloc_dmabuf_exit;
}
ret = bus_dmamap_load(dma_buf->dma_tag,
dma_buf->dma_map,
dma_buf->dma_b,
dma_buf->size,
qla_dmamap_callback,
&b_addr, BUS_DMA_NOWAIT);
if (ret || !b_addr) {
bus_dma_tag_destroy(dma_buf->dma_tag);
bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b,
dma_buf->dma_map);
ret = -1;
goto ql_alloc_dmabuf_exit;
}
dma_buf->dma_addr = b_addr;
ql_alloc_dmabuf_exit:
QL_DPRINT2(ha, (dev, "%s: exit ret 0x%08x tag %p map %p b %p sz 0x%x\n",
__func__, ret, (void *)dma_buf->dma_tag,
(void *)dma_buf->dma_map, (void *)dma_buf->dma_b,
dma_buf->size));
return ret;
}
void
ql_free_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
{
bus_dmamap_unload(dma_buf->dma_tag, dma_buf->dma_map);
bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, dma_buf->dma_map);
bus_dma_tag_destroy(dma_buf->dma_tag);
}
static int
qla_alloc_parent_dma_tag(qla_host_t *ha)
{
int ret;
device_t dev;
dev = ha->pci_dev;
/*
* Allocate parent DMA Tag
*/
ret = bus_dma_tag_create(
bus_get_dma_tag(dev), /* parent */
1,((bus_size_t)(1ULL << 32)),/* alignment, boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
0, /* nsegments */
BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&ha->parent_tag);
if (ret) {
device_printf(dev, "%s: could not create parent dma tag\n",
__func__);
return (-1);
}
ha->flags.parent_tag = 1;
return (0);
}
static void
qla_free_parent_dma_tag(qla_host_t *ha)
{
if (ha->flags.parent_tag) {
bus_dma_tag_destroy(ha->parent_tag);
ha->flags.parent_tag = 0;
}
}
/*
* Name: qla_init_ifnet
* Function: Creates the Network Device Interface and Registers it with the O.S
*/
static void
qla_init_ifnet(device_t dev, qla_host_t *ha)
{
struct ifnet *ifp;
QL_DPRINT2(ha, (dev, "%s: enter\n", __func__));
ifp = ha->ifp = if_alloc(IFT_ETHER);
if (ifp == NULL)
panic("%s: cannot if_alloc()\n", device_get_nameunit(dev));
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_baudrate = IF_Gbps(10);
ifp->if_capabilities = IFCAP_LINKSTATE;
ifp->if_init = qla_init;
ifp->if_softc = ha;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = qla_ioctl;
ifp->if_start = qla_start;
IFQ_SET_MAXLEN(&ifp->if_snd, qla_get_ifq_snd_maxlen(ha));
ifp->if_snd.ifq_drv_maxlen = qla_get_ifq_snd_maxlen(ha);
IFQ_SET_READY(&ifp->if_snd);
ha->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
ether_ifattach(ifp, qla_get_mac_addr(ha));
ifp->if_capabilities = IFCAP_HWCSUM |
IFCAP_TSO4 |
IFCAP_JUMBO_MTU;
ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
ifp->if_capenable = ifp->if_capabilities;
ifp->if_hdrlen = sizeof(struct ether_vlan_header);
ifmedia_init(&ha->media, IFM_IMASK, qla_media_change, qla_media_status);
ifmedia_add(&ha->media, (IFM_ETHER | qla_get_optics(ha) | IFM_FDX), 0,
NULL);
ifmedia_add(&ha->media, (IFM_ETHER | IFM_AUTO), 0, NULL);
ifmedia_set(&ha->media, (IFM_ETHER | IFM_AUTO));
QL_DPRINT2(ha, (dev, "%s: exit\n", __func__));
return;
}
static void
qla_init_locked(qla_host_t *ha)
{
struct ifnet *ifp = ha->ifp;
qla_stop(ha);
if (qla_alloc_xmt_bufs(ha) != 0)
return;
if (qla_alloc_rcv_bufs(ha) != 0)
return;
bcopy(IF_LLADDR(ha->ifp), ha->hw.mac_addr, ETHER_ADDR_LEN);
ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO;
ha->flags.stop_rcv = 0;
if (ql_init_hw_if(ha) == 0) {
ifp = ha->ifp;
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
ha->flags.qla_watchdog_pause = 0;
ha->hw_vlan_tx_frames = 0;
ha->tx_tso_frames = 0;
}
return;
}
static void
qla_init(void *arg)
{
qla_host_t *ha;
ha = (qla_host_t *)arg;
QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
(void)QLA_LOCK(ha, __func__, 0);
qla_init_locked(ha);
QLA_UNLOCK(ha, __func__);
QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
}
static int
qla_set_multi(qla_host_t *ha, uint32_t add_multi)
{
uint8_t mta[Q8_MAX_NUM_MULTICAST_ADDRS * Q8_MAC_ADDR_LEN];
struct ifmultiaddr *ifma;
int mcnt = 0;
struct ifnet *ifp = ha->ifp;
int ret = 0;
if_maddr_rlock(ifp);
TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
if (ifma->ifma_addr->sa_family != AF_LINK)
continue;
if (mcnt == Q8_MAX_NUM_MULTICAST_ADDRS)
break;
bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
&mta[mcnt * Q8_MAC_ADDR_LEN], Q8_MAC_ADDR_LEN);
mcnt++;
}
if_maddr_runlock(ifp);
if (QLA_LOCK(ha, __func__, 1) == 0) {
ret = ql_hw_set_multi(ha, mta, mcnt, add_multi);
QLA_UNLOCK(ha, __func__);
}
return (ret);
}
static int
qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
int ret = 0;
struct ifreq *ifr = (struct ifreq *)data;
struct ifaddr *ifa = (struct ifaddr *)data;
qla_host_t *ha;
ha = (qla_host_t *)ifp->if_softc;
switch (cmd) {
case SIOCSIFADDR:
QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFADDR (0x%lx)\n",
__func__, cmd));
if (ifa->ifa_addr->sa_family == AF_INET) {
ifp->if_flags |= IFF_UP;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
(void)QLA_LOCK(ha, __func__, 0);
qla_init_locked(ha);
QLA_UNLOCK(ha, __func__);
}
QL_DPRINT4(ha, (ha->pci_dev,
"%s: SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n",
__func__, cmd,
ntohl(IA_SIN(ifa)->sin_addr.s_addr)));
arp_ifinit(ifp, ifa);
} else {
ether_ioctl(ifp, cmd, data);
}
break;
case SIOCSIFMTU:
QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFMTU (0x%lx)\n",
__func__, cmd));
if (ifr->ifr_mtu > QLA_MAX_MTU) {
ret = EINVAL;
} else {
(void) QLA_LOCK(ha, __func__, 0);
ifp->if_mtu = ifr->ifr_mtu;
ha->max_frame_size =
ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
ret = ql_set_max_mtu(ha, ha->max_frame_size,
ha->hw.rcv_cntxt_id);
}
if (ifp->if_mtu > ETHERMTU)
ha->std_replenish = QL_JUMBO_REPLENISH_THRES;
else
ha->std_replenish = QL_STD_REPLENISH_THRES;
QLA_UNLOCK(ha, __func__);
if (ret)
ret = EINVAL;
}
break;
case SIOCSIFFLAGS:
QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFFLAGS (0x%lx)\n",
__func__, cmd));
(void)QLA_LOCK(ha, __func__, 0);
if (ifp->if_flags & IFF_UP) {
if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
if ((ifp->if_flags ^ ha->if_flags) &
IFF_PROMISC) {
ret = ql_set_promisc(ha);
} else if ((ifp->if_flags ^ ha->if_flags) &
IFF_ALLMULTI) {
ret = ql_set_allmulti(ha);
}
} else {
qla_init_locked(ha);
ha->max_frame_size = ifp->if_mtu +
ETHER_HDR_LEN + ETHER_CRC_LEN;
ret = ql_set_max_mtu(ha, ha->max_frame_size,
ha->hw.rcv_cntxt_id);
}
} else {
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
qla_stop(ha);
ha->if_flags = ifp->if_flags;
}
QLA_UNLOCK(ha, __func__);
break;
case SIOCADDMULTI:
QL_DPRINT4(ha, (ha->pci_dev,
"%s: %s (0x%lx)\n", __func__, "SIOCADDMULTI", cmd));
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
if (qla_set_multi(ha, 1))
ret = EINVAL;
}
break;
case SIOCDELMULTI:
QL_DPRINT4(ha, (ha->pci_dev,
"%s: %s (0x%lx)\n", __func__, "SIOCDELMULTI", cmd));
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
if (qla_set_multi(ha, 0))
ret = EINVAL;
}
break;
case SIOCSIFMEDIA:
case SIOCGIFMEDIA:
QL_DPRINT4(ha, (ha->pci_dev,
"%s: SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n",
__func__, cmd));
ret = ifmedia_ioctl(ifp, ifr, &ha->media, cmd);
break;
case SIOCSIFCAP:
{
int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFCAP (0x%lx)\n",
__func__, cmd));
if (mask & IFCAP_HWCSUM)
ifp->if_capenable ^= IFCAP_HWCSUM;
if (mask & IFCAP_TSO4)
ifp->if_capenable ^= IFCAP_TSO4;
if (mask & IFCAP_VLAN_HWTAGGING)
ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
if (mask & IFCAP_VLAN_HWTSO)
ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
qla_init(ha);
VLAN_CAPABILITIES(ifp);
break;
}
default:
QL_DPRINT4(ha, (ha->pci_dev, "%s: default (0x%lx)\n",
__func__, cmd));
ret = ether_ioctl(ifp, cmd, data);
break;
}
return (ret);
}
static int
qla_media_change(struct ifnet *ifp)
{
qla_host_t *ha;
struct ifmedia *ifm;
int ret = 0;
ha = (qla_host_t *)ifp->if_softc;
QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
ifm = &ha->media;
if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
ret = EINVAL;
QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
return (ret);
}
static void
qla_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
qla_host_t *ha;
ha = (qla_host_t *)ifp->if_softc;
QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
ifmr->ifm_status = IFM_AVALID;
ifmr->ifm_active = IFM_ETHER;
ql_update_link_state(ha);
if (ha->hw.link_up) {
ifmr->ifm_status |= IFM_ACTIVE;
ifmr->ifm_active |= (IFM_FDX | qla_get_optics(ha));
}
QL_DPRINT2(ha, (ha->pci_dev, "%s: exit (%s)\n", __func__,\
(ha->hw.link_up ? "link_up" : "link_down")));
return;
}
static void
qla_start(struct ifnet *ifp)
{
struct mbuf *m_head;
qla_host_t *ha = (qla_host_t *)ifp->if_softc;
QL_DPRINT8(ha, (ha->pci_dev, "%s: enter\n", __func__));
if (!mtx_trylock(&ha->tx_lock)) {
QL_DPRINT8(ha, (ha->pci_dev,
"%s: mtx_trylock(&ha->tx_lock) failed\n", __func__));
return;
}
if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
IFF_DRV_RUNNING) {
QL_DPRINT8(ha,
(ha->pci_dev, "%s: !IFF_DRV_RUNNING\n", __func__));
QLA_TX_UNLOCK(ha);
return;
}
if (!ha->watchdog_ticks)
ql_update_link_state(ha);
if (!ha->hw.link_up) {
QL_DPRINT8(ha, (ha->pci_dev, "%s: link down\n", __func__));
QLA_TX_UNLOCK(ha);
return;
}
while (ifp->if_snd.ifq_head != NULL) {
IF_DEQUEUE(&ifp->if_snd, m_head);
if (m_head == NULL) {
QL_DPRINT8(ha, (ha->pci_dev, "%s: m_head == NULL\n",
__func__));
break;
}
if (qla_send(ha, &m_head)) {
if (m_head == NULL)
break;
QL_DPRINT8(ha, (ha->pci_dev, "%s: PREPEND\n", __func__));
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IF_PREPEND(&ifp->if_snd, m_head);
break;
}
/* Send a copy of the frame to the BPF listener */
ETHER_BPF_MTAP(ifp, m_head);
}
QLA_TX_UNLOCK(ha);
QL_DPRINT8(ha, (ha->pci_dev, "%s: exit\n", __func__));
return;
}
static int
qla_send(qla_host_t *ha, struct mbuf **m_headp)
{
bus_dma_segment_t segs[QLA_MAX_SEGMENTS];
bus_dmamap_t map;
int nsegs;
int ret = -1;
uint32_t tx_idx;
struct mbuf *m_head = *m_headp;
uint32_t txr_idx = ha->txr_idx;
QL_DPRINT8(ha, (ha->pci_dev, "%s: enter\n", __func__));
/* check if flowid is set */
if (M_HASHTYPE_GET(m_head) != M_HASHTYPE_NONE)
txr_idx = m_head->m_pkthdr.flowid & (ha->hw.num_tx_rings - 1);
tx_idx = ha->hw.tx_cntxt[txr_idx].txr_next;
map = ha->tx_ring[txr_idx].tx_buf[tx_idx].map;
ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs,
BUS_DMA_NOWAIT);
if (ret == EFBIG) {
struct mbuf *m;
QL_DPRINT8(ha, (ha->pci_dev, "%s: EFBIG [%d]\n", __func__,
m_head->m_pkthdr.len));
m = m_defrag(m_head, M_NOWAIT);
if (m == NULL) {
ha->err_tx_defrag++;
m_freem(m_head);
*m_headp = NULL;
device_printf(ha->pci_dev,
"%s: m_defrag() = NULL [%d]\n",
__func__, ret);
return (ENOBUFS);
}
m_head = m;
*m_headp = m_head;
if ((ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head,
segs, &nsegs, BUS_DMA_NOWAIT))) {
ha->err_tx_dmamap_load++;
device_printf(ha->pci_dev,
"%s: bus_dmamap_load_mbuf_sg failed0[%d, %d]\n",
__func__, ret, m_head->m_pkthdr.len);
if (ret != ENOMEM) {
m_freem(m_head);
*m_headp = NULL;
}
return (ret);
}
} else if (ret) {
ha->err_tx_dmamap_load++;
device_printf(ha->pci_dev,
"%s: bus_dmamap_load_mbuf_sg failed1[%d, %d]\n",
__func__, ret, m_head->m_pkthdr.len);
if (ret != ENOMEM) {
m_freem(m_head);
*m_headp = NULL;
}
return (ret);
}
QL_ASSERT(ha, (nsegs != 0), ("qla_send: empty packet"));
bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_PREWRITE);
if (!(ret = ql_hw_send(ha, segs, nsegs, tx_idx, m_head, txr_idx))) {
ha->tx_ring[txr_idx].count++;
ha->tx_ring[txr_idx].tx_buf[tx_idx].m_head = m_head;
} else {
if (ret == EINVAL) {
if (m_head)
m_freem(m_head);
*m_headp = NULL;
}
}
QL_DPRINT8(ha, (ha->pci_dev, "%s: exit\n", __func__));
return (ret);
}
static void
qla_stop(qla_host_t *ha)
{
struct ifnet *ifp = ha->ifp;
device_t dev;
dev = ha->pci_dev;
ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
ha->flags.qla_watchdog_pause = 1;
while (!ha->qla_watchdog_paused)
qla_mdelay(__func__, 1);
ha->flags.stop_rcv = 1;
ql_hw_stop_rcv(ha);
ql_del_hw_if(ha);
qla_free_xmt_bufs(ha);
qla_free_rcv_bufs(ha);
return;
}
/*
* Buffer Management Functions for Transmit and Receive Rings
*/
static int
qla_alloc_xmt_bufs(qla_host_t *ha)
{
int ret = 0;
uint32_t i, j;
qla_tx_buf_t *txb;
if (bus_dma_tag_create(NULL, /* parent */
1, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
QLA_MAX_TSO_FRAME_SIZE, /* maxsize */
QLA_MAX_SEGMENTS, /* nsegments */
PAGE_SIZE, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, /* lockfunc */
NULL, /* lockfuncarg */
&ha->tx_tag)) {
device_printf(ha->pci_dev, "%s: tx_tag alloc failed\n",
__func__);
return (ENOMEM);
}
for (i = 0; i < ha->hw.num_tx_rings; i++) {
bzero((void *)ha->tx_ring[i].tx_buf,
(sizeof(qla_tx_buf_t) * NUM_TX_DESCRIPTORS));
}
for (j = 0; j < ha->hw.num_tx_rings; j++) {
for (i = 0; i < NUM_TX_DESCRIPTORS; i++) {
txb = &ha->tx_ring[j].tx_buf[i];
if ((ret = bus_dmamap_create(ha->tx_tag,
BUS_DMA_NOWAIT, &txb->map))) {
ha->err_tx_dmamap_create++;
device_printf(ha->pci_dev,
"%s: bus_dmamap_create failed[%d]\n",
__func__, ret);
qla_free_xmt_bufs(ha);
return (ret);
}
}
}
return 0;
}
/*
* Release mbuf after it sent on the wire
*/
static void
qla_clear_tx_buf(qla_host_t *ha, qla_tx_buf_t *txb)
{
QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
if (txb->m_head && txb->map) {
bus_dmamap_unload(ha->tx_tag, txb->map);
m_freem(txb->m_head);
txb->m_head = NULL;
}
if (txb->map)
bus_dmamap_destroy(ha->tx_tag, txb->map);
QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
}
static void
qla_free_xmt_bufs(qla_host_t *ha)
{
int i, j;
for (j = 0; j < ha->hw.num_tx_rings; j++) {
for (i = 0; i < NUM_TX_DESCRIPTORS; i++)
qla_clear_tx_buf(ha, &ha->tx_ring[j].tx_buf[i]);
}
if (ha->tx_tag != NULL) {
bus_dma_tag_destroy(ha->tx_tag);
ha->tx_tag = NULL;
}
for (i = 0; i < ha->hw.num_tx_rings; i++) {
bzero((void *)ha->tx_ring[i].tx_buf,
(sizeof(qla_tx_buf_t) * NUM_TX_DESCRIPTORS));
}
return;
}
static int
qla_alloc_rcv_std(qla_host_t *ha)
{
int i, j, k, r, ret = 0;
qla_rx_buf_t *rxb;
qla_rx_ring_t *rx_ring;
for (r = 0; r < ha->hw.num_rds_rings; r++) {
rx_ring = &ha->rx_ring[r];
for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
rxb = &rx_ring->rx_buf[i];
ret = bus_dmamap_create(ha->rx_tag, BUS_DMA_NOWAIT,
&rxb->map);
if (ret) {
device_printf(ha->pci_dev,
"%s: dmamap[%d, %d] failed\n",
__func__, r, i);
for (k = 0; k < r; k++) {
for (j = 0; j < NUM_RX_DESCRIPTORS;
j++) {
rxb = &ha->rx_ring[k].rx_buf[j];
bus_dmamap_destroy(ha->rx_tag,
rxb->map);
}
}
for (j = 0; j < i; j++) {
bus_dmamap_destroy(ha->rx_tag,
rx_ring->rx_buf[j].map);
}
goto qla_alloc_rcv_std_err;
}
}
}
qla_init_hw_rcv_descriptors(ha);
for (r = 0; r < ha->hw.num_rds_rings; r++) {
rx_ring = &ha->rx_ring[r];
for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
rxb = &rx_ring->rx_buf[i];
rxb->handle = i;
if (!(ret = ql_get_mbuf(ha, rxb, NULL))) {
/*
* set the physical address in the
* corresponding descriptor entry in the
* receive ring/queue for the hba
*/
qla_set_hw_rcv_desc(ha, r, i, rxb->handle,
rxb->paddr,
(rxb->m_head)->m_pkthdr.len);
} else {
device_printf(ha->pci_dev,
"%s: ql_get_mbuf [%d, %d] failed\n",
__func__, r, i);
bus_dmamap_destroy(ha->rx_tag, rxb->map);
goto qla_alloc_rcv_std_err;
}
}
}
return 0;
qla_alloc_rcv_std_err:
return (-1);
}
static void
qla_free_rcv_std(qla_host_t *ha)
{
int i, r;
qla_rx_buf_t *rxb;
for (r = 0; r < ha->hw.num_rds_rings; r++) {
for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
rxb = &ha->rx_ring[r].rx_buf[i];
if (rxb->m_head != NULL) {
bus_dmamap_unload(ha->rx_tag, rxb->map);
bus_dmamap_destroy(ha->rx_tag, rxb->map);
m_freem(rxb->m_head);
rxb->m_head = NULL;
}
}
}
return;
}
static int
qla_alloc_rcv_bufs(qla_host_t *ha)
{
int i, ret = 0;
if (bus_dma_tag_create(NULL, /* parent */
1, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
MJUM9BYTES, /* maxsize */
1, /* nsegments */
MJUM9BYTES, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, /* lockfunc */
NULL, /* lockfuncarg */
&ha->rx_tag)) {
device_printf(ha->pci_dev, "%s: rx_tag alloc failed\n",
__func__);
return (ENOMEM);
}
bzero((void *)ha->rx_ring, (sizeof(qla_rx_ring_t) * MAX_RDS_RINGS));
for (i = 0; i < ha->hw.num_sds_rings; i++) {
ha->hw.sds[i].sdsr_next = 0;
ha->hw.sds[i].rxb_free = NULL;
ha->hw.sds[i].rx_free = 0;
}
ret = qla_alloc_rcv_std(ha);
return (ret);
}
static void
qla_free_rcv_bufs(qla_host_t *ha)
{
int i;
qla_free_rcv_std(ha);
if (ha->rx_tag != NULL) {
bus_dma_tag_destroy(ha->rx_tag);
ha->rx_tag = NULL;
}
bzero((void *)ha->rx_ring, (sizeof(qla_rx_ring_t) * MAX_RDS_RINGS));
for (i = 0; i < ha->hw.num_sds_rings; i++) {
ha->hw.sds[i].sdsr_next = 0;
ha->hw.sds[i].rxb_free = NULL;
ha->hw.sds[i].rx_free = 0;
}
return;
}
int
ql_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp)
{
register struct mbuf *mp = nmp;
struct ifnet *ifp;
int ret = 0;
uint32_t offset;
bus_dma_segment_t segs[1];
int nsegs;
QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
ifp = ha->ifp;
if (mp == NULL) {
mp = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
if (mp == NULL) {
ha->err_m_getcl++;
ret = ENOBUFS;
device_printf(ha->pci_dev,
"%s: m_getcl failed\n", __func__);
goto exit_ql_get_mbuf;
}
mp->m_len = mp->m_pkthdr.len = MCLBYTES;
} else {
mp->m_len = mp->m_pkthdr.len = MCLBYTES;
mp->m_data = mp->m_ext.ext_buf;
mp->m_next = NULL;
}
offset = (uint32_t)((unsigned long long)mp->m_data & 0x7ULL);
if (offset) {
offset = 8 - offset;
m_adj(mp, offset);
}
/*
* Using memory from the mbuf cluster pool, invoke the bus_dma
* machinery to arrange the memory mapping.
*/
ret = bus_dmamap_load_mbuf_sg(ha->rx_tag, rxb->map,
mp, segs, &nsegs, BUS_DMA_NOWAIT);
rxb->paddr = segs[0].ds_addr;
if (ret || !rxb->paddr || (nsegs != 1)) {
m_free(mp);
rxb->m_head = NULL;
device_printf(ha->pci_dev,
"%s: bus_dmamap_load failed[%d, 0x%016llx, %d]\n",
__func__, ret, (long long unsigned int)rxb->paddr,
nsegs);
ret = -1;
goto exit_ql_get_mbuf;
}
rxb->m_head = mp;
bus_dmamap_sync(ha->rx_tag, rxb->map, BUS_DMASYNC_PREREAD);
exit_ql_get_mbuf:
QL_DPRINT2(ha, (ha->pci_dev, "%s: exit ret = 0x%08x\n", __func__, ret));
return (ret);
}
static void
qla_tx_done(void *context, int pending)
{
qla_host_t *ha = context;
struct ifnet *ifp;
ifp = ha->ifp;
if (!ifp)
return;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
QL_DPRINT8(ha, (ha->pci_dev, "%s: !IFF_DRV_RUNNING\n", __func__));
return;
}
ql_hw_tx_done(ha);
qla_start(ha->ifp);
}
static void
qla_get_peer(qla_host_t *ha)
{
device_t *peers;
int count, i, slot;
int my_slot = pci_get_slot(ha->pci_dev);
if (device_get_children(device_get_parent(ha->pci_dev), &peers, &count))
return;
for (i = 0; i < count; i++) {
slot = pci_get_slot(peers[i]);
if ((slot >= 0) && (slot == my_slot) &&
(pci_get_device(peers[i]) ==
pci_get_device(ha->pci_dev))) {
if (ha->pci_dev != peers[i])
ha->peer_dev = peers[i];
}
}
}
static void
qla_send_msg_to_peer(qla_host_t *ha, uint32_t msg_to_peer)
{
qla_host_t *ha_peer;
if (ha->peer_dev) {
if ((ha_peer = device_get_softc(ha->peer_dev)) != NULL) {
ha_peer->msg_from_peer = msg_to_peer;
}
}
}
static void
qla_error_recovery(void *context, int pending)
{
qla_host_t *ha = context;
uint32_t msecs_100 = 100;
struct ifnet *ifp = ha->ifp;
(void)QLA_LOCK(ha, __func__, 0);
ha->flags.stop_rcv = 1;
ql_hw_stop_rcv(ha);
ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
QLA_UNLOCK(ha, __func__);
if ((ha->pci_func & 0x1) == 0) {
if (!ha->msg_from_peer) {
qla_send_msg_to_peer(ha, QL_PEER_MSG_RESET);
while ((ha->msg_from_peer != QL_PEER_MSG_ACK) &&
msecs_100--)
qla_mdelay(__func__, 100);
}
ha->msg_from_peer = 0;
ql_minidump(ha);
(void) ql_init_hw(ha);
qla_free_xmt_bufs(ha);
qla_free_rcv_bufs(ha);
qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK);
} else {
if (ha->msg_from_peer == QL_PEER_MSG_RESET) {
ha->msg_from_peer = 0;
qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK);
} else {
qla_send_msg_to_peer(ha, QL_PEER_MSG_RESET);
}
while ((ha->msg_from_peer != QL_PEER_MSG_ACK) && msecs_100--)
qla_mdelay(__func__, 100);
ha->msg_from_peer = 0;
(void) ql_init_hw(ha);
qla_free_xmt_bufs(ha);
qla_free_rcv_bufs(ha);
}
(void)QLA_LOCK(ha, __func__, 0);
if (qla_alloc_xmt_bufs(ha) != 0) {
QLA_UNLOCK(ha, __func__);
return;
}
if (qla_alloc_rcv_bufs(ha) != 0) {
QLA_UNLOCK(ha, __func__);
return;
}
ha->flags.stop_rcv = 0;
if (ql_init_hw_if(ha) == 0) {
ifp = ha->ifp;
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
ha->flags.qla_watchdog_pause = 0;
}
QLA_UNLOCK(ha, __func__);
}
Index: head/sys/dev/qlxge/qls_os.c
===================================================================
--- head/sys/dev/qlxge/qls_os.c (revision 283290)
+++ head/sys/dev/qlxge/qls_os.c (revision 283291)
@@ -1,1532 +1,1532 @@
/*
* Copyright (c) 2013-2014 Qlogic Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* and ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* File: qls_os.c
* Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "qls_os.h"
#include "qls_hw.h"
#include "qls_def.h"
#include "qls_inline.h"
#include "qls_ver.h"
#include "qls_glbl.h"
#include "qls_dbg.h"
#include <sys/smp.h>
/*
* Some PCI Configuration Space Related Defines
*/
#ifndef PCI_VENDOR_QLOGIC
#define PCI_VENDOR_QLOGIC 0x1077
#endif
#ifndef PCI_DEVICE_QLOGIC_8000
#define PCI_DEVICE_QLOGIC_8000 0x8000
#endif
#define PCI_QLOGIC_DEV8000 \
((PCI_DEVICE_QLOGIC_8000 << 16) | PCI_VENDOR_QLOGIC)
/*
* static functions
*/
static int qls_alloc_parent_dma_tag(qla_host_t *ha);
static void qls_free_parent_dma_tag(qla_host_t *ha);
static void qls_flush_xmt_bufs(qla_host_t *ha);
static int qls_alloc_rcv_bufs(qla_host_t *ha);
static void qls_free_rcv_bufs(qla_host_t *ha);
static void qls_init_ifnet(device_t dev, qla_host_t *ha);
static void qls_release(qla_host_t *ha);
static void qls_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
int error);
static void qls_stop(qla_host_t *ha);
static int qls_send(qla_host_t *ha, struct mbuf **m_headp);
static void qls_tx_done(void *context, int pending);
static int qls_config_lro(qla_host_t *ha);
static void qls_free_lro(qla_host_t *ha);
static void qls_error_recovery(void *context, int pending);
/*
* Hooks to the Operating Systems
*/
static int qls_pci_probe (device_t);
static int qls_pci_attach (device_t);
static int qls_pci_detach (device_t);
static void qls_start(struct ifnet *ifp);
static void qls_init(void *arg);
static int qls_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
static int qls_media_change(struct ifnet *ifp);
static void qls_media_status(struct ifnet *ifp, struct ifmediareq *ifmr);
static device_method_t qla_pci_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, qls_pci_probe),
DEVMETHOD(device_attach, qls_pci_attach),
DEVMETHOD(device_detach, qls_pci_detach),
{ 0, 0 }
};
static driver_t qla_pci_driver = {
"ql", qla_pci_methods, sizeof (qla_host_t),
};
static devclass_t qla8000_devclass;
DRIVER_MODULE(qla8000, pci, qla_pci_driver, qla8000_devclass, 0, 0);
MODULE_DEPEND(qla8000, pci, 1, 1, 1);
MODULE_DEPEND(qla8000, ether, 1, 1, 1);
MALLOC_DEFINE(M_QLA8000BUF, "qla8000buf", "Buffers for qla8000 driver");
static char dev_str[64];
static char ver_str[64];
/*
* Name: qls_pci_probe
* Function: Validate the PCI device to be a QLA80XX device
*/
static int
qls_pci_probe(device_t dev)
{
switch ((pci_get_device(dev) << 16) | (pci_get_vendor(dev))) {
case PCI_QLOGIC_DEV8000:
snprintf(dev_str, sizeof(dev_str), "%s v%d.%d.%d",
"Qlogic ISP 8000 PCI CNA Adapter-Ethernet Function",
QLA_VERSION_MAJOR, QLA_VERSION_MINOR,
QLA_VERSION_BUILD);
snprintf(ver_str, sizeof(ver_str), "v%d.%d.%d",
QLA_VERSION_MAJOR, QLA_VERSION_MINOR,
QLA_VERSION_BUILD);
device_set_desc(dev, dev_str);
break;
default:
return (ENXIO);
}
if (bootverbose)
printf("%s: %s\n ", __func__, dev_str);
return (BUS_PROBE_DEFAULT);
}
static int
qls_sysctl_get_drvr_stats(SYSCTL_HANDLER_ARGS)
{
int err = 0, ret;
qla_host_t *ha;
uint32_t i;
err = sysctl_handle_int(oidp, &ret, 0, req);
if (err || !req->newptr)
return (err);
if (ret == 1) {
ha = (qla_host_t *)arg1;
for (i = 0; i < ha->num_tx_rings; i++) {
device_printf(ha->pci_dev,
"%s: tx_ring[%d].tx_frames= %p\n",
__func__, i,
(void *)ha->tx_ring[i].tx_frames);
device_printf(ha->pci_dev,
"%s: tx_ring[%d].tx_tso_frames= %p\n",
__func__, i,
(void *)ha->tx_ring[i].tx_tso_frames);
device_printf(ha->pci_dev,
"%s: tx_ring[%d].tx_vlan_frames= %p\n",
__func__, i,
(void *)ha->tx_ring[i].tx_vlan_frames);
device_printf(ha->pci_dev,
"%s: tx_ring[%d].txr_free= 0x%08x\n",
__func__, i,
ha->tx_ring[i].txr_free);
device_printf(ha->pci_dev,
"%s: tx_ring[%d].txr_next= 0x%08x\n",
__func__, i,
ha->tx_ring[i].txr_next);
device_printf(ha->pci_dev,
"%s: tx_ring[%d].txr_done= 0x%08x\n",
__func__, i,
ha->tx_ring[i].txr_done);
device_printf(ha->pci_dev,
"%s: tx_ring[%d].txr_cons_idx= 0x%08x\n",
__func__, i,
*(ha->tx_ring[i].txr_cons_vaddr));
}
for (i = 0; i < ha->num_rx_rings; i++) {
device_printf(ha->pci_dev,
"%s: rx_ring[%d].rx_int= %p\n",
__func__, i,
(void *)ha->rx_ring[i].rx_int);
device_printf(ha->pci_dev,
"%s: rx_ring[%d].rss_int= %p\n",
__func__, i,
(void *)ha->rx_ring[i].rss_int);
device_printf(ha->pci_dev,
"%s: rx_ring[%d].lbq_next= 0x%08x\n",
__func__, i,
ha->rx_ring[i].lbq_next);
device_printf(ha->pci_dev,
"%s: rx_ring[%d].lbq_free= 0x%08x\n",
__func__, i,
ha->rx_ring[i].lbq_free);
device_printf(ha->pci_dev,
"%s: rx_ring[%d].lbq_in= 0x%08x\n",
__func__, i,
ha->rx_ring[i].lbq_in);
device_printf(ha->pci_dev,
"%s: rx_ring[%d].sbq_next= 0x%08x\n",
__func__, i,
ha->rx_ring[i].sbq_next);
device_printf(ha->pci_dev,
"%s: rx_ring[%d].sbq_free= 0x%08x\n",
__func__, i,
ha->rx_ring[i].sbq_free);
device_printf(ha->pci_dev,
"%s: rx_ring[%d].sbq_in= 0x%08x\n",
__func__, i,
ha->rx_ring[i].sbq_in);
}
device_printf(ha->pci_dev, "%s: err_m_getcl = 0x%08x\n",
__func__, ha->err_m_getcl);
device_printf(ha->pci_dev, "%s: err_m_getjcl = 0x%08x\n",
__func__, ha->err_m_getjcl);
device_printf(ha->pci_dev,
"%s: err_tx_dmamap_create = 0x%08x\n",
__func__, ha->err_tx_dmamap_create);
device_printf(ha->pci_dev,
"%s: err_tx_dmamap_load = 0x%08x\n",
__func__, ha->err_tx_dmamap_load);
device_printf(ha->pci_dev,
"%s: err_tx_defrag = 0x%08x\n",
__func__, ha->err_tx_defrag);
}
return (err);
}
static void
qls_add_sysctls(qla_host_t *ha)
{
device_t dev = ha->pci_dev;
SYSCTL_ADD_STRING(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "version", CTLFLAG_RD,
ver_str, 0, "Driver Version");
qls_dbg_level = 0;
SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "debug", CTLFLAG_RW,
&qls_dbg_level, qls_dbg_level, "Debug Level");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "drvr_stats", CTLTYPE_INT | CTLFLAG_RW,
(void *)ha, 0,
qls_sysctl_get_drvr_stats, "I", "Driver Maintained Statistics");
return;
}
static void
qls_watchdog(void *arg)
{
qla_host_t *ha = arg;
struct ifnet *ifp;
ifp = ha->ifp;
if (ha->flags.qla_watchdog_exit) {
ha->qla_watchdog_exited = 1;
return;
}
ha->qla_watchdog_exited = 0;
if (!ha->flags.qla_watchdog_pause) {
if (ha->qla_initiate_recovery) {
ha->qla_watchdog_paused = 1;
ha->qla_initiate_recovery = 0;
ha->err_inject = 0;
taskqueue_enqueue(ha->err_tq, &ha->err_task);
} else if ((ifp->if_snd.ifq_head != NULL) && QL_RUNNING(ifp)) {
taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
}
ha->qla_watchdog_paused = 0;
} else {
ha->qla_watchdog_paused = 1;
}
ha->watchdog_ticks = ha->watchdog_ticks++ % 1000;
callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
qls_watchdog, ha);
return;
}
/*
* Name: qls_pci_attach
* Function: attaches the device to the operating system
*/
static int
qls_pci_attach(device_t dev)
{
qla_host_t *ha = NULL;
int i;
QL_DPRINT2((dev, "%s: enter\n", __func__));
if ((ha = device_get_softc(dev)) == NULL) {
device_printf(dev, "cannot get softc\n");
return (ENOMEM);
}
memset(ha, 0, sizeof (qla_host_t));
if (pci_get_device(dev) != PCI_DEVICE_QLOGIC_8000) {
device_printf(dev, "device is not QLE8000\n");
return (ENXIO);
}
ha->pci_func = pci_get_function(dev);
ha->pci_dev = dev;
pci_enable_busmaster(dev);
ha->reg_rid = PCIR_BAR(1);
ha->pci_reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->reg_rid,
RF_ACTIVE);
if (ha->pci_reg == NULL) {
device_printf(dev, "unable to map any ports\n");
goto qls_pci_attach_err;
}
ha->reg_rid1 = PCIR_BAR(3);
ha->pci_reg1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
&ha->reg_rid1, RF_ACTIVE);
if (ha->pci_reg1 == NULL) {
device_printf(dev, "unable to map any ports\n");
goto qls_pci_attach_err;
}
mtx_init(&ha->hw_lock, "qla80xx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF);
mtx_init(&ha->tx_lock, "qla80xx_tx_lock", MTX_NETWORK_LOCK, MTX_DEF);
qls_add_sysctls(ha);
qls_hw_add_sysctls(ha);
ha->flags.lock_init = 1;
ha->msix_count = pci_msix_count(dev);
if (ha->msix_count < qls_get_msix_count(ha)) {
device_printf(dev, "%s: msix_count[%d] not enough\n", __func__,
ha->msix_count);
goto qls_pci_attach_err;
}
ha->msix_count = qls_get_msix_count(ha);
device_printf(dev, "\n%s: ha %p pci_func 0x%x msix_count 0x%x"
" pci_reg %p pci_reg1 %p\n", __func__, ha,
ha->pci_func, ha->msix_count, ha->pci_reg, ha->pci_reg1);
if (pci_alloc_msix(dev, &ha->msix_count)) {
device_printf(dev, "%s: pci_alloc_msi[%d] failed\n", __func__,
ha->msix_count);
ha->msix_count = 0;
goto qls_pci_attach_err;
}
for (i = 0; i < ha->num_rx_rings; i++) {
ha->irq_vec[i].cq_idx = i;
ha->irq_vec[i].ha = ha;
ha->irq_vec[i].irq_rid = 1 + i;
ha->irq_vec[i].irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
&ha->irq_vec[i].irq_rid,
(RF_ACTIVE | RF_SHAREABLE));
if (ha->irq_vec[i].irq == NULL) {
device_printf(dev, "could not allocate interrupt\n");
goto qls_pci_attach_err;
}
if (bus_setup_intr(dev, ha->irq_vec[i].irq,
(INTR_TYPE_NET | INTR_MPSAFE), NULL, qls_isr,
&ha->irq_vec[i], &ha->irq_vec[i].handle)) {
device_printf(dev,
"could not setup interrupt\n");
goto qls_pci_attach_err;
}
}
qls_rd_nic_params(ha);
/* allocate parent dma tag */
if (qls_alloc_parent_dma_tag(ha)) {
device_printf(dev, "%s: qls_alloc_parent_dma_tag failed\n",
__func__);
goto qls_pci_attach_err;
}
/* alloc all dma buffers */
if (qls_alloc_dma(ha)) {
device_printf(dev, "%s: qls_alloc_dma failed\n", __func__);
goto qls_pci_attach_err;
}
/* create the o.s ethernet interface */
qls_init_ifnet(dev, ha);
ha->flags.qla_watchdog_active = 1;
ha->flags.qla_watchdog_pause = 1;
TASK_INIT(&ha->tx_task, 0, qls_tx_done, ha);
ha->tx_tq = taskqueue_create_fast("qla_txq", M_NOWAIT,
taskqueue_thread_enqueue, &ha->tx_tq);
taskqueue_start_threads(&ha->tx_tq, 1, PI_NET, "%s txq",
device_get_nameunit(ha->pci_dev));
- callout_init(&ha->tx_callout, TRUE);
+ callout_init(&ha->tx_callout, 1);
ha->flags.qla_callout_init = 1;
/* create ioctl device interface */
if (qls_make_cdev(ha)) {
device_printf(dev, "%s: qls_make_cdev failed\n", __func__);
goto qls_pci_attach_err;
}
callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
qls_watchdog, ha);
TASK_INIT(&ha->err_task, 0, qls_error_recovery, ha);
ha->err_tq = taskqueue_create_fast("qla_errq", M_NOWAIT,
taskqueue_thread_enqueue, &ha->err_tq);
taskqueue_start_threads(&ha->err_tq, 1, PI_NET, "%s errq",
device_get_nameunit(ha->pci_dev));
QL_DPRINT2((dev, "%s: exit 0\n", __func__));
return (0);
qls_pci_attach_err:
qls_release(ha);
QL_DPRINT2((dev, "%s: exit ENXIO\n", __func__));
return (ENXIO);
}
/*
* Name: qls_pci_detach
* Function: Unhooks the device from the operating system
*/
static int
qls_pci_detach(device_t dev)
{
qla_host_t *ha = NULL;
struct ifnet *ifp;
QL_DPRINT2((dev, "%s: enter\n", __func__));
if ((ha = device_get_softc(dev)) == NULL) {
device_printf(dev, "cannot get softc\n");
return (ENOMEM);
}
ifp = ha->ifp;
(void)QLA_LOCK(ha, __func__, 0);
qls_stop(ha);
QLA_UNLOCK(ha, __func__);
qls_release(ha);
QL_DPRINT2((dev, "%s: exit\n", __func__));
return (0);
}
/*
* Name: qls_release
* Function: Releases the resources allocated for the device
*/
static void
qls_release(qla_host_t *ha)
{
device_t dev;
int i;
dev = ha->pci_dev;
if (ha->err_tq) {
taskqueue_drain(ha->err_tq, &ha->err_task);
taskqueue_free(ha->err_tq);
}
if (ha->tx_tq) {
taskqueue_drain(ha->tx_tq, &ha->tx_task);
taskqueue_free(ha->tx_tq);
}
qls_del_cdev(ha);
if (ha->flags.qla_watchdog_active) {
ha->flags.qla_watchdog_exit = 1;
while (ha->qla_watchdog_exited == 0)
qls_mdelay(__func__, 1);
}
if (ha->flags.qla_callout_init)
callout_stop(&ha->tx_callout);
if (ha->ifp != NULL)
ether_ifdetach(ha->ifp);
qls_free_dma(ha);
qls_free_parent_dma_tag(ha);
for (i = 0; i < ha->num_rx_rings; i++) {
if (ha->irq_vec[i].handle) {
(void)bus_teardown_intr(dev, ha->irq_vec[i].irq,
ha->irq_vec[i].handle);
}
if (ha->irq_vec[i].irq) {
(void)bus_release_resource(dev, SYS_RES_IRQ,
ha->irq_vec[i].irq_rid,
ha->irq_vec[i].irq);
}
}
if (ha->msix_count)
pci_release_msi(dev);
if (ha->flags.lock_init) {
mtx_destroy(&ha->tx_lock);
mtx_destroy(&ha->hw_lock);
}
if (ha->pci_reg)
(void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid,
ha->pci_reg);
if (ha->pci_reg1)
(void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid1,
ha->pci_reg1);
}
/*
* DMA Related Functions
*/
static void
qls_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
{
*((bus_addr_t *)arg) = 0;
if (error) {
printf("%s: bus_dmamap_load failed (%d)\n", __func__, error);
return;
}
*((bus_addr_t *)arg) = segs[0].ds_addr;
return;
}
int
qls_alloc_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
{
int ret = 0;
device_t dev;
bus_addr_t b_addr;
dev = ha->pci_dev;
QL_DPRINT2((dev, "%s: enter\n", __func__));
ret = bus_dma_tag_create(
ha->parent_tag,/* parent */
dma_buf->alignment,
((bus_size_t)(1ULL << 32)),/* boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
dma_buf->size, /* maxsize */
1, /* nsegments */
dma_buf->size, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&dma_buf->dma_tag);
if (ret) {
device_printf(dev, "%s: could not create dma tag\n", __func__);
goto qls_alloc_dmabuf_exit;
}
ret = bus_dmamem_alloc(dma_buf->dma_tag,
(void **)&dma_buf->dma_b,
(BUS_DMA_ZERO | BUS_DMA_COHERENT | BUS_DMA_NOWAIT),
&dma_buf->dma_map);
if (ret) {
bus_dma_tag_destroy(dma_buf->dma_tag);
device_printf(dev, "%s: bus_dmamem_alloc failed\n", __func__);
goto qls_alloc_dmabuf_exit;
}
ret = bus_dmamap_load(dma_buf->dma_tag,
dma_buf->dma_map,
dma_buf->dma_b,
dma_buf->size,
qls_dmamap_callback,
&b_addr, BUS_DMA_NOWAIT);
if (ret || !b_addr) {
bus_dma_tag_destroy(dma_buf->dma_tag);
bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b,
dma_buf->dma_map);
ret = -1;
goto qls_alloc_dmabuf_exit;
}
dma_buf->dma_addr = b_addr;
qls_alloc_dmabuf_exit:
QL_DPRINT2((dev, "%s: exit ret 0x%08x tag %p map %p b %p sz 0x%x\n",
__func__, ret, (void *)dma_buf->dma_tag,
(void *)dma_buf->dma_map, (void *)dma_buf->dma_b,
dma_buf->size));
return ret;
}
void
qls_free_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
{
bus_dmamap_unload(dma_buf->dma_tag, dma_buf->dma_map);
bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, dma_buf->dma_map);
bus_dma_tag_destroy(dma_buf->dma_tag);
}
static int
qls_alloc_parent_dma_tag(qla_host_t *ha)
{
int ret;
device_t dev;
dev = ha->pci_dev;
/*
* Allocate parent DMA Tag
*/
ret = bus_dma_tag_create(
bus_get_dma_tag(dev), /* parent */
1,((bus_size_t)(1ULL << 32)),/* alignment, boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
0, /* nsegments */
BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&ha->parent_tag);
if (ret) {
device_printf(dev, "%s: could not create parent dma tag\n",
__func__);
return (-1);
}
ha->flags.parent_tag = 1;
return (0);
}
static void
qls_free_parent_dma_tag(qla_host_t *ha)
{
if (ha->flags.parent_tag) {
bus_dma_tag_destroy(ha->parent_tag);
ha->flags.parent_tag = 0;
}
}
/*
* Name: qls_init_ifnet
* Function: Creates the Network Device Interface and Registers it with the O.S
*/
static void
qls_init_ifnet(device_t dev, qla_host_t *ha)
{
struct ifnet *ifp;
QL_DPRINT2((dev, "%s: enter\n", __func__));
ifp = ha->ifp = if_alloc(IFT_ETHER);
if (ifp == NULL)
panic("%s: cannot if_alloc()\n", device_get_nameunit(dev));
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_baudrate = IF_Gbps(10);
ifp->if_init = qls_init;
ifp->if_softc = ha;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = qls_ioctl;
ifp->if_start = qls_start;
IFQ_SET_MAXLEN(&ifp->if_snd, qls_get_ifq_snd_maxlen(ha));
ifp->if_snd.ifq_drv_maxlen = qls_get_ifq_snd_maxlen(ha);
IFQ_SET_READY(&ifp->if_snd);
ha->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
if (ha->max_frame_size <= MCLBYTES) {
ha->msize = MCLBYTES;
} else if (ha->max_frame_size <= MJUMPAGESIZE) {
ha->msize = MJUMPAGESIZE;
} else
ha->msize = MJUM9BYTES;
ether_ifattach(ifp, qls_get_mac_addr(ha));
ifp->if_capabilities = IFCAP_JUMBO_MTU;
ifp->if_capabilities |= IFCAP_HWCSUM;
ifp->if_capabilities |= IFCAP_VLAN_MTU;
ifp->if_capabilities |= IFCAP_TSO4;
ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING;
ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
ifp->if_capabilities |= IFCAP_LINKSTATE;
ifp->if_capenable = ifp->if_capabilities;
ifp->if_hdrlen = sizeof(struct ether_vlan_header);
ifmedia_init(&ha->media, IFM_IMASK, qls_media_change, qls_media_status);
ifmedia_add(&ha->media, (IFM_ETHER | qls_get_optics(ha) | IFM_FDX), 0,
NULL);
ifmedia_add(&ha->media, (IFM_ETHER | IFM_AUTO), 0, NULL);
ifmedia_set(&ha->media, (IFM_ETHER | IFM_AUTO));
QL_DPRINT2((dev, "%s: exit\n", __func__));
return;
}
static void
qls_init_locked(qla_host_t *ha)
{
struct ifnet *ifp = ha->ifp;
qls_stop(ha);
qls_flush_xmt_bufs(ha);
if (qls_alloc_rcv_bufs(ha) != 0)
return;
if (qls_config_lro(ha))
return;
bcopy(IF_LLADDR(ha->ifp), ha->mac_addr, ETHER_ADDR_LEN);
ifp->if_hwassist = CSUM_IP;
ifp->if_hwassist |= CSUM_TCP;
ifp->if_hwassist |= CSUM_UDP;
ifp->if_hwassist |= CSUM_TSO;
if (qls_init_hw_if(ha) == 0) {
ifp = ha->ifp;
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
ha->flags.qla_watchdog_pause = 0;
}
return;
}
static void
qls_init(void *arg)
{
qla_host_t *ha;
ha = (qla_host_t *)arg;
QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
(void)QLA_LOCK(ha, __func__, 0);
qls_init_locked(ha);
QLA_UNLOCK(ha, __func__);
QL_DPRINT2((ha->pci_dev, "%s: exit\n", __func__));
}
static void
qls_set_multi(qla_host_t *ha, uint32_t add_multi)
{
uint8_t mta[Q8_MAX_NUM_MULTICAST_ADDRS * Q8_MAC_ADDR_LEN];
struct ifmultiaddr *ifma;
int mcnt = 0;
struct ifnet *ifp = ha->ifp;
if_maddr_rlock(ifp);
TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
if (ifma->ifma_addr->sa_family != AF_LINK)
continue;
if (mcnt == Q8_MAX_NUM_MULTICAST_ADDRS)
break;
bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
&mta[mcnt * Q8_MAC_ADDR_LEN], Q8_MAC_ADDR_LEN);
mcnt++;
}
if_maddr_runlock(ifp);
if (QLA_LOCK(ha, __func__, 1) == 0) {
qls_hw_set_multi(ha, mta, mcnt, add_multi);
QLA_UNLOCK(ha, __func__);
}
return;
}
static int
qls_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
int ret = 0;
struct ifreq *ifr = (struct ifreq *)data;
struct ifaddr *ifa = (struct ifaddr *)data;
qla_host_t *ha;
ha = (qla_host_t *)ifp->if_softc;
switch (cmd) {
case SIOCSIFADDR:
QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFADDR (0x%lx)\n",
__func__, cmd));
if (ifa->ifa_addr->sa_family == AF_INET) {
ifp->if_flags |= IFF_UP;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
(void)QLA_LOCK(ha, __func__, 0);
qls_init_locked(ha);
QLA_UNLOCK(ha, __func__);
}
QL_DPRINT4((ha->pci_dev,
"%s: SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n",
__func__, cmd,
ntohl(IA_SIN(ifa)->sin_addr.s_addr)));
arp_ifinit(ifp, ifa);
} else {
ether_ioctl(ifp, cmd, data);
}
break;
case SIOCSIFMTU:
QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFMTU (0x%lx)\n",
__func__, cmd));
if (ifr->ifr_mtu > QLA_MAX_MTU) {
ret = EINVAL;
} else {
(void) QLA_LOCK(ha, __func__, 0);
ifp->if_mtu = ifr->ifr_mtu;
ha->max_frame_size =
ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
QLA_UNLOCK(ha, __func__);
if (ret)
ret = EINVAL;
}
break;
case SIOCSIFFLAGS:
QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFFLAGS (0x%lx)\n",
__func__, cmd));
(void)QLA_LOCK(ha, __func__, 0);
if (ifp->if_flags & IFF_UP) {
if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
if ((ifp->if_flags ^ ha->if_flags) &
IFF_PROMISC) {
ret = qls_set_promisc(ha);
} else if ((ifp->if_flags ^ ha->if_flags) &
IFF_ALLMULTI) {
ret = qls_set_allmulti(ha);
}
} else {
ha->max_frame_size = ifp->if_mtu +
ETHER_HDR_LEN + ETHER_CRC_LEN;
qls_init_locked(ha);
}
} else {
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
qls_stop(ha);
ha->if_flags = ifp->if_flags;
}
QLA_UNLOCK(ha, __func__);
break;
case SIOCADDMULTI:
QL_DPRINT4((ha->pci_dev,
"%s: %s (0x%lx)\n", __func__, "SIOCADDMULTI", cmd));
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
qls_set_multi(ha, 1);
}
break;
case SIOCDELMULTI:
QL_DPRINT4((ha->pci_dev,
"%s: %s (0x%lx)\n", __func__, "SIOCDELMULTI", cmd));
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
qls_set_multi(ha, 0);
}
break;
case SIOCSIFMEDIA:
case SIOCGIFMEDIA:
QL_DPRINT4((ha->pci_dev,
"%s: SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n",
__func__, cmd));
ret = ifmedia_ioctl(ifp, ifr, &ha->media, cmd);
break;
case SIOCSIFCAP:
{
int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFCAP (0x%lx)\n",
__func__, cmd));
if (mask & IFCAP_HWCSUM)
ifp->if_capenable ^= IFCAP_HWCSUM;
if (mask & IFCAP_TSO4)
ifp->if_capenable ^= IFCAP_TSO4;
if (mask & IFCAP_VLAN_HWTAGGING)
ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
if (mask & IFCAP_VLAN_HWTSO)
ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
qls_init(ha);
VLAN_CAPABILITIES(ifp);
break;
}
default:
QL_DPRINT4((ha->pci_dev, "%s: default (0x%lx)\n",
__func__, cmd));
ret = ether_ioctl(ifp, cmd, data);
break;
}
return (ret);
}
static int
qls_media_change(struct ifnet *ifp)
{
qla_host_t *ha;
struct ifmedia *ifm;
int ret = 0;
ha = (qla_host_t *)ifp->if_softc;
QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
ifm = &ha->media;
if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
ret = EINVAL;
QL_DPRINT2((ha->pci_dev, "%s: exit\n", __func__));
return (ret);
}
static void
qls_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
qla_host_t *ha;
ha = (qla_host_t *)ifp->if_softc;
QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
ifmr->ifm_status = IFM_AVALID;
ifmr->ifm_active = IFM_ETHER;
qls_update_link_state(ha);
if (ha->link_up) {
ifmr->ifm_status |= IFM_ACTIVE;
ifmr->ifm_active |= (IFM_FDX | qls_get_optics(ha));
}
QL_DPRINT2((ha->pci_dev, "%s: exit (%s)\n", __func__,\
(ha->link_up ? "link_up" : "link_down")));
return;
}
static void
qls_start(struct ifnet *ifp)
{
int i, ret = 0;
struct mbuf *m_head;
qla_host_t *ha = (qla_host_t *)ifp->if_softc;
QL_DPRINT8((ha->pci_dev, "%s: enter\n", __func__));
if (!mtx_trylock(&ha->tx_lock)) {
QL_DPRINT8((ha->pci_dev,
"%s: mtx_trylock(&ha->tx_lock) failed\n", __func__));
return;
}
if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) ==
IFF_DRV_RUNNING) {
for (i = 0; i < ha->num_tx_rings; i++) {
ret |= qls_hw_tx_done(ha, i);
}
if (ret == 0)
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
}
if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
IFF_DRV_RUNNING) {
QL_DPRINT8((ha->pci_dev, "%s: !IFF_DRV_RUNNING\n", __func__));
QLA_TX_UNLOCK(ha);
return;
}
if (!ha->link_up) {
qls_update_link_state(ha);
if (!ha->link_up) {
QL_DPRINT8((ha->pci_dev, "%s: link down\n", __func__));
QLA_TX_UNLOCK(ha);
return;
}
}
while (ifp->if_snd.ifq_head != NULL) {
IF_DEQUEUE(&ifp->if_snd, m_head);
if (m_head == NULL) {
QL_DPRINT8((ha->pci_dev, "%s: m_head == NULL\n",
__func__));
break;
}
if (qls_send(ha, &m_head)) {
if (m_head == NULL)
break;
QL_DPRINT8((ha->pci_dev, "%s: PREPEND\n", __func__));
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IF_PREPEND(&ifp->if_snd, m_head);
break;
}
/* Send a copy of the frame to the BPF listener */
ETHER_BPF_MTAP(ifp, m_head);
}
QLA_TX_UNLOCK(ha);
QL_DPRINT8((ha->pci_dev, "%s: exit\n", __func__));
return;
}
static int
qls_send(qla_host_t *ha, struct mbuf **m_headp)
{
bus_dma_segment_t segs[QLA_MAX_SEGMENTS];
bus_dmamap_t map;
int nsegs;
int ret = -1;
uint32_t tx_idx;
struct mbuf *m_head = *m_headp;
uint32_t txr_idx = 0;
QL_DPRINT8((ha->pci_dev, "%s: enter\n", __func__));
/* check if flowid is set */
if (M_HASHTYPE_GET(m_head) != M_HASHTYPE_NONE)
txr_idx = m_head->m_pkthdr.flowid & (ha->num_tx_rings - 1);
tx_idx = ha->tx_ring[txr_idx].txr_next;
map = ha->tx_ring[txr_idx].tx_buf[tx_idx].map;
ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs,
BUS_DMA_NOWAIT);
if (ret == EFBIG) {
struct mbuf *m;
QL_DPRINT8((ha->pci_dev, "%s: EFBIG [%d]\n", __func__,
m_head->m_pkthdr.len));
m = m_defrag(m_head, M_NOWAIT);
if (m == NULL) {
ha->err_tx_defrag++;
m_freem(m_head);
*m_headp = NULL;
device_printf(ha->pci_dev,
"%s: m_defrag() = NULL [%d]\n",
__func__, ret);
return (ENOBUFS);
}
m_head = m;
*m_headp = m_head;
if ((ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head,
segs, &nsegs, BUS_DMA_NOWAIT))) {
ha->err_tx_dmamap_load++;
device_printf(ha->pci_dev,
"%s: bus_dmamap_load_mbuf_sg failed0[%d, %d]\n",
__func__, ret, m_head->m_pkthdr.len);
if (ret != ENOMEM) {
m_freem(m_head);
*m_headp = NULL;
}
return (ret);
}
} else if (ret) {
ha->err_tx_dmamap_load++;
device_printf(ha->pci_dev,
"%s: bus_dmamap_load_mbuf_sg failed1[%d, %d]\n",
__func__, ret, m_head->m_pkthdr.len);
if (ret != ENOMEM) {
m_freem(m_head);
*m_headp = NULL;
}
return (ret);
}
QL_ASSERT(ha, (nsegs != 0), ("qls_send: empty packet"));
bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_PREWRITE);
if (!(ret = qls_hw_send(ha, segs, nsegs, tx_idx, m_head, txr_idx))) {
ha->tx_ring[txr_idx].count++;
ha->tx_ring[txr_idx].tx_buf[tx_idx].m_head = m_head;
ha->tx_ring[txr_idx].tx_buf[tx_idx].map = map;
} else {
if (ret == EINVAL) {
if (m_head)
m_freem(m_head);
*m_headp = NULL;
}
}
QL_DPRINT8((ha->pci_dev, "%s: exit\n", __func__));
return (ret);
}
static void
qls_stop(qla_host_t *ha)
{
struct ifnet *ifp = ha->ifp;
device_t dev;
dev = ha->pci_dev;
ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
ha->flags.qla_watchdog_pause = 1;
while (!ha->qla_watchdog_paused)
qls_mdelay(__func__, 1);
qls_del_hw_if(ha);
qls_free_lro(ha);
qls_flush_xmt_bufs(ha);
qls_free_rcv_bufs(ha);
return;
}
/*
* Buffer Management Functions for Transmit and Receive Rings
*/
/*
* Release mbuf after it sent on the wire
*/
static void
qls_flush_tx_buf(qla_host_t *ha, qla_tx_buf_t *txb)
{
QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
if (txb->m_head) {
bus_dmamap_unload(ha->tx_tag, txb->map);
m_freem(txb->m_head);
txb->m_head = NULL;
}
QL_DPRINT2((ha->pci_dev, "%s: exit\n", __func__));
}
static void
qls_flush_xmt_bufs(qla_host_t *ha)
{
int i, j;
for (j = 0; j < ha->num_tx_rings; j++) {
for (i = 0; i < NUM_TX_DESCRIPTORS; i++)
qls_flush_tx_buf(ha, &ha->tx_ring[j].tx_buf[i]);
}
return;
}
static int
qls_alloc_rcv_mbufs(qla_host_t *ha, int r)
{
int i, j, ret = 0;
qla_rx_buf_t *rxb;
qla_rx_ring_t *rx_ring;
volatile q81_bq_addr_e_t *sbq_e;
rx_ring = &ha->rx_ring[r];
for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
rxb = &rx_ring->rx_buf[i];
ret = bus_dmamap_create(ha->rx_tag, BUS_DMA_NOWAIT, &rxb->map);
if (ret) {
device_printf(ha->pci_dev,
"%s: dmamap[%d, %d] failed\n", __func__, r, i);
for (j = 0; j < i; j++) {
rxb = &rx_ring->rx_buf[j];
bus_dmamap_destroy(ha->rx_tag, rxb->map);
}
goto qls_alloc_rcv_mbufs_err;
}
}
rx_ring = &ha->rx_ring[r];
sbq_e = rx_ring->sbq_vaddr;
rxb = &rx_ring->rx_buf[0];
for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
if (!(ret = qls_get_mbuf(ha, rxb, NULL))) {
/*
* set the physical address in the
* corresponding descriptor entry in the
* receive ring/queue for the hba
*/
sbq_e->addr_lo = rxb->paddr & 0xFFFFFFFF;
sbq_e->addr_hi = (rxb->paddr >> 32) & 0xFFFFFFFF;
} else {
device_printf(ha->pci_dev,
"%s: qls_get_mbuf [%d, %d] failed\n",
__func__, r, i);
bus_dmamap_destroy(ha->rx_tag, rxb->map);
goto qls_alloc_rcv_mbufs_err;
}
rxb++;
sbq_e++;
}
return 0;
qls_alloc_rcv_mbufs_err:
return (-1);
}
static void
qls_free_rcv_bufs(qla_host_t *ha)
{
int i, r;
qla_rx_buf_t *rxb;
qla_rx_ring_t *rxr;
for (r = 0; r < ha->num_rx_rings; r++) {
rxr = &ha->rx_ring[r];
for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
rxb = &rxr->rx_buf[i];
if (rxb->m_head != NULL) {
bus_dmamap_unload(ha->rx_tag, rxb->map);
bus_dmamap_destroy(ha->rx_tag, rxb->map);
m_freem(rxb->m_head);
}
}
bzero(rxr->rx_buf, (sizeof(qla_rx_buf_t) * NUM_RX_DESCRIPTORS));
}
return;
}
static int
qls_alloc_rcv_bufs(qla_host_t *ha)
{
int r, ret = 0;
qla_rx_ring_t *rxr;
for (r = 0; r < ha->num_rx_rings; r++) {
rxr = &ha->rx_ring[r];
bzero(rxr->rx_buf, (sizeof(qla_rx_buf_t) * NUM_RX_DESCRIPTORS));
}
for (r = 0; r < ha->num_rx_rings; r++) {
ret = qls_alloc_rcv_mbufs(ha, r);
if (ret)
qls_free_rcv_bufs(ha);
}
return (ret);
}
int
qls_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp)
{
register struct mbuf *mp = nmp;
struct ifnet *ifp;
int ret = 0;
uint32_t offset;
bus_dma_segment_t segs[1];
int nsegs;
QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
ifp = ha->ifp;
if (mp == NULL) {
mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ha->msize);
if (mp == NULL) {
if (ha->msize == MCLBYTES)
ha->err_m_getcl++;
else
ha->err_m_getjcl++;
ret = ENOBUFS;
device_printf(ha->pci_dev,
"%s: m_getcl failed\n", __func__);
goto exit_qls_get_mbuf;
}
mp->m_len = mp->m_pkthdr.len = ha->msize;
} else {
mp->m_len = mp->m_pkthdr.len = ha->msize;
mp->m_data = mp->m_ext.ext_buf;
mp->m_next = NULL;
}
/* align the receive buffers to 8 byte boundary */
offset = (uint32_t)((unsigned long long)mp->m_data & 0x7ULL);
if (offset) {
offset = 8 - offset;
m_adj(mp, offset);
}
/*
* Using memory from the mbuf cluster pool, invoke the bus_dma
* machinery to arrange the memory mapping.
*/
ret = bus_dmamap_load_mbuf_sg(ha->rx_tag, rxb->map,
mp, segs, &nsegs, BUS_DMA_NOWAIT);
rxb->paddr = segs[0].ds_addr;
if (ret || !rxb->paddr || (nsegs != 1)) {
m_freem(mp);
rxb->m_head = NULL;
device_printf(ha->pci_dev,
"%s: bus_dmamap_load failed[%d, 0x%016llx, %d]\n",
__func__, ret, (long long unsigned int)rxb->paddr,
nsegs);
ret = -1;
goto exit_qls_get_mbuf;
}
rxb->m_head = mp;
bus_dmamap_sync(ha->rx_tag, rxb->map, BUS_DMASYNC_PREREAD);
exit_qls_get_mbuf:
QL_DPRINT2((ha->pci_dev, "%s: exit ret = 0x%08x\n", __func__, ret));
return (ret);
}
static void
qls_tx_done(void *context, int pending)
{
qla_host_t *ha = context;
struct ifnet *ifp;
ifp = ha->ifp;
if (!ifp)
return;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
QL_DPRINT8((ha->pci_dev, "%s: !IFF_DRV_RUNNING\n", __func__));
return;
}
qls_start(ha->ifp);
return;
}
static int
qls_config_lro(qla_host_t *ha)
{
int i;
struct lro_ctrl *lro;
for (i = 0; i < ha->num_rx_rings; i++) {
lro = &ha->rx_ring[i].lro;
if (tcp_lro_init(lro)) {
device_printf(ha->pci_dev, "%s: tcp_lro_init failed\n",
__func__);
return (-1);
}
lro->ifp = ha->ifp;
}
ha->flags.lro_init = 1;
QL_DPRINT2((ha->pci_dev, "%s: LRO initialized\n", __func__));
return (0);
}
static void
qls_free_lro(qla_host_t *ha)
{
int i;
struct lro_ctrl *lro;
if (!ha->flags.lro_init)
return;
for (i = 0; i < ha->num_rx_rings; i++) {
lro = &ha->rx_ring[i].lro;
tcp_lro_free(lro);
}
ha->flags.lro_init = 0;
}
static void
qls_error_recovery(void *context, int pending)
{
qla_host_t *ha = context;
qls_init(ha);
return;
}
Index: head/sys/dev/rndtest/rndtest.c
===================================================================
--- head/sys/dev/rndtest/rndtest.c (revision 283290)
+++ head/sys/dev/rndtest/rndtest.c (revision 283291)
@@ -1,408 +1,408 @@
/* $OpenBSD$ */
/*-
* Copyright (c) 2002 Jason L. Wright (jason@thought.net)
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by Jason L. Wright
* 4. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/callout.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/random.h>
#include <sys/sysctl.h>
#include <machine/stdarg.h>
#include <dev/rndtest/rndtest.h>
static void rndtest_test(struct rndtest_state *);
static void rndtest_timeout(void *);
/* The tests themselves */
static int rndtest_monobit(struct rndtest_state *);
static int rndtest_runs(struct rndtest_state *);
static int rndtest_longruns(struct rndtest_state *);
static int rndtest_chi_4(struct rndtest_state *);
static int rndtest_runs_check(struct rndtest_state *, int, int *);
static void rndtest_runs_record(struct rndtest_state *, int, int *);
static const struct rndtest_testfunc {
int (*test)(struct rndtest_state *);
} rndtest_funcs[] = {
{ rndtest_monobit },
{ rndtest_runs },
{ rndtest_chi_4 },
{ rndtest_longruns },
};
#define RNDTEST_NTESTS (sizeof(rndtest_funcs)/sizeof(rndtest_funcs[0]))
static SYSCTL_NODE(_kern, OID_AUTO, rndtest, CTLFLAG_RD, 0,
"RNG test parameters");
static int rndtest_retest = 120; /* interval in seconds */
SYSCTL_INT(_kern_rndtest, OID_AUTO, retest, CTLFLAG_RW, &rndtest_retest,
0, "retest interval (seconds)");
static struct rndtest_stats rndstats;
SYSCTL_STRUCT(_kern_rndtest, OID_AUTO, stats, CTLFLAG_RD, &rndstats,
rndtest_stats, "RNG test statistics");
static int rndtest_verbose = 1; /* report only failures */
SYSCTL_INT(_kern_rndtest, OID_AUTO, verbose, CTLFLAG_RW, &rndtest_verbose,
0, "display results on console");
struct rndtest_state *
rndtest_attach(device_t dev)
{
struct rndtest_state *rsp;
rsp = malloc(sizeof (*rsp), M_DEVBUF, M_NOWAIT);
if (rsp != NULL) {
rsp->rs_begin = rsp->rs_buf;
rsp->rs_end = rsp->rs_buf + sizeof(rsp->rs_buf);
rsp->rs_current = rsp->rs_begin;
rsp->rs_discard = 1;
rsp->rs_collect = 1;
rsp->rs_parent = dev;
#if __FreeBSD_version < 500000
callout_init(&rsp->rs_to);
#else
- callout_init(&rsp->rs_to, CALLOUT_MPSAFE);
+ callout_init(&rsp->rs_to, 1);
#endif
} else
device_printf(dev, "rndtest_init: no memory for state block\n");
return (rsp);
}
void
rndtest_detach(struct rndtest_state *rsp)
{
callout_stop(&rsp->rs_to);
free(rsp, M_DEVBUF);
}
void
rndtest_harvest(struct rndtest_state *rsp, void *buf, u_int len)
{
size_t i;
/*
* If enabled, collect data and run tests when we have enough.
*/
if (rsp->rs_collect) {
for (i = 0; i < len; i++) {
*rsp->rs_current = ((u_char *) buf)[i];
if (++rsp->rs_current == rsp->rs_end) {
rndtest_test(rsp);
rsp->rs_current = rsp->rs_begin;
/*
* If tests passed, turn off collection and
* schedule another test. Otherwise we keep
* testing until the data looks ok.
*/
if (!rsp->rs_discard && rndtest_retest != 0) {
rsp->rs_collect = 0;
callout_reset(&rsp->rs_to,
hz * rndtest_retest,
rndtest_timeout, rsp);
break;
}
}
}
}
/*
* Only stir entropy that passes muster into the pool.
*/
if (rsp->rs_discard)
rndstats.rst_discard += len;
else {
#if __FreeBSD_version < 500000
/* XXX verify buffer is word aligned */
u_int32_t *p = buf;
for (len /= sizeof (u_int32_t); len; len--)
add_true_randomness(*p++);
#else
random_harvest(buf, len, len*NBBY/2, RANDOM_PURE_RNDTEST);
#endif
}
}
static void
rndtest_test(struct rndtest_state *rsp)
{
int i, rv = 0;
rndstats.rst_tests++;
for (i = 0; i < RNDTEST_NTESTS; i++)
rv |= (*rndtest_funcs[i].test)(rsp);
rsp->rs_discard = (rv != 0);
}
static void
rndtest_report(struct rndtest_state *rsp, int failure, const char *fmt, ...)
{
char buf[80];
va_list ap;
if (rndtest_verbose == 0)
return;
if (!failure && rndtest_verbose == 1) /* don't report successes */
return;
va_start(ap, fmt);
vsnprintf(buf, sizeof (buf), fmt, ap);
va_end(ap);
device_printf(rsp->rs_parent, "rndtest: %s\n", buf);
}
#define RNDTEST_MONOBIT_MINONES 9725
#define RNDTEST_MONOBIT_MAXONES 10275
static int
rndtest_monobit(struct rndtest_state *rsp)
{
int i, ones = 0, j;
u_int8_t r;
for (i = 0; i < RNDTEST_NBYTES; i++) {
r = rsp->rs_buf[i];
for (j = 0; j < 8; j++, r <<= 1)
if (r & 0x80)
ones++;
}
if (ones > RNDTEST_MONOBIT_MINONES &&
ones < RNDTEST_MONOBIT_MAXONES) {
if (rndtest_verbose > 1)
rndtest_report(rsp, 0, "monobit pass (%d < %d < %d)",
RNDTEST_MONOBIT_MINONES, ones,
RNDTEST_MONOBIT_MAXONES);
return (0);
} else {
if (rndtest_verbose)
rndtest_report(rsp, 1,
"monobit failed (%d ones)", ones);
rndstats.rst_monobit++;
return (-1);
}
}
#define RNDTEST_RUNS_NINTERVAL 6
static const struct rndtest_runs_tabs {
u_int16_t min, max;
} rndtest_runs_tab[] = {
{ 2343, 2657 },
{ 1135, 1365 },
{ 542, 708 },
{ 251, 373 },
{ 111, 201 },
{ 111, 201 },
};
static int
rndtest_runs(struct rndtest_state *rsp)
{
int i, j, ones, zeros, rv = 0;
int onei[RNDTEST_RUNS_NINTERVAL], zeroi[RNDTEST_RUNS_NINTERVAL];
u_int8_t c;
bzero(onei, sizeof(onei));
bzero(zeroi, sizeof(zeroi));
ones = zeros = 0;
for (i = 0; i < RNDTEST_NBYTES; i++) {
c = rsp->rs_buf[i];
for (j = 0; j < 8; j++, c <<= 1) {
if (c & 0x80) {
ones++;
rndtest_runs_record(rsp, zeros, zeroi);
zeros = 0;
} else {
zeros++;
rndtest_runs_record(rsp, ones, onei);
ones = 0;
}
}
}
rndtest_runs_record(rsp, ones, onei);
rndtest_runs_record(rsp, zeros, zeroi);
rv |= rndtest_runs_check(rsp, 0, zeroi);
rv |= rndtest_runs_check(rsp, 1, onei);
if (rv)
rndstats.rst_runs++;
return (rv);
}
static void
rndtest_runs_record(struct rndtest_state *rsp, int len, int *intrv)
{
if (len == 0)
return;
if (len > RNDTEST_RUNS_NINTERVAL)
len = RNDTEST_RUNS_NINTERVAL;
len -= 1;
intrv[len]++;
}
static int
rndtest_runs_check(struct rndtest_state *rsp, int val, int *src)
{
int i, rv = 0;
for (i = 0; i < RNDTEST_RUNS_NINTERVAL; i++) {
if (src[i] < rndtest_runs_tab[i].min ||
src[i] > rndtest_runs_tab[i].max) {
rndtest_report(rsp, 1,
"%s interval %d failed (%d, %d-%d)",
val ? "ones" : "zeros",
i + 1, src[i], rndtest_runs_tab[i].min,
rndtest_runs_tab[i].max);
rv = -1;
} else {
rndtest_report(rsp, 0,
"runs pass %s interval %d (%d < %d < %d)",
val ? "ones" : "zeros",
i + 1, rndtest_runs_tab[i].min, src[i],
rndtest_runs_tab[i].max);
}
}
return (rv);
}
static int
rndtest_longruns(struct rndtest_state *rsp)
{
int i, j, ones = 0, zeros = 0, maxones = 0, maxzeros = 0;
u_int8_t c;
for (i = 0; i < RNDTEST_NBYTES; i++) {
c = rsp->rs_buf[i];
for (j = 0; j < 8; j++, c <<= 1) {
if (c & 0x80) {
zeros = 0;
ones++;
if (ones > maxones)
maxones = ones;
} else {
ones = 0;
zeros++;
if (zeros > maxzeros)
maxzeros = zeros;
}
}
}
if (maxones < 26 && maxzeros < 26) {
rndtest_report(rsp, 0, "longruns pass (%d ones, %d zeros)",
maxones, maxzeros);
return (0);
} else {
rndtest_report(rsp, 1, "longruns fail (%d ones, %d zeros)",
maxones, maxzeros);
rndstats.rst_longruns++;
return (-1);
}
}
/*
* chi^2 test over 4 bits: (this is called the poker test in FIPS 140-2,
* but it is really the chi^2 test over 4 bits (the poker test as described
* by Knuth vol 2 is something different, and I take him as authoritative
* on nomenclature over NIST).
*/
#define RNDTEST_CHI4_K 16
#define RNDTEST_CHI4_K_MASK (RNDTEST_CHI4_K - 1)
/*
* The unnormalized values are used so that we don't have to worry about
* fractional precision. The "real" value is found by:
* (V - 1562500) * (16 / 5000) = Vn (where V is the unnormalized value)
*/
#define RNDTEST_CHI4_VMIN 1563181 /* 2.1792 */
#define RNDTEST_CHI4_VMAX 1576929 /* 46.1728 */
static int
rndtest_chi_4(struct rndtest_state *rsp)
{
unsigned int freq[RNDTEST_CHI4_K], i, sum;
for (i = 0; i < RNDTEST_CHI4_K; i++)
freq[i] = 0;
/* Get number of occurrences of each 4 bit pattern */
for (i = 0; i < RNDTEST_NBYTES; i++) {
freq[(rsp->rs_buf[i] >> 4) & RNDTEST_CHI4_K_MASK]++;
freq[(rsp->rs_buf[i] >> 0) & RNDTEST_CHI4_K_MASK]++;
}
for (i = 0, sum = 0; i < RNDTEST_CHI4_K; i++)
sum += freq[i] * freq[i];
if (sum >= 1563181 && sum <= 1576929) {
rndtest_report(rsp, 0, "chi^2(4): pass (sum %u)", sum);
return (0);
} else {
rndtest_report(rsp, 1, "chi^2(4): failed (sum %u)", sum);
rndstats.rst_chi++;
return (-1);
}
}
static void
rndtest_timeout(void *xrsp)
{
struct rndtest_state *rsp = xrsp;
rsp->rs_collect = 1;
}
static int
rndtest_modevent(module_t mod, int type, void *unused)
{
switch (type) {
case MOD_LOAD:
return 0;
case MOD_UNLOAD:
return 0;
}
return EINVAL;
}
static moduledata_t rndtest_mod = {
"rndtest",
rndtest_modevent,
0
};
DECLARE_MODULE(rndtest, rndtest_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
MODULE_VERSION(rndtest, 1);
Index: head/sys/dev/safe/safe.c
===================================================================
--- head/sys/dev/safe/safe.c (revision 283290)
+++ head/sys/dev/safe/safe.c (revision 283291)
@@ -1,2230 +1,2230 @@
/*-
* Copyright (c) 2003 Sam Leffler, Errno Consulting
* Copyright (c) 2003 Global Technology Associates, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* SafeNet SafeXcel-1141 hardware crypto accelerator
*/
#include "opt_safe.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/errno.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <sys/endian.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <crypto/sha1.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/cryptosoft.h>
#include <sys/md5.h>
#include <sys/random.h>
#include <sys/kobj.h>
#include "cryptodev_if.h"
#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
#ifdef SAFE_RNDTEST
#include <dev/rndtest/rndtest.h>
#endif
#include <dev/safe/safereg.h>
#include <dev/safe/safevar.h>
#ifndef bswap32
#define bswap32 NTOHL
#endif
/*
* Prototypes and count for the pci_device structure
*/
static int safe_probe(device_t);
static int safe_attach(device_t);
static int safe_detach(device_t);
static int safe_suspend(device_t);
static int safe_resume(device_t);
static int safe_shutdown(device_t);
static int safe_newsession(device_t, u_int32_t *, struct cryptoini *);
static int safe_freesession(device_t, u_int64_t);
static int safe_process(device_t, struct cryptop *, int);
static device_method_t safe_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, safe_probe),
DEVMETHOD(device_attach, safe_attach),
DEVMETHOD(device_detach, safe_detach),
DEVMETHOD(device_suspend, safe_suspend),
DEVMETHOD(device_resume, safe_resume),
DEVMETHOD(device_shutdown, safe_shutdown),
/* crypto device methods */
DEVMETHOD(cryptodev_newsession, safe_newsession),
DEVMETHOD(cryptodev_freesession,safe_freesession),
DEVMETHOD(cryptodev_process, safe_process),
DEVMETHOD_END
};
static driver_t safe_driver = {
"safe",
safe_methods,
sizeof (struct safe_softc)
};
static devclass_t safe_devclass;
DRIVER_MODULE(safe, pci, safe_driver, safe_devclass, 0, 0);
MODULE_DEPEND(safe, crypto, 1, 1, 1);
#ifdef SAFE_RNDTEST
MODULE_DEPEND(safe, rndtest, 1, 1, 1);
#endif
static void safe_intr(void *);
static void safe_callback(struct safe_softc *, struct safe_ringentry *);
static void safe_feed(struct safe_softc *, struct safe_ringentry *);
static void safe_mcopy(struct mbuf *, struct mbuf *, u_int);
#ifndef SAFE_NO_RNG
static void safe_rng_init(struct safe_softc *);
static void safe_rng(void *);
#endif /* SAFE_NO_RNG */
static int safe_dma_malloc(struct safe_softc *, bus_size_t,
struct safe_dma_alloc *, int);
#define safe_dma_sync(_dma, _flags) \
bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
static void safe_dma_free(struct safe_softc *, struct safe_dma_alloc *);
static int safe_dmamap_aligned(const struct safe_operand *);
static int safe_dmamap_uniform(const struct safe_operand *);
static void safe_reset_board(struct safe_softc *);
static void safe_init_board(struct safe_softc *);
static void safe_init_pciregs(device_t dev);
static void safe_cleanchip(struct safe_softc *);
static void safe_totalreset(struct safe_softc *);
static int safe_free_entry(struct safe_softc *, struct safe_ringentry *);
static SYSCTL_NODE(_hw, OID_AUTO, safe, CTLFLAG_RD, 0,
"SafeNet driver parameters");
#ifdef SAFE_DEBUG
static void safe_dump_dmastatus(struct safe_softc *, const char *);
static void safe_dump_ringstate(struct safe_softc *, const char *);
static void safe_dump_intrstate(struct safe_softc *, const char *);
static void safe_dump_request(struct safe_softc *, const char *,
struct safe_ringentry *);
static struct safe_softc *safec; /* for use by hw.safe.dump */
static int safe_debug = 0;
SYSCTL_INT(_hw_safe, OID_AUTO, debug, CTLFLAG_RW, &safe_debug,
0, "control debugging msgs");
#define DPRINTF(_x) if (safe_debug) printf _x
#else
#define DPRINTF(_x)
#endif
#define READ_REG(sc,r) \
bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
#define WRITE_REG(sc,reg,val) \
bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
struct safe_stats safestats;
SYSCTL_STRUCT(_hw_safe, OID_AUTO, stats, CTLFLAG_RD, &safestats,
safe_stats, "driver statistics");
#ifndef SAFE_NO_RNG
static int safe_rnginterval = 1; /* poll once a second */
SYSCTL_INT(_hw_safe, OID_AUTO, rnginterval, CTLFLAG_RW, &safe_rnginterval,
0, "RNG polling interval (secs)");
static int safe_rngbufsize = 16; /* 64 bytes each poll */
SYSCTL_INT(_hw_safe, OID_AUTO, rngbufsize, CTLFLAG_RW, &safe_rngbufsize,
0, "RNG polling buffer size (32-bit words)");
static int safe_rngmaxalarm = 8; /* max alarms before reset */
SYSCTL_INT(_hw_safe, OID_AUTO, rngmaxalarm, CTLFLAG_RW, &safe_rngmaxalarm,
0, "RNG max alarms before reset");
#endif /* SAFE_NO_RNG */
static int
safe_probe(device_t dev)
{
if (pci_get_vendor(dev) == PCI_VENDOR_SAFENET &&
pci_get_device(dev) == PCI_PRODUCT_SAFEXCEL)
return (BUS_PROBE_DEFAULT);
return (ENXIO);
}
static const char*
safe_partname(struct safe_softc *sc)
{
/* XXX sprintf numbers when not decoded */
switch (pci_get_vendor(sc->sc_dev)) {
case PCI_VENDOR_SAFENET:
switch (pci_get_device(sc->sc_dev)) {
case PCI_PRODUCT_SAFEXCEL: return "SafeNet SafeXcel-1141";
}
return "SafeNet unknown-part";
}
return "Unknown-vendor unknown-part";
}
#ifndef SAFE_NO_RNG
static void
default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
{
random_harvest(buf, count, count*NBBY/2, RANDOM_PURE_SAFE);
}
#endif /* SAFE_NO_RNG */
static int
safe_attach(device_t dev)
{
struct safe_softc *sc = device_get_softc(dev);
u_int32_t raddr;
u_int32_t i, devinfo;
int rid;
bzero(sc, sizeof (*sc));
sc->sc_dev = dev;
/* XXX handle power management */
pci_enable_busmaster(dev);
/*
* Setup memory-mapping of PCI registers.
*/
rid = BS_BAR;
sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (sc->sc_sr == NULL) {
device_printf(dev, "cannot map register space\n");
goto bad;
}
sc->sc_st = rman_get_bustag(sc->sc_sr);
sc->sc_sh = rman_get_bushandle(sc->sc_sr);
/*
* Arrange interrupt line.
*/
rid = 0;
sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_SHAREABLE|RF_ACTIVE);
if (sc->sc_irq == NULL) {
device_printf(dev, "could not map interrupt\n");
goto bad1;
}
/*
* NB: Network code assumes we are blocked with splimp()
* so make sure the IRQ is mapped appropriately.
*/
if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
NULL, safe_intr, sc, &sc->sc_ih)) {
device_printf(dev, "could not establish interrupt\n");
goto bad2;
}
sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
if (sc->sc_cid < 0) {
device_printf(dev, "could not get crypto driver id\n");
goto bad3;
}
sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
(SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
/*
* Setup DMA descriptor area.
*/
if (bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1, /* alignment */
SAFE_DMA_BOUNDARY, /* boundary */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
SAFE_MAX_DMA, /* maxsize */
SAFE_MAX_PART, /* nsegments */
SAFE_MAX_SSIZE, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, NULL, /* locking */
&sc->sc_srcdmat)) {
device_printf(dev, "cannot allocate DMA tag\n");
goto bad4;
}
if (bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1, /* alignment */
SAFE_MAX_DSIZE, /* boundary */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
SAFE_MAX_DMA, /* maxsize */
SAFE_MAX_PART, /* nsegments */
SAFE_MAX_DSIZE, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, NULL, /* locking */
&sc->sc_dstdmat)) {
device_printf(dev, "cannot allocate DMA tag\n");
goto bad4;
}
/*
* Allocate packet engine descriptors.
*/
if (safe_dma_malloc(sc,
SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
&sc->sc_ringalloc, 0)) {
device_printf(dev, "cannot allocate PE descriptor ring\n");
bus_dma_tag_destroy(sc->sc_srcdmat);
goto bad4;
}
/*
* Hookup the static portion of all our data structures.
*/
sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
sc->sc_front = sc->sc_ring;
sc->sc_back = sc->sc_ring;
raddr = sc->sc_ringalloc.dma_paddr;
bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
struct safe_ringentry *re = &sc->sc_ring[i];
re->re_desc.d_sa = raddr +
offsetof(struct safe_ringentry, re_sa);
re->re_sa.sa_staterec = raddr +
offsetof(struct safe_ringentry, re_sastate);
raddr += sizeof (struct safe_ringentry);
}
mtx_init(&sc->sc_ringmtx, device_get_nameunit(dev),
"packet engine ring", MTX_DEF);
/*
* Allocate scatter and gather particle descriptors.
*/
if (safe_dma_malloc(sc, SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
&sc->sc_spalloc, 0)) {
device_printf(dev, "cannot allocate source particle "
"descriptor ring\n");
mtx_destroy(&sc->sc_ringmtx);
safe_dma_free(sc, &sc->sc_ringalloc);
bus_dma_tag_destroy(sc->sc_srcdmat);
goto bad4;
}
sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
sc->sc_spfree = sc->sc_spring;
bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
if (safe_dma_malloc(sc, SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
&sc->sc_dpalloc, 0)) {
device_printf(dev, "cannot allocate destination particle "
"descriptor ring\n");
mtx_destroy(&sc->sc_ringmtx);
safe_dma_free(sc, &sc->sc_spalloc);
safe_dma_free(sc, &sc->sc_ringalloc);
bus_dma_tag_destroy(sc->sc_dstdmat);
goto bad4;
}
sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
sc->sc_dpfree = sc->sc_dpring;
bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
device_printf(sc->sc_dev, "%s", safe_partname(sc));
devinfo = READ_REG(sc, SAFE_DEVINFO);
if (devinfo & SAFE_DEVINFO_RNG) {
sc->sc_flags |= SAFE_FLAGS_RNG;
printf(" rng");
}
if (devinfo & SAFE_DEVINFO_PKEY) {
#if 0
printf(" key");
sc->sc_flags |= SAFE_FLAGS_KEY;
crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
#endif
}
if (devinfo & SAFE_DEVINFO_DES) {
printf(" des/3des");
crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
}
if (devinfo & SAFE_DEVINFO_AES) {
printf(" aes");
crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
}
if (devinfo & SAFE_DEVINFO_MD5) {
printf(" md5");
crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
}
if (devinfo & SAFE_DEVINFO_SHA1) {
printf(" sha1");
crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
}
printf(" null");
crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
/* XXX other supported algorithms */
printf("\n");
safe_reset_board(sc); /* reset h/w */
safe_init_pciregs(dev); /* init pci settings */
safe_init_board(sc); /* init h/w */
#ifndef SAFE_NO_RNG
if (sc->sc_flags & SAFE_FLAGS_RNG) {
#ifdef SAFE_RNDTEST
sc->sc_rndtest = rndtest_attach(dev);
if (sc->sc_rndtest)
sc->sc_harvest = rndtest_harvest;
else
sc->sc_harvest = default_harvest;
#else
sc->sc_harvest = default_harvest;
#endif
safe_rng_init(sc);
- callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_rngto, 1);
callout_reset(&sc->sc_rngto, hz*safe_rnginterval, safe_rng, sc);
}
#endif /* SAFE_NO_RNG */
#ifdef SAFE_DEBUG
safec = sc; /* for use by hw.safe.dump */
#endif
return (0);
bad4:
crypto_unregister_all(sc->sc_cid);
bad3:
bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
bad2:
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
bad1:
bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
bad:
return (ENXIO);
}
/*
* Detach a device that successfully probed.
*/
static int
safe_detach(device_t dev)
{
struct safe_softc *sc = device_get_softc(dev);
/* XXX wait/abort active ops */
WRITE_REG(sc, SAFE_HI_MASK, 0); /* disable interrupts */
callout_stop(&sc->sc_rngto);
crypto_unregister_all(sc->sc_cid);
#ifdef SAFE_RNDTEST
if (sc->sc_rndtest)
rndtest_detach(sc->sc_rndtest);
#endif
safe_cleanchip(sc);
safe_dma_free(sc, &sc->sc_dpalloc);
safe_dma_free(sc, &sc->sc_spalloc);
mtx_destroy(&sc->sc_ringmtx);
safe_dma_free(sc, &sc->sc_ringalloc);
bus_generic_detach(dev);
bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
bus_dma_tag_destroy(sc->sc_srcdmat);
bus_dma_tag_destroy(sc->sc_dstdmat);
bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
return (0);
}
/*
* Stop all chip i/o so that the kernel's probe routines don't
* get confused by errant DMAs when rebooting.
*/
static int
safe_shutdown(device_t dev)
{
#ifdef notyet
safe_stop(device_get_softc(dev));
#endif
return (0);
}
/*
* Device suspend routine.
*/
static int
safe_suspend(device_t dev)
{
struct safe_softc *sc = device_get_softc(dev);
#ifdef notyet
/* XXX stop the device and save PCI settings */
#endif
sc->sc_suspended = 1;
return (0);
}
static int
safe_resume(device_t dev)
{
struct safe_softc *sc = device_get_softc(dev);
#ifdef notyet
/* XXX retore PCI settings and start the device */
#endif
sc->sc_suspended = 0;
return (0);
}
/*
* SafeXcel Interrupt routine
*/
static void
safe_intr(void *arg)
{
struct safe_softc *sc = arg;
volatile u_int32_t stat;
stat = READ_REG(sc, SAFE_HM_STAT);
if (stat == 0) /* shared irq, not for us */
return;
WRITE_REG(sc, SAFE_HI_CLR, stat); /* IACK */
if ((stat & SAFE_INT_PE_DDONE)) {
/*
* Descriptor(s) done; scan the ring and
* process completed operations.
*/
mtx_lock(&sc->sc_ringmtx);
while (sc->sc_back != sc->sc_front) {
struct safe_ringentry *re = sc->sc_back;
#ifdef SAFE_DEBUG
if (safe_debug) {
safe_dump_ringstate(sc, __func__);
safe_dump_request(sc, __func__, re);
}
#endif
/*
* safe_process marks ring entries that were allocated
* but not used with a csr of zero. This insures the
* ring front pointer never needs to be set backwards
* in the event that an entry is allocated but not used
* because of a setup error.
*/
if (re->re_desc.d_csr != 0) {
if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))
break;
if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))
break;
sc->sc_nqchip--;
safe_callback(sc, re);
}
if (++(sc->sc_back) == sc->sc_ringtop)
sc->sc_back = sc->sc_ring;
}
mtx_unlock(&sc->sc_ringmtx);
}
/*
* Check to see if we got any DMA Error
*/
if (stat & SAFE_INT_PE_ERROR) {
DPRINTF(("dmaerr dmastat %08x\n",
READ_REG(sc, SAFE_PE_DMASTAT)));
safestats.st_dmaerr++;
safe_totalreset(sc);
#if 0
safe_feed(sc);
#endif
}
if (sc->sc_needwakeup) { /* XXX check high watermark */
int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
DPRINTF(("%s: wakeup crypto %x\n", __func__,
sc->sc_needwakeup));
sc->sc_needwakeup &= ~wakeup;
crypto_unblock(sc->sc_cid, wakeup);
}
}
/*
* safe_feed() - post a request to chip
*/
static void
safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
{
bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_PREWRITE);
if (re->re_dst_map != NULL)
bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
BUS_DMASYNC_PREREAD);
/* XXX have no smaller granularity */
safe_dma_sync(&sc->sc_ringalloc,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
safe_dma_sync(&sc->sc_spalloc, BUS_DMASYNC_PREWRITE);
safe_dma_sync(&sc->sc_dpalloc, BUS_DMASYNC_PREWRITE);
#ifdef SAFE_DEBUG
if (safe_debug) {
safe_dump_ringstate(sc, __func__);
safe_dump_request(sc, __func__, re);
}
#endif
sc->sc_nqchip++;
if (sc->sc_nqchip > safestats.st_maxqchip)
safestats.st_maxqchip = sc->sc_nqchip;
/* poke h/w to check descriptor ring, any value can be written */
WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
}
#define N(a) (sizeof(a) / sizeof (a[0]))
static void
safe_setup_enckey(struct safe_session *ses, caddr_t key)
{
int i;
bcopy(key, ses->ses_key, ses->ses_klen / 8);
/* PE is little-endian, insure proper byte order */
for (i = 0; i < N(ses->ses_key); i++)
ses->ses_key[i] = htole32(ses->ses_key[i]);
}
static void
safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
{
MD5_CTX md5ctx;
SHA1_CTX sha1ctx;
int i;
for (i = 0; i < klen; i++)
key[i] ^= HMAC_IPAD_VAL;
if (algo == CRYPTO_MD5_HMAC) {
MD5Init(&md5ctx);
MD5Update(&md5ctx, key, klen);
MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
} else {
SHA1Init(&sha1ctx);
SHA1Update(&sha1ctx, key, klen);
SHA1Update(&sha1ctx, hmac_ipad_buffer,
SHA1_HMAC_BLOCK_LEN - klen);
bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
}
for (i = 0; i < klen; i++)
key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
if (algo == CRYPTO_MD5_HMAC) {
MD5Init(&md5ctx);
MD5Update(&md5ctx, key, klen);
MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
} else {
SHA1Init(&sha1ctx);
SHA1Update(&sha1ctx, key, klen);
SHA1Update(&sha1ctx, hmac_opad_buffer,
SHA1_HMAC_BLOCK_LEN - klen);
bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
}
for (i = 0; i < klen; i++)
key[i] ^= HMAC_OPAD_VAL;
/* PE is little-endian, insure proper byte order */
for (i = 0; i < N(ses->ses_hminner); i++) {
ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
}
}
#undef N
/*
* Allocate a new 'session' and return an encoded session id. 'sidp'
* contains our registration id, and should contain an encoded session
* id on successful allocation.
*/
static int
safe_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
{
struct safe_softc *sc = device_get_softc(dev);
struct cryptoini *c, *encini = NULL, *macini = NULL;
struct safe_session *ses = NULL;
int sesn;
if (sidp == NULL || cri == NULL || sc == NULL)
return (EINVAL);
for (c = cri; c != NULL; c = c->cri_next) {
if (c->cri_alg == CRYPTO_MD5_HMAC ||
c->cri_alg == CRYPTO_SHA1_HMAC ||
c->cri_alg == CRYPTO_NULL_HMAC) {
if (macini)
return (EINVAL);
macini = c;
} else if (c->cri_alg == CRYPTO_DES_CBC ||
c->cri_alg == CRYPTO_3DES_CBC ||
c->cri_alg == CRYPTO_AES_CBC ||
c->cri_alg == CRYPTO_NULL_CBC) {
if (encini)
return (EINVAL);
encini = c;
} else
return (EINVAL);
}
if (encini == NULL && macini == NULL)
return (EINVAL);
if (encini) { /* validate key length */
switch (encini->cri_alg) {
case CRYPTO_DES_CBC:
if (encini->cri_klen != 64)
return (EINVAL);
break;
case CRYPTO_3DES_CBC:
if (encini->cri_klen != 192)
return (EINVAL);
break;
case CRYPTO_AES_CBC:
if (encini->cri_klen != 128 &&
encini->cri_klen != 192 &&
encini->cri_klen != 256)
return (EINVAL);
break;
}
}
if (sc->sc_sessions == NULL) {
ses = sc->sc_sessions = (struct safe_session *)malloc(
sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
if (ses == NULL)
return (ENOMEM);
sesn = 0;
sc->sc_nsessions = 1;
} else {
for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
if (sc->sc_sessions[sesn].ses_used == 0) {
ses = &sc->sc_sessions[sesn];
break;
}
}
if (ses == NULL) {
sesn = sc->sc_nsessions;
ses = (struct safe_session *)malloc((sesn + 1) *
sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
if (ses == NULL)
return (ENOMEM);
bcopy(sc->sc_sessions, ses, sesn *
sizeof(struct safe_session));
bzero(sc->sc_sessions, sesn *
sizeof(struct safe_session));
free(sc->sc_sessions, M_DEVBUF);
sc->sc_sessions = ses;
ses = &sc->sc_sessions[sesn];
sc->sc_nsessions++;
}
}
bzero(ses, sizeof(struct safe_session));
ses->ses_used = 1;
if (encini) {
/* get an IV */
/* XXX may read fewer than requested */
read_random(ses->ses_iv, sizeof(ses->ses_iv));
ses->ses_klen = encini->cri_klen;
if (encini->cri_key != NULL)
safe_setup_enckey(ses, encini->cri_key);
}
if (macini) {
ses->ses_mlen = macini->cri_mlen;
if (ses->ses_mlen == 0) {
if (macini->cri_alg == CRYPTO_MD5_HMAC)
ses->ses_mlen = MD5_HASH_LEN;
else
ses->ses_mlen = SHA1_HASH_LEN;
}
if (macini->cri_key != NULL) {
safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
macini->cri_klen / 8);
}
}
*sidp = SAFE_SID(device_get_unit(sc->sc_dev), sesn);
return (0);
}
/*
* Deallocate a session.
*/
static int
safe_freesession(device_t dev, u_int64_t tid)
{
struct safe_softc *sc = device_get_softc(dev);
int session, ret;
u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
if (sc == NULL)
return (EINVAL);
session = SAFE_SESSION(sid);
if (session < sc->sc_nsessions) {
bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
ret = 0;
} else
ret = EINVAL;
return (ret);
}
static void
safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
{
struct safe_operand *op = arg;
DPRINTF(("%s: mapsize %u nsegs %d error %d\n", __func__,
(u_int) mapsize, nsegs, error));
if (error != 0)
return;
op->mapsize = mapsize;
op->nsegs = nsegs;
bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
}
static int
safe_process(device_t dev, struct cryptop *crp, int hint)
{
struct safe_softc *sc = device_get_softc(dev);
int err = 0, i, nicealign, uniform;
struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
int bypass, oplen, ivsize;
caddr_t iv;
int16_t coffset;
struct safe_session *ses;
struct safe_ringentry *re;
struct safe_sarec *sa;
struct safe_pdesc *pd;
u_int32_t cmd0, cmd1, staterec;
if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
safestats.st_invalid++;
return (EINVAL);
}
if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
safestats.st_badsession++;
return (EINVAL);
}
mtx_lock(&sc->sc_ringmtx);
if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
safestats.st_ringfull++;
sc->sc_needwakeup |= CRYPTO_SYMQ;
mtx_unlock(&sc->sc_ringmtx);
return (ERESTART);
}
re = sc->sc_front;
staterec = re->re_sa.sa_staterec; /* save */
/* NB: zero everything but the PE descriptor */
bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
re->re_sa.sa_staterec = staterec; /* restore */
re->re_crp = crp;
re->re_sesn = SAFE_SESSION(crp->crp_sid);
if (crp->crp_flags & CRYPTO_F_IMBUF) {
re->re_src_m = (struct mbuf *)crp->crp_buf;
re->re_dst_m = (struct mbuf *)crp->crp_buf;
} else if (crp->crp_flags & CRYPTO_F_IOV) {
re->re_src_io = (struct uio *)crp->crp_buf;
re->re_dst_io = (struct uio *)crp->crp_buf;
} else {
safestats.st_badflags++;
err = EINVAL;
goto errout; /* XXX we don't handle contiguous blocks! */
}
sa = &re->re_sa;
ses = &sc->sc_sessions[re->re_sesn];
crd1 = crp->crp_desc;
if (crd1 == NULL) {
safestats.st_nodesc++;
err = EINVAL;
goto errout;
}
crd2 = crd1->crd_next;
cmd0 = SAFE_SA_CMD0_BASIC; /* basic group operation */
cmd1 = 0;
if (crd2 == NULL) {
if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC ||
crd1->crd_alg == CRYPTO_NULL_HMAC) {
maccrd = crd1;
enccrd = NULL;
cmd0 |= SAFE_SA_CMD0_OP_HASH;
} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
crd1->crd_alg == CRYPTO_3DES_CBC ||
crd1->crd_alg == CRYPTO_AES_CBC ||
crd1->crd_alg == CRYPTO_NULL_CBC) {
maccrd = NULL;
enccrd = crd1;
cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
} else {
safestats.st_badalg++;
err = EINVAL;
goto errout;
}
} else {
if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC ||
crd1->crd_alg == CRYPTO_NULL_HMAC) &&
(crd2->crd_alg == CRYPTO_DES_CBC ||
crd2->crd_alg == CRYPTO_3DES_CBC ||
crd2->crd_alg == CRYPTO_AES_CBC ||
crd2->crd_alg == CRYPTO_NULL_CBC) &&
((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
maccrd = crd1;
enccrd = crd2;
} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
crd1->crd_alg == CRYPTO_3DES_CBC ||
crd1->crd_alg == CRYPTO_AES_CBC ||
crd1->crd_alg == CRYPTO_NULL_CBC) &&
(crd2->crd_alg == CRYPTO_MD5_HMAC ||
crd2->crd_alg == CRYPTO_SHA1_HMAC ||
crd2->crd_alg == CRYPTO_NULL_HMAC) &&
(crd1->crd_flags & CRD_F_ENCRYPT)) {
enccrd = crd1;
maccrd = crd2;
} else {
safestats.st_badalg++;
err = EINVAL;
goto errout;
}
cmd0 |= SAFE_SA_CMD0_OP_BOTH;
}
if (enccrd) {
if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
safe_setup_enckey(ses, enccrd->crd_key);
if (enccrd->crd_alg == CRYPTO_DES_CBC) {
cmd0 |= SAFE_SA_CMD0_DES;
cmd1 |= SAFE_SA_CMD1_CBC;
ivsize = 2*sizeof(u_int32_t);
} else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
cmd0 |= SAFE_SA_CMD0_3DES;
cmd1 |= SAFE_SA_CMD1_CBC;
ivsize = 2*sizeof(u_int32_t);
} else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
cmd0 |= SAFE_SA_CMD0_AES;
cmd1 |= SAFE_SA_CMD1_CBC;
if (ses->ses_klen == 128)
cmd1 |= SAFE_SA_CMD1_AES128;
else if (ses->ses_klen == 192)
cmd1 |= SAFE_SA_CMD1_AES192;
else
cmd1 |= SAFE_SA_CMD1_AES256;
ivsize = 4*sizeof(u_int32_t);
} else {
cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
ivsize = 0;
}
/*
* Setup encrypt/decrypt state. When using basic ops
* we can't use an inline IV because hash/crypt offset
* must be from the end of the IV to the start of the
* crypt data and this leaves out the preceding header
* from the hash calculation. Instead we place the IV
* in the state record and set the hash/crypt offset to
* copy both the header+IV.
*/
if (enccrd->crd_flags & CRD_F_ENCRYPT) {
cmd0 |= SAFE_SA_CMD0_OUTBOUND;
if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
iv = enccrd->crd_iv;
else
iv = (caddr_t) ses->ses_iv;
if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
crypto_copyback(crp->crp_flags, crp->crp_buf,
enccrd->crd_inject, ivsize, iv);
}
bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
} else {
cmd0 |= SAFE_SA_CMD0_INBOUND;
if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
bcopy(enccrd->crd_iv,
re->re_sastate.sa_saved_iv, ivsize);
} else {
crypto_copydata(crp->crp_flags, crp->crp_buf,
enccrd->crd_inject, ivsize,
(caddr_t)re->re_sastate.sa_saved_iv);
}
cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
}
/*
* For basic encryption use the zero pad algorithm.
* This pads results to an 8-byte boundary and
* suppresses padding verification for inbound (i.e.
* decrypt) operations.
*
* NB: Not sure if the 8-byte pad boundary is a problem.
*/
cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
/* XXX assert key bufs have the same size */
bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
}
if (maccrd) {
if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
safe_setup_mackey(ses, maccrd->crd_alg,
maccrd->crd_key, maccrd->crd_klen / 8);
}
if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
cmd0 |= SAFE_SA_CMD0_MD5;
cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
} else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
cmd0 |= SAFE_SA_CMD0_SHA1;
cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
} else {
cmd0 |= SAFE_SA_CMD0_HASH_NULL;
}
/*
* Digest data is loaded from the SA and the hash
* result is saved to the state block where we
* retrieve it for return to the caller.
*/
/* XXX assert digest bufs have the same size */
bcopy(ses->ses_hminner, sa->sa_indigest,
sizeof(sa->sa_indigest));
bcopy(ses->ses_hmouter, sa->sa_outdigest,
sizeof(sa->sa_outdigest));
cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
}
if (enccrd && maccrd) {
/*
* The offset from hash data to the start of
* crypt data is the difference in the skips.
*/
bypass = maccrd->crd_skip;
coffset = enccrd->crd_skip - maccrd->crd_skip;
if (coffset < 0) {
DPRINTF(("%s: hash does not precede crypt; "
"mac skip %u enc skip %u\n",
__func__, maccrd->crd_skip, enccrd->crd_skip));
safestats.st_skipmismatch++;
err = EINVAL;
goto errout;
}
oplen = enccrd->crd_skip + enccrd->crd_len;
if (maccrd->crd_skip + maccrd->crd_len != oplen) {
DPRINTF(("%s: hash amount %u != crypt amount %u\n",
__func__, maccrd->crd_skip + maccrd->crd_len,
oplen));
safestats.st_lenmismatch++;
err = EINVAL;
goto errout;
}
#ifdef SAFE_DEBUG
if (safe_debug) {
printf("mac: skip %d, len %d, inject %d\n",
maccrd->crd_skip, maccrd->crd_len,
maccrd->crd_inject);
printf("enc: skip %d, len %d, inject %d\n",
enccrd->crd_skip, enccrd->crd_len,
enccrd->crd_inject);
printf("bypass %d coffset %d oplen %d\n",
bypass, coffset, oplen);
}
#endif
if (coffset & 3) { /* offset must be 32-bit aligned */
DPRINTF(("%s: coffset %u misaligned\n",
__func__, coffset));
safestats.st_coffmisaligned++;
err = EINVAL;
goto errout;
}
coffset >>= 2;
if (coffset > 255) { /* offset must be <256 dwords */
DPRINTF(("%s: coffset %u too big\n",
__func__, coffset));
safestats.st_cofftoobig++;
err = EINVAL;
goto errout;
}
/*
* Tell the hardware to copy the header to the output.
* The header is defined as the data from the end of
* the bypass to the start of data to be encrypted.
* Typically this is the inline IV. Note that you need
* to do this even if src+dst are the same; it appears
* that w/o this bit the crypted data is written
* immediately after the bypass data.
*/
cmd1 |= SAFE_SA_CMD1_HDRCOPY;
/*
* Disable IP header mutable bit handling. This is
* needed to get correct HMAC calculations.
*/
cmd1 |= SAFE_SA_CMD1_MUTABLE;
} else {
if (enccrd) {
bypass = enccrd->crd_skip;
oplen = bypass + enccrd->crd_len;
} else {
bypass = maccrd->crd_skip;
oplen = bypass + maccrd->crd_len;
}
coffset = 0;
}
/* XXX verify multiple of 4 when using s/g */
if (bypass > 96) { /* bypass offset must be <= 96 bytes */
DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
safestats.st_bypasstoobig++;
err = EINVAL;
goto errout;
}
if (bus_dmamap_create(sc->sc_srcdmat, BUS_DMA_NOWAIT, &re->re_src_map)) {
safestats.st_nomap++;
err = ENOMEM;
goto errout;
}
if (crp->crp_flags & CRYPTO_F_IMBUF) {
if (bus_dmamap_load_mbuf(sc->sc_srcdmat, re->re_src_map,
re->re_src_m, safe_op_cb,
&re->re_src, BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
re->re_src_map = NULL;
safestats.st_noload++;
err = ENOMEM;
goto errout;
}
} else if (crp->crp_flags & CRYPTO_F_IOV) {
if (bus_dmamap_load_uio(sc->sc_srcdmat, re->re_src_map,
re->re_src_io, safe_op_cb,
&re->re_src, BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
re->re_src_map = NULL;
safestats.st_noload++;
err = ENOMEM;
goto errout;
}
}
nicealign = safe_dmamap_aligned(&re->re_src);
uniform = safe_dmamap_uniform(&re->re_src);
DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
nicealign, uniform, re->re_src.nsegs));
if (re->re_src.nsegs > 1) {
re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
for (i = 0; i < re->re_src_nsegs; i++) {
/* NB: no need to check if there's space */
pd = sc->sc_spfree;
if (++(sc->sc_spfree) == sc->sc_springtop)
sc->sc_spfree = sc->sc_spring;
KASSERT((pd->pd_flags&3) == 0 ||
(pd->pd_flags&3) == SAFE_PD_DONE,
("bogus source particle descriptor; flags %x",
pd->pd_flags));
pd->pd_addr = re->re_src_segs[i].ds_addr;
pd->pd_size = re->re_src_segs[i].ds_len;
pd->pd_flags = SAFE_PD_READY;
}
cmd0 |= SAFE_SA_CMD0_IGATHER;
} else {
/*
* No need for gather, reference the operand directly.
*/
re->re_desc.d_src = re->re_src_segs[0].ds_addr;
}
if (enccrd == NULL && maccrd != NULL) {
/*
* Hash op; no destination needed.
*/
} else {
if (crp->crp_flags & CRYPTO_F_IOV) {
if (!nicealign) {
safestats.st_iovmisaligned++;
err = EINVAL;
goto errout;
}
if (uniform != 1) {
/*
* Source is not suitable for direct use as
* the destination. Create a new scatter/gather
* list based on the destination requirements
* and check if that's ok.
*/
if (bus_dmamap_create(sc->sc_dstdmat,
BUS_DMA_NOWAIT, &re->re_dst_map)) {
safestats.st_nomap++;
err = ENOMEM;
goto errout;
}
if (bus_dmamap_load_uio(sc->sc_dstdmat,
re->re_dst_map, re->re_dst_io,
safe_op_cb, &re->re_dst,
BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dstdmat,
re->re_dst_map);
re->re_dst_map = NULL;
safestats.st_noload++;
err = ENOMEM;
goto errout;
}
uniform = safe_dmamap_uniform(&re->re_dst);
if (!uniform) {
/*
* There's no way to handle the DMA
* requirements with this uio. We
* could create a separate DMA area for
* the result and then copy it back,
* but for now we just bail and return
* an error. Note that uio requests
* > SAFE_MAX_DSIZE are handled because
* the DMA map and segment list for the
* destination wil result in a
* destination particle list that does
* the necessary scatter DMA.
*/
safestats.st_iovnotuniform++;
err = EINVAL;
goto errout;
}
} else
re->re_dst = re->re_src;
} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
if (nicealign && uniform == 1) {
/*
* Source layout is suitable for direct
* sharing of the DMA map and segment list.
*/
re->re_dst = re->re_src;
} else if (nicealign && uniform == 2) {
/*
* The source is properly aligned but requires a
* different particle list to handle DMA of the
* result. Create a new map and do the load to
* create the segment list. The particle
* descriptor setup code below will handle the
* rest.
*/
if (bus_dmamap_create(sc->sc_dstdmat,
BUS_DMA_NOWAIT, &re->re_dst_map)) {
safestats.st_nomap++;
err = ENOMEM;
goto errout;
}
if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
re->re_dst_map, re->re_dst_m,
safe_op_cb, &re->re_dst,
BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dstdmat,
re->re_dst_map);
re->re_dst_map = NULL;
safestats.st_noload++;
err = ENOMEM;
goto errout;
}
} else { /* !(aligned and/or uniform) */
int totlen, len;
struct mbuf *m, *top, **mp;
/*
* DMA constraints require that we allocate a
* new mbuf chain for the destination. We
* allocate an entire new set of mbufs of
* optimal/required size and then tell the
* hardware to copy any bits that are not
* created as a byproduct of the operation.
*/
if (!nicealign)
safestats.st_unaligned++;
if (!uniform)
safestats.st_notuniform++;
totlen = re->re_src_mapsize;
if (re->re_src_m->m_flags & M_PKTHDR) {
len = MHLEN;
MGETHDR(m, M_NOWAIT, MT_DATA);
if (m && !m_dup_pkthdr(m, re->re_src_m,
M_NOWAIT)) {
m_free(m);
m = NULL;
}
} else {
len = MLEN;
MGET(m, M_NOWAIT, MT_DATA);
}
if (m == NULL) {
safestats.st_nombuf++;
err = sc->sc_nqchip ? ERESTART : ENOMEM;
goto errout;
}
if (totlen >= MINCLSIZE) {
if (!(MCLGET(m, M_NOWAIT))) {
m_free(m);
safestats.st_nomcl++;
err = sc->sc_nqchip ?
ERESTART : ENOMEM;
goto errout;
}
len = MCLBYTES;
}
m->m_len = len;
top = NULL;
mp = &top;
while (totlen > 0) {
if (top) {
MGET(m, M_NOWAIT, MT_DATA);
if (m == NULL) {
m_freem(top);
safestats.st_nombuf++;
err = sc->sc_nqchip ?
ERESTART : ENOMEM;
goto errout;
}
len = MLEN;
}
if (top && totlen >= MINCLSIZE) {
if (!(MCLGET(m, M_NOWAIT))) {
*mp = m;
m_freem(top);
safestats.st_nomcl++;
err = sc->sc_nqchip ?
ERESTART : ENOMEM;
goto errout;
}
len = MCLBYTES;
}
m->m_len = len = min(totlen, len);
totlen -= len;
*mp = m;
mp = &m->m_next;
}
re->re_dst_m = top;
if (bus_dmamap_create(sc->sc_dstdmat,
BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
safestats.st_nomap++;
err = ENOMEM;
goto errout;
}
if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
re->re_dst_map, re->re_dst_m,
safe_op_cb, &re->re_dst,
BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dstdmat,
re->re_dst_map);
re->re_dst_map = NULL;
safestats.st_noload++;
err = ENOMEM;
goto errout;
}
if (re->re_src.mapsize > oplen) {
/*
* There's data following what the
* hardware will copy for us. If this
* isn't just the ICV (that's going to
* be written on completion), copy it
* to the new mbufs
*/
if (!(maccrd &&
(re->re_src.mapsize-oplen) == 12 &&
maccrd->crd_inject == oplen))
safe_mcopy(re->re_src_m,
re->re_dst_m,
oplen);
else
safestats.st_noicvcopy++;
}
}
} else {
safestats.st_badflags++;
err = EINVAL;
goto errout;
}
if (re->re_dst.nsegs > 1) {
re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
for (i = 0; i < re->re_dst_nsegs; i++) {
pd = sc->sc_dpfree;
KASSERT((pd->pd_flags&3) == 0 ||
(pd->pd_flags&3) == SAFE_PD_DONE,
("bogus dest particle descriptor; flags %x",
pd->pd_flags));
if (++(sc->sc_dpfree) == sc->sc_dpringtop)
sc->sc_dpfree = sc->sc_dpring;
pd->pd_addr = re->re_dst_segs[i].ds_addr;
pd->pd_flags = SAFE_PD_READY;
}
cmd0 |= SAFE_SA_CMD0_OSCATTER;
} else {
/*
* No need for scatter, reference the operand directly.
*/
re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
}
}
/*
* All done with setup; fillin the SA command words
* and the packet engine descriptor. The operation
* is now ready for submission to the hardware.
*/
sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
sa->sa_cmd1 = cmd1
| (coffset << SAFE_SA_CMD1_OFFSET_S)
| SAFE_SA_CMD1_SAREV1 /* Rev 1 SA data structure */
| SAFE_SA_CMD1_SRPCI
;
/*
* NB: the order of writes is important here. In case the
* chip is scanning the ring because of an outstanding request
* it might nab this one too. In that case we need to make
* sure the setup is complete before we write the length
* field of the descriptor as it signals the descriptor is
* ready for processing.
*/
re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
if (maccrd)
re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
re->re_desc.d_len = oplen
| SAFE_PE_LEN_READY
| (bypass << SAFE_PE_LEN_BYPASS_S)
;
safestats.st_ipackets++;
safestats.st_ibytes += oplen;
if (++(sc->sc_front) == sc->sc_ringtop)
sc->sc_front = sc->sc_ring;
/* XXX honor batching */
safe_feed(sc, re);
mtx_unlock(&sc->sc_ringmtx);
return (0);
errout:
if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
m_freem(re->re_dst_m);
if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
}
if (re->re_src_map != NULL) {
bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
}
mtx_unlock(&sc->sc_ringmtx);
if (err != ERESTART) {
crp->crp_etype = err;
crypto_done(crp);
} else {
sc->sc_needwakeup |= CRYPTO_SYMQ;
}
return (err);
}
static void
safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
{
struct cryptop *crp = (struct cryptop *)re->re_crp;
struct cryptodesc *crd;
safestats.st_opackets++;
safestats.st_obytes += re->re_dst.mapsize;
safe_dma_sync(&sc->sc_ringalloc,
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
re->re_desc.d_csr,
re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
safestats.st_peoperr++;
crp->crp_etype = EIO; /* something more meaningful? */
}
if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
}
bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
/*
* If result was written to a differet mbuf chain, swap
* it in as the return value and reclaim the original.
*/
if ((crp->crp_flags & CRYPTO_F_IMBUF) && re->re_src_m != re->re_dst_m) {
m_freem(re->re_src_m);
crp->crp_buf = (caddr_t)re->re_dst_m;
}
if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
/* copy out IV for future use */
for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
int ivsize;
if (crd->crd_alg == CRYPTO_DES_CBC ||
crd->crd_alg == CRYPTO_3DES_CBC) {
ivsize = 2*sizeof(u_int32_t);
} else if (crd->crd_alg == CRYPTO_AES_CBC) {
ivsize = 4*sizeof(u_int32_t);
} else
continue;
crypto_copydata(crp->crp_flags, crp->crp_buf,
crd->crd_skip + crd->crd_len - ivsize, ivsize,
(caddr_t)sc->sc_sessions[re->re_sesn].ses_iv);
break;
}
}
if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
/* copy out ICV result */
for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
crd->crd_alg == CRYPTO_SHA1_HMAC ||
crd->crd_alg == CRYPTO_NULL_HMAC))
continue;
if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
/*
* SHA-1 ICV's are byte-swapped; fix 'em up
* before copy them to their destination.
*/
re->re_sastate.sa_saved_indigest[0] =
bswap32(re->re_sastate.sa_saved_indigest[0]);
re->re_sastate.sa_saved_indigest[1] =
bswap32(re->re_sastate.sa_saved_indigest[1]);
re->re_sastate.sa_saved_indigest[2] =
bswap32(re->re_sastate.sa_saved_indigest[2]);
}
crypto_copyback(crp->crp_flags, crp->crp_buf,
crd->crd_inject,
sc->sc_sessions[re->re_sesn].ses_mlen,
(caddr_t)re->re_sastate.sa_saved_indigest);
break;
}
}
crypto_done(crp);
}
/*
* Copy all data past offset from srcm to dstm.
*/
static void
safe_mcopy(struct mbuf *srcm, struct mbuf *dstm, u_int offset)
{
u_int j, dlen, slen;
caddr_t dptr, sptr;
/*
* Advance src and dst to offset.
*/
j = offset;
while (j >= 0) {
if (srcm->m_len > j)
break;
j -= srcm->m_len;
srcm = srcm->m_next;
if (srcm == NULL)
return;
}
sptr = mtod(srcm, caddr_t) + j;
slen = srcm->m_len - j;
j = offset;
while (j >= 0) {
if (dstm->m_len > j)
break;
j -= dstm->m_len;
dstm = dstm->m_next;
if (dstm == NULL)
return;
}
dptr = mtod(dstm, caddr_t) + j;
dlen = dstm->m_len - j;
/*
* Copy everything that remains.
*/
for (;;) {
j = min(slen, dlen);
bcopy(sptr, dptr, j);
if (slen == j) {
srcm = srcm->m_next;
if (srcm == NULL)
return;
sptr = srcm->m_data;
slen = srcm->m_len;
} else
sptr += j, slen -= j;
if (dlen == j) {
dstm = dstm->m_next;
if (dstm == NULL)
return;
dptr = dstm->m_data;
dlen = dstm->m_len;
} else
dptr += j, dlen -= j;
}
}
#ifndef SAFE_NO_RNG
#define SAFE_RNG_MAXWAIT 1000
static void
safe_rng_init(struct safe_softc *sc)
{
u_int32_t w, v;
int i;
WRITE_REG(sc, SAFE_RNG_CTRL, 0);
/* use default value according to the manual */
WRITE_REG(sc, SAFE_RNG_CNFG, 0x834); /* magic from SafeNet */
WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
/*
* There is a bug in rev 1.0 of the 1140 that when the RNG
* is brought out of reset the ready status flag does not
* work until the RNG has finished its internal initialization.
*
* So in order to determine the device is through its
* initialization we must read the data register, using the
* status reg in the read in case it is initialized. Then read
* the data register until it changes from the first read.
* Once it changes read the data register until it changes
* again. At this time the RNG is considered initialized.
* This could take between 750ms - 1000ms in time.
*/
i = 0;
w = READ_REG(sc, SAFE_RNG_OUT);
do {
v = READ_REG(sc, SAFE_RNG_OUT);
if (v != w) {
w = v;
break;
}
DELAY(10);
} while (++i < SAFE_RNG_MAXWAIT);
/* Wait Until data changes again */
i = 0;
do {
v = READ_REG(sc, SAFE_RNG_OUT);
if (v != w)
break;
DELAY(10);
} while (++i < SAFE_RNG_MAXWAIT);
}
static __inline void
safe_rng_disable_short_cycle(struct safe_softc *sc)
{
WRITE_REG(sc, SAFE_RNG_CTRL,
READ_REG(sc, SAFE_RNG_CTRL) &~ SAFE_RNG_CTRL_SHORTEN);
}
static __inline void
safe_rng_enable_short_cycle(struct safe_softc *sc)
{
WRITE_REG(sc, SAFE_RNG_CTRL,
READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
}
static __inline u_int32_t
safe_rng_read(struct safe_softc *sc)
{
int i;
i = 0;
while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
;
return READ_REG(sc, SAFE_RNG_OUT);
}
static void
safe_rng(void *arg)
{
struct safe_softc *sc = arg;
u_int32_t buf[SAFE_RNG_MAXBUFSIZ]; /* NB: maybe move to softc */
u_int maxwords;
int i;
safestats.st_rng++;
/*
* Fetch the next block of data.
*/
maxwords = safe_rngbufsize;
if (maxwords > SAFE_RNG_MAXBUFSIZ)
maxwords = SAFE_RNG_MAXBUFSIZ;
retry:
for (i = 0; i < maxwords; i++)
buf[i] = safe_rng_read(sc);
/*
* Check the comparator alarm count and reset the h/w if
* it exceeds our threshold. This guards against the
* hardware oscillators resonating with external signals.
*/
if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
u_int32_t freq_inc, w;
DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
safestats.st_rngalarm++;
safe_rng_enable_short_cycle(sc);
freq_inc = 18;
for (i = 0; i < 64; i++) {
w = READ_REG(sc, SAFE_RNG_CNFG);
freq_inc = ((w + freq_inc) & 0x3fL);
w = ((w & ~0x3fL) | freq_inc);
WRITE_REG(sc, SAFE_RNG_CNFG, w);
WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
(void) safe_rng_read(sc);
DELAY(25);
if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
safe_rng_disable_short_cycle(sc);
goto retry;
}
freq_inc = 1;
}
safe_rng_disable_short_cycle(sc);
} else
WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
(*sc->sc_harvest)(sc->sc_rndtest, buf, maxwords*sizeof (u_int32_t));
callout_reset(&sc->sc_rngto,
hz * (safe_rnginterval ? safe_rnginterval : 1), safe_rng, sc);
}
#endif /* SAFE_NO_RNG */
static void
safe_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
bus_addr_t *paddr = (bus_addr_t*) arg;
*paddr = segs->ds_addr;
}
static int
safe_dma_malloc(
struct safe_softc *sc,
bus_size_t size,
struct safe_dma_alloc *dma,
int mapflags
)
{
int r;
r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
sizeof(u_int32_t), 0, /* alignment, bounds */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
size, /* maxsize */
1, /* nsegments */
size, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, NULL, /* locking */
&dma->dma_tag);
if (r != 0) {
device_printf(sc->sc_dev, "safe_dma_malloc: "
"bus_dma_tag_create failed; error %u\n", r);
goto fail_0;
}
r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
BUS_DMA_NOWAIT, &dma->dma_map);
if (r != 0) {
device_printf(sc->sc_dev, "safe_dma_malloc: "
"bus_dmammem_alloc failed; size %ju, error %u\n",
(uintmax_t)size, r);
goto fail_1;
}
r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
size,
safe_dmamap_cb,
&dma->dma_paddr,
mapflags | BUS_DMA_NOWAIT);
if (r != 0) {
device_printf(sc->sc_dev, "safe_dma_malloc: "
"bus_dmamap_load failed; error %u\n", r);
goto fail_2;
}
dma->dma_size = size;
return (0);
bus_dmamap_unload(dma->dma_tag, dma->dma_map);
fail_2:
bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
fail_1:
bus_dma_tag_destroy(dma->dma_tag);
fail_0:
dma->dma_tag = NULL;
return (r);
}
static void
safe_dma_free(struct safe_softc *sc, struct safe_dma_alloc *dma)
{
bus_dmamap_unload(dma->dma_tag, dma->dma_map);
bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
bus_dma_tag_destroy(dma->dma_tag);
}
/*
* Resets the board. Values in the regesters are left as is
* from the reset (i.e. initial values are assigned elsewhere).
*/
static void
safe_reset_board(struct safe_softc *sc)
{
u_int32_t v;
/*
* Reset the device. The manual says no delay
* is needed between marking and clearing reset.
*/
v = READ_REG(sc, SAFE_PE_DMACFG) &~
(SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
SAFE_PE_DMACFG_SGRESET);
WRITE_REG(sc, SAFE_PE_DMACFG, v
| SAFE_PE_DMACFG_PERESET
| SAFE_PE_DMACFG_PDRRESET
| SAFE_PE_DMACFG_SGRESET);
WRITE_REG(sc, SAFE_PE_DMACFG, v);
}
/*
* Initialize registers we need to touch only once.
*/
static void
safe_init_board(struct safe_softc *sc)
{
u_int32_t v, dwords;
v = READ_REG(sc, SAFE_PE_DMACFG);
v &=~ SAFE_PE_DMACFG_PEMODE;
v |= SAFE_PE_DMACFG_FSENA /* failsafe enable */
| SAFE_PE_DMACFG_GPRPCI /* gather ring on PCI */
| SAFE_PE_DMACFG_SPRPCI /* scatter ring on PCI */
| SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
| SAFE_PE_DMACFG_ESSA /* endian-swap SA's */
| SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
;
WRITE_REG(sc, SAFE_PE_DMACFG, v);
#if 0
/* XXX select byte swap based on host byte order */
WRITE_REG(sc, SAFE_ENDIAN, 0x1b);
#endif
if (sc->sc_chiprev == SAFE_REV(1,0)) {
/*
* Avoid large PCI DMA transfers. Rev 1.0 has a bug where
* "target mode transfers" done while the chip is DMA'ing
* >1020 bytes cause the hardware to lockup. To avoid this
* we reduce the max PCI transfer size and use small source
* particle descriptors (<= 256 bytes).
*/
WRITE_REG(sc, SAFE_DMA_CFG, 256);
device_printf(sc->sc_dev,
"Reduce max DMA size to %u words for rev %u.%u WAR\n",
(READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff,
SAFE_REV_MAJ(sc->sc_chiprev),
SAFE_REV_MIN(sc->sc_chiprev));
}
/* NB: operands+results are overlaid */
WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
/*
* Configure ring entry size and number of items in the ring.
*/
KASSERT((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
("PE ring entry not 32-bit aligned!"));
dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
WRITE_REG(sc, SAFE_PE_RINGCFG,
(dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
WRITE_REG(sc, SAFE_PE_RINGPOLL, 0); /* disable polling */
WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
WRITE_REG(sc, SAFE_PE_PARTSIZE,
(SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
/*
* NB: destination particles are fixed size. We use
* an mbuf cluster and require all results go to
* clusters or smaller.
*/
WRITE_REG(sc, SAFE_PE_PARTCFG, SAFE_MAX_DSIZE);
/* it's now safe to enable PE mode, do it */
WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
/*
* Configure hardware to use level-triggered interrupts and
* to interrupt after each descriptor is processed.
*/
WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
}
/*
* Init PCI registers
*/
static void
safe_init_pciregs(device_t dev)
{
}
/*
* Clean up after a chip crash.
* It is assumed that the caller in splimp()
*/
static void
safe_cleanchip(struct safe_softc *sc)
{
if (sc->sc_nqchip != 0) {
struct safe_ringentry *re = sc->sc_back;
while (re != sc->sc_front) {
if (re->re_desc.d_csr != 0)
safe_free_entry(sc, re);
if (++re == sc->sc_ringtop)
re = sc->sc_ring;
}
sc->sc_back = re;
sc->sc_nqchip = 0;
}
}
/*
* free a safe_q
* It is assumed that the caller is within splimp().
*/
static int
safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
{
struct cryptop *crp;
/*
* Free header MCR
*/
if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
m_freem(re->re_dst_m);
crp = (struct cryptop *)re->re_crp;
re->re_desc.d_csr = 0;
crp->crp_etype = EFAULT;
crypto_done(crp);
return(0);
}
/*
* Routine to reset the chip and clean up.
* It is assumed that the caller is in splimp()
*/
static void
safe_totalreset(struct safe_softc *sc)
{
safe_reset_board(sc);
safe_init_board(sc);
safe_cleanchip(sc);
}
/*
* Is the operand suitable aligned for direct DMA. Each
* segment must be aligned on a 32-bit boundary and all
* but the last segment must be a multiple of 4 bytes.
*/
static int
safe_dmamap_aligned(const struct safe_operand *op)
{
int i;
for (i = 0; i < op->nsegs; i++) {
if (op->segs[i].ds_addr & 3)
return (0);
if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
return (0);
}
return (1);
}
/*
* Is the operand suitable for direct DMA as the destination
* of an operation. The hardware requires that each ``particle''
* but the last in an operation result have the same size. We
* fix that size at SAFE_MAX_DSIZE bytes. This routine returns
* 0 if some segment is not a multiple of of this size, 1 if all
* segments are exactly this size, or 2 if segments are at worst
* a multple of this size.
*/
static int
safe_dmamap_uniform(const struct safe_operand *op)
{
int result = 1;
if (op->nsegs > 0) {
int i;
for (i = 0; i < op->nsegs-1; i++) {
if (op->segs[i].ds_len % SAFE_MAX_DSIZE)
return (0);
if (op->segs[i].ds_len != SAFE_MAX_DSIZE)
result = 2;
}
}
return (result);
}
#ifdef SAFE_DEBUG
static void
safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
{
printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n"
, tag
, READ_REG(sc, SAFE_DMA_ENDIAN)
, READ_REG(sc, SAFE_DMA_SRCADDR)
, READ_REG(sc, SAFE_DMA_DSTADDR)
, READ_REG(sc, SAFE_DMA_STAT)
);
}
static void
safe_dump_intrstate(struct safe_softc *sc, const char *tag)
{
printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n"
, tag
, READ_REG(sc, SAFE_HI_CFG)
, READ_REG(sc, SAFE_HI_MASK)
, READ_REG(sc, SAFE_HI_DESC_CNT)
, READ_REG(sc, SAFE_HU_STAT)
, READ_REG(sc, SAFE_HM_STAT)
);
}
static void
safe_dump_ringstate(struct safe_softc *sc, const char *tag)
{
u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
/* NB: assume caller has lock on ring */
printf("%s: ERNGSTAT %x (next %u) back %lu front %lu\n",
tag,
estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
(unsigned long)(sc->sc_back - sc->sc_ring),
(unsigned long)(sc->sc_front - sc->sc_ring));
}
static void
safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
{
int ix, nsegs;
ix = re - sc->sc_ring;
printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n"
, tag
, re, ix
, re->re_desc.d_csr
, re->re_desc.d_src
, re->re_desc.d_dst
, re->re_desc.d_sa
, re->re_desc.d_len
);
if (re->re_src.nsegs > 1) {
ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
sizeof(struct safe_pdesc);
for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
printf(" spd[%u] %p: %p size %u flags %x"
, ix, &sc->sc_spring[ix]
, (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
, sc->sc_spring[ix].pd_size
, sc->sc_spring[ix].pd_flags
);
if (sc->sc_spring[ix].pd_size == 0)
printf(" (zero!)");
printf("\n");
if (++ix == SAFE_TOTAL_SPART)
ix = 0;
}
}
if (re->re_dst.nsegs > 1) {
ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
sizeof(struct safe_pdesc);
for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
printf(" dpd[%u] %p: %p flags %x\n"
, ix, &sc->sc_dpring[ix]
, (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
, sc->sc_dpring[ix].pd_flags
);
if (++ix == SAFE_TOTAL_DPART)
ix = 0;
}
}
printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
printf("sa: key %x %x %x %x %x %x %x %x\n"
, re->re_sa.sa_key[0]
, re->re_sa.sa_key[1]
, re->re_sa.sa_key[2]
, re->re_sa.sa_key[3]
, re->re_sa.sa_key[4]
, re->re_sa.sa_key[5]
, re->re_sa.sa_key[6]
, re->re_sa.sa_key[7]
);
printf("sa: indigest %x %x %x %x %x\n"
, re->re_sa.sa_indigest[0]
, re->re_sa.sa_indigest[1]
, re->re_sa.sa_indigest[2]
, re->re_sa.sa_indigest[3]
, re->re_sa.sa_indigest[4]
);
printf("sa: outdigest %x %x %x %x %x\n"
, re->re_sa.sa_outdigest[0]
, re->re_sa.sa_outdigest[1]
, re->re_sa.sa_outdigest[2]
, re->re_sa.sa_outdigest[3]
, re->re_sa.sa_outdigest[4]
);
printf("sr: iv %x %x %x %x\n"
, re->re_sastate.sa_saved_iv[0]
, re->re_sastate.sa_saved_iv[1]
, re->re_sastate.sa_saved_iv[2]
, re->re_sastate.sa_saved_iv[3]
);
printf("sr: hashbc %u indigest %x %x %x %x %x\n"
, re->re_sastate.sa_saved_hashbc
, re->re_sastate.sa_saved_indigest[0]
, re->re_sastate.sa_saved_indigest[1]
, re->re_sastate.sa_saved_indigest[2]
, re->re_sastate.sa_saved_indigest[3]
, re->re_sastate.sa_saved_indigest[4]
);
}
static void
safe_dump_ring(struct safe_softc *sc, const char *tag)
{
mtx_lock(&sc->sc_ringmtx);
printf("\nSafeNet Ring State:\n");
safe_dump_intrstate(sc, tag);
safe_dump_dmastatus(sc, tag);
safe_dump_ringstate(sc, tag);
if (sc->sc_nqchip) {
struct safe_ringentry *re = sc->sc_back;
do {
safe_dump_request(sc, tag, re);
if (++re == sc->sc_ringtop)
re = sc->sc_ring;
} while (re != sc->sc_front);
}
mtx_unlock(&sc->sc_ringmtx);
}
static int
sysctl_hw_safe_dump(SYSCTL_HANDLER_ARGS)
{
char dmode[64];
int error;
strncpy(dmode, "", sizeof(dmode) - 1);
dmode[sizeof(dmode) - 1] = '\0';
error = sysctl_handle_string(oidp, &dmode[0], sizeof(dmode), req);
if (error == 0 && req->newptr != NULL) {
struct safe_softc *sc = safec;
if (!sc)
return EINVAL;
if (strncmp(dmode, "dma", 3) == 0)
safe_dump_dmastatus(sc, "safe0");
else if (strncmp(dmode, "int", 3) == 0)
safe_dump_intrstate(sc, "safe0");
else if (strncmp(dmode, "ring", 4) == 0)
safe_dump_ring(sc, "safe0");
else
return EINVAL;
}
return error;
}
SYSCTL_PROC(_hw_safe, OID_AUTO, dump, CTLTYPE_STRING | CTLFLAG_RW,
0, 0, sysctl_hw_safe_dump, "A", "Dump driver state");
#endif /* SAFE_DEBUG */
Index: head/sys/dev/sfxge/sfxge_rx.c
===================================================================
--- head/sys/dev/sfxge/sfxge_rx.c (revision 283290)
+++ head/sys/dev/sfxge/sfxge_rx.c (revision 283291)
@@ -1,1339 +1,1339 @@
/*-
* Copyright (c) 2010-2011 Solarflare Communications, Inc.
* All rights reserved.
*
* This software was developed in part by Philip Paeps under contract for
* Solarflare Communications, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/types.h>
#include <sys/mbuf.h>
#include <sys/smp.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <sys/limits.h>
#include <sys/syslog.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_vlan_var.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>
#include <netinet/tcp.h>
#include <machine/in_cksum.h>
#include "common/efx.h"
#include "sfxge.h"
#include "sfxge_rx.h"
#define RX_REFILL_THRESHOLD(_entries) (EFX_RXQ_LIMIT(_entries) * 9 / 10)
#ifdef SFXGE_LRO
SYSCTL_NODE(_hw_sfxge, OID_AUTO, lro, CTLFLAG_RD, NULL,
"Large receive offload (LRO) parameters");
#define SFXGE_LRO_PARAM(_param) SFXGE_PARAM(lro._param)
/* Size of the LRO hash table. Must be a power of 2. A larger table
* means we can accelerate a larger number of streams.
*/
static unsigned lro_table_size = 128;
TUNABLE_INT(SFXGE_LRO_PARAM(table_size), &lro_table_size);
SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, table_size, CTLFLAG_RDTUN,
&lro_table_size, 0,
"Size of the LRO hash table (must be a power of 2)");
/* Maximum length of a hash chain. If chains get too long then the lookup
* time increases and may exceed the benefit of LRO.
*/
static unsigned lro_chain_max = 20;
TUNABLE_INT(SFXGE_LRO_PARAM(chain_max), &lro_chain_max);
SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, chain_max, CTLFLAG_RDTUN,
&lro_chain_max, 0,
"The maximum length of a hash chain");
/* Maximum time (in ticks) that a connection can be idle before it's LRO
* state is discarded.
*/
static unsigned lro_idle_ticks; /* initialised in sfxge_rx_init() */
TUNABLE_INT(SFXGE_LRO_PARAM(idle_ticks), &lro_idle_ticks);
SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, idle_ticks, CTLFLAG_RDTUN,
&lro_idle_ticks, 0,
"The maximum time (in ticks) that a connection can be idle "
"before it's LRO state is discarded");
/* Number of packets with payload that must arrive in-order before a
* connection is eligible for LRO. The idea is we should avoid coalescing
* segments when the sender is in slow-start because reducing the ACK rate
* can damage performance.
*/
static int lro_slow_start_packets = 2000;
TUNABLE_INT(SFXGE_LRO_PARAM(slow_start_packets), &lro_slow_start_packets);
SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, slow_start_packets, CTLFLAG_RDTUN,
&lro_slow_start_packets, 0,
"Number of packets with payload that must arrive in-order before "
"a connection is eligible for LRO");
/* Number of packets with payload that must arrive in-order following loss
* before a connection is eligible for LRO. The idea is we should avoid
* coalescing segments when the sender is recovering from loss, because
* reducing the ACK rate can damage performance.
*/
static int lro_loss_packets = 20;
TUNABLE_INT(SFXGE_LRO_PARAM(loss_packets), &lro_loss_packets);
SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, loss_packets, CTLFLAG_RDTUN,
&lro_loss_packets, 0,
"Number of packets with payload that must arrive in-order "
"following loss before a connection is eligible for LRO");
/* Flags for sfxge_lro_conn::l2_id; must not collide with EVL_VLID_MASK */
#define SFXGE_LRO_L2_ID_VLAN 0x4000
#define SFXGE_LRO_L2_ID_IPV6 0x8000
#define SFXGE_LRO_CONN_IS_VLAN_ENCAP(c) ((c)->l2_id & SFXGE_LRO_L2_ID_VLAN)
#define SFXGE_LRO_CONN_IS_TCPIPV4(c) (!((c)->l2_id & SFXGE_LRO_L2_ID_IPV6))
/* Compare IPv6 addresses, avoiding conditional branches */
static unsigned long ipv6_addr_cmp(const struct in6_addr *left,
const struct in6_addr *right)
{
#if LONG_BIT == 64
const uint64_t *left64 = (const uint64_t *)left;
const uint64_t *right64 = (const uint64_t *)right;
return (left64[0] - right64[0]) | (left64[1] - right64[1]);
#else
return (left->s6_addr32[0] - right->s6_addr32[0]) |
(left->s6_addr32[1] - right->s6_addr32[1]) |
(left->s6_addr32[2] - right->s6_addr32[2]) |
(left->s6_addr32[3] - right->s6_addr32[3]);
#endif
}
#endif /* SFXGE_LRO */
void
sfxge_rx_qflush_done(struct sfxge_rxq *rxq)
{
rxq->flush_state = SFXGE_FLUSH_DONE;
}
void
sfxge_rx_qflush_failed(struct sfxge_rxq *rxq)
{
rxq->flush_state = SFXGE_FLUSH_FAILED;
}
static uint8_t toep_key[] = {
0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa
};
static void
sfxge_rx_post_refill(void *arg)
{
struct sfxge_rxq *rxq = arg;
struct sfxge_softc *sc;
unsigned int index;
struct sfxge_evq *evq;
uint16_t magic;
sc = rxq->sc;
index = rxq->index;
evq = sc->evq[index];
magic = SFXGE_MAGIC_RX_QREFILL | index;
/* This is guaranteed due to the start/stop order of rx and ev */
KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
("evq not started"));
KASSERT(rxq->init_state == SFXGE_RXQ_STARTED,
("rxq not started"));
efx_ev_qpost(evq->common, magic);
}
static void
sfxge_rx_schedule_refill(struct sfxge_rxq *rxq, boolean_t retrying)
{
/* Initially retry after 100 ms, but back off in case of
* repeated failures as we probably have to wait for the
* administrator to raise the pool limit. */
if (retrying)
rxq->refill_delay = min(rxq->refill_delay * 2, 10 * hz);
else
rxq->refill_delay = hz / 10;
callout_reset_curcpu(&rxq->refill_callout, rxq->refill_delay,
sfxge_rx_post_refill, rxq);
}
static struct mbuf *sfxge_rx_alloc_mbuf(struct sfxge_softc *sc)
{
struct mb_args args;
struct mbuf *m;
/* Allocate mbuf structure */
args.flags = M_PKTHDR;
args.type = MT_DATA;
m = (struct mbuf *)uma_zalloc_arg(zone_mbuf, &args, M_NOWAIT);
/* Allocate (and attach) packet buffer */
if (m != NULL && !uma_zalloc_arg(sc->rx_buffer_zone, m, M_NOWAIT)) {
uma_zfree(zone_mbuf, m);
m = NULL;
}
return (m);
}
#define SFXGE_REFILL_BATCH 64
static void
sfxge_rx_qfill(struct sfxge_rxq *rxq, unsigned int target, boolean_t retrying)
{
struct sfxge_softc *sc;
unsigned int index;
struct sfxge_evq *evq;
unsigned int batch;
unsigned int rxfill;
unsigned int mblksize;
int ntodo;
efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
sc = rxq->sc;
index = rxq->index;
evq = sc->evq[index];
prefetch_read_many(sc->enp);
prefetch_read_many(rxq->common);
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
return;
rxfill = rxq->added - rxq->completed;
KASSERT(rxfill <= EFX_RXQ_LIMIT(rxq->entries),
("rxfill > EFX_RXQ_LIMIT(rxq->entries)"));
ntodo = min(EFX_RXQ_LIMIT(rxq->entries) - rxfill, target);
KASSERT(ntodo <= EFX_RXQ_LIMIT(rxq->entries),
("ntodo > EFX_RQX_LIMIT(rxq->entries)"));
if (ntodo == 0)
return;
batch = 0;
mblksize = sc->rx_buffer_size;
while (ntodo-- > 0) {
unsigned int id;
struct sfxge_rx_sw_desc *rx_desc;
bus_dma_segment_t seg;
struct mbuf *m;
id = (rxq->added + batch) & rxq->ptr_mask;
rx_desc = &rxq->queue[id];
KASSERT(rx_desc->mbuf == NULL, ("rx_desc->mbuf != NULL"));
rx_desc->flags = EFX_DISCARD;
m = rx_desc->mbuf = sfxge_rx_alloc_mbuf(sc);
if (m == NULL)
break;
sfxge_map_mbuf_fast(rxq->mem.esm_tag, rxq->mem.esm_map, m, &seg);
addr[batch++] = seg.ds_addr;
if (batch == SFXGE_REFILL_BATCH) {
efx_rx_qpost(rxq->common, addr, mblksize, batch,
rxq->completed, rxq->added);
rxq->added += batch;
batch = 0;
}
}
if (ntodo != 0)
sfxge_rx_schedule_refill(rxq, retrying);
if (batch != 0) {
efx_rx_qpost(rxq->common, addr, mblksize, batch,
rxq->completed, rxq->added);
rxq->added += batch;
}
/* Make the descriptors visible to the hardware */
bus_dmamap_sync(rxq->mem.esm_tag, rxq->mem.esm_map,
BUS_DMASYNC_PREWRITE);
efx_rx_qpush(rxq->common, rxq->added);
}
void
sfxge_rx_qrefill(struct sfxge_rxq *rxq)
{
if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
return;
/* Make sure the queue is full */
sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(rxq->entries), B_TRUE);
}
static void __sfxge_rx_deliver(struct sfxge_softc *sc, struct mbuf *m)
{
struct ifnet *ifp = sc->ifnet;
m->m_pkthdr.rcvif = ifp;
m->m_pkthdr.csum_data = 0xffff;
ifp->if_input(ifp, m);
}
static void
sfxge_rx_deliver(struct sfxge_softc *sc, struct sfxge_rx_sw_desc *rx_desc)
{
struct mbuf *m = rx_desc->mbuf;
int flags = rx_desc->flags;
int csum_flags;
/* Convert checksum flags */
csum_flags = (flags & EFX_CKSUM_IPV4) ?
(CSUM_IP_CHECKED | CSUM_IP_VALID) : 0;
if (flags & EFX_CKSUM_TCPUDP)
csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
if (flags & (EFX_PKT_IPV4 | EFX_PKT_IPV6)) {
m->m_pkthdr.flowid = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_TOEPLITZ,
mtod(m, uint8_t *));
/* The hash covers a 4-tuple for TCP only */
M_HASHTYPE_SET(m,
(flags & EFX_PKT_IPV4) ?
((flags & EFX_PKT_TCP) ?
M_HASHTYPE_RSS_TCP_IPV4 : M_HASHTYPE_RSS_IPV4) :
((flags & EFX_PKT_TCP) ?
M_HASHTYPE_RSS_TCP_IPV6 : M_HASHTYPE_RSS_IPV6));
}
m->m_data += sc->rx_prefix_size;
m->m_len = rx_desc->size - sc->rx_prefix_size;
m->m_pkthdr.len = m->m_len;
m->m_pkthdr.csum_flags = csum_flags;
__sfxge_rx_deliver(sc, rx_desc->mbuf);
rx_desc->flags = EFX_DISCARD;
rx_desc->mbuf = NULL;
}
#ifdef SFXGE_LRO
static void
sfxge_lro_deliver(struct sfxge_lro_state *st, struct sfxge_lro_conn *c)
{
struct sfxge_softc *sc = st->sc;
struct mbuf *m = c->mbuf;
struct tcphdr *c_th;
int csum_flags;
KASSERT(m, ("no mbuf to deliver"));
++st->n_bursts;
/* Finish off packet munging and recalculate IP header checksum. */
if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
struct ip *iph = c->nh;
iph->ip_len = htons(iph->ip_len);
iph->ip_sum = 0;
iph->ip_sum = in_cksum_hdr(iph);
c_th = (struct tcphdr *)(iph + 1);
csum_flags = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
CSUM_IP_CHECKED | CSUM_IP_VALID);
} else {
struct ip6_hdr *iph = c->nh;
iph->ip6_plen = htons(iph->ip6_plen);
c_th = (struct tcphdr *)(iph + 1);
csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
}
c_th->th_win = c->th_last->th_win;
c_th->th_ack = c->th_last->th_ack;
if (c_th->th_off == c->th_last->th_off) {
/* Copy TCP options (take care to avoid going negative). */
int optlen = ((c_th->th_off - 5) & 0xf) << 2u;
memcpy(c_th + 1, c->th_last + 1, optlen);
}
m->m_pkthdr.flowid = c->conn_hash;
M_HASHTYPE_SET(m,
SFXGE_LRO_CONN_IS_TCPIPV4(c) ?
M_HASHTYPE_RSS_TCP_IPV4 : M_HASHTYPE_RSS_TCP_IPV6);
m->m_pkthdr.csum_flags = csum_flags;
__sfxge_rx_deliver(sc, m);
c->mbuf = NULL;
c->delivered = 1;
}
/* Drop the given connection, and add it to the free list. */
static void sfxge_lro_drop(struct sfxge_rxq *rxq, struct sfxge_lro_conn *c)
{
unsigned bucket;
KASSERT(!c->mbuf, ("found orphaned mbuf"));
if (c->next_buf.mbuf != NULL) {
sfxge_rx_deliver(rxq->sc, &c->next_buf);
LIST_REMOVE(c, active_link);
}
bucket = c->conn_hash & rxq->lro.conns_mask;
KASSERT(rxq->lro.conns_n[bucket] > 0, ("LRO: bucket fill level wrong"));
--rxq->lro.conns_n[bucket];
TAILQ_REMOVE(&rxq->lro.conns[bucket], c, link);
TAILQ_INSERT_HEAD(&rxq->lro.free_conns, c, link);
}
/* Stop tracking connections that have gone idle in order to keep hash
* chains short.
*/
static void sfxge_lro_purge_idle(struct sfxge_rxq *rxq, unsigned now)
{
struct sfxge_lro_conn *c;
unsigned i;
KASSERT(LIST_EMPTY(&rxq->lro.active_conns),
("found active connections"));
rxq->lro.last_purge_ticks = now;
for (i = 0; i <= rxq->lro.conns_mask; ++i) {
if (TAILQ_EMPTY(&rxq->lro.conns[i]))
continue;
c = TAILQ_LAST(&rxq->lro.conns[i], sfxge_lro_tailq);
if (now - c->last_pkt_ticks > lro_idle_ticks) {
++rxq->lro.n_drop_idle;
sfxge_lro_drop(rxq, c);
}
}
}
static void
sfxge_lro_merge(struct sfxge_lro_state *st, struct sfxge_lro_conn *c,
struct mbuf *mbuf, struct tcphdr *th)
{
struct tcphdr *c_th;
/* Tack the new mbuf onto the chain. */
KASSERT(!mbuf->m_next, ("mbuf already chained"));
c->mbuf_tail->m_next = mbuf;
c->mbuf_tail = mbuf;
/* Increase length appropriately */
c->mbuf->m_pkthdr.len += mbuf->m_len;
/* Update the connection state flags */
if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
struct ip *iph = c->nh;
iph->ip_len += mbuf->m_len;
c_th = (struct tcphdr *)(iph + 1);
} else {
struct ip6_hdr *iph = c->nh;
iph->ip6_plen += mbuf->m_len;
c_th = (struct tcphdr *)(iph + 1);
}
c_th->th_flags |= (th->th_flags & TH_PUSH);
c->th_last = th;
++st->n_merges;
/* Pass packet up now if another segment could overflow the IP
* length.
*/
if (c->mbuf->m_pkthdr.len > 65536 - 9200)
sfxge_lro_deliver(st, c);
}
static void
sfxge_lro_start(struct sfxge_lro_state *st, struct sfxge_lro_conn *c,
struct mbuf *mbuf, void *nh, struct tcphdr *th)
{
/* Start the chain */
c->mbuf = mbuf;
c->mbuf_tail = c->mbuf;
c->nh = nh;
c->th_last = th;
mbuf->m_pkthdr.len = mbuf->m_len;
/* Mangle header fields for later processing */
if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
struct ip *iph = nh;
iph->ip_len = ntohs(iph->ip_len);
} else {
struct ip6_hdr *iph = nh;
iph->ip6_plen = ntohs(iph->ip6_plen);
}
}
/* Try to merge or otherwise hold or deliver (as appropriate) the
* packet buffered for this connection (c->next_buf). Return a flag
* indicating whether the connection is still active for LRO purposes.
*/
static int
sfxge_lro_try_merge(struct sfxge_rxq *rxq, struct sfxge_lro_conn *c)
{
struct sfxge_rx_sw_desc *rx_buf = &c->next_buf;
char *eh = c->next_eh;
int data_length, hdr_length, dont_merge;
unsigned th_seq, pkt_length;
struct tcphdr *th;
unsigned now;
if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
struct ip *iph = c->next_nh;
th = (struct tcphdr *)(iph + 1);
pkt_length = ntohs(iph->ip_len) + (char *) iph - eh;
} else {
struct ip6_hdr *iph = c->next_nh;
th = (struct tcphdr *)(iph + 1);
pkt_length = ntohs(iph->ip6_plen) + (char *) th - eh;
}
hdr_length = (char *) th + th->th_off * 4 - eh;
data_length = (min(pkt_length, rx_buf->size - rxq->sc->rx_prefix_size) -
hdr_length);
th_seq = ntohl(th->th_seq);
dont_merge = ((data_length <= 0)
| (th->th_flags & (TH_URG | TH_SYN | TH_RST | TH_FIN)));
/* Check for options other than aligned timestamp. */
if (th->th_off != 5) {
const uint32_t *opt_ptr = (const uint32_t *) (th + 1);
if (th->th_off == 8 &&
opt_ptr[0] == ntohl((TCPOPT_NOP << 24) |
(TCPOPT_NOP << 16) |
(TCPOPT_TIMESTAMP << 8) |
TCPOLEN_TIMESTAMP)) {
/* timestamp option -- okay */
} else {
dont_merge = 1;
}
}
if (__predict_false(th_seq != c->next_seq)) {
/* Out-of-order, so start counting again. */
if (c->mbuf != NULL)
sfxge_lro_deliver(&rxq->lro, c);
c->n_in_order_pkts -= lro_loss_packets;
c->next_seq = th_seq + data_length;
++rxq->lro.n_misorder;
goto deliver_buf_out;
}
c->next_seq = th_seq + data_length;
now = ticks;
if (now - c->last_pkt_ticks > lro_idle_ticks) {
++rxq->lro.n_drop_idle;
if (c->mbuf != NULL)
sfxge_lro_deliver(&rxq->lro, c);
sfxge_lro_drop(rxq, c);
return (0);
}
c->last_pkt_ticks = ticks;
if (c->n_in_order_pkts < lro_slow_start_packets) {
/* May be in slow-start, so don't merge. */
++rxq->lro.n_slow_start;
++c->n_in_order_pkts;
goto deliver_buf_out;
}
if (__predict_false(dont_merge)) {
if (c->mbuf != NULL)
sfxge_lro_deliver(&rxq->lro, c);
if (th->th_flags & (TH_FIN | TH_RST)) {
++rxq->lro.n_drop_closed;
sfxge_lro_drop(rxq, c);
return (0);
}
goto deliver_buf_out;
}
rx_buf->mbuf->m_data += rxq->sc->rx_prefix_size;
if (__predict_true(c->mbuf != NULL)) {
/* Remove headers and any padding */
rx_buf->mbuf->m_data += hdr_length;
rx_buf->mbuf->m_len = data_length;
sfxge_lro_merge(&rxq->lro, c, rx_buf->mbuf, th);
} else {
/* Remove any padding */
rx_buf->mbuf->m_len = pkt_length;
sfxge_lro_start(&rxq->lro, c, rx_buf->mbuf, c->next_nh, th);
}
rx_buf->mbuf = NULL;
return (1);
deliver_buf_out:
sfxge_rx_deliver(rxq->sc, rx_buf);
return (1);
}
static void sfxge_lro_new_conn(struct sfxge_lro_state *st, uint32_t conn_hash,
uint16_t l2_id, void *nh, struct tcphdr *th)
{
unsigned bucket = conn_hash & st->conns_mask;
struct sfxge_lro_conn *c;
if (st->conns_n[bucket] >= lro_chain_max) {
++st->n_too_many;
return;
}
if (!TAILQ_EMPTY(&st->free_conns)) {
c = TAILQ_FIRST(&st->free_conns);
TAILQ_REMOVE(&st->free_conns, c, link);
} else {
c = malloc(sizeof(*c), M_SFXGE, M_NOWAIT);
if (c == NULL)
return;
c->mbuf = NULL;
c->next_buf.mbuf = NULL;
}
/* Create the connection tracking data */
++st->conns_n[bucket];
TAILQ_INSERT_HEAD(&st->conns[bucket], c, link);
c->l2_id = l2_id;
c->conn_hash = conn_hash;
c->source = th->th_sport;
c->dest = th->th_dport;
c->n_in_order_pkts = 0;
c->last_pkt_ticks = *(volatile int *)&ticks;
c->delivered = 0;
++st->n_new_stream;
/* NB. We don't initialise c->next_seq, and it doesn't matter what
* value it has. Most likely the next packet received for this
* connection will not match -- no harm done.
*/
}
/* Process mbuf and decide whether to dispatch it to the stack now or
* later.
*/
static void
sfxge_lro(struct sfxge_rxq *rxq, struct sfxge_rx_sw_desc *rx_buf)
{
struct sfxge_softc *sc = rxq->sc;
struct mbuf *m = rx_buf->mbuf;
struct ether_header *eh;
struct sfxge_lro_conn *c;
uint16_t l2_id;
uint16_t l3_proto;
void *nh;
struct tcphdr *th;
uint32_t conn_hash;
unsigned bucket;
/* Get the hardware hash */
conn_hash = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_TOEPLITZ,
mtod(m, uint8_t *));
eh = (struct ether_header *)(m->m_data + sc->rx_prefix_size);
if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
struct ether_vlan_header *veh = (struct ether_vlan_header *)eh;
l2_id = EVL_VLANOFTAG(ntohs(veh->evl_tag)) |
SFXGE_LRO_L2_ID_VLAN;
l3_proto = veh->evl_proto;
nh = veh + 1;
} else {
l2_id = 0;
l3_proto = eh->ether_type;
nh = eh + 1;
}
/* Check whether this is a suitable packet (unfragmented
* TCP/IPv4 or TCP/IPv6). If so, find the TCP header and
* length, and compute a hash if necessary. If not, return.
*/
if (l3_proto == htons(ETHERTYPE_IP)) {
struct ip *iph = nh;
KASSERT(iph->ip_p == IPPROTO_TCP,
("IPv4 protocol is not TCP, but packet marker is set"));
if ((iph->ip_hl - (sizeof(*iph) >> 2u)) |
(iph->ip_off & htons(IP_MF | IP_OFFMASK)))
goto deliver_now;
th = (struct tcphdr *)(iph + 1);
} else if (l3_proto == htons(ETHERTYPE_IPV6)) {
struct ip6_hdr *iph = nh;
KASSERT(iph->ip6_nxt == IPPROTO_TCP,
("IPv6 next header is not TCP, but packet marker is set"));
l2_id |= SFXGE_LRO_L2_ID_IPV6;
th = (struct tcphdr *)(iph + 1);
} else {
goto deliver_now;
}
bucket = conn_hash & rxq->lro.conns_mask;
TAILQ_FOREACH(c, &rxq->lro.conns[bucket], link) {
if ((c->l2_id - l2_id) | (c->conn_hash - conn_hash))
continue;
if ((c->source - th->th_sport) | (c->dest - th->th_dport))
continue;
if (c->mbuf != NULL) {
if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
struct ip *c_iph, *iph = nh;
c_iph = c->nh;
if ((c_iph->ip_src.s_addr - iph->ip_src.s_addr) |
(c_iph->ip_dst.s_addr - iph->ip_dst.s_addr))
continue;
} else {
struct ip6_hdr *c_iph, *iph = nh;
c_iph = c->nh;
if (ipv6_addr_cmp(&c_iph->ip6_src, &iph->ip6_src) |
ipv6_addr_cmp(&c_iph->ip6_dst, &iph->ip6_dst))
continue;
}
}
/* Re-insert at head of list to reduce lookup time. */
TAILQ_REMOVE(&rxq->lro.conns[bucket], c, link);
TAILQ_INSERT_HEAD(&rxq->lro.conns[bucket], c, link);
if (c->next_buf.mbuf != NULL) {
if (!sfxge_lro_try_merge(rxq, c))
goto deliver_now;
} else {
LIST_INSERT_HEAD(&rxq->lro.active_conns, c,
active_link);
}
c->next_buf = *rx_buf;
c->next_eh = eh;
c->next_nh = nh;
rx_buf->mbuf = NULL;
rx_buf->flags = EFX_DISCARD;
return;
}
sfxge_lro_new_conn(&rxq->lro, conn_hash, l2_id, nh, th);
deliver_now:
sfxge_rx_deliver(sc, rx_buf);
}
static void sfxge_lro_end_of_burst(struct sfxge_rxq *rxq)
{
struct sfxge_lro_state *st = &rxq->lro;
struct sfxge_lro_conn *c;
unsigned t;
while (!LIST_EMPTY(&st->active_conns)) {
c = LIST_FIRST(&st->active_conns);
if (!c->delivered && c->mbuf != NULL)
sfxge_lro_deliver(st, c);
if (sfxge_lro_try_merge(rxq, c)) {
if (c->mbuf != NULL)
sfxge_lro_deliver(st, c);
LIST_REMOVE(c, active_link);
}
c->delivered = 0;
}
t = *(volatile int *)&ticks;
if (__predict_false(t != st->last_purge_ticks))
sfxge_lro_purge_idle(rxq, t);
}
#else /* !SFXGE_LRO */
static void
sfxge_lro(struct sfxge_rxq *rxq, struct sfxge_rx_sw_desc *rx_buf)
{
}
static void
sfxge_lro_end_of_burst(struct sfxge_rxq *rxq)
{
}
#endif /* SFXGE_LRO */
void
sfxge_rx_qcomplete(struct sfxge_rxq *rxq, boolean_t eop)
{
struct sfxge_softc *sc = rxq->sc;
int if_capenable = sc->ifnet->if_capenable;
int lro_enabled = if_capenable & IFCAP_LRO;
unsigned int index;
struct sfxge_evq *evq;
unsigned int completed;
unsigned int level;
struct mbuf *m;
struct sfxge_rx_sw_desc *prev = NULL;
index = rxq->index;
evq = sc->evq[index];
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
completed = rxq->completed;
while (completed != rxq->pending) {
unsigned int id;
struct sfxge_rx_sw_desc *rx_desc;
id = completed++ & rxq->ptr_mask;
rx_desc = &rxq->queue[id];
m = rx_desc->mbuf;
if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
goto discard;
if (rx_desc->flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
goto discard;
prefetch_read_many(mtod(m, caddr_t));
switch (rx_desc->flags & (EFX_PKT_IPV4 | EFX_PKT_IPV6)) {
case EFX_PKT_IPV4:
if (~if_capenable & IFCAP_RXCSUM)
rx_desc->flags &=
~(EFX_CKSUM_IPV4 | EFX_CKSUM_TCPUDP);
break;
case EFX_PKT_IPV6:
if (~if_capenable & IFCAP_RXCSUM_IPV6)
rx_desc->flags &= ~EFX_CKSUM_TCPUDP;
break;
case 0:
/* Check for loopback packets */
{
struct ether_header *etherhp;
/*LINTED*/
etherhp = mtod(m, struct ether_header *);
if (etherhp->ether_type ==
htons(SFXGE_ETHERTYPE_LOOPBACK)) {
EFSYS_PROBE(loopback);
rxq->loopback++;
goto discard;
}
}
break;
default:
KASSERT(B_FALSE,
("Rx descriptor with both IPv4 and IPv6 flags"));
goto discard;
}
/* Pass packet up the stack or into LRO (pipelined) */
if (prev != NULL) {
if (lro_enabled &&
((prev->flags & (EFX_PKT_TCP | EFX_CKSUM_TCPUDP)) ==
(EFX_PKT_TCP | EFX_CKSUM_TCPUDP)))
sfxge_lro(rxq, prev);
else
sfxge_rx_deliver(sc, prev);
}
prev = rx_desc;
continue;
discard:
/* Return the packet to the pool */
m_free(m);
rx_desc->mbuf = NULL;
}
rxq->completed = completed;
level = rxq->added - rxq->completed;
/* Pass last packet up the stack or into LRO */
if (prev != NULL) {
if (lro_enabled &&
((prev->flags & (EFX_PKT_TCP | EFX_CKSUM_TCPUDP)) ==
(EFX_PKT_TCP | EFX_CKSUM_TCPUDP)))
sfxge_lro(rxq, prev);
else
sfxge_rx_deliver(sc, prev);
}
/*
* If there are any pending flows and this is the end of the
* poll then they must be completed.
*/
if (eop)
sfxge_lro_end_of_burst(rxq);
/* Top up the queue if necessary */
if (level < rxq->refill_threshold)
sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(rxq->entries), B_FALSE);
}
static void
sfxge_rx_qstop(struct sfxge_softc *sc, unsigned int index)
{
struct sfxge_rxq *rxq;
struct sfxge_evq *evq;
unsigned int count;
rxq = sc->rxq[index];
evq = sc->evq[index];
SFXGE_EVQ_LOCK(evq);
KASSERT(rxq->init_state == SFXGE_RXQ_STARTED,
("rxq not started"));
rxq->init_state = SFXGE_RXQ_INITIALIZED;
callout_stop(&rxq->refill_callout);
again:
rxq->flush_state = SFXGE_FLUSH_PENDING;
/* Flush the receive queue */
efx_rx_qflush(rxq->common);
SFXGE_EVQ_UNLOCK(evq);
count = 0;
do {
/* Spin for 100 ms */
DELAY(100000);
if (rxq->flush_state != SFXGE_FLUSH_PENDING)
break;
} while (++count < 20);
SFXGE_EVQ_LOCK(evq);
if (rxq->flush_state == SFXGE_FLUSH_FAILED)
goto again;
rxq->flush_state = SFXGE_FLUSH_DONE;
rxq->pending = rxq->added;
sfxge_rx_qcomplete(rxq, B_TRUE);
KASSERT(rxq->completed == rxq->pending,
("rxq->completed != rxq->pending"));
rxq->added = 0;
rxq->pending = 0;
rxq->completed = 0;
rxq->loopback = 0;
/* Destroy the common code receive queue. */
efx_rx_qdestroy(rxq->common);
efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
EFX_RXQ_NBUFS(sc->rxq_entries));
SFXGE_EVQ_UNLOCK(evq);
}
static int
sfxge_rx_qstart(struct sfxge_softc *sc, unsigned int index)
{
struct sfxge_rxq *rxq;
efsys_mem_t *esmp;
struct sfxge_evq *evq;
int rc;
rxq = sc->rxq[index];
esmp = &rxq->mem;
evq = sc->evq[index];
KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED,
("rxq->init_state != SFXGE_RXQ_INITIALIZED"));
KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
("evq->init_state != SFXGE_EVQ_STARTED"));
/* Program the buffer table. */
if ((rc = efx_sram_buf_tbl_set(sc->enp, rxq->buf_base_id, esmp,
EFX_RXQ_NBUFS(sc->rxq_entries))) != 0)
return (rc);
/* Create the common code receive queue. */
if ((rc = efx_rx_qcreate(sc->enp, index, index, EFX_RXQ_TYPE_DEFAULT,
esmp, sc->rxq_entries, rxq->buf_base_id, evq->common,
&rxq->common)) != 0)
goto fail;
SFXGE_EVQ_LOCK(evq);
/* Enable the receive queue. */
efx_rx_qenable(rxq->common);
rxq->init_state = SFXGE_RXQ_STARTED;
/* Try to fill the queue from the pool. */
sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(sc->rxq_entries), B_FALSE);
SFXGE_EVQ_UNLOCK(evq);
return (0);
fail:
efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
EFX_RXQ_NBUFS(sc->rxq_entries));
return (rc);
}
void
sfxge_rx_stop(struct sfxge_softc *sc)
{
int index;
/* Stop the receive queue(s) */
index = sc->rxq_count;
while (--index >= 0)
sfxge_rx_qstop(sc, index);
sc->rx_prefix_size = 0;
sc->rx_buffer_size = 0;
efx_rx_fini(sc->enp);
}
int
sfxge_rx_start(struct sfxge_softc *sc)
{
struct sfxge_intr *intr;
int index;
int rc;
intr = &sc->intr;
/* Initialize the common code receive module. */
if ((rc = efx_rx_init(sc->enp)) != 0)
return (rc);
/* Calculate the receive packet buffer size. */
sc->rx_prefix_size = EFX_RX_PREFIX_SIZE;
sc->rx_buffer_size = (EFX_MAC_PDU(sc->ifnet->if_mtu) +
sc->rx_prefix_size);
/* Select zone for packet buffers */
if (sc->rx_buffer_size <= MCLBYTES)
sc->rx_buffer_zone = zone_clust;
else if (sc->rx_buffer_size <= MJUMPAGESIZE)
sc->rx_buffer_zone = zone_jumbop;
else if (sc->rx_buffer_size <= MJUM9BYTES)
sc->rx_buffer_zone = zone_jumbo9;
else
sc->rx_buffer_zone = zone_jumbo16;
/*
* Set up the scale table. Enable all hash types and hash insertion.
*/
for (index = 0; index < SFXGE_RX_SCALE_MAX; index++)
sc->rx_indir_table[index] = index % sc->rxq_count;
if ((rc = efx_rx_scale_tbl_set(sc->enp, sc->rx_indir_table,
SFXGE_RX_SCALE_MAX)) != 0)
goto fail;
(void)efx_rx_scale_mode_set(sc->enp, EFX_RX_HASHALG_TOEPLITZ,
(1 << EFX_RX_HASH_IPV4) | (1 << EFX_RX_HASH_TCPIPV4) |
(1 << EFX_RX_HASH_IPV6) | (1 << EFX_RX_HASH_TCPIPV6), B_TRUE);
if ((rc = efx_rx_scale_toeplitz_ipv4_key_set(sc->enp, toep_key,
sizeof(toep_key))) != 0)
goto fail;
/* Start the receive queue(s). */
for (index = 0; index < sc->rxq_count; index++) {
if ((rc = sfxge_rx_qstart(sc, index)) != 0)
goto fail2;
}
return (0);
fail2:
while (--index >= 0)
sfxge_rx_qstop(sc, index);
fail:
efx_rx_fini(sc->enp);
return (rc);
}
#ifdef SFXGE_LRO
static void sfxge_lro_init(struct sfxge_rxq *rxq)
{
struct sfxge_lro_state *st = &rxq->lro;
unsigned i;
st->conns_mask = lro_table_size - 1;
KASSERT(!((st->conns_mask + 1) & st->conns_mask),
("lro_table_size must be a power of 2"));
st->sc = rxq->sc;
st->conns = malloc((st->conns_mask + 1) * sizeof(st->conns[0]),
M_SFXGE, M_WAITOK);
st->conns_n = malloc((st->conns_mask + 1) * sizeof(st->conns_n[0]),
M_SFXGE, M_WAITOK);
for (i = 0; i <= st->conns_mask; ++i) {
TAILQ_INIT(&st->conns[i]);
st->conns_n[i] = 0;
}
LIST_INIT(&st->active_conns);
TAILQ_INIT(&st->free_conns);
}
static void sfxge_lro_fini(struct sfxge_rxq *rxq)
{
struct sfxge_lro_state *st = &rxq->lro;
struct sfxge_lro_conn *c;
unsigned i;
/* Return cleanly if sfxge_lro_init() has not been called. */
if (st->conns == NULL)
return;
KASSERT(LIST_EMPTY(&st->active_conns), ("found active connections"));
for (i = 0; i <= st->conns_mask; ++i) {
while (!TAILQ_EMPTY(&st->conns[i])) {
c = TAILQ_LAST(&st->conns[i], sfxge_lro_tailq);
sfxge_lro_drop(rxq, c);
}
}
while (!TAILQ_EMPTY(&st->free_conns)) {
c = TAILQ_FIRST(&st->free_conns);
TAILQ_REMOVE(&st->free_conns, c, link);
KASSERT(!c->mbuf, ("found orphaned mbuf"));
free(c, M_SFXGE);
}
free(st->conns_n, M_SFXGE);
free(st->conns, M_SFXGE);
st->conns = NULL;
}
#else
static void
sfxge_lro_init(struct sfxge_rxq *rxq)
{
}
static void
sfxge_lro_fini(struct sfxge_rxq *rxq)
{
}
#endif /* SFXGE_LRO */
static void
sfxge_rx_qfini(struct sfxge_softc *sc, unsigned int index)
{
struct sfxge_rxq *rxq;
rxq = sc->rxq[index];
KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED,
("rxq->init_state != SFXGE_RXQ_INITIALIZED"));
/* Free the context array and the flow table. */
free(rxq->queue, M_SFXGE);
sfxge_lro_fini(rxq);
/* Release DMA memory. */
sfxge_dma_free(&rxq->mem);
sc->rxq[index] = NULL;
free(rxq, M_SFXGE);
}
static int
sfxge_rx_qinit(struct sfxge_softc *sc, unsigned int index)
{
struct sfxge_rxq *rxq;
struct sfxge_evq *evq;
efsys_mem_t *esmp;
int rc;
KASSERT(index < sc->rxq_count, ("index >= %d", sc->rxq_count));
rxq = malloc(sizeof(struct sfxge_rxq), M_SFXGE, M_ZERO | M_WAITOK);
rxq->sc = sc;
rxq->index = index;
rxq->entries = sc->rxq_entries;
rxq->ptr_mask = rxq->entries - 1;
rxq->refill_threshold = RX_REFILL_THRESHOLD(rxq->entries);
sc->rxq[index] = rxq;
esmp = &rxq->mem;
evq = sc->evq[index];
/* Allocate and zero DMA space. */
if ((rc = sfxge_dma_alloc(sc, EFX_RXQ_SIZE(sc->rxq_entries), esmp)) != 0)
return (rc);
/* Allocate buffer table entries. */
sfxge_sram_buf_tbl_alloc(sc, EFX_RXQ_NBUFS(sc->rxq_entries),
&rxq->buf_base_id);
/* Allocate the context array and the flow table. */
rxq->queue = malloc(sizeof(struct sfxge_rx_sw_desc) * sc->rxq_entries,
M_SFXGE, M_WAITOK | M_ZERO);
sfxge_lro_init(rxq);
- callout_init(&rxq->refill_callout, B_TRUE);
+ callout_init(&rxq->refill_callout, 1);
rxq->init_state = SFXGE_RXQ_INITIALIZED;
return (0);
}
static const struct {
const char *name;
size_t offset;
} sfxge_rx_stats[] = {
#define SFXGE_RX_STAT(name, member) \
{ #name, offsetof(struct sfxge_rxq, member) }
#ifdef SFXGE_LRO
SFXGE_RX_STAT(lro_merges, lro.n_merges),
SFXGE_RX_STAT(lro_bursts, lro.n_bursts),
SFXGE_RX_STAT(lro_slow_start, lro.n_slow_start),
SFXGE_RX_STAT(lro_misorder, lro.n_misorder),
SFXGE_RX_STAT(lro_too_many, lro.n_too_many),
SFXGE_RX_STAT(lro_new_stream, lro.n_new_stream),
SFXGE_RX_STAT(lro_drop_idle, lro.n_drop_idle),
SFXGE_RX_STAT(lro_drop_closed, lro.n_drop_closed)
#endif
};
static int
sfxge_rx_stat_handler(SYSCTL_HANDLER_ARGS)
{
struct sfxge_softc *sc = arg1;
unsigned int id = arg2;
unsigned int sum, index;
/* Sum across all RX queues */
sum = 0;
for (index = 0; index < sc->rxq_count; index++)
sum += *(unsigned int *)((caddr_t)sc->rxq[index] +
sfxge_rx_stats[id].offset);
return (SYSCTL_OUT(req, &sum, sizeof(sum)));
}
static void
sfxge_rx_stat_init(struct sfxge_softc *sc)
{
struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
struct sysctl_oid_list *stat_list;
unsigned int id;
stat_list = SYSCTL_CHILDREN(sc->stats_node);
for (id = 0; id < nitems(sfxge_rx_stats); id++) {
SYSCTL_ADD_PROC(
ctx, stat_list,
OID_AUTO, sfxge_rx_stats[id].name,
CTLTYPE_UINT|CTLFLAG_RD,
sc, id, sfxge_rx_stat_handler, "IU",
"");
}
}
void
sfxge_rx_fini(struct sfxge_softc *sc)
{
int index;
index = sc->rxq_count;
while (--index >= 0)
sfxge_rx_qfini(sc, index);
sc->rxq_count = 0;
}
int
sfxge_rx_init(struct sfxge_softc *sc)
{
struct sfxge_intr *intr;
int index;
int rc;
#ifdef SFXGE_LRO
if (!ISP2(lro_table_size)) {
log(LOG_ERR, "%s=%u must be power of 2",
SFXGE_LRO_PARAM(table_size), lro_table_size);
rc = EINVAL;
goto fail_lro_table_size;
}
if (lro_idle_ticks == 0)
lro_idle_ticks = hz / 10 + 1; /* 100 ms */
#endif
intr = &sc->intr;
sc->rxq_count = intr->n_alloc;
KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
("intr->state != SFXGE_INTR_INITIALIZED"));
/* Initialize the receive queue(s) - one per interrupt. */
for (index = 0; index < sc->rxq_count; index++) {
if ((rc = sfxge_rx_qinit(sc, index)) != 0)
goto fail;
}
sfxge_rx_stat_init(sc);
return (0);
fail:
/* Tear down the receive queue(s). */
while (--index >= 0)
sfxge_rx_qfini(sc, index);
sc->rxq_count = 0;
#ifdef SFXGE_LRO
fail_lro_table_size:
#endif
return (rc);
}
Index: head/sys/dev/sound/midi/mpu401.c
===================================================================
--- head/sys/dev/sound/midi/mpu401.c (revision 283290)
+++ head/sys/dev/sound/midi/mpu401.c (revision 283291)
@@ -1,298 +1,298 @@
/*-
* Copyright (c) 2003 Mathew Kanner
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/types.h>
#include <sys/param.h>
#include <sys/queue.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <sys/kobj.h>
#include <sys/malloc.h>
#include <sys/bus.h> /* to get driver_intr_t */
#ifdef HAVE_KERNEL_OPTION_HEADERS
#include "opt_snd.h"
#endif
#include <dev/sound/midi/mpu401.h>
#include <dev/sound/midi/midi.h>
#include "mpu_if.h"
#include "mpufoi_if.h"
#ifndef KOBJMETHOD_END
#define KOBJMETHOD_END { NULL, NULL }
#endif
#define MPU_DATAPORT 0
#define MPU_CMDPORT 1
#define MPU_STATPORT 1
#define MPU_RESET 0xff
#define MPU_UART 0x3f
#define MPU_ACK 0xfe
#define MPU_STATMASK 0xc0
#define MPU_OUTPUTBUSY 0x40
#define MPU_INPUTBUSY 0x80
#define MPU_TRYDATA 50
#define MPU_DELAY 2500
#define CMD(m,d) MPUFOI_WRITE(m, m->cookie, MPU_CMDPORT,d)
#define STATUS(m) MPUFOI_READ(m, m->cookie, MPU_STATPORT)
#define READ(m) MPUFOI_READ(m, m->cookie, MPU_DATAPORT)
#define WRITE(m,d) MPUFOI_WRITE(m, m->cookie, MPU_DATAPORT,d)
struct mpu401 {
KOBJ_FIELDS;
struct snd_midi *mid;
int flags;
driver_intr_t *si;
void *cookie;
struct callout timer;
};
static void mpu401_timeout(void *m);
static mpu401_intr_t mpu401_intr;
static int mpu401_minit(struct snd_midi *, void *);
static int mpu401_muninit(struct snd_midi *, void *);
static int mpu401_minqsize(struct snd_midi *, void *);
static int mpu401_moutqsize(struct snd_midi *, void *);
static void mpu401_mcallback(struct snd_midi *, void *, int);
static void mpu401_mcallbackp(struct snd_midi *, void *, int);
static const char *mpu401_mdescr(struct snd_midi *, void *, int);
static const char *mpu401_mprovider(struct snd_midi *, void *);
static kobj_method_t mpu401_methods[] = {
KOBJMETHOD(mpu_init, mpu401_minit),
KOBJMETHOD(mpu_uninit, mpu401_muninit),
KOBJMETHOD(mpu_inqsize, mpu401_minqsize),
KOBJMETHOD(mpu_outqsize, mpu401_moutqsize),
KOBJMETHOD(mpu_callback, mpu401_mcallback),
KOBJMETHOD(mpu_callbackp, mpu401_mcallbackp),
KOBJMETHOD(mpu_descr, mpu401_mdescr),
KOBJMETHOD(mpu_provider, mpu401_mprovider),
KOBJMETHOD_END
};
DEFINE_CLASS(mpu401, mpu401_methods, 0);
void
mpu401_timeout(void *a)
{
struct mpu401 *m = (struct mpu401 *)a;
if (m->si)
(m->si)(m->cookie);
}
static int
mpu401_intr(struct mpu401 *m)
{
#define MPU_INTR_BUF 16
MIDI_TYPE b[MPU_INTR_BUF];
int i;
int s;
/*
printf("mpu401_intr\n");
*/
#define RXRDY(m) ( (STATUS(m) & MPU_INPUTBUSY) == 0)
#define TXRDY(m) ( (STATUS(m) & MPU_OUTPUTBUSY) == 0)
#if 0
#define D(x,l) printf("mpu401_intr %d %x %s %s\n",l, x, x&MPU_INPUTBUSY?"RX":"", x&MPU_OUTPUTBUSY?"TX":"")
#else
#define D(x,l)
#endif
i = 0;
s = STATUS(m);
D(s, 1);
while ((s & MPU_INPUTBUSY) == 0 && i < MPU_INTR_BUF) {
b[i] = READ(m);
/*
printf("mpu401_intr in i %d d %d\n", i, b[i]);
*/
i++;
s = STATUS(m);
}
if (i)
midi_in(m->mid, b, i);
i = 0;
while (!(s & MPU_OUTPUTBUSY) && i < MPU_INTR_BUF) {
if (midi_out(m->mid, b, 1)) {
/*
printf("mpu401_intr out i %d d %d\n", i, b[0]);
*/
WRITE(m, *b);
} else {
/*
printf("mpu401_intr write: no output\n");
*/
return 0;
}
i++;
/* DELAY(100); */
s = STATUS(m);
}
if ((m->flags & M_TXEN) && (m->si)) {
callout_reset(&m->timer, 1, mpu401_timeout, m);
}
return (m->flags & M_TXEN) == M_TXEN;
}
struct mpu401 *
mpu401_init(kobj_class_t cls, void *cookie, driver_intr_t softintr,
mpu401_intr_t ** cb)
{
struct mpu401 *m;
*cb = NULL;
m = malloc(sizeof(*m), M_MIDI, M_NOWAIT | M_ZERO);
if (!m)
return NULL;
kobj_init((kobj_t)m, cls);
- callout_init(&m->timer, CALLOUT_MPSAFE);
+ callout_init(&m->timer, 1);
m->si = softintr;
m->cookie = cookie;
m->flags = 0;
m->mid = midi_init(&mpu401_class, 0, 0, m);
if (!m->mid)
goto err;
*cb = mpu401_intr;
return m;
err:
printf("mpu401_init error\n");
free(m, M_MIDI);
return NULL;
}
int
mpu401_uninit(struct mpu401 *m)
{
int retval;
CMD(m, MPU_RESET);
retval = midi_uninit(m->mid);
if (retval)
return retval;
free(m, M_MIDI);
return 0;
}
static int
mpu401_minit(struct snd_midi *sm, void *arg)
{
struct mpu401 *m = arg;
int i;
CMD(m, MPU_RESET);
CMD(m, MPU_UART);
return 0;
i = 0;
while (++i < 2000) {
if (RXRDY(m))
if (READ(m) == MPU_ACK)
break;
}
if (i < 2000) {
CMD(m, MPU_UART);
return 0;
}
printf("mpu401_minit failed active sensing\n");
return 1;
}
int
mpu401_muninit(struct snd_midi *sm, void *arg)
{
struct mpu401 *m = arg;
return MPUFOI_UNINIT(m, m->cookie);
}
int
mpu401_minqsize(struct snd_midi *sm, void *arg)
{
return 128;
}
int
mpu401_moutqsize(struct snd_midi *sm, void *arg)
{
return 128;
}
static void
mpu401_mcallback(struct snd_midi *sm, void *arg, int flags)
{
struct mpu401 *m = arg;
#if 0
printf("mpu401_callback %s %s %s %s\n",
flags & M_RX ? "M_RX" : "",
flags & M_TX ? "M_TX" : "",
flags & M_RXEN ? "M_RXEN" : "",
flags & M_TXEN ? "M_TXEN" : "");
#endif
if (flags & M_TXEN && m->si) {
callout_reset(&m->timer, 1, mpu401_timeout, m);
}
m->flags = flags;
}
static void
mpu401_mcallbackp(struct snd_midi *sm, void *arg, int flags)
{
/* printf("mpu401_callbackp\n"); */
mpu401_mcallback(sm, arg, flags);
}
static const char *
mpu401_mdescr(struct snd_midi *sm, void *arg, int verbosity)
{
return "descr mpu401";
}
static const char *
mpu401_mprovider(struct snd_midi *m, void *arg)
{
return "provider mpu401";
}
Index: head/sys/dev/sound/pci/atiixp.c
===================================================================
--- head/sys/dev/sound/pci/atiixp.c (revision 283290)
+++ head/sys/dev/sound/pci/atiixp.c (revision 283291)
@@ -1,1424 +1,1424 @@
/*-
* Copyright (c) 2005 Ariff Abdullah <ariff@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHERIN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THEPOSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* FreeBSD pcm driver for ATI IXP 150/200/250/300 AC97 controllers
*
* Features
* * 16bit playback / recording
* * 32bit native playback - yay!
* * 32bit native recording (seems broken on few hardwares)
*
* Issues / TODO:
* * SPDIF
* * Support for more than 2 channels.
* * VRA ? VRM ? DRA ?
* * 32bit native recording seems broken on few hardwares, most
* probably because of incomplete VRA/DRA cleanup.
*
*
* Thanks goes to:
*
* Shaharil @ SCAN Associates whom relentlessly providing me the
* mind blowing Acer Ferrari 4002 WLMi with this ATI IXP hardware.
*
* Reinoud Zandijk <reinoud@NetBSD.org> (auixp), which this driver is
* largely based upon although large part of it has been reworked. His
* driver is the primary reference and pretty much well documented.
*
* Takashi Iwai (ALSA snd-atiixp), for register definitions and some
* random ninja hackery.
*/
#ifdef HAVE_KERNEL_OPTION_HEADERS
#include "opt_snd.h"
#endif
#include <dev/sound/pcm/sound.h>
#include <dev/sound/pcm/ac97.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <sys/sysctl.h>
#include <sys/endian.h>
#include <dev/sound/pci/atiixp.h>
SND_DECLARE_FILE("$FreeBSD$");
#define ATI_IXP_DMA_RETRY_MAX 100
#define ATI_IXP_BUFSZ_MIN 4096
#define ATI_IXP_BUFSZ_MAX 65536
#define ATI_IXP_BUFSZ_DEFAULT 16384
#define ATI_IXP_BLK_MIN 32
#define ATI_IXP_BLK_ALIGN (~(ATI_IXP_BLK_MIN - 1))
#define ATI_IXP_CHN_RUNNING 0x00000001
#define ATI_IXP_CHN_SUSPEND 0x00000002
struct atiixp_dma_op {
volatile uint32_t addr;
volatile uint16_t status;
volatile uint16_t size;
volatile uint32_t next;
};
struct atiixp_info;
struct atiixp_chinfo {
struct snd_dbuf *buffer;
struct pcm_channel *channel;
struct atiixp_info *parent;
struct atiixp_dma_op *sgd_table;
bus_addr_t sgd_addr;
uint32_t enable_bit, flush_bit, linkptr_bit, dt_cur_bit;
uint32_t blksz, blkcnt;
uint32_t ptr, prevptr;
uint32_t fmt;
uint32_t flags;
int caps_32bit, dir;
};
struct atiixp_info {
device_t dev;
bus_space_tag_t st;
bus_space_handle_t sh;
bus_dma_tag_t parent_dmat;
bus_dma_tag_t sgd_dmat;
bus_dmamap_t sgd_dmamap;
bus_addr_t sgd_addr;
struct resource *reg, *irq;
int regtype, regid, irqid;
void *ih;
struct ac97_info *codec;
struct atiixp_chinfo pch;
struct atiixp_chinfo rch;
struct atiixp_dma_op *sgd_table;
struct intr_config_hook delayed_attach;
uint32_t bufsz;
uint32_t codec_not_ready_bits, codec_idx, codec_found;
uint32_t blkcnt;
int registered_channels;
struct mtx *lock;
struct callout poll_timer;
int poll_ticks, polling;
};
#define atiixp_rd(_sc, _reg) \
bus_space_read_4((_sc)->st, (_sc)->sh, _reg)
#define atiixp_wr(_sc, _reg, _val) \
bus_space_write_4((_sc)->st, (_sc)->sh, _reg, _val)
#define atiixp_lock(_sc) snd_mtxlock((_sc)->lock)
#define atiixp_unlock(_sc) snd_mtxunlock((_sc)->lock)
#define atiixp_assert(_sc) snd_mtxassert((_sc)->lock)
static uint32_t atiixp_fmt_32bit[] = {
SND_FORMAT(AFMT_S16_LE, 2, 0),
SND_FORMAT(AFMT_S32_LE, 2, 0),
0
};
static uint32_t atiixp_fmt[] = {
SND_FORMAT(AFMT_S16_LE, 2, 0),
0
};
static struct pcmchan_caps atiixp_caps_32bit = {
ATI_IXP_BASE_RATE,
ATI_IXP_BASE_RATE,
atiixp_fmt_32bit, 0
};
static struct pcmchan_caps atiixp_caps = {
ATI_IXP_BASE_RATE,
ATI_IXP_BASE_RATE,
atiixp_fmt, 0
};
static const struct {
uint16_t vendor;
uint16_t devid;
char *desc;
} atiixp_hw[] = {
{ ATI_VENDOR_ID, ATI_IXP_200_ID, "ATI IXP 200" },
{ ATI_VENDOR_ID, ATI_IXP_300_ID, "ATI IXP 300" },
{ ATI_VENDOR_ID, ATI_IXP_400_ID, "ATI IXP 400" },
{ ATI_VENDOR_ID, ATI_IXP_SB600_ID, "ATI IXP SB600" },
};
static void atiixp_enable_interrupts(struct atiixp_info *);
static void atiixp_disable_interrupts(struct atiixp_info *);
static void atiixp_reset_aclink(struct atiixp_info *);
static void atiixp_flush_dma(struct atiixp_chinfo *);
static void atiixp_enable_dma(struct atiixp_chinfo *);
static void atiixp_disable_dma(struct atiixp_chinfo *);
static int atiixp_waitready_codec(struct atiixp_info *);
static int atiixp_rdcd(kobj_t, void *, int);
static int atiixp_wrcd(kobj_t, void *, int, uint32_t);
static void *atiixp_chan_init(kobj_t, void *, struct snd_dbuf *,
struct pcm_channel *, int);
static int atiixp_chan_setformat(kobj_t, void *, uint32_t);
static uint32_t atiixp_chan_setspeed(kobj_t, void *, uint32_t);
static int atiixp_chan_setfragments(kobj_t, void *, uint32_t, uint32_t);
static uint32_t atiixp_chan_setblocksize(kobj_t, void *, uint32_t);
static void atiixp_buildsgdt(struct atiixp_chinfo *);
static int atiixp_chan_trigger(kobj_t, void *, int);
static __inline uint32_t atiixp_dmapos(struct atiixp_chinfo *);
static uint32_t atiixp_chan_getptr(kobj_t, void *);
static struct pcmchan_caps *atiixp_chan_getcaps(kobj_t, void *);
static void atiixp_intr(void *);
static void atiixp_dma_cb(void *, bus_dma_segment_t *, int, int);
static void atiixp_chip_pre_init(struct atiixp_info *);
static void atiixp_chip_post_init(void *);
static void atiixp_release_resource(struct atiixp_info *);
static int atiixp_pci_probe(device_t);
static int atiixp_pci_attach(device_t);
static int atiixp_pci_detach(device_t);
static int atiixp_pci_suspend(device_t);
static int atiixp_pci_resume(device_t);
/*
* ATI IXP helper functions
*/
static void
atiixp_enable_interrupts(struct atiixp_info *sc)
{
uint32_t value;
/* clear all pending */
atiixp_wr(sc, ATI_REG_ISR, 0xffffffff);
/* enable all relevant interrupt sources we can handle */
value = atiixp_rd(sc, ATI_REG_IER);
value |= ATI_REG_IER_IO_STATUS_EN;
/*
* Disable / ignore internal xrun/spdf interrupt flags
* since it doesn't interest us (for now).
*/
#if 1
value &= ~(ATI_REG_IER_IN_XRUN_EN | ATI_REG_IER_OUT_XRUN_EN |
ATI_REG_IER_SPDF_XRUN_EN | ATI_REG_IER_SPDF_STATUS_EN);
#else
value |= ATI_REG_IER_IN_XRUN_EN;
value |= ATI_REG_IER_OUT_XRUN_EN;
value |= ATI_REG_IER_SPDF_XRUN_EN;
value |= ATI_REG_IER_SPDF_STATUS_EN;
#endif
atiixp_wr(sc, ATI_REG_IER, value);
}
static void
atiixp_disable_interrupts(struct atiixp_info *sc)
{
/* disable all interrupt sources */
atiixp_wr(sc, ATI_REG_IER, 0);
/* clear all pending */
atiixp_wr(sc, ATI_REG_ISR, 0xffffffff);
}
static void
atiixp_reset_aclink(struct atiixp_info *sc)
{
uint32_t value, timeout;
/* if power is down, power it up */
value = atiixp_rd(sc, ATI_REG_CMD);
if (value & ATI_REG_CMD_POWERDOWN) {
/* explicitly enable power */
value &= ~ATI_REG_CMD_POWERDOWN;
atiixp_wr(sc, ATI_REG_CMD, value);
/* have to wait at least 10 usec for it to initialise */
DELAY(20);
}
/* perform a soft reset */
value = atiixp_rd(sc, ATI_REG_CMD);
value |= ATI_REG_CMD_AC_SOFT_RESET;
atiixp_wr(sc, ATI_REG_CMD, value);
/* need to read the CMD reg and wait aprox. 10 usec to init */
value = atiixp_rd(sc, ATI_REG_CMD);
DELAY(20);
/* clear soft reset flag again */
value = atiixp_rd(sc, ATI_REG_CMD);
value &= ~ATI_REG_CMD_AC_SOFT_RESET;
atiixp_wr(sc, ATI_REG_CMD, value);
/* check if the ac-link is working; reset device otherwise */
timeout = 10;
value = atiixp_rd(sc, ATI_REG_CMD);
while (!(value & ATI_REG_CMD_ACLINK_ACTIVE) && --timeout) {
#if 0
device_printf(sc->dev, "not up; resetting aclink hardware\n");
#endif
/* dip aclink reset but keep the acsync */
value &= ~ATI_REG_CMD_AC_RESET;
value |= ATI_REG_CMD_AC_SYNC;
atiixp_wr(sc, ATI_REG_CMD, value);
/* need to read CMD again and wait again (clocking in issue?) */
value = atiixp_rd(sc, ATI_REG_CMD);
DELAY(20);
/* assert aclink reset again */
value = atiixp_rd(sc, ATI_REG_CMD);
value |= ATI_REG_CMD_AC_RESET;
atiixp_wr(sc, ATI_REG_CMD, value);
/* check if its active now */
value = atiixp_rd(sc, ATI_REG_CMD);
}
if (timeout == 0)
device_printf(sc->dev, "giving up aclink reset\n");
#if 0
if (timeout != 10)
device_printf(sc->dev, "aclink hardware reset successful\n");
#endif
/* assert reset and sync for safety */
value = atiixp_rd(sc, ATI_REG_CMD);
value |= ATI_REG_CMD_AC_SYNC | ATI_REG_CMD_AC_RESET;
atiixp_wr(sc, ATI_REG_CMD, value);
}
static void
atiixp_flush_dma(struct atiixp_chinfo *ch)
{
atiixp_wr(ch->parent, ATI_REG_FIFO_FLUSH, ch->flush_bit);
}
static void
atiixp_enable_dma(struct atiixp_chinfo *ch)
{
uint32_t value;
value = atiixp_rd(ch->parent, ATI_REG_CMD);
if (!(value & ch->enable_bit)) {
value |= ch->enable_bit;
atiixp_wr(ch->parent, ATI_REG_CMD, value);
}
}
static void
atiixp_disable_dma(struct atiixp_chinfo *ch)
{
uint32_t value;
value = atiixp_rd(ch->parent, ATI_REG_CMD);
if (value & ch->enable_bit) {
value &= ~ch->enable_bit;
atiixp_wr(ch->parent, ATI_REG_CMD, value);
}
}
/*
* AC97 interface
*/
static int
atiixp_waitready_codec(struct atiixp_info *sc)
{
int timeout = 500;
do {
if ((atiixp_rd(sc, ATI_REG_PHYS_OUT_ADDR) &
ATI_REG_PHYS_OUT_ADDR_EN) == 0)
return (0);
DELAY(1);
} while (--timeout);
return (-1);
}
static int
atiixp_rdcd(kobj_t obj, void *devinfo, int reg)
{
struct atiixp_info *sc = devinfo;
uint32_t data;
int timeout;
if (atiixp_waitready_codec(sc))
return (-1);
data = (reg << ATI_REG_PHYS_OUT_ADDR_SHIFT) |
ATI_REG_PHYS_OUT_ADDR_EN | ATI_REG_PHYS_OUT_RW | sc->codec_idx;
atiixp_wr(sc, ATI_REG_PHYS_OUT_ADDR, data);
if (atiixp_waitready_codec(sc))
return (-1);
timeout = 500;
do {
data = atiixp_rd(sc, ATI_REG_PHYS_IN_ADDR);
if (data & ATI_REG_PHYS_IN_READ_FLAG)
return (data >> ATI_REG_PHYS_IN_DATA_SHIFT);
DELAY(1);
} while (--timeout);
if (reg < 0x7c)
device_printf(sc->dev, "codec read timeout! (reg 0x%x)\n", reg);
return (-1);
}
static int
atiixp_wrcd(kobj_t obj, void *devinfo, int reg, uint32_t data)
{
struct atiixp_info *sc = devinfo;
if (atiixp_waitready_codec(sc))
return (-1);
data = (data << ATI_REG_PHYS_OUT_DATA_SHIFT) |
(((uint32_t)reg) << ATI_REG_PHYS_OUT_ADDR_SHIFT) |
ATI_REG_PHYS_OUT_ADDR_EN | sc->codec_idx;
atiixp_wr(sc, ATI_REG_PHYS_OUT_ADDR, data);
return (0);
}
static kobj_method_t atiixp_ac97_methods[] = {
KOBJMETHOD(ac97_read, atiixp_rdcd),
KOBJMETHOD(ac97_write, atiixp_wrcd),
KOBJMETHOD_END
};
AC97_DECLARE(atiixp_ac97);
/*
* Playback / Record channel interface
*/
static void *
atiixp_chan_init(kobj_t obj, void *devinfo, struct snd_dbuf *b,
struct pcm_channel *c, int dir)
{
struct atiixp_info *sc = devinfo;
struct atiixp_chinfo *ch;
int num;
atiixp_lock(sc);
if (dir == PCMDIR_PLAY) {
ch = &sc->pch;
ch->linkptr_bit = ATI_REG_OUT_DMA_LINKPTR;
ch->enable_bit = ATI_REG_CMD_OUT_DMA_EN | ATI_REG_CMD_SEND_EN;
ch->flush_bit = ATI_REG_FIFO_OUT_FLUSH;
ch->dt_cur_bit = ATI_REG_OUT_DMA_DT_CUR;
/* Native 32bit playback working properly */
ch->caps_32bit = 1;
} else {
ch = &sc->rch;
ch->linkptr_bit = ATI_REG_IN_DMA_LINKPTR;
ch->enable_bit = ATI_REG_CMD_IN_DMA_EN | ATI_REG_CMD_RECEIVE_EN;
ch->flush_bit = ATI_REG_FIFO_IN_FLUSH;
ch->dt_cur_bit = ATI_REG_IN_DMA_DT_CUR;
/* XXX Native 32bit recording appear to be broken */
ch->caps_32bit = 1;
}
ch->buffer = b;
ch->parent = sc;
ch->channel = c;
ch->dir = dir;
ch->blkcnt = sc->blkcnt;
ch->blksz = sc->bufsz / ch->blkcnt;
atiixp_unlock(sc);
if (sndbuf_alloc(ch->buffer, sc->parent_dmat, 0, sc->bufsz) == -1)
return (NULL);
atiixp_lock(sc);
num = sc->registered_channels++;
ch->sgd_table = &sc->sgd_table[num * ATI_IXP_DMA_CHSEGS_MAX];
ch->sgd_addr = sc->sgd_addr + (num * ATI_IXP_DMA_CHSEGS_MAX *
sizeof(struct atiixp_dma_op));
atiixp_disable_dma(ch);
atiixp_unlock(sc);
return (ch);
}
static int
atiixp_chan_setformat(kobj_t obj, void *data, uint32_t format)
{
struct atiixp_chinfo *ch = data;
struct atiixp_info *sc = ch->parent;
uint32_t value;
atiixp_lock(sc);
if (ch->dir == PCMDIR_REC) {
value = atiixp_rd(sc, ATI_REG_CMD);
value &= ~ATI_REG_CMD_INTERLEAVE_IN;
if ((format & AFMT_32BIT) == 0)
value |= ATI_REG_CMD_INTERLEAVE_IN;
atiixp_wr(sc, ATI_REG_CMD, value);
} else {
value = atiixp_rd(sc, ATI_REG_OUT_DMA_SLOT);
value &= ~ATI_REG_OUT_DMA_SLOT_MASK;
/* We do not have support for more than 2 channels, _yet_. */
value |= ATI_REG_OUT_DMA_SLOT_BIT(3) |
ATI_REG_OUT_DMA_SLOT_BIT(4);
value |= 0x04 << ATI_REG_OUT_DMA_THRESHOLD_SHIFT;
atiixp_wr(sc, ATI_REG_OUT_DMA_SLOT, value);
value = atiixp_rd(sc, ATI_REG_CMD);
value &= ~ATI_REG_CMD_INTERLEAVE_OUT;
if ((format & AFMT_32BIT) == 0)
value |= ATI_REG_CMD_INTERLEAVE_OUT;
atiixp_wr(sc, ATI_REG_CMD, value);
value = atiixp_rd(sc, ATI_REG_6CH_REORDER);
value &= ~ATI_REG_6CH_REORDER_EN;
atiixp_wr(sc, ATI_REG_6CH_REORDER, value);
}
ch->fmt = format;
atiixp_unlock(sc);
return (0);
}
static uint32_t
atiixp_chan_setspeed(kobj_t obj, void *data, uint32_t spd)
{
/* XXX We're supposed to do VRA/DRA processing right here */
return (ATI_IXP_BASE_RATE);
}
static int
atiixp_chan_setfragments(kobj_t obj, void *data,
uint32_t blksz, uint32_t blkcnt)
{
struct atiixp_chinfo *ch = data;
struct atiixp_info *sc = ch->parent;
blksz &= ATI_IXP_BLK_ALIGN;
if (blksz > (sndbuf_getmaxsize(ch->buffer) / ATI_IXP_DMA_CHSEGS_MIN))
blksz = sndbuf_getmaxsize(ch->buffer) / ATI_IXP_DMA_CHSEGS_MIN;
if (blksz < ATI_IXP_BLK_MIN)
blksz = ATI_IXP_BLK_MIN;
if (blkcnt > ATI_IXP_DMA_CHSEGS_MAX)
blkcnt = ATI_IXP_DMA_CHSEGS_MAX;
if (blkcnt < ATI_IXP_DMA_CHSEGS_MIN)
blkcnt = ATI_IXP_DMA_CHSEGS_MIN;
while ((blksz * blkcnt) > sndbuf_getmaxsize(ch->buffer)) {
if ((blkcnt >> 1) >= ATI_IXP_DMA_CHSEGS_MIN)
blkcnt >>= 1;
else if ((blksz >> 1) >= ATI_IXP_BLK_MIN)
blksz >>= 1;
else
break;
}
if ((sndbuf_getblksz(ch->buffer) != blksz ||
sndbuf_getblkcnt(ch->buffer) != blkcnt) &&
sndbuf_resize(ch->buffer, blkcnt, blksz) != 0)
device_printf(sc->dev, "%s: failed blksz=%u blkcnt=%u\n",
__func__, blksz, blkcnt);
ch->blksz = sndbuf_getblksz(ch->buffer);
ch->blkcnt = sndbuf_getblkcnt(ch->buffer);
return (0);
}
static uint32_t
atiixp_chan_setblocksize(kobj_t obj, void *data, uint32_t blksz)
{
struct atiixp_chinfo *ch = data;
struct atiixp_info *sc = ch->parent;
atiixp_chan_setfragments(obj, data, blksz, sc->blkcnt);
return (ch->blksz);
}
static void
atiixp_buildsgdt(struct atiixp_chinfo *ch)
{
struct atiixp_info *sc = ch->parent;
uint32_t addr, blksz, blkcnt;
int i;
addr = sndbuf_getbufaddr(ch->buffer);
if (sc->polling != 0) {
blksz = ch->blksz * ch->blkcnt;
blkcnt = 1;
} else {
blksz = ch->blksz;
blkcnt = ch->blkcnt;
}
for (i = 0; i < blkcnt; i++) {
ch->sgd_table[i].addr = htole32(addr + (i * blksz));
ch->sgd_table[i].status = htole16(0);
ch->sgd_table[i].size = htole16(blksz >> 2);
ch->sgd_table[i].next = htole32((uint32_t)ch->sgd_addr +
(((i + 1) % blkcnt) * sizeof(struct atiixp_dma_op)));
}
}
static __inline uint32_t
atiixp_dmapos(struct atiixp_chinfo *ch)
{
struct atiixp_info *sc = ch->parent;
uint32_t reg, addr, sz, retry;
volatile uint32_t ptr;
reg = ch->dt_cur_bit;
addr = sndbuf_getbufaddr(ch->buffer);
sz = ch->blkcnt * ch->blksz;
retry = ATI_IXP_DMA_RETRY_MAX;
do {
ptr = atiixp_rd(sc, reg);
if (ptr < addr)
continue;
ptr -= addr;
if (ptr < sz) {
#if 0
#ifdef ATI_IXP_DEBUG
if ((ptr & ~(ch->blksz - 1)) != ch->ptr) {
uint32_t delta;
delta = (sz + ptr - ch->prevptr) % sz;
#ifndef ATI_IXP_DEBUG_VERBOSE
if (delta < ch->blksz)
#endif
device_printf(sc->dev,
"PCMDIR_%s: incoherent DMA "
"prevptr=%u ptr=%u "
"ptr=%u blkcnt=%u "
"[delta=%u != blksz=%u] "
"(%s)\n",
(ch->dir == PCMDIR_PLAY) ?
"PLAY" : "REC",
ch->prevptr, ptr,
ch->ptr, ch->blkcnt,
delta, ch->blksz,
(delta < ch->blksz) ?
"OVERLAPPED!" : "Ok");
ch->ptr = ptr & ~(ch->blksz - 1);
}
ch->prevptr = ptr;
#endif
#endif
return (ptr);
}
} while (--retry);
device_printf(sc->dev, "PCMDIR_%s: invalid DMA pointer ptr=%u\n",
(ch->dir == PCMDIR_PLAY) ? "PLAY" : "REC", ptr);
return (0);
}
static __inline int
atiixp_poll_channel(struct atiixp_chinfo *ch)
{
uint32_t sz, delta;
volatile uint32_t ptr;
if (!(ch->flags & ATI_IXP_CHN_RUNNING))
return (0);
sz = ch->blksz * ch->blkcnt;
ptr = atiixp_dmapos(ch);
ch->ptr = ptr;
ptr %= sz;
ptr &= ~(ch->blksz - 1);
delta = (sz + ptr - ch->prevptr) % sz;
if (delta < ch->blksz)
return (0);
ch->prevptr = ptr;
return (1);
}
#define atiixp_chan_active(sc) (((sc)->pch.flags | (sc)->rch.flags) & \
ATI_IXP_CHN_RUNNING)
static void
atiixp_poll_callback(void *arg)
{
struct atiixp_info *sc = arg;
uint32_t trigger = 0;
if (sc == NULL)
return;
atiixp_lock(sc);
if (sc->polling == 0 || atiixp_chan_active(sc) == 0) {
atiixp_unlock(sc);
return;
}
trigger |= (atiixp_poll_channel(&sc->pch) != 0) ? 1 : 0;
trigger |= (atiixp_poll_channel(&sc->rch) != 0) ? 2 : 0;
/* XXX */
callout_reset(&sc->poll_timer, 1/*sc->poll_ticks*/,
atiixp_poll_callback, sc);
atiixp_unlock(sc);
if (trigger & 1)
chn_intr(sc->pch.channel);
if (trigger & 2)
chn_intr(sc->rch.channel);
}
static int
atiixp_chan_trigger(kobj_t obj, void *data, int go)
{
struct atiixp_chinfo *ch = data;
struct atiixp_info *sc = ch->parent;
uint32_t value;
int pollticks;
if (!PCMTRIG_COMMON(go))
return (0);
atiixp_lock(sc);
switch (go) {
case PCMTRIG_START:
atiixp_flush_dma(ch);
atiixp_buildsgdt(ch);
atiixp_wr(sc, ch->linkptr_bit, 0);
atiixp_enable_dma(ch);
atiixp_wr(sc, ch->linkptr_bit,
(uint32_t)ch->sgd_addr | ATI_REG_LINKPTR_EN);
if (sc->polling != 0) {
ch->ptr = 0;
ch->prevptr = 0;
pollticks = ((uint64_t)hz * ch->blksz) /
((uint64_t)sndbuf_getalign(ch->buffer) *
sndbuf_getspd(ch->buffer));
pollticks >>= 2;
if (pollticks > hz)
pollticks = hz;
if (pollticks < 1)
pollticks = 1;
if (atiixp_chan_active(sc) == 0 ||
pollticks < sc->poll_ticks) {
if (bootverbose) {
if (atiixp_chan_active(sc) == 0)
device_printf(sc->dev,
"%s: pollticks=%d\n",
__func__, pollticks);
else
device_printf(sc->dev,
"%s: pollticks %d -> %d\n",
__func__, sc->poll_ticks,
pollticks);
}
sc->poll_ticks = pollticks;
callout_reset(&sc->poll_timer, 1,
atiixp_poll_callback, sc);
}
}
ch->flags |= ATI_IXP_CHN_RUNNING;
break;
case PCMTRIG_STOP:
case PCMTRIG_ABORT:
atiixp_disable_dma(ch);
atiixp_flush_dma(ch);
ch->flags &= ~ATI_IXP_CHN_RUNNING;
if (sc->polling != 0) {
if (atiixp_chan_active(sc) == 0) {
callout_stop(&sc->poll_timer);
sc->poll_ticks = 1;
} else {
if (sc->pch.flags & ATI_IXP_CHN_RUNNING)
ch = &sc->pch;
else
ch = &sc->rch;
pollticks = ((uint64_t)hz * ch->blksz) /
((uint64_t)sndbuf_getalign(ch->buffer) *
sndbuf_getspd(ch->buffer));
pollticks >>= 2;
if (pollticks > hz)
pollticks = hz;
if (pollticks < 1)
pollticks = 1;
if (pollticks > sc->poll_ticks) {
if (bootverbose)
device_printf(sc->dev,
"%s: pollticks %d -> %d\n",
__func__, sc->poll_ticks,
pollticks);
sc->poll_ticks = pollticks;
callout_reset(&sc->poll_timer,
1, atiixp_poll_callback,
sc);
}
}
}
break;
default:
atiixp_unlock(sc);
return (0);
break;
}
/* Update bus busy status */
value = atiixp_rd(sc, ATI_REG_IER);
if (atiixp_rd(sc, ATI_REG_CMD) & (ATI_REG_CMD_SEND_EN |
ATI_REG_CMD_RECEIVE_EN | ATI_REG_CMD_SPDF_OUT_EN))
value |= ATI_REG_IER_SET_BUS_BUSY;
else
value &= ~ATI_REG_IER_SET_BUS_BUSY;
atiixp_wr(sc, ATI_REG_IER, value);
atiixp_unlock(sc);
return (0);
}
static uint32_t
atiixp_chan_getptr(kobj_t obj, void *data)
{
struct atiixp_chinfo *ch = data;
struct atiixp_info *sc = ch->parent;
uint32_t ptr;
atiixp_lock(sc);
if (sc->polling != 0)
ptr = ch->ptr;
else
ptr = atiixp_dmapos(ch);
atiixp_unlock(sc);
return (ptr);
}
static struct pcmchan_caps *
atiixp_chan_getcaps(kobj_t obj, void *data)
{
struct atiixp_chinfo *ch = data;
if (ch->caps_32bit)
return (&atiixp_caps_32bit);
return (&atiixp_caps);
}
static kobj_method_t atiixp_chan_methods[] = {
KOBJMETHOD(channel_init, atiixp_chan_init),
KOBJMETHOD(channel_setformat, atiixp_chan_setformat),
KOBJMETHOD(channel_setspeed, atiixp_chan_setspeed),
KOBJMETHOD(channel_setblocksize, atiixp_chan_setblocksize),
KOBJMETHOD(channel_setfragments, atiixp_chan_setfragments),
KOBJMETHOD(channel_trigger, atiixp_chan_trigger),
KOBJMETHOD(channel_getptr, atiixp_chan_getptr),
KOBJMETHOD(channel_getcaps, atiixp_chan_getcaps),
KOBJMETHOD_END
};
CHANNEL_DECLARE(atiixp_chan);
/*
* PCI driver interface
*/
static void
atiixp_intr(void *p)
{
struct atiixp_info *sc = p;
uint32_t status, enable, detected_codecs;
uint32_t trigger = 0;
atiixp_lock(sc);
if (sc->polling != 0) {
atiixp_unlock(sc);
return;
}
status = atiixp_rd(sc, ATI_REG_ISR);
if (status == 0) {
atiixp_unlock(sc);
return;
}
if ((status & ATI_REG_ISR_OUT_STATUS) &&
(sc->pch.flags & ATI_IXP_CHN_RUNNING))
trigger |= 1;
if ((status & ATI_REG_ISR_IN_STATUS) &&
(sc->rch.flags & ATI_IXP_CHN_RUNNING))
trigger |= 2;
#if 0
if (status & ATI_REG_ISR_IN_XRUN) {
device_printf(sc->dev,
"Recieve IN XRUN interrupt\n");
}
if (status & ATI_REG_ISR_OUT_XRUN) {
device_printf(sc->dev,
"Recieve OUT XRUN interrupt\n");
}
#endif
if (status & CODEC_CHECK_BITS) {
/* mark missing codecs as not ready */
detected_codecs = status & CODEC_CHECK_BITS;
sc->codec_not_ready_bits |= detected_codecs;
/* disable detected interrupt sources */
enable = atiixp_rd(sc, ATI_REG_IER);
enable &= ~detected_codecs;
atiixp_wr(sc, ATI_REG_IER, enable);
wakeup(sc);
}
/* acknowledge */
atiixp_wr(sc, ATI_REG_ISR, status);
atiixp_unlock(sc);
if (trigger & 1)
chn_intr(sc->pch.channel);
if (trigger & 2)
chn_intr(sc->rch.channel);
}
static void
atiixp_dma_cb(void *p, bus_dma_segment_t *bds, int a, int b)
{
struct atiixp_info *sc = (struct atiixp_info *)p;
sc->sgd_addr = bds->ds_addr;
}
static void
atiixp_chip_pre_init(struct atiixp_info *sc)
{
uint32_t value;
atiixp_lock(sc);
/* disable interrupts */
atiixp_disable_interrupts(sc);
/* clear all DMA enables (preserving rest of settings) */
value = atiixp_rd(sc, ATI_REG_CMD);
value &= ~(ATI_REG_CMD_IN_DMA_EN | ATI_REG_CMD_OUT_DMA_EN |
ATI_REG_CMD_SPDF_OUT_EN );
atiixp_wr(sc, ATI_REG_CMD, value);
/* reset aclink */
atiixp_reset_aclink(sc);
sc->codec_not_ready_bits = 0;
/* enable all codecs to interrupt as well as the new frame interrupt */
atiixp_wr(sc, ATI_REG_IER, CODEC_CHECK_BITS);
atiixp_unlock(sc);
}
static int
sysctl_atiixp_polling(SYSCTL_HANDLER_ARGS)
{
struct atiixp_info *sc;
device_t dev;
int err, val;
dev = oidp->oid_arg1;
sc = pcm_getdevinfo(dev);
if (sc == NULL)
return (EINVAL);
atiixp_lock(sc);
val = sc->polling;
atiixp_unlock(sc);
err = sysctl_handle_int(oidp, &val, 0, req);
if (err || req->newptr == NULL)
return (err);
if (val < 0 || val > 1)
return (EINVAL);
atiixp_lock(sc);
if (val != sc->polling) {
if (atiixp_chan_active(sc) != 0)
err = EBUSY;
else if (val == 0) {
atiixp_enable_interrupts(sc);
sc->polling = 0;
DELAY(1000);
} else {
atiixp_disable_interrupts(sc);
sc->polling = 1;
DELAY(1000);
}
}
atiixp_unlock(sc);
return (err);
}
static void
atiixp_chip_post_init(void *arg)
{
struct atiixp_info *sc = (struct atiixp_info *)arg;
uint32_t subdev;
int i, timeout, found, polling;
char status[SND_STATUSLEN];
atiixp_lock(sc);
if (sc->delayed_attach.ich_func) {
config_intrhook_disestablish(&sc->delayed_attach);
sc->delayed_attach.ich_func = NULL;
}
polling = sc->polling;
sc->polling = 0;
timeout = 10;
if (sc->codec_not_ready_bits == 0) {
/* wait for the interrupts to happen */
do {
msleep(sc, sc->lock, PWAIT, "ixpslp", max(hz / 10, 1));
if (sc->codec_not_ready_bits != 0)
break;
} while (--timeout);
}
sc->polling = polling;
atiixp_disable_interrupts(sc);
if (sc->codec_not_ready_bits == 0 && timeout == 0) {
device_printf(sc->dev,
"WARNING: timeout during codec detection; "
"codecs might be present but haven't interrupted\n");
atiixp_unlock(sc);
goto postinitbad;
}
found = 0;
/*
* ATI IXP can have upto 3 codecs, but single codec should be
* suffice for now.
*/
if (!(sc->codec_not_ready_bits & ATI_REG_ISR_CODEC0_NOT_READY)) {
/* codec 0 present */
sc->codec_found++;
sc->codec_idx = 0;
found++;
}
if (!(sc->codec_not_ready_bits & ATI_REG_ISR_CODEC1_NOT_READY)) {
/* codec 1 present */
sc->codec_found++;
}
if (!(sc->codec_not_ready_bits & ATI_REG_ISR_CODEC2_NOT_READY)) {
/* codec 2 present */
sc->codec_found++;
}
atiixp_unlock(sc);
if (found == 0)
goto postinitbad;
/* create/init mixer */
sc->codec = AC97_CREATE(sc->dev, sc, atiixp_ac97);
if (sc->codec == NULL)
goto postinitbad;
subdev = (pci_get_subdevice(sc->dev) << 16) |
pci_get_subvendor(sc->dev);
switch (subdev) {
case 0x11831043: /* ASUS A6R */
case 0x2043161f: /* Maxselect x710s - http://maxselect.ru/ */
ac97_setflags(sc->codec, ac97_getflags(sc->codec) |
AC97_F_EAPD_INV);
break;
default:
break;
}
mixer_init(sc->dev, ac97_getmixerclass(), sc->codec);
if (pcm_register(sc->dev, sc, ATI_IXP_NPCHAN, ATI_IXP_NRCHAN))
goto postinitbad;
for (i = 0; i < ATI_IXP_NPCHAN; i++)
pcm_addchan(sc->dev, PCMDIR_PLAY, &atiixp_chan_class, sc);
for (i = 0; i < ATI_IXP_NRCHAN; i++)
pcm_addchan(sc->dev, PCMDIR_REC, &atiixp_chan_class, sc);
SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO,
"polling", CTLTYPE_INT | CTLFLAG_RW, sc->dev, sizeof(sc->dev),
sysctl_atiixp_polling, "I", "Enable polling mode");
snprintf(status, SND_STATUSLEN, "at memory 0x%lx irq %ld %s",
rman_get_start(sc->reg), rman_get_start(sc->irq),
PCM_KLDSTRING(snd_atiixp));
pcm_setstatus(sc->dev, status);
atiixp_lock(sc);
if (sc->polling == 0)
atiixp_enable_interrupts(sc);
atiixp_unlock(sc);
return;
postinitbad:
atiixp_release_resource(sc);
}
static void
atiixp_release_resource(struct atiixp_info *sc)
{
if (sc == NULL)
return;
if (sc->registered_channels != 0) {
atiixp_lock(sc);
sc->polling = 0;
callout_stop(&sc->poll_timer);
atiixp_unlock(sc);
callout_drain(&sc->poll_timer);
}
if (sc->codec) {
ac97_destroy(sc->codec);
sc->codec = NULL;
}
if (sc->ih) {
bus_teardown_intr(sc->dev, sc->irq, sc->ih);
sc->ih = NULL;
}
if (sc->reg) {
bus_release_resource(sc->dev, sc->regtype, sc->regid, sc->reg);
sc->reg = NULL;
}
if (sc->irq) {
bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irqid, sc->irq);
sc->irq = NULL;
}
if (sc->parent_dmat) {
bus_dma_tag_destroy(sc->parent_dmat);
sc->parent_dmat = NULL;
}
if (sc->sgd_addr) {
bus_dmamap_unload(sc->sgd_dmat, sc->sgd_dmamap);
sc->sgd_addr = 0;
}
if (sc->sgd_table) {
bus_dmamem_free(sc->sgd_dmat, sc->sgd_table, sc->sgd_dmamap);
sc->sgd_table = NULL;
}
if (sc->sgd_dmat) {
bus_dma_tag_destroy(sc->sgd_dmat);
sc->sgd_dmat = NULL;
}
if (sc->lock) {
snd_mtxfree(sc->lock);
sc->lock = NULL;
}
free(sc, M_DEVBUF);
}
static int
atiixp_pci_probe(device_t dev)
{
int i;
uint16_t devid, vendor;
vendor = pci_get_vendor(dev);
devid = pci_get_device(dev);
for (i = 0; i < sizeof(atiixp_hw) / sizeof(atiixp_hw[0]); i++) {
if (vendor == atiixp_hw[i].vendor &&
devid == atiixp_hw[i].devid) {
device_set_desc(dev, atiixp_hw[i].desc);
return (BUS_PROBE_DEFAULT);
}
}
return (ENXIO);
}
static int
atiixp_pci_attach(device_t dev)
{
struct atiixp_info *sc;
int i;
sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK | M_ZERO);
sc->lock = snd_mtxcreate(device_get_nameunit(dev), "snd_atiixp softc");
sc->dev = dev;
- callout_init(&sc->poll_timer, CALLOUT_MPSAFE);
+ callout_init(&sc->poll_timer, 1);
sc->poll_ticks = 1;
if (resource_int_value(device_get_name(sc->dev),
device_get_unit(sc->dev), "polling", &i) == 0 && i != 0)
sc->polling = 1;
else
sc->polling = 0;
pci_enable_busmaster(dev);
sc->regid = PCIR_BAR(0);
sc->regtype = SYS_RES_MEMORY;
sc->reg = bus_alloc_resource_any(dev, sc->regtype,
&sc->regid, RF_ACTIVE);
if (!sc->reg) {
device_printf(dev, "unable to allocate register space\n");
goto bad;
}
sc->st = rman_get_bustag(sc->reg);
sc->sh = rman_get_bushandle(sc->reg);
sc->bufsz = pcm_getbuffersize(dev, ATI_IXP_BUFSZ_MIN,
ATI_IXP_BUFSZ_DEFAULT, ATI_IXP_BUFSZ_MAX);
sc->irqid = 0;
sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irqid,
RF_ACTIVE | RF_SHAREABLE);
if (!sc->irq || snd_setup_intr(dev, sc->irq, INTR_MPSAFE,
atiixp_intr, sc, &sc->ih)) {
device_printf(dev, "unable to map interrupt\n");
goto bad;
}
/*
* Let the user choose the best DMA segments.
*/
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "blocksize", &i) == 0 && i > 0) {
i &= ATI_IXP_BLK_ALIGN;
if (i < ATI_IXP_BLK_MIN)
i = ATI_IXP_BLK_MIN;
sc->blkcnt = sc->bufsz / i;
i = 0;
while (sc->blkcnt >> i)
i++;
sc->blkcnt = 1 << (i - 1);
if (sc->blkcnt < ATI_IXP_DMA_CHSEGS_MIN)
sc->blkcnt = ATI_IXP_DMA_CHSEGS_MIN;
else if (sc->blkcnt > ATI_IXP_DMA_CHSEGS_MAX)
sc->blkcnt = ATI_IXP_DMA_CHSEGS_MAX;
} else
sc->blkcnt = ATI_IXP_DMA_CHSEGS;
/*
* DMA tag for scatter-gather buffers and link pointers
*/
if (bus_dma_tag_create(/*parent*/bus_get_dma_tag(dev), /*alignment*/2,
/*boundary*/0,
/*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
/*highaddr*/BUS_SPACE_MAXADDR,
/*filter*/NULL, /*filterarg*/NULL,
/*maxsize*/sc->bufsz, /*nsegments*/1, /*maxsegz*/0x3ffff,
/*flags*/0, /*lockfunc*/NULL,
/*lockarg*/NULL, &sc->parent_dmat) != 0) {
device_printf(dev, "unable to create dma tag\n");
goto bad;
}
if (bus_dma_tag_create(/*parent*/bus_get_dma_tag(dev), /*alignment*/2,
/*boundary*/0,
/*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
/*highaddr*/BUS_SPACE_MAXADDR,
/*filter*/NULL, /*filterarg*/NULL,
/*maxsize*/ATI_IXP_DMA_CHSEGS_MAX * ATI_IXP_NCHANS *
sizeof(struct atiixp_dma_op),
/*nsegments*/1, /*maxsegz*/0x3ffff,
/*flags*/0, /*lockfunc*/NULL,
/*lockarg*/NULL, &sc->sgd_dmat) != 0) {
device_printf(dev, "unable to create dma tag\n");
goto bad;
}
if (bus_dmamem_alloc(sc->sgd_dmat, (void **)&sc->sgd_table,
BUS_DMA_NOWAIT, &sc->sgd_dmamap) == -1)
goto bad;
if (bus_dmamap_load(sc->sgd_dmat, sc->sgd_dmamap, sc->sgd_table,
ATI_IXP_DMA_CHSEGS_MAX * ATI_IXP_NCHANS *
sizeof(struct atiixp_dma_op), atiixp_dma_cb, sc, 0))
goto bad;
atiixp_chip_pre_init(sc);
sc->delayed_attach.ich_func = atiixp_chip_post_init;
sc->delayed_attach.ich_arg = sc;
if (cold == 0 ||
config_intrhook_establish(&sc->delayed_attach) != 0) {
sc->delayed_attach.ich_func = NULL;
atiixp_chip_post_init(sc);
}
return (0);
bad:
atiixp_release_resource(sc);
return (ENXIO);
}
static int
atiixp_pci_detach(device_t dev)
{
int r;
struct atiixp_info *sc;
sc = pcm_getdevinfo(dev);
if (sc != NULL) {
if (sc->codec != NULL) {
r = pcm_unregister(dev);
if (r)
return (r);
}
sc->codec = NULL;
if (sc->st != 0 && sc->sh != 0)
atiixp_disable_interrupts(sc);
atiixp_release_resource(sc);
}
return (0);
}
static int
atiixp_pci_suspend(device_t dev)
{
struct atiixp_info *sc = pcm_getdevinfo(dev);
uint32_t value;
/* quickly disable interrupts and save channels active state */
atiixp_lock(sc);
atiixp_disable_interrupts(sc);
atiixp_unlock(sc);
/* stop everything */
if (sc->pch.flags & ATI_IXP_CHN_RUNNING) {
atiixp_chan_trigger(NULL, &sc->pch, PCMTRIG_STOP);
sc->pch.flags |= ATI_IXP_CHN_SUSPEND;
}
if (sc->rch.flags & ATI_IXP_CHN_RUNNING) {
atiixp_chan_trigger(NULL, &sc->rch, PCMTRIG_STOP);
sc->rch.flags |= ATI_IXP_CHN_SUSPEND;
}
/* power down aclink and pci bus */
atiixp_lock(sc);
value = atiixp_rd(sc, ATI_REG_CMD);
value |= ATI_REG_CMD_POWERDOWN | ATI_REG_CMD_AC_RESET;
atiixp_wr(sc, ATI_REG_CMD, ATI_REG_CMD_POWERDOWN);
atiixp_unlock(sc);
return (0);
}
static int
atiixp_pci_resume(device_t dev)
{
struct atiixp_info *sc = pcm_getdevinfo(dev);
atiixp_lock(sc);
/* reset / power up aclink */
atiixp_reset_aclink(sc);
atiixp_unlock(sc);
if (mixer_reinit(dev) == -1) {
device_printf(dev, "unable to reinitialize the mixer\n");
return (ENXIO);
}
/*
* Resume channel activities. Reset channel format regardless
* of its previous state.
*/
if (sc->pch.channel != NULL) {
if (sc->pch.fmt != 0)
atiixp_chan_setformat(NULL, &sc->pch, sc->pch.fmt);
if (sc->pch.flags & ATI_IXP_CHN_SUSPEND) {
sc->pch.flags &= ~ATI_IXP_CHN_SUSPEND;
atiixp_chan_trigger(NULL, &sc->pch, PCMTRIG_START);
}
}
if (sc->rch.channel != NULL) {
if (sc->rch.fmt != 0)
atiixp_chan_setformat(NULL, &sc->rch, sc->rch.fmt);
if (sc->rch.flags & ATI_IXP_CHN_SUSPEND) {
sc->rch.flags &= ~ATI_IXP_CHN_SUSPEND;
atiixp_chan_trigger(NULL, &sc->rch, PCMTRIG_START);
}
}
/* enable interrupts */
atiixp_lock(sc);
if (sc->polling == 0)
atiixp_enable_interrupts(sc);
atiixp_unlock(sc);
return (0);
}
static device_method_t atiixp_methods[] = {
DEVMETHOD(device_probe, atiixp_pci_probe),
DEVMETHOD(device_attach, atiixp_pci_attach),
DEVMETHOD(device_detach, atiixp_pci_detach),
DEVMETHOD(device_suspend, atiixp_pci_suspend),
DEVMETHOD(device_resume, atiixp_pci_resume),
{ 0, 0 }
};
static driver_t atiixp_driver = {
"pcm",
atiixp_methods,
PCM_SOFTC_SIZE,
};
DRIVER_MODULE(snd_atiixp, pci, atiixp_driver, pcm_devclass, 0, 0);
MODULE_DEPEND(snd_atiixp, sound, SOUND_MINVER, SOUND_PREFVER, SOUND_MAXVER);
MODULE_VERSION(snd_atiixp, 1);
Index: head/sys/dev/sound/pci/es137x.c
===================================================================
--- head/sys/dev/sound/pci/es137x.c (revision 283290)
+++ head/sys/dev/sound/pci/es137x.c (revision 283291)
@@ -1,1945 +1,1945 @@
/*-
* Copyright (c) 1999 Russell Cattelan <cattelan@thebarn.com>
* Copyright (c) 1998 Joachim Kuebart <joachim.kuebart@gmx.net>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*-
* Copyright (c) 1999 Cameron Grant <cg@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. All advertising materials mentioning features or use of this
* software must display the following acknowledgement:
* This product includes software developed by Joachim Kuebart.
*
* 4. The name of the author may not be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Support the ENSONIQ AudioPCI board and Creative Labs SoundBlaster PCI
* boards based on the ES1370, ES1371 and ES1373 chips.
*
* Part of this code was heavily inspired by the linux driver from
* Thomas Sailer (sailer@ife.ee.ethz.ch)
* Just about everything has been touched and reworked in some way but
* the all the underlying sequences/timing/register values are from
* Thomas' code.
*/
#ifdef HAVE_KERNEL_OPTION_HEADERS
#include "opt_snd.h"
#endif
#include <dev/sound/pcm/sound.h>
#include <dev/sound/pcm/ac97.h>
#include <dev/sound/pci/es137x.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <sys/sysctl.h>
#include "mixer_if.h"
SND_DECLARE_FILE("$FreeBSD$");
#define MEM_MAP_REG 0x14
/* PCI IDs of supported chips */
#define ES1370_PCI_ID 0x50001274
#define ES1371_PCI_ID 0x13711274
#define ES1371_PCI_ID2 0x13713274
#define CT5880_PCI_ID 0x58801274
#define CT4730_PCI_ID 0x89381102
#define ES1371REV_ES1371_A 0x02
#define ES1371REV_ES1371_B 0x09
#define ES1371REV_ES1373_8 0x08
#define ES1371REV_ES1373_A 0x04
#define ES1371REV_ES1373_B 0x06
#define ES1371REV_CT5880_A 0x07
#define CT5880REV_CT5880_C 0x02
#define CT5880REV_CT5880_D 0x03
#define CT5880REV_CT5880_E 0x04
#define CT4730REV_CT4730_A 0x00
#define ES_DEFAULT_BUFSZ 4096
/* 2 DAC for playback, 1 ADC for record */
#define ES_DAC1 0
#define ES_DAC2 1
#define ES_ADC 2
#define ES_NCHANS 3
#define ES_DMA_SEGS_MIN 2
#define ES_DMA_SEGS_MAX 256
#define ES_BLK_MIN 64
#define ES_BLK_ALIGN (~(ES_BLK_MIN - 1))
#define ES1370_DAC1_MINSPEED 5512
#define ES1370_DAC1_MAXSPEED 44100
/* device private data */
struct es_info;
struct es_chinfo {
struct es_info *parent;
struct pcm_channel *channel;
struct snd_dbuf *buffer;
struct pcmchan_caps caps;
int dir, num, index;
uint32_t fmt, blksz, blkcnt, bufsz;
uint32_t ptr, prevptr;
int active;
};
/*
* 32bit Ensoniq Configuration (es->escfg).
* ----------------------------------------
*
* +-------+--------+------+------+---------+--------+---------+---------+
* len | 16 | 1 | 1 | 1 | 2 | 2 | 1 | 8 |
* +-------+--------+------+------+---------+--------+---------+---------+
* | fixed | single | | | | | is | general |
* | rate | pcm | DACx | DACy | numplay | numrec | es1370? | purpose |
* | | mixer | | | | | | |
* +-------+--------+------+------+---------+--------+---------+---------+
*/
#define ES_FIXED_RATE(cfgv) \
(((cfgv) & 0xffff0000) >> 16)
#define ES_SET_FIXED_RATE(cfgv, nv) \
(((cfgv) & ~0xffff0000) | (((nv) & 0xffff) << 16))
#define ES_SINGLE_PCM_MIX(cfgv) \
(((cfgv) & 0x8000) >> 15)
#define ES_SET_SINGLE_PCM_MIX(cfgv, nv) \
(((cfgv) & ~0x8000) | (((nv) ? 1 : 0) << 15))
#define ES_DAC_FIRST(cfgv) \
(((cfgv) & 0x4000) >> 14)
#define ES_SET_DAC_FIRST(cfgv, nv) \
(((cfgv) & ~0x4000) | (((nv) & 0x1) << 14))
#define ES_DAC_SECOND(cfgv) \
(((cfgv) & 0x2000) >> 13)
#define ES_SET_DAC_SECOND(cfgv, nv) \
(((cfgv) & ~0x2000) | (((nv) & 0x1) << 13))
#define ES_NUMPLAY(cfgv) \
(((cfgv) & 0x1800) >> 11)
#define ES_SET_NUMPLAY(cfgv, nv) \
(((cfgv) & ~0x1800) | (((nv) & 0x3) << 11))
#define ES_NUMREC(cfgv) \
(((cfgv) & 0x600) >> 9)
#define ES_SET_NUMREC(cfgv, nv) \
(((cfgv) & ~0x600) | (((nv) & 0x3) << 9))
#define ES_IS_ES1370(cfgv) \
(((cfgv) & 0x100) >> 8)
#define ES_SET_IS_ES1370(cfgv, nv) \
(((cfgv) & ~0x100) | (((nv) ? 1 : 0) << 8))
#define ES_GP(cfgv) \
((cfgv) & 0xff)
#define ES_SET_GP(cfgv, nv) \
(((cfgv) & ~0xff) | ((nv) & 0xff))
#define ES_DAC1_ENABLED(cfgv) \
(ES_NUMPLAY(cfgv) > 1 || \
(ES_NUMPLAY(cfgv) == 1 && ES_DAC_FIRST(cfgv) == ES_DAC1))
#define ES_DAC2_ENABLED(cfgv) \
(ES_NUMPLAY(cfgv) > 1 || \
(ES_NUMPLAY(cfgv) == 1 && ES_DAC_FIRST(cfgv) == ES_DAC2))
/*
* DAC 1/2 configuration through kernel hint - hint.pcm.<unit>.dac="val"
*
* 0 = Enable both DACs - Default
* 1 = Enable single DAC (DAC1)
* 2 = Enable single DAC (DAC2)
* 3 = Enable both DACs, swap position (DAC2 comes first instead of DAC1)
*/
#define ES_DEFAULT_DAC_CFG 0
struct es_info {
bus_space_tag_t st;
bus_space_handle_t sh;
bus_dma_tag_t parent_dmat;
struct resource *reg, *irq;
int regtype, regid, irqid;
void *ih;
device_t dev;
int num;
unsigned int bufsz, blkcnt;
/* Contents of board's registers */
uint32_t ctrl;
uint32_t sctrl;
uint32_t escfg;
struct es_chinfo ch[ES_NCHANS];
struct mtx *lock;
struct callout poll_timer;
int poll_ticks, polling;
};
#define ES_LOCK(sc) snd_mtxlock((sc)->lock)
#define ES_UNLOCK(sc) snd_mtxunlock((sc)->lock)
#define ES_LOCK_ASSERT(sc) snd_mtxassert((sc)->lock)
/* prototypes */
static void es_intr(void *);
static uint32_t es1371_wait_src_ready(struct es_info *);
static void es1371_src_write(struct es_info *,
unsigned short, unsigned short);
static unsigned int es1371_adc_rate(struct es_info *, unsigned int, int);
static unsigned int es1371_dac_rate(struct es_info *, unsigned int, int);
static int es1371_init(struct es_info *);
static int es1370_init(struct es_info *);
static int es1370_wrcodec(struct es_info *, unsigned char, unsigned char);
static uint32_t es_fmt[] = {
SND_FORMAT(AFMT_U8, 1, 0),
SND_FORMAT(AFMT_U8, 2, 0),
SND_FORMAT(AFMT_S16_LE, 1, 0),
SND_FORMAT(AFMT_S16_LE, 2, 0),
0
};
static struct pcmchan_caps es_caps = {4000, 48000, es_fmt, 0};
static const struct {
unsigned volidx:4;
unsigned left:4;
unsigned right:4;
unsigned stereo:1;
unsigned recmask:13;
unsigned avail:1;
} mixtable[SOUND_MIXER_NRDEVICES] = {
[SOUND_MIXER_VOLUME] = { 0, 0x0, 0x1, 1, 0x1f7f, 1 },
[SOUND_MIXER_PCM] = { 1, 0x2, 0x3, 1, 0x0400, 1 },
[SOUND_MIXER_SYNTH] = { 2, 0x4, 0x5, 1, 0x0060, 1 },
[SOUND_MIXER_CD] = { 3, 0x6, 0x7, 1, 0x0006, 1 },
[SOUND_MIXER_LINE] = { 4, 0x8, 0x9, 1, 0x0018, 1 },
[SOUND_MIXER_LINE1] = { 5, 0xa, 0xb, 1, 0x1800, 1 },
[SOUND_MIXER_LINE2] = { 6, 0xc, 0x0, 0, 0x0100, 1 },
[SOUND_MIXER_LINE3] = { 7, 0xd, 0x0, 0, 0x0200, 1 },
[SOUND_MIXER_MIC] = { 8, 0xe, 0x0, 0, 0x0001, 1 },
[SOUND_MIXER_OGAIN] = { 9, 0xf, 0x0, 0, 0x0000, 1 }
};
static __inline uint32_t
es_rd(struct es_info *es, int regno, int size)
{
switch (size) {
case 1:
return (bus_space_read_1(es->st, es->sh, regno));
case 2:
return (bus_space_read_2(es->st, es->sh, regno));
case 4:
return (bus_space_read_4(es->st, es->sh, regno));
default:
return (0xFFFFFFFF);
}
}
static __inline void
es_wr(struct es_info *es, int regno, uint32_t data, int size)
{
switch (size) {
case 1:
bus_space_write_1(es->st, es->sh, regno, data);
break;
case 2:
bus_space_write_2(es->st, es->sh, regno, data);
break;
case 4:
bus_space_write_4(es->st, es->sh, regno, data);
break;
}
}
/* -------------------------------------------------------------------- */
/* The es1370 mixer interface */
static int
es1370_mixinit(struct snd_mixer *m)
{
struct es_info *es;
int i;
uint32_t v;
es = mix_getdevinfo(m);
v = 0;
for (i = 0; i < SOUND_MIXER_NRDEVICES; i++) {
if (mixtable[i].avail)
v |= (1 << i);
}
/*
* Each DAC1/2 for ES1370 can be controlled independently
* DAC1 = controlled by synth
* DAC2 = controlled by pcm
* This is indeed can confuse user if DAC1 become primary playback
* channel. Try to be smart and combine both if necessary.
*/
if (ES_SINGLE_PCM_MIX(es->escfg))
v &= ~(1 << SOUND_MIXER_SYNTH);
mix_setdevs(m, v);
v = 0;
for (i = 0; i < SOUND_MIXER_NRDEVICES; i++) {
if (mixtable[i].recmask)
v |= (1 << i);
}
if (ES_SINGLE_PCM_MIX(es->escfg)) /* ditto */
v &= ~(1 << SOUND_MIXER_SYNTH);
mix_setrecdevs(m, v);
return (0);
}
static int
es1370_mixset(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right)
{
struct es_info *es;
int l, r, rl, rr, set_dac1;
if (!mixtable[dev].avail)
return (-1);
l = left;
r = (mixtable[dev].stereo) ? right : l;
if (mixtable[dev].left == 0xf)
rl = (l < 2) ? 0x80 : 7 - (l - 2) / 14;
else
rl = (l < 7) ? 0x80 : 31 - (l - 7) / 3;
es = mix_getdevinfo(m);
ES_LOCK(es);
if (dev == SOUND_MIXER_PCM && (ES_SINGLE_PCM_MIX(es->escfg)) &&
ES_DAC1_ENABLED(es->escfg))
set_dac1 = 1;
else
set_dac1 = 0;
if (mixtable[dev].stereo) {
rr = (r < 7) ? 0x80 : 31 - (r - 7) / 3;
es1370_wrcodec(es, mixtable[dev].right, rr);
if (set_dac1 && mixtable[SOUND_MIXER_SYNTH].stereo)
es1370_wrcodec(es,
mixtable[SOUND_MIXER_SYNTH].right, rr);
}
es1370_wrcodec(es, mixtable[dev].left, rl);
if (set_dac1)
es1370_wrcodec(es, mixtable[SOUND_MIXER_SYNTH].left, rl);
ES_UNLOCK(es);
return (l | (r << 8));
}
static uint32_t
es1370_mixsetrecsrc(struct snd_mixer *m, uint32_t src)
{
struct es_info *es;
int i, j = 0;
es = mix_getdevinfo(m);
if (src == 0) src = 1 << SOUND_MIXER_MIC;
src &= mix_getrecdevs(m);
for (i = 0; i < SOUND_MIXER_NRDEVICES; i++)
if ((src & (1 << i)) != 0) j |= mixtable[i].recmask;
ES_LOCK(es);
if ((src & (1 << SOUND_MIXER_PCM)) && ES_SINGLE_PCM_MIX(es->escfg) &&
ES_DAC1_ENABLED(es->escfg))
j |= mixtable[SOUND_MIXER_SYNTH].recmask;
es1370_wrcodec(es, CODEC_LIMIX1, j & 0x55);
es1370_wrcodec(es, CODEC_RIMIX1, j & 0xaa);
es1370_wrcodec(es, CODEC_LIMIX2, (j >> 8) & 0x17);
es1370_wrcodec(es, CODEC_RIMIX2, (j >> 8) & 0x0f);
es1370_wrcodec(es, CODEC_OMIX1, 0x7f);
es1370_wrcodec(es, CODEC_OMIX2, 0x3f);
ES_UNLOCK(es);
return (src);
}
static kobj_method_t es1370_mixer_methods[] = {
KOBJMETHOD(mixer_init, es1370_mixinit),
KOBJMETHOD(mixer_set, es1370_mixset),
KOBJMETHOD(mixer_setrecsrc, es1370_mixsetrecsrc),
KOBJMETHOD_END
};
MIXER_DECLARE(es1370_mixer);
/* -------------------------------------------------------------------- */
static int
es1370_wrcodec(struct es_info *es, unsigned char i, unsigned char data)
{
unsigned int t;
ES_LOCK_ASSERT(es);
for (t = 0; t < 0x1000; t++) {
if ((es_rd(es, ES1370_REG_STATUS, 4) &
STAT_CSTAT) == 0) {
es_wr(es, ES1370_REG_CODEC,
((unsigned short)i << CODEC_INDEX_SHIFT) | data, 2);
return (0);
}
DELAY(1);
}
device_printf(es->dev, "%s: timed out\n", __func__);
return (-1);
}
/* -------------------------------------------------------------------- */
/* channel interface */
static void *
eschan_init(kobj_t obj, void *devinfo, struct snd_dbuf *b,
struct pcm_channel *c, int dir)
{
struct es_info *es = devinfo;
struct es_chinfo *ch;
uint32_t index;
ES_LOCK(es);
if (dir == PCMDIR_PLAY) {
index = ES_GP(es->escfg);
es->escfg = ES_SET_GP(es->escfg, index + 1);
if (index == 0)
index = ES_DAC_FIRST(es->escfg);
else if (index == 1)
index = ES_DAC_SECOND(es->escfg);
else {
device_printf(es->dev,
"Invalid ES_GP index: %d\n", index);
ES_UNLOCK(es);
return (NULL);
}
if (!(index == ES_DAC1 || index == ES_DAC2)) {
device_printf(es->dev, "Unknown DAC: %d\n", index + 1);
ES_UNLOCK(es);
return (NULL);
}
if (es->ch[index].channel != NULL) {
device_printf(es->dev, "DAC%d already initialized!\n",
index + 1);
ES_UNLOCK(es);
return (NULL);
}
} else
index = ES_ADC;
ch = &es->ch[index];
ch->index = index;
ch->num = es->num++;
ch->caps = es_caps;
if (ES_IS_ES1370(es->escfg)) {
if (ch->index == ES_DAC1) {
ch->caps.maxspeed = ES1370_DAC1_MAXSPEED;
ch->caps.minspeed = ES1370_DAC1_MINSPEED;
} else {
uint32_t fixed_rate = ES_FIXED_RATE(es->escfg);
if (!(fixed_rate < es_caps.minspeed ||
fixed_rate > es_caps.maxspeed)) {
ch->caps.maxspeed = fixed_rate;
ch->caps.minspeed = fixed_rate;
}
}
}
ch->parent = es;
ch->channel = c;
ch->buffer = b;
ch->bufsz = es->bufsz;
ch->blkcnt = es->blkcnt;
ch->blksz = ch->bufsz / ch->blkcnt;
ch->dir = dir;
ES_UNLOCK(es);
if (sndbuf_alloc(ch->buffer, es->parent_dmat, 0, ch->bufsz) != 0)
return (NULL);
ES_LOCK(es);
if (dir == PCMDIR_PLAY) {
if (ch->index == ES_DAC1) {
es_wr(es, ES1370_REG_MEMPAGE,
ES1370_REG_DAC1_FRAMEADR >> 8, 1);
es_wr(es, ES1370_REG_DAC1_FRAMEADR & 0xff,
sndbuf_getbufaddr(ch->buffer), 4);
es_wr(es, ES1370_REG_DAC1_FRAMECNT & 0xff,
(ch->bufsz >> 2) - 1, 4);
} else {
es_wr(es, ES1370_REG_MEMPAGE,
ES1370_REG_DAC2_FRAMEADR >> 8, 1);
es_wr(es, ES1370_REG_DAC2_FRAMEADR & 0xff,
sndbuf_getbufaddr(ch->buffer), 4);
es_wr(es, ES1370_REG_DAC2_FRAMECNT & 0xff,
(ch->bufsz >> 2) - 1, 4);
}
} else {
es_wr(es, ES1370_REG_MEMPAGE, ES1370_REG_ADC_FRAMEADR >> 8, 1);
es_wr(es, ES1370_REG_ADC_FRAMEADR & 0xff,
sndbuf_getbufaddr(ch->buffer), 4);
es_wr(es, ES1370_REG_ADC_FRAMECNT & 0xff,
(ch->bufsz >> 2) - 1, 4);
}
ES_UNLOCK(es);
return (ch);
}
static int
eschan_setformat(kobj_t obj, void *data, uint32_t format)
{
struct es_chinfo *ch = data;
struct es_info *es = ch->parent;
ES_LOCK(es);
if (ch->dir == PCMDIR_PLAY) {
if (ch->index == ES_DAC1) {
es->sctrl &= ~SCTRL_P1FMT;
if (format & AFMT_S16_LE)
es->sctrl |= SCTRL_P1SEB;
if (AFMT_CHANNEL(format) > 1)
es->sctrl |= SCTRL_P1SMB;
} else {
es->sctrl &= ~SCTRL_P2FMT;
if (format & AFMT_S16_LE)
es->sctrl |= SCTRL_P2SEB;
if (AFMT_CHANNEL(format) > 1)
es->sctrl |= SCTRL_P2SMB;
}
} else {
es->sctrl &= ~SCTRL_R1FMT;
if (format & AFMT_S16_LE)
es->sctrl |= SCTRL_R1SEB;
if (AFMT_CHANNEL(format) > 1)
es->sctrl |= SCTRL_R1SMB;
}
es_wr(es, ES1370_REG_SERIAL_CONTROL, es->sctrl, 4);
ES_UNLOCK(es);
ch->fmt = format;
return (0);
}
static uint32_t
eschan1370_setspeed(kobj_t obj, void *data, uint32_t speed)
{
struct es_chinfo *ch = data;
struct es_info *es = ch->parent;
ES_LOCK(es);
/* Fixed rate , do nothing. */
if (ch->caps.minspeed == ch->caps.maxspeed) {
ES_UNLOCK(es);
return (ch->caps.maxspeed);
}
if (speed < ch->caps.minspeed)
speed = ch->caps.minspeed;
if (speed > ch->caps.maxspeed)
speed = ch->caps.maxspeed;
if (ch->index == ES_DAC1) {
/*
* DAC1 does not support continuous rate settings.
* Pick the nearest and use it since FEEDER_RATE will
* do the proper conversion for us.
*/
es->ctrl &= ~CTRL_WTSRSEL;
if (speed < 8268) {
speed = 5512;
es->ctrl |= 0 << CTRL_SH_WTSRSEL;
} else if (speed < 16537) {
speed = 11025;
es->ctrl |= 1 << CTRL_SH_WTSRSEL;
} else if (speed < 33075) {
speed = 22050;
es->ctrl |= 2 << CTRL_SH_WTSRSEL;
} else {
speed = 44100;
es->ctrl |= 3 << CTRL_SH_WTSRSEL;
}
} else {
es->ctrl &= ~CTRL_PCLKDIV;
es->ctrl |= DAC2_SRTODIV(speed) << CTRL_SH_PCLKDIV;
}
es_wr(es, ES1370_REG_CONTROL, es->ctrl, 4);
ES_UNLOCK(es);
return (speed);
}
static uint32_t
eschan1371_setspeed(kobj_t obj, void *data, uint32_t speed)
{
struct es_chinfo *ch = data;
struct es_info *es = ch->parent;
uint32_t i;
int delta;
ES_LOCK(es);
if (ch->dir == PCMDIR_PLAY)
i = es1371_dac_rate(es, speed, ch->index); /* play */
else
i = es1371_adc_rate(es, speed, ch->index); /* record */
ES_UNLOCK(es);
delta = (speed > i) ? (speed - i) : (i - speed);
if (delta < 2)
return (speed);
return (i);
}
static int
eschan_setfragments(kobj_t obj, void *data, uint32_t blksz, uint32_t blkcnt)
{
struct es_chinfo *ch = data;
struct es_info *es = ch->parent;
blksz &= ES_BLK_ALIGN;
if (blksz > (sndbuf_getmaxsize(ch->buffer) / ES_DMA_SEGS_MIN))
blksz = sndbuf_getmaxsize(ch->buffer) / ES_DMA_SEGS_MIN;
if (blksz < ES_BLK_MIN)
blksz = ES_BLK_MIN;
if (blkcnt > ES_DMA_SEGS_MAX)
blkcnt = ES_DMA_SEGS_MAX;
if (blkcnt < ES_DMA_SEGS_MIN)
blkcnt = ES_DMA_SEGS_MIN;
while ((blksz * blkcnt) > sndbuf_getmaxsize(ch->buffer)) {
if ((blkcnt >> 1) >= ES_DMA_SEGS_MIN)
blkcnt >>= 1;
else if ((blksz >> 1) >= ES_BLK_MIN)
blksz >>= 1;
else
break;
}
if ((sndbuf_getblksz(ch->buffer) != blksz ||
sndbuf_getblkcnt(ch->buffer) != blkcnt) &&
sndbuf_resize(ch->buffer, blkcnt, blksz) != 0)
device_printf(es->dev, "%s: failed blksz=%u blkcnt=%u\n",
__func__, blksz, blkcnt);
ch->bufsz = sndbuf_getsize(ch->buffer);
ch->blksz = sndbuf_getblksz(ch->buffer);
ch->blkcnt = sndbuf_getblkcnt(ch->buffer);
return (0);
}
static uint32_t
eschan_setblocksize(kobj_t obj, void *data, uint32_t blksz)
{
struct es_chinfo *ch = data;
struct es_info *es = ch->parent;
eschan_setfragments(obj, data, blksz, es->blkcnt);
return (ch->blksz);
}
#define es_chan_active(es) ((es)->ch[ES_DAC1].active + \
(es)->ch[ES_DAC2].active + \
(es)->ch[ES_ADC].active)
static __inline int
es_poll_channel(struct es_chinfo *ch)
{
struct es_info *es;
uint32_t sz, delta;
uint32_t reg, ptr;
if (ch == NULL || ch->channel == NULL || ch->active == 0)
return (0);
es = ch->parent;
if (ch->dir == PCMDIR_PLAY) {
if (ch->index == ES_DAC1)
reg = ES1370_REG_DAC1_FRAMECNT;
else
reg = ES1370_REG_DAC2_FRAMECNT;
} else
reg = ES1370_REG_ADC_FRAMECNT;
sz = ch->blksz * ch->blkcnt;
es_wr(es, ES1370_REG_MEMPAGE, reg >> 8, 4);
ptr = es_rd(es, reg & 0x000000ff, 4) >> 16;
ptr <<= 2;
ch->ptr = ptr;
ptr %= sz;
ptr &= ~(ch->blksz - 1);
delta = (sz + ptr - ch->prevptr) % sz;
if (delta < ch->blksz)
return (0);
ch->prevptr = ptr;
return (1);
}
static void
es_poll_callback(void *arg)
{
struct es_info *es = arg;
uint32_t trigger = 0;
int i;
if (es == NULL)
return;
ES_LOCK(es);
if (es->polling == 0 || es_chan_active(es) == 0) {
ES_UNLOCK(es);
return;
}
for (i = 0; i < ES_NCHANS; i++) {
if (es_poll_channel(&es->ch[i]) != 0)
trigger |= 1 << i;
}
/* XXX */
callout_reset(&es->poll_timer, 1/*es->poll_ticks*/,
es_poll_callback, es);
ES_UNLOCK(es);
for (i = 0; i < ES_NCHANS; i++) {
if (trigger & (1 << i))
chn_intr(es->ch[i].channel);
}
}
static int
eschan_trigger(kobj_t obj, void *data, int go)
{
struct es_chinfo *ch = data;
struct es_info *es = ch->parent;
uint32_t cnt, b = 0;
if (!PCMTRIG_COMMON(go))
return 0;
ES_LOCK(es);
cnt = (ch->blksz / sndbuf_getalign(ch->buffer)) - 1;
if (ch->fmt & AFMT_16BIT)
b |= 0x02;
if (AFMT_CHANNEL(ch->fmt) > 1)
b |= 0x01;
if (ch->dir == PCMDIR_PLAY) {
if (go == PCMTRIG_START) {
if (ch->index == ES_DAC1) {
es->ctrl |= CTRL_DAC1_EN;
es->sctrl &= ~(SCTRL_P1LOOPSEL |
SCTRL_P1PAUSE | SCTRL_P1SCTRLD);
if (es->polling == 0)
es->sctrl |= SCTRL_P1INTEN;
else
es->sctrl &= ~SCTRL_P1INTEN;
es->sctrl |= b;
es_wr(es, ES1370_REG_DAC1_SCOUNT, cnt, 4);
/* start at beginning of buffer */
es_wr(es, ES1370_REG_MEMPAGE,
ES1370_REG_DAC1_FRAMECNT >> 8, 4);
es_wr(es, ES1370_REG_DAC1_FRAMECNT & 0xff,
(ch->bufsz >> 2) - 1, 4);
} else {
es->ctrl |= CTRL_DAC2_EN;
es->sctrl &= ~(SCTRL_P2ENDINC | SCTRL_P2STINC |
SCTRL_P2LOOPSEL | SCTRL_P2PAUSE |
SCTRL_P2DACSEN);
if (es->polling == 0)
es->sctrl |= SCTRL_P2INTEN;
else
es->sctrl &= ~SCTRL_P2INTEN;
es->sctrl |= (b << 2) |
((((b >> 1) & 1) + 1) << SCTRL_SH_P2ENDINC);
es_wr(es, ES1370_REG_DAC2_SCOUNT, cnt, 4);
/* start at beginning of buffer */
es_wr(es, ES1370_REG_MEMPAGE,
ES1370_REG_DAC2_FRAMECNT >> 8, 4);
es_wr(es, ES1370_REG_DAC2_FRAMECNT & 0xff,
(ch->bufsz >> 2) - 1, 4);
}
} else
es->ctrl &= ~((ch->index == ES_DAC1) ?
CTRL_DAC1_EN : CTRL_DAC2_EN);
} else {
if (go == PCMTRIG_START) {
es->ctrl |= CTRL_ADC_EN;
es->sctrl &= ~SCTRL_R1LOOPSEL;
if (es->polling == 0)
es->sctrl |= SCTRL_R1INTEN;
else
es->sctrl &= ~SCTRL_R1INTEN;
es->sctrl |= b << 4;
es_wr(es, ES1370_REG_ADC_SCOUNT, cnt, 4);
/* start at beginning of buffer */
es_wr(es, ES1370_REG_MEMPAGE,
ES1370_REG_ADC_FRAMECNT >> 8, 4);
es_wr(es, ES1370_REG_ADC_FRAMECNT & 0xff,
(ch->bufsz >> 2) - 1, 4);
} else
es->ctrl &= ~CTRL_ADC_EN;
}
es_wr(es, ES1370_REG_SERIAL_CONTROL, es->sctrl, 4);
es_wr(es, ES1370_REG_CONTROL, es->ctrl, 4);
if (go == PCMTRIG_START) {
if (es->polling != 0) {
ch->ptr = 0;
ch->prevptr = 0;
if (es_chan_active(es) == 0) {
es->poll_ticks = 1;
callout_reset(&es->poll_timer, 1,
es_poll_callback, es);
}
}
ch->active = 1;
} else {
ch->active = 0;
if (es->polling != 0) {
if (es_chan_active(es) == 0) {
callout_stop(&es->poll_timer);
es->poll_ticks = 1;
}
}
}
ES_UNLOCK(es);
return (0);
}
static uint32_t
eschan_getptr(kobj_t obj, void *data)
{
struct es_chinfo *ch = data;
struct es_info *es = ch->parent;
uint32_t reg, cnt;
ES_LOCK(es);
if (es->polling != 0)
cnt = ch->ptr;
else {
if (ch->dir == PCMDIR_PLAY) {
if (ch->index == ES_DAC1)
reg = ES1370_REG_DAC1_FRAMECNT;
else
reg = ES1370_REG_DAC2_FRAMECNT;
} else
reg = ES1370_REG_ADC_FRAMECNT;
es_wr(es, ES1370_REG_MEMPAGE, reg >> 8, 4);
cnt = es_rd(es, reg & 0x000000ff, 4) >> 16;
/* cnt is longwords */
cnt <<= 2;
}
ES_UNLOCK(es);
cnt &= ES_BLK_ALIGN;
return (cnt);
}
static struct pcmchan_caps *
eschan_getcaps(kobj_t obj, void *data)
{
struct es_chinfo *ch = data;
return (&ch->caps);
}
static kobj_method_t eschan1370_methods[] = {
KOBJMETHOD(channel_init, eschan_init),
KOBJMETHOD(channel_setformat, eschan_setformat),
KOBJMETHOD(channel_setspeed, eschan1370_setspeed),
KOBJMETHOD(channel_setblocksize, eschan_setblocksize),
KOBJMETHOD(channel_setfragments, eschan_setfragments),
KOBJMETHOD(channel_trigger, eschan_trigger),
KOBJMETHOD(channel_getptr, eschan_getptr),
KOBJMETHOD(channel_getcaps, eschan_getcaps),
KOBJMETHOD_END
};
CHANNEL_DECLARE(eschan1370);
static kobj_method_t eschan1371_methods[] = {
KOBJMETHOD(channel_init, eschan_init),
KOBJMETHOD(channel_setformat, eschan_setformat),
KOBJMETHOD(channel_setspeed, eschan1371_setspeed),
KOBJMETHOD(channel_setblocksize, eschan_setblocksize),
KOBJMETHOD(channel_setfragments, eschan_setfragments),
KOBJMETHOD(channel_trigger, eschan_trigger),
KOBJMETHOD(channel_getptr, eschan_getptr),
KOBJMETHOD(channel_getcaps, eschan_getcaps),
KOBJMETHOD_END
};
CHANNEL_DECLARE(eschan1371);
/* -------------------------------------------------------------------- */
/* The interrupt handler */
static void
es_intr(void *p)
{
struct es_info *es = p;
uint32_t intsrc, sctrl;
ES_LOCK(es);
if (es->polling != 0) {
ES_UNLOCK(es);
return;
}
intsrc = es_rd(es, ES1370_REG_STATUS, 4);
if ((intsrc & STAT_INTR) == 0) {
ES_UNLOCK(es);
return;
}
sctrl = es->sctrl;
if (intsrc & STAT_ADC)
sctrl &= ~SCTRL_R1INTEN;
if (intsrc & STAT_DAC1)
sctrl &= ~SCTRL_P1INTEN;
if (intsrc & STAT_DAC2)
sctrl &= ~SCTRL_P2INTEN;
es_wr(es, ES1370_REG_SERIAL_CONTROL, sctrl, 4);
es_wr(es, ES1370_REG_SERIAL_CONTROL, es->sctrl, 4);
ES_UNLOCK(es);
if (intsrc & STAT_ADC)
chn_intr(es->ch[ES_ADC].channel);
if (intsrc & STAT_DAC1)
chn_intr(es->ch[ES_DAC1].channel);
if (intsrc & STAT_DAC2)
chn_intr(es->ch[ES_DAC2].channel);
}
/* ES1370 specific */
static int
es1370_init(struct es_info *es)
{
uint32_t fixed_rate;
int r, single_pcm;
/* ES1370 default to fixed rate operation */
if (resource_int_value(device_get_name(es->dev),
device_get_unit(es->dev), "fixed_rate", &r) == 0) {
fixed_rate = r;
if (fixed_rate) {
if (fixed_rate < es_caps.minspeed)
fixed_rate = es_caps.minspeed;
if (fixed_rate > es_caps.maxspeed)
fixed_rate = es_caps.maxspeed;
}
} else
fixed_rate = es_caps.maxspeed;
if (resource_int_value(device_get_name(es->dev),
device_get_unit(es->dev), "single_pcm_mixer", &r) == 0)
single_pcm = (r != 0) ? 1 : 0;
else
single_pcm = 1;
ES_LOCK(es);
if (ES_NUMPLAY(es->escfg) == 1)
single_pcm = 1;
/* This is ES1370 */
es->escfg = ES_SET_IS_ES1370(es->escfg, 1);
if (fixed_rate)
es->escfg = ES_SET_FIXED_RATE(es->escfg, fixed_rate);
else {
es->escfg = ES_SET_FIXED_RATE(es->escfg, 0);
fixed_rate = DSP_DEFAULT_SPEED;
}
if (single_pcm)
es->escfg = ES_SET_SINGLE_PCM_MIX(es->escfg, 1);
else
es->escfg = ES_SET_SINGLE_PCM_MIX(es->escfg, 0);
es->ctrl = CTRL_CDC_EN | CTRL_JYSTK_EN | CTRL_SERR_DIS |
(DAC2_SRTODIV(fixed_rate) << CTRL_SH_PCLKDIV);
es->ctrl |= 3 << CTRL_SH_WTSRSEL;
es_wr(es, ES1370_REG_CONTROL, es->ctrl, 4);
es->sctrl = 0;
es_wr(es, ES1370_REG_SERIAL_CONTROL, es->sctrl, 4);
/* No RST, PD */
es1370_wrcodec(es, CODEC_RES_PD, 3);
/*
* CODEC ADC and CODEC DAC use {LR,B}CLK2 and run off the LRCLK2 PLL;
* program DAC_SYNC=0!
*/
es1370_wrcodec(es, CODEC_CSEL, 0);
/* Recording source is mixer */
es1370_wrcodec(es, CODEC_ADSEL, 0);
/* MIC amp is 0db */
es1370_wrcodec(es, CODEC_MGAIN, 0);
ES_UNLOCK(es);
return (0);
}
/* ES1371 specific */
int
es1371_init(struct es_info *es)
{
uint32_t cssr, devid, revid, subdev;
int idx;
ES_LOCK(es);
/* This is NOT ES1370 */
es->escfg = ES_SET_IS_ES1370(es->escfg, 0);
es->num = 0;
es->sctrl = 0;
cssr = 0;
devid = pci_get_devid(es->dev);
revid = pci_get_revid(es->dev);
subdev = (pci_get_subdevice(es->dev) << 16) |
pci_get_subvendor(es->dev);
/*
* Joyport blacklist. Either we're facing with broken hardware
* or because this hardware need special (unknown) initialization
* procedures.
*/
switch (subdev) {
case 0x20001274: /* old Ensoniq */
es->ctrl = 0;
break;
default:
es->ctrl = CTRL_JYSTK_EN;
break;
}
if (devid == CT4730_PCI_ID) {
/* XXX amplifier hack? */
es->ctrl |= (1 << 16);
}
/* initialize the chips */
es_wr(es, ES1370_REG_CONTROL, es->ctrl, 4);
es_wr(es, ES1370_REG_SERIAL_CONTROL, es->sctrl, 4);
es_wr(es, ES1371_REG_LEGACY, 0, 4);
if ((devid == ES1371_PCI_ID && revid == ES1371REV_ES1373_8) ||
(devid == ES1371_PCI_ID && revid == ES1371REV_CT5880_A) ||
(devid == CT5880_PCI_ID && revid == CT5880REV_CT5880_C) ||
(devid == CT5880_PCI_ID && revid == CT5880REV_CT5880_D) ||
(devid == CT5880_PCI_ID && revid == CT5880REV_CT5880_E)) {
cssr = 1 << 29;
es_wr(es, ES1370_REG_STATUS, cssr, 4);
DELAY(20000);
}
/* AC'97 warm reset to start the bitclk */
es_wr(es, ES1370_REG_CONTROL, es->ctrl, 4);
es_wr(es, ES1371_REG_LEGACY, ES1371_SYNC_RES, 4);
DELAY(2000);
es_wr(es, ES1370_REG_CONTROL, es->sctrl, 4);
es1371_wait_src_ready(es);
/* Init the sample rate converter */
es_wr(es, ES1371_REG_SMPRATE, ES1371_DIS_SRC, 4);
for (idx = 0; idx < 0x80; idx++)
es1371_src_write(es, idx, 0);
es1371_src_write(es, ES_SMPREG_DAC1 + ES_SMPREG_TRUNC_N, 16 << 4);
es1371_src_write(es, ES_SMPREG_DAC1 + ES_SMPREG_INT_REGS, 16 << 10);
es1371_src_write(es, ES_SMPREG_DAC2 + ES_SMPREG_TRUNC_N, 16 << 4);
es1371_src_write(es, ES_SMPREG_DAC2 + ES_SMPREG_INT_REGS, 16 << 10);
es1371_src_write(es, ES_SMPREG_VOL_ADC, 1 << 12);
es1371_src_write(es, ES_SMPREG_VOL_ADC + 1, 1 << 12);
es1371_src_write(es, ES_SMPREG_VOL_DAC1, 1 << 12);
es1371_src_write(es, ES_SMPREG_VOL_DAC1 + 1, 1 << 12);
es1371_src_write(es, ES_SMPREG_VOL_DAC2, 1 << 12);
es1371_src_write(es, ES_SMPREG_VOL_DAC2 + 1, 1 << 12);
es1371_adc_rate(es, 22050, ES_ADC);
es1371_dac_rate(es, 22050, ES_DAC1);
es1371_dac_rate(es, 22050, ES_DAC2);
/*
* WARNING:
* enabling the sample rate converter without properly programming
* its parameters causes the chip to lock up (the SRC busy bit will
* be stuck high, and I've found no way to rectify this other than
* power cycle)
*/
es1371_wait_src_ready(es);
es_wr(es, ES1371_REG_SMPRATE, 0, 4);
/* try to reset codec directly */
es_wr(es, ES1371_REG_CODEC, 0, 4);
es_wr(es, ES1370_REG_STATUS, cssr, 4);
ES_UNLOCK(es);
return (0);
}
/* -------------------------------------------------------------------- */
static int
es1371_wrcd(kobj_t obj, void *s, int addr, uint32_t data)
{
uint32_t t, x, orig;
struct es_info *es = (struct es_info*)s;
for (t = 0; t < 0x1000; t++) {
if (!es_rd(es, ES1371_REG_CODEC & CODEC_WIP, 4))
break;
}
/* save the current state for later */
x = orig = es_rd(es, ES1371_REG_SMPRATE, 4);
/* enable SRC state data in SRC mux */
es_wr(es, ES1371_REG_SMPRATE, (x & (ES1371_DIS_SRC | ES1371_DIS_P1 |
ES1371_DIS_P2 | ES1371_DIS_R1)) | 0x00010000, 4);
/* busy wait */
for (t = 0; t < 0x1000; t++) {
if ((es_rd(es, ES1371_REG_SMPRATE, 4) & 0x00870000) ==
0x00000000)
break;
}
/* wait for a SAFE time to write addr/data and then do it, dammit */
for (t = 0; t < 0x1000; t++) {
if ((es_rd(es, ES1371_REG_SMPRATE, 4) & 0x00870000) ==
0x00010000)
break;
}
es_wr(es, ES1371_REG_CODEC, ((addr << CODEC_POADD_SHIFT) &
CODEC_POADD_MASK) | ((data << CODEC_PODAT_SHIFT) &
CODEC_PODAT_MASK), 4);
/* restore SRC reg */
es1371_wait_src_ready(s);
es_wr(es, ES1371_REG_SMPRATE, orig, 4);
return (0);
}
static int
es1371_rdcd(kobj_t obj, void *s, int addr)
{
uint32_t t, x, orig;
struct es_info *es = (struct es_info *)s;
for (t = 0; t < 0x1000; t++) {
if (!(x = es_rd(es, ES1371_REG_CODEC, 4) & CODEC_WIP))
break;
}
/* save the current state for later */
x = orig = es_rd(es, ES1371_REG_SMPRATE, 4);
/* enable SRC state data in SRC mux */
es_wr(es, ES1371_REG_SMPRATE, (x & (ES1371_DIS_SRC | ES1371_DIS_P1 |
ES1371_DIS_P2 | ES1371_DIS_R1)) | 0x00010000, 4);
/* busy wait */
for (t = 0; t < 0x1000; t++) {
if ((x = es_rd(es, ES1371_REG_SMPRATE, 4) & 0x00870000) ==
0x00000000)
break;
}
/* wait for a SAFE time to write addr/data and then do it, dammit */
for (t = 0; t < 0x1000; t++) {
if ((x = es_rd(es, ES1371_REG_SMPRATE, 4) & 0x00870000) ==
0x00010000)
break;
}
es_wr(es, ES1371_REG_CODEC, ((addr << CODEC_POADD_SHIFT) &
CODEC_POADD_MASK) | CODEC_PORD, 4);
/* restore SRC reg */
es1371_wait_src_ready(s);
es_wr(es, ES1371_REG_SMPRATE, orig, 4);
/* now wait for the stinkin' data (RDY) */
for (t = 0; t < 0x1000; t++) {
if ((x = es_rd(es, ES1371_REG_CODEC, 4)) & CODEC_RDY)
break;
}
return ((x & CODEC_PIDAT_MASK) >> CODEC_PIDAT_SHIFT);
}
static kobj_method_t es1371_ac97_methods[] = {
KOBJMETHOD(ac97_read, es1371_rdcd),
KOBJMETHOD(ac97_write, es1371_wrcd),
KOBJMETHOD_END
};
AC97_DECLARE(es1371_ac97);
/* -------------------------------------------------------------------- */
static unsigned int
es1371_src_read(struct es_info *es, unsigned short reg)
{
uint32_t r;
r = es1371_wait_src_ready(es) & (ES1371_DIS_SRC | ES1371_DIS_P1 |
ES1371_DIS_P2 | ES1371_DIS_R1);
r |= ES1371_SRC_RAM_ADDRO(reg);
es_wr(es, ES1371_REG_SMPRATE, r, 4);
return (ES1371_SRC_RAM_DATAI(es1371_wait_src_ready(es)));
}
static void
es1371_src_write(struct es_info *es, unsigned short reg, unsigned short data)
{
uint32_t r;
r = es1371_wait_src_ready(es) & (ES1371_DIS_SRC | ES1371_DIS_P1 |
ES1371_DIS_P2 | ES1371_DIS_R1);
r |= ES1371_SRC_RAM_ADDRO(reg) | ES1371_SRC_RAM_DATAO(data);
es_wr(es, ES1371_REG_SMPRATE, r | ES1371_SRC_RAM_WE, 4);
}
static unsigned int
es1371_adc_rate(struct es_info *es, unsigned int rate, int set)
{
unsigned int n, truncm, freq, result;
ES_LOCK_ASSERT(es);
if (rate > 48000)
rate = 48000;
if (rate < 4000)
rate = 4000;
n = rate / 3000;
if ((1 << n) & ((1 << 15) | (1 << 13) | (1 << 11) | (1 << 9)))
n--;
truncm = (21 * n - 1) | 1;
freq = ((48000UL << 15) / rate) * n;
result = (48000UL << 15) / (freq / n);
if (set) {
if (rate >= 24000) {
if (truncm > 239)
truncm = 239;
es1371_src_write(es, ES_SMPREG_ADC + ES_SMPREG_TRUNC_N,
(((239 - truncm) >> 1) << 9) | (n << 4));
} else {
if (truncm > 119)
truncm = 119;
es1371_src_write(es, ES_SMPREG_ADC + ES_SMPREG_TRUNC_N,
0x8000 | (((119 - truncm) >> 1) << 9) | (n << 4));
}
es1371_src_write(es, ES_SMPREG_ADC + ES_SMPREG_INT_REGS,
(es1371_src_read(es, ES_SMPREG_ADC + ES_SMPREG_INT_REGS) &
0x00ff) | ((freq >> 5) & 0xfc00));
es1371_src_write(es, ES_SMPREG_ADC + ES_SMPREG_VFREQ_FRAC,
freq & 0x7fff);
es1371_src_write(es, ES_SMPREG_VOL_ADC, n << 8);
es1371_src_write(es, ES_SMPREG_VOL_ADC + 1, n << 8);
}
return (result);
}
static unsigned int
es1371_dac_rate(struct es_info *es, unsigned int rate, int set)
{
unsigned int freq, r, result, dac, dis;
ES_LOCK_ASSERT(es);
if (rate > 48000)
rate = 48000;
if (rate < 4000)
rate = 4000;
freq = ((rate << 15) + 1500) / 3000;
result = (freq * 3000) >> 15;
dac = (set == ES_DAC1) ? ES_SMPREG_DAC1 : ES_SMPREG_DAC2;
dis = (set == ES_DAC1) ? ES1371_DIS_P2 : ES1371_DIS_P1;
r = (es1371_wait_src_ready(es) & (ES1371_DIS_SRC | ES1371_DIS_P1 |
ES1371_DIS_P2 | ES1371_DIS_R1));
es_wr(es, ES1371_REG_SMPRATE, r, 4);
es1371_src_write(es, dac + ES_SMPREG_INT_REGS,
(es1371_src_read(es, dac + ES_SMPREG_INT_REGS) & 0x00ff) |
((freq >> 5) & 0xfc00));
es1371_src_write(es, dac + ES_SMPREG_VFREQ_FRAC, freq & 0x7fff);
r = (es1371_wait_src_ready(es) &
(ES1371_DIS_SRC | dis | ES1371_DIS_R1));
es_wr(es, ES1371_REG_SMPRATE, r, 4);
return (result);
}
static uint32_t
es1371_wait_src_ready(struct es_info *es)
{
uint32_t t, r;
for (t = 0; t < 0x1000; t++) {
if (!((r = es_rd(es, ES1371_REG_SMPRATE, 4)) &
ES1371_SRC_RAM_BUSY))
return (r);
DELAY(1);
}
device_printf(es->dev, "%s: timed out 0x%x [0x%x]\n", __func__,
ES1371_REG_SMPRATE, r);
return (0);
}
/* -------------------------------------------------------------------- */
/*
* Probe and attach the card
*/
static int
es_pci_probe(device_t dev)
{
switch(pci_get_devid(dev)) {
case ES1370_PCI_ID:
device_set_desc(dev, "AudioPCI ES1370");
return (BUS_PROBE_DEFAULT);
case ES1371_PCI_ID:
switch(pci_get_revid(dev)) {
case ES1371REV_ES1371_A:
device_set_desc(dev, "AudioPCI ES1371-A");
return (BUS_PROBE_DEFAULT);
case ES1371REV_ES1371_B:
device_set_desc(dev, "AudioPCI ES1371-B");
return (BUS_PROBE_DEFAULT);
case ES1371REV_ES1373_A:
device_set_desc(dev, "AudioPCI ES1373-A");
return (BUS_PROBE_DEFAULT);
case ES1371REV_ES1373_B:
device_set_desc(dev, "AudioPCI ES1373-B");
return (BUS_PROBE_DEFAULT);
case ES1371REV_ES1373_8:
device_set_desc(dev, "AudioPCI ES1373-8");
return (BUS_PROBE_DEFAULT);
case ES1371REV_CT5880_A:
device_set_desc(dev, "Creative CT5880-A");
return (BUS_PROBE_DEFAULT);
default:
device_set_desc(dev, "AudioPCI ES1371-?");
device_printf(dev,
"unknown revision %d -- please report to "
"freebsd-multimedia@freebsd.org\n",
pci_get_revid(dev));
return (BUS_PROBE_DEFAULT);
}
case ES1371_PCI_ID2:
device_set_desc(dev, "Strange AudioPCI ES1371-? (vid=3274)");
device_printf(dev,
"unknown revision %d -- please report to "
"freebsd-multimedia@freebsd.org\n", pci_get_revid(dev));
return (BUS_PROBE_DEFAULT);
case CT4730_PCI_ID:
switch(pci_get_revid(dev)) {
case CT4730REV_CT4730_A:
device_set_desc(dev,
"Creative SB AudioPCI CT4730/EV1938");
return (BUS_PROBE_DEFAULT);
default:
device_set_desc(dev, "Creative SB AudioPCI CT4730-?");
device_printf(dev,
"unknown revision %d -- please report to "
"freebsd-multimedia@freebsd.org\n",
pci_get_revid(dev));
return (BUS_PROBE_DEFAULT);
}
case CT5880_PCI_ID:
switch(pci_get_revid(dev)) {
case CT5880REV_CT5880_C:
device_set_desc(dev, "Creative CT5880-C");
return (BUS_PROBE_DEFAULT);
case CT5880REV_CT5880_D:
device_set_desc(dev, "Creative CT5880-D");
return (BUS_PROBE_DEFAULT);
case CT5880REV_CT5880_E:
device_set_desc(dev, "Creative CT5880-E");
return (BUS_PROBE_DEFAULT);
default:
device_set_desc(dev, "Creative CT5880-?");
device_printf(dev,
"unknown revision %d -- please report to "
"freebsd-multimedia@freebsd.org\n",
pci_get_revid(dev));
return (BUS_PROBE_DEFAULT);
}
default:
return (ENXIO);
}
}
static int
sysctl_es137x_spdif_enable(SYSCTL_HANDLER_ARGS)
{
struct es_info *es;
device_t dev;
uint32_t r;
int err, new_en;
dev = oidp->oid_arg1;
es = pcm_getdevinfo(dev);
ES_LOCK(es);
r = es_rd(es, ES1370_REG_STATUS, 4);
ES_UNLOCK(es);
new_en = (r & ENABLE_SPDIF) ? 1 : 0;
err = sysctl_handle_int(oidp, &new_en, 0, req);
if (err || req->newptr == NULL)
return (err);
if (new_en < 0 || new_en > 1)
return (EINVAL);
ES_LOCK(es);
if (new_en) {
r |= ENABLE_SPDIF;
es->ctrl |= SPDIFEN_B;
es->ctrl |= RECEN_B;
} else {
r &= ~ENABLE_SPDIF;
es->ctrl &= ~SPDIFEN_B;
es->ctrl &= ~RECEN_B;
}
es_wr(es, ES1370_REG_CONTROL, es->ctrl, 4);
es_wr(es, ES1370_REG_STATUS, r, 4);
ES_UNLOCK(es);
return (0);
}
static int
sysctl_es137x_latency_timer(SYSCTL_HANDLER_ARGS)
{
struct es_info *es;
device_t dev;
uint32_t val;
int err;
dev = oidp->oid_arg1;
es = pcm_getdevinfo(dev);
ES_LOCK(es);
val = pci_read_config(dev, PCIR_LATTIMER, 1);
ES_UNLOCK(es);
err = sysctl_handle_int(oidp, &val, 0, req);
if (err || req->newptr == NULL)
return (err);
if (val > 255)
return (EINVAL);
ES_LOCK(es);
pci_write_config(dev, PCIR_LATTIMER, val, 1);
ES_UNLOCK(es);
return (0);
}
static int
sysctl_es137x_fixed_rate(SYSCTL_HANDLER_ARGS)
{
struct es_info *es;
device_t dev;
uint32_t val;
int err;
dev = oidp->oid_arg1;
es = pcm_getdevinfo(dev);
ES_LOCK(es);
val = ES_FIXED_RATE(es->escfg);
if (val < es_caps.minspeed)
val = 0;
ES_UNLOCK(es);
err = sysctl_handle_int(oidp, &val, 0, req);
if (err || req->newptr == NULL)
return (err);
if (val != 0 && (val < es_caps.minspeed || val > es_caps.maxspeed))
return (EINVAL);
ES_LOCK(es);
if (es->ctrl & (CTRL_DAC2_EN|CTRL_ADC_EN)) {
ES_UNLOCK(es);
return (EBUSY);
}
if (val) {
if (val != ES_FIXED_RATE(es->escfg)) {
es->escfg = ES_SET_FIXED_RATE(es->escfg, val);
es->ch[ES_DAC2].caps.maxspeed = val;
es->ch[ES_DAC2].caps.minspeed = val;
es->ch[ES_ADC].caps.maxspeed = val;
es->ch[ES_ADC].caps.minspeed = val;
es->ctrl &= ~CTRL_PCLKDIV;
es->ctrl |= DAC2_SRTODIV(val) << CTRL_SH_PCLKDIV;
es_wr(es, ES1370_REG_CONTROL, es->ctrl, 4);
}
} else {
es->escfg = ES_SET_FIXED_RATE(es->escfg, 0);
es->ch[ES_DAC2].caps = es_caps;
es->ch[ES_ADC].caps = es_caps;
}
ES_UNLOCK(es);
return (0);
}
static int
sysctl_es137x_single_pcm_mixer(SYSCTL_HANDLER_ARGS)
{
struct es_info *es;
struct snddev_info *d;
struct snd_mixer *m;
device_t dev;
uint32_t val, set;
int recsrc, level, err;
dev = oidp->oid_arg1;
d = device_get_softc(dev);
if (!PCM_REGISTERED(d) || d->mixer_dev == NULL ||
d->mixer_dev->si_drv1 == NULL)
return (EINVAL);
es = d->devinfo;
if (es == NULL)
return (EINVAL);
ES_LOCK(es);
set = ES_SINGLE_PCM_MIX(es->escfg);
val = set;
ES_UNLOCK(es);
err = sysctl_handle_int(oidp, &val, 0, req);
if (err || req->newptr == NULL)
return (err);
if (!(val == 0 || val == 1))
return (EINVAL);
if (val == set)
return (0);
PCM_ACQUIRE_QUICK(d);
m = (d->mixer_dev != NULL) ? d->mixer_dev->si_drv1 : NULL;
if (m == NULL) {
PCM_RELEASE_QUICK(d);
return (ENODEV);
}
if (mixer_busy(m) != 0) {
PCM_RELEASE_QUICK(d);
return (EBUSY);
}
level = mix_get(m, SOUND_MIXER_PCM);
recsrc = mix_getrecsrc(m);
if (level < 0 || recsrc < 0) {
PCM_RELEASE_QUICK(d);
return (ENXIO);
}
ES_LOCK(es);
if (es->ctrl & (CTRL_ADC_EN | CTRL_DAC1_EN | CTRL_DAC2_EN)) {
ES_UNLOCK(es);
PCM_RELEASE_QUICK(d);
return (EBUSY);
}
if (val)
es->escfg = ES_SET_SINGLE_PCM_MIX(es->escfg, 1);
else
es->escfg = ES_SET_SINGLE_PCM_MIX(es->escfg, 0);
ES_UNLOCK(es);
if (!val) {
mix_setdevs(m, mix_getdevs(m) | (1 << SOUND_MIXER_SYNTH));
mix_setrecdevs(m, mix_getrecdevs(m) | (1 << SOUND_MIXER_SYNTH));
err = mix_set(m, SOUND_MIXER_SYNTH, level & 0x7f,
(level >> 8) & 0x7f);
} else {
err = mix_set(m, SOUND_MIXER_SYNTH, level & 0x7f,
(level >> 8) & 0x7f);
mix_setdevs(m, mix_getdevs(m) & ~(1 << SOUND_MIXER_SYNTH));
mix_setrecdevs(m, mix_getrecdevs(m) &
~(1 << SOUND_MIXER_SYNTH));
}
if (!err) {
level = recsrc;
if (recsrc & (1 << SOUND_MIXER_PCM))
recsrc |= 1 << SOUND_MIXER_SYNTH;
else if (recsrc & (1 << SOUND_MIXER_SYNTH))
recsrc |= 1 << SOUND_MIXER_PCM;
if (level != recsrc)
err = mix_setrecsrc(m, recsrc);
}
PCM_RELEASE_QUICK(d);
return (err);
}
static int
sysctl_es_polling(SYSCTL_HANDLER_ARGS)
{
struct es_info *es;
device_t dev;
int err, val;
dev = oidp->oid_arg1;
es = pcm_getdevinfo(dev);
if (es == NULL)
return (EINVAL);
ES_LOCK(es);
val = es->polling;
ES_UNLOCK(es);
err = sysctl_handle_int(oidp, &val, 0, req);
if (err || req->newptr == NULL)
return (err);
if (val < 0 || val > 1)
return (EINVAL);
ES_LOCK(es);
if (val != es->polling) {
if (es_chan_active(es) != 0)
err = EBUSY;
else if (val == 0)
es->polling = 0;
else
es->polling = 1;
}
ES_UNLOCK(es);
return (err);
}
static void
es_init_sysctls(device_t dev)
{
struct es_info *es;
int r, devid, revid;
devid = pci_get_devid(dev);
revid = pci_get_revid(dev);
es = pcm_getdevinfo(dev);
if ((devid == ES1371_PCI_ID && revid == ES1371REV_ES1373_8) ||
(devid == ES1371_PCI_ID && revid == ES1371REV_CT5880_A) ||
(devid == CT5880_PCI_ID && revid == CT5880REV_CT5880_C) ||
(devid == CT5880_PCI_ID && revid == CT5880REV_CT5880_D) ||
(devid == CT5880_PCI_ID && revid == CT5880REV_CT5880_E)) {
/* XXX: an user should be able to set this with a control tool,
if not done before 7.0-RELEASE, this needs to be converted
to a device specific sysctl "dev.pcm.X.yyy" via
device_get_sysctl_*() as discussed on multimedia@ in msg-id
<861wujij2q.fsf@xps.des.no> */
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"spdif_enabled", CTLTYPE_INT | CTLFLAG_RW, dev, sizeof(dev),
sysctl_es137x_spdif_enable, "I",
"Enable S/PDIF output on primary playback channel");
} else if (devid == ES1370_PCI_ID) {
/*
* Enable fixed rate sysctl if both DAC2 / ADC enabled.
*/
if (es->ch[ES_DAC2].channel != NULL &&
es->ch[ES_ADC].channel != NULL) {
/* XXX: an user should be able to set this with a control tool,
if not done before 7.0-RELEASE, this needs to be converted
to a device specific sysctl "dev.pcm.X.yyy" via
device_get_sysctl_*() as discussed on multimedia@ in msg-id
<861wujij2q.fsf@xps.des.no> */
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "fixed_rate", CTLTYPE_INT | CTLFLAG_RW,
dev, sizeof(dev), sysctl_es137x_fixed_rate, "I",
"Enable fixed rate playback/recording");
}
/*
* Enable single pcm mixer sysctl if both DAC1/2 enabled.
*/
if (es->ch[ES_DAC1].channel != NULL &&
es->ch[ES_DAC2].channel != NULL) {
/* XXX: an user should be able to set this with a control tool,
if not done before 7.0-RELEASE, this needs to be converted
to a device specific sysctl "dev.pcm.X.yyy" via
device_get_sysctl_*() as discussed on multimedia@ in msg-id
<861wujij2q.fsf@xps.des.no> */
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "single_pcm_mixer",
CTLTYPE_INT | CTLFLAG_RW, dev, sizeof(dev),
sysctl_es137x_single_pcm_mixer, "I",
"Single PCM mixer controller for both DAC1/DAC2");
}
}
if (resource_int_value(device_get_name(dev), device_get_unit(dev),
"latency_timer", &r) == 0 && !(r < 0 || r > 255))
pci_write_config(dev, PCIR_LATTIMER, r, 1);
/* XXX: this needs to be converted to a device specific sysctl
"dev.pcm.X.yyy" via device_get_sysctl_*() as discussed on
multimedia@ in msg-id <861wujij2q.fsf@xps.des.no> */
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"latency_timer", CTLTYPE_INT | CTLFLAG_RW, dev, sizeof(dev),
sysctl_es137x_latency_timer, "I",
"PCI Latency Timer configuration");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"polling", CTLTYPE_INT | CTLFLAG_RW, dev, sizeof(dev),
sysctl_es_polling, "I",
"Enable polling mode");
}
static int
es_pci_attach(device_t dev)
{
struct es_info *es = NULL;
int mapped, i, numplay, dac_cfg;
char status[SND_STATUSLEN];
struct ac97_info *codec = NULL;
kobj_class_t ct = NULL;
uint32_t devid;
es = malloc(sizeof *es, M_DEVBUF, M_WAITOK | M_ZERO);
es->lock = snd_mtxcreate(device_get_nameunit(dev), "snd_es137x softc");
es->dev = dev;
es->escfg = 0;
mapped = 0;
pci_enable_busmaster(dev);
if (mapped == 0) {
es->regid = MEM_MAP_REG;
es->regtype = SYS_RES_MEMORY;
es->reg = bus_alloc_resource_any(dev, es->regtype, &es->regid,
RF_ACTIVE);
if (es->reg)
mapped++;
}
if (mapped == 0) {
es->regid = PCIR_BAR(0);
es->regtype = SYS_RES_IOPORT;
es->reg = bus_alloc_resource_any(dev, es->regtype, &es->regid,
RF_ACTIVE);
if (es->reg)
mapped++;
}
if (mapped == 0) {
device_printf(dev, "unable to map register space\n");
goto bad;
}
es->st = rman_get_bustag(es->reg);
es->sh = rman_get_bushandle(es->reg);
- callout_init(&es->poll_timer, CALLOUT_MPSAFE);
+ callout_init(&es->poll_timer, 1);
es->poll_ticks = 1;
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "polling", &i) == 0 && i != 0)
es->polling = 1;
else
es->polling = 0;
es->bufsz = pcm_getbuffersize(dev, 4096, ES_DEFAULT_BUFSZ, 65536);
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "blocksize", &i) == 0 && i > 0) {
i &= ES_BLK_ALIGN;
if (i < ES_BLK_MIN)
i = ES_BLK_MIN;
es->blkcnt = es->bufsz / i;
i = 0;
while (es->blkcnt >> i)
i++;
es->blkcnt = 1 << (i - 1);
if (es->blkcnt < ES_DMA_SEGS_MIN)
es->blkcnt = ES_DMA_SEGS_MIN;
else if (es->blkcnt > ES_DMA_SEGS_MAX)
es->blkcnt = ES_DMA_SEGS_MAX;
} else
es->blkcnt = 2;
if (resource_int_value(device_get_name(dev), device_get_unit(dev),
"dac", &dac_cfg) == 0) {
if (dac_cfg < 0 || dac_cfg > 3)
dac_cfg = ES_DEFAULT_DAC_CFG;
} else
dac_cfg = ES_DEFAULT_DAC_CFG;
switch (dac_cfg) {
case 0: /* Enable all DAC: DAC1, DAC2 */
numplay = 2;
es->escfg = ES_SET_DAC_FIRST(es->escfg, ES_DAC1);
es->escfg = ES_SET_DAC_SECOND(es->escfg, ES_DAC2);
break;
case 1: /* Only DAC1 */
numplay = 1;
es->escfg = ES_SET_DAC_FIRST(es->escfg, ES_DAC1);
break;
case 3: /* Enable all DAC / swap position: DAC2, DAC1 */
numplay = 2;
es->escfg = ES_SET_DAC_FIRST(es->escfg, ES_DAC2);
es->escfg = ES_SET_DAC_SECOND(es->escfg, ES_DAC1);
break;
case 2: /* Only DAC2 */
default:
numplay = 1;
es->escfg = ES_SET_DAC_FIRST(es->escfg, ES_DAC2);
break;
}
es->escfg = ES_SET_NUMPLAY(es->escfg, numplay);
es->escfg = ES_SET_NUMREC(es->escfg, 1);
devid = pci_get_devid(dev);
switch (devid) {
case ES1371_PCI_ID:
case ES1371_PCI_ID2:
case CT5880_PCI_ID:
case CT4730_PCI_ID:
es1371_init(es);
codec = AC97_CREATE(dev, es, es1371_ac97);
if (codec == NULL)
goto bad;
/* our init routine does everything for us */
/* set to NULL; flag mixer_init not to run the ac97_init */
/* ac97_mixer.init = NULL; */
if (mixer_init(dev, ac97_getmixerclass(), codec))
goto bad;
ct = &eschan1371_class;
break;
case ES1370_PCI_ID:
es1370_init(es);
/*
* Disable fixed rate operation if DAC2 disabled.
* This is a special case for es1370 only, where the
* speed of both ADC and DAC2 locked together.
*/
if (!ES_DAC2_ENABLED(es->escfg))
es->escfg = ES_SET_FIXED_RATE(es->escfg, 0);
if (mixer_init(dev, &es1370_mixer_class, es))
goto bad;
ct = &eschan1370_class;
break;
default:
goto bad;
/* NOTREACHED */
}
es->irqid = 0;
es->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &es->irqid,
RF_ACTIVE | RF_SHAREABLE);
if (!es->irq || snd_setup_intr(dev, es->irq, INTR_MPSAFE, es_intr,
es, &es->ih)) {
device_printf(dev, "unable to map interrupt\n");
goto bad;
}
if (bus_dma_tag_create(/*parent*/bus_get_dma_tag(dev),
/*alignment*/2, /*boundary*/0,
/*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
/*highaddr*/BUS_SPACE_MAXADDR,
/*filter*/NULL, /*filterarg*/NULL,
/*maxsize*/es->bufsz, /*nsegments*/1, /*maxsegz*/0x3ffff,
/*flags*/0, /*lockfunc*/NULL,
/*lockarg*/NULL, &es->parent_dmat) != 0) {
device_printf(dev, "unable to create dma tag\n");
goto bad;
}
snprintf(status, SND_STATUSLEN, "at %s 0x%lx irq %ld %s",
(es->regtype == SYS_RES_IOPORT)? "io" : "memory",
rman_get_start(es->reg), rman_get_start(es->irq),
PCM_KLDSTRING(snd_es137x));
if (pcm_register(dev, es, numplay, 1))
goto bad;
for (i = 0; i < numplay; i++)
pcm_addchan(dev, PCMDIR_PLAY, ct, es);
pcm_addchan(dev, PCMDIR_REC, ct, es);
es_init_sysctls(dev);
pcm_setstatus(dev, status);
es->escfg = ES_SET_GP(es->escfg, 0);
if (numplay == 1)
device_printf(dev, "<Playback: DAC%d / Record: ADC>\n",
ES_DAC_FIRST(es->escfg) + 1);
else if (numplay == 2)
device_printf(dev, "<Playback: DAC%d,DAC%d / Record: ADC>\n",
ES_DAC_FIRST(es->escfg) + 1, ES_DAC_SECOND(es->escfg) + 1);
return (0);
bad:
if (es->parent_dmat)
bus_dma_tag_destroy(es->parent_dmat);
if (es->ih)
bus_teardown_intr(dev, es->irq, es->ih);
if (es->irq)
bus_release_resource(dev, SYS_RES_IRQ, es->irqid, es->irq);
if (codec)
ac97_destroy(codec);
if (es->reg)
bus_release_resource(dev, es->regtype, es->regid, es->reg);
if (es->lock)
snd_mtxfree(es->lock);
if (es)
free(es, M_DEVBUF);
return (ENXIO);
}
static int
es_pci_detach(device_t dev)
{
int r;
struct es_info *es;
r = pcm_unregister(dev);
if (r)
return (r);
es = pcm_getdevinfo(dev);
if (es != NULL && es->num != 0) {
ES_LOCK(es);
es->polling = 0;
callout_stop(&es->poll_timer);
ES_UNLOCK(es);
callout_drain(&es->poll_timer);
}
bus_teardown_intr(dev, es->irq, es->ih);
bus_release_resource(dev, SYS_RES_IRQ, es->irqid, es->irq);
bus_release_resource(dev, es->regtype, es->regid, es->reg);
bus_dma_tag_destroy(es->parent_dmat);
snd_mtxfree(es->lock);
free(es, M_DEVBUF);
return (0);
}
static device_method_t es_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, es_pci_probe),
DEVMETHOD(device_attach, es_pci_attach),
DEVMETHOD(device_detach, es_pci_detach),
{ 0, 0 }
};
static driver_t es_driver = {
"pcm",
es_methods,
PCM_SOFTC_SIZE,
};
DRIVER_MODULE(snd_es137x, pci, es_driver, pcm_devclass, 0, 0);
MODULE_DEPEND(snd_es137x, sound, SOUND_MINVER, SOUND_PREFVER, SOUND_MAXVER);
MODULE_VERSION(snd_es137x, 1);
Index: head/sys/dev/sound/pci/hda/hdaa.c
===================================================================
--- head/sys/dev/sound/pci/hda/hdaa.c (revision 283290)
+++ head/sys/dev/sound/pci/hda/hdaa.c (revision 283291)
@@ -1,7155 +1,7155 @@
/*-
* Copyright (c) 2006 Stephane E. Potvin <sepotvin@videotron.ca>
* Copyright (c) 2006 Ariff Abdullah <ariff@FreeBSD.org>
* Copyright (c) 2008-2012 Alexander Motin <mav@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Intel High Definition Audio (Audio function) driver for FreeBSD.
*/
#ifdef HAVE_KERNEL_OPTION_HEADERS
#include "opt_snd.h"
#endif
#include <dev/sound/pcm/sound.h>
#include <sys/ctype.h>
#include <sys/taskqueue.h>
#include <dev/sound/pci/hda/hdac.h>
#include <dev/sound/pci/hda/hdaa.h>
#include <dev/sound/pci/hda/hda_reg.h>
#include "mixer_if.h"
SND_DECLARE_FILE("$FreeBSD$");
#define hdaa_lock(devinfo) snd_mtxlock((devinfo)->lock)
#define hdaa_unlock(devinfo) snd_mtxunlock((devinfo)->lock)
#define hdaa_lockassert(devinfo) snd_mtxassert((devinfo)->lock)
#define hdaa_lockowned(devinfo) mtx_owned((devinfo)->lock)
static const struct {
const char *key;
uint32_t value;
} hdaa_quirks_tab[] = {
{ "softpcmvol", HDAA_QUIRK_SOFTPCMVOL },
{ "fixedrate", HDAA_QUIRK_FIXEDRATE },
{ "forcestereo", HDAA_QUIRK_FORCESTEREO },
{ "eapdinv", HDAA_QUIRK_EAPDINV },
{ "senseinv", HDAA_QUIRK_SENSEINV },
{ "ivref50", HDAA_QUIRK_IVREF50 },
{ "ivref80", HDAA_QUIRK_IVREF80 },
{ "ivref100", HDAA_QUIRK_IVREF100 },
{ "ovref50", HDAA_QUIRK_OVREF50 },
{ "ovref80", HDAA_QUIRK_OVREF80 },
{ "ovref100", HDAA_QUIRK_OVREF100 },
{ "ivref", HDAA_QUIRK_IVREF },
{ "ovref", HDAA_QUIRK_OVREF },
{ "vref", HDAA_QUIRK_VREF },
};
#define HDA_PARSE_MAXDEPTH 10
MALLOC_DEFINE(M_HDAA, "hdaa", "HDA Audio");
static const char *HDA_COLORS[16] = {"Unknown", "Black", "Grey", "Blue",
"Green", "Red", "Orange", "Yellow", "Purple", "Pink", "Res.A", "Res.B",
"Res.C", "Res.D", "White", "Other"};
static const char *HDA_DEVS[16] = {"Line-out", "Speaker", "Headphones", "CD",
"SPDIF-out", "Digital-out", "Modem-line", "Modem-handset", "Line-in",
"AUX", "Mic", "Telephony", "SPDIF-in", "Digital-in", "Res.E", "Other"};
static const char *HDA_CONNS[4] = {"Jack", "None", "Fixed", "Both"};
static const char *HDA_CONNECTORS[16] = {
"Unknown", "1/8", "1/4", "ATAPI", "RCA", "Optical", "Digital", "Analog",
"DIN", "XLR", "RJ-11", "Combo", "0xc", "0xd", "0xe", "Other" };
static const char *HDA_LOCS[64] = {
"0x00", "Rear", "Front", "Left", "Right", "Top", "Bottom", "Rear-panel",
"Drive-bay", "0x09", "0x0a", "0x0b", "0x0c", "0x0d", "0x0e", "0x0f",
"Internal", "0x11", "0x12", "0x13", "0x14", "0x15", "0x16", "Riser",
"0x18", "Onboard", "0x1a", "0x1b", "0x1c", "0x1d", "0x1e", "0x1f",
"External", "Ext-Rear", "Ext-Front", "Ext-Left", "Ext-Right", "Ext-Top", "Ext-Bottom", "0x07",
"0x28", "0x29", "0x2a", "0x2b", "0x2c", "0x2d", "0x2e", "0x2f",
"Other", "0x31", "0x32", "0x33", "0x34", "0x35", "Other-Bott", "Lid-In",
"Lid-Out", "0x39", "0x3a", "0x3b", "0x3c", "0x3d", "0x3e", "0x3f" };
static const char *HDA_GPIO_ACTIONS[8] = {
"keep", "set", "clear", "disable", "input", "0x05", "0x06", "0x07"};
static const char *HDA_HDMI_CODING_TYPES[18] = {
"undefined", "LPCM", "AC-3", "MPEG1", "MP3", "MPEG2", "AAC-LC", "DTS",
"ATRAC", "DSD", "E-AC-3", "DTS-HD", "MLP", "DST", "WMAPro", "HE-AAC",
"HE-AACv2", "MPEG-Surround"
};
/* Default */
static uint32_t hdaa_fmt[] = {
SND_FORMAT(AFMT_S16_LE, 2, 0),
0
};
static struct pcmchan_caps hdaa_caps = {48000, 48000, hdaa_fmt, 0};
static const struct {
uint32_t rate;
int valid;
uint16_t base;
uint16_t mul;
uint16_t div;
} hda_rate_tab[] = {
{ 8000, 1, 0x0000, 0x0000, 0x0500 }, /* (48000 * 1) / 6 */
{ 9600, 0, 0x0000, 0x0000, 0x0400 }, /* (48000 * 1) / 5 */
{ 12000, 0, 0x0000, 0x0000, 0x0300 }, /* (48000 * 1) / 4 */
{ 16000, 1, 0x0000, 0x0000, 0x0200 }, /* (48000 * 1) / 3 */
{ 18000, 0, 0x0000, 0x1000, 0x0700 }, /* (48000 * 3) / 8 */
{ 19200, 0, 0x0000, 0x0800, 0x0400 }, /* (48000 * 2) / 5 */
{ 24000, 0, 0x0000, 0x0000, 0x0100 }, /* (48000 * 1) / 2 */
{ 28800, 0, 0x0000, 0x1000, 0x0400 }, /* (48000 * 3) / 5 */
{ 32000, 1, 0x0000, 0x0800, 0x0200 }, /* (48000 * 2) / 3 */
{ 36000, 0, 0x0000, 0x1000, 0x0300 }, /* (48000 * 3) / 4 */
{ 38400, 0, 0x0000, 0x1800, 0x0400 }, /* (48000 * 4) / 5 */
{ 48000, 1, 0x0000, 0x0000, 0x0000 }, /* (48000 * 1) / 1 */
{ 64000, 0, 0x0000, 0x1800, 0x0200 }, /* (48000 * 4) / 3 */
{ 72000, 0, 0x0000, 0x1000, 0x0100 }, /* (48000 * 3) / 2 */
{ 96000, 1, 0x0000, 0x0800, 0x0000 }, /* (48000 * 2) / 1 */
{ 144000, 0, 0x0000, 0x1000, 0x0000 }, /* (48000 * 3) / 1 */
{ 192000, 1, 0x0000, 0x1800, 0x0000 }, /* (48000 * 4) / 1 */
{ 8820, 0, 0x4000, 0x0000, 0x0400 }, /* (44100 * 1) / 5 */
{ 11025, 1, 0x4000, 0x0000, 0x0300 }, /* (44100 * 1) / 4 */
{ 12600, 0, 0x4000, 0x0800, 0x0600 }, /* (44100 * 2) / 7 */
{ 14700, 0, 0x4000, 0x0000, 0x0200 }, /* (44100 * 1) / 3 */
{ 17640, 0, 0x4000, 0x0800, 0x0400 }, /* (44100 * 2) / 5 */
{ 18900, 0, 0x4000, 0x1000, 0x0600 }, /* (44100 * 3) / 7 */
{ 22050, 1, 0x4000, 0x0000, 0x0100 }, /* (44100 * 1) / 2 */
{ 25200, 0, 0x4000, 0x1800, 0x0600 }, /* (44100 * 4) / 7 */
{ 26460, 0, 0x4000, 0x1000, 0x0400 }, /* (44100 * 3) / 5 */
{ 29400, 0, 0x4000, 0x0800, 0x0200 }, /* (44100 * 2) / 3 */
{ 33075, 0, 0x4000, 0x1000, 0x0300 }, /* (44100 * 3) / 4 */
{ 35280, 0, 0x4000, 0x1800, 0x0400 }, /* (44100 * 4) / 5 */
{ 44100, 1, 0x4000, 0x0000, 0x0000 }, /* (44100 * 1) / 1 */
{ 58800, 0, 0x4000, 0x1800, 0x0200 }, /* (44100 * 4) / 3 */
{ 66150, 0, 0x4000, 0x1000, 0x0100 }, /* (44100 * 3) / 2 */
{ 88200, 1, 0x4000, 0x0800, 0x0000 }, /* (44100 * 2) / 1 */
{ 132300, 0, 0x4000, 0x1000, 0x0000 }, /* (44100 * 3) / 1 */
{ 176400, 1, 0x4000, 0x1800, 0x0000 }, /* (44100 * 4) / 1 */
};
#define HDA_RATE_TAB_LEN (sizeof(hda_rate_tab) / sizeof(hda_rate_tab[0]))
const static char *ossnames[] = SOUND_DEVICE_NAMES;
/****************************************************************************
* Function prototypes
****************************************************************************/
static int hdaa_pcmchannel_setup(struct hdaa_chan *);
static void hdaa_widget_connection_select(struct hdaa_widget *, uint8_t);
static void hdaa_audio_ctl_amp_set(struct hdaa_audio_ctl *,
uint32_t, int, int);
static struct hdaa_audio_ctl *hdaa_audio_ctl_amp_get(struct hdaa_devinfo *,
nid_t, int, int, int);
static void hdaa_audio_ctl_amp_set_internal(struct hdaa_devinfo *,
nid_t, int, int, int, int, int, int);
static void hdaa_dump_pin_config(struct hdaa_widget *w, uint32_t conf);
static char *
hdaa_audio_ctl_ossmixer_mask2allname(uint32_t mask, char *buf, size_t len)
{
int i, first = 1;
bzero(buf, len);
for (i = 0; i < SOUND_MIXER_NRDEVICES; i++) {
if (mask & (1 << i)) {
if (first == 0)
strlcat(buf, ", ", len);
strlcat(buf, ossnames[i], len);
first = 0;
}
}
return (buf);
}
static struct hdaa_audio_ctl *
hdaa_audio_ctl_each(struct hdaa_devinfo *devinfo, int *index)
{
if (devinfo == NULL ||
index == NULL || devinfo->ctl == NULL ||
devinfo->ctlcnt < 1 ||
*index < 0 || *index >= devinfo->ctlcnt)
return (NULL);
return (&devinfo->ctl[(*index)++]);
}
static struct hdaa_audio_ctl *
hdaa_audio_ctl_amp_get(struct hdaa_devinfo *devinfo, nid_t nid, int dir,
int index, int cnt)
{
struct hdaa_audio_ctl *ctl;
int i, found = 0;
if (devinfo == NULL || devinfo->ctl == NULL)
return (NULL);
i = 0;
while ((ctl = hdaa_audio_ctl_each(devinfo, &i)) != NULL) {
if (ctl->enable == 0)
continue;
if (ctl->widget->nid != nid)
continue;
if (dir && ctl->ndir != dir)
continue;
if (index >= 0 && ctl->ndir == HDAA_CTL_IN &&
ctl->dir == ctl->ndir && ctl->index != index)
continue;
found++;
if (found == cnt || cnt <= 0)
return (ctl);
}
return (NULL);
}
static const struct matrix {
struct pcmchan_matrix m;
int analog;
} matrixes[] = {
{ SND_CHN_MATRIX_MAP_1_0, 1 },
{ SND_CHN_MATRIX_MAP_2_0, 1 },
{ SND_CHN_MATRIX_MAP_2_1, 0 },
{ SND_CHN_MATRIX_MAP_3_0, 0 },
{ SND_CHN_MATRIX_MAP_3_1, 0 },
{ SND_CHN_MATRIX_MAP_4_0, 1 },
{ SND_CHN_MATRIX_MAP_4_1, 0 },
{ SND_CHN_MATRIX_MAP_5_0, 0 },
{ SND_CHN_MATRIX_MAP_5_1, 1 },
{ SND_CHN_MATRIX_MAP_6_0, 0 },
{ SND_CHN_MATRIX_MAP_6_1, 0 },
{ SND_CHN_MATRIX_MAP_7_0, 0 },
{ SND_CHN_MATRIX_MAP_7_1, 1 },
};
static const char *channel_names[] = SND_CHN_T_NAMES;
/*
* Connected channels change handler.
*/
static void
hdaa_channels_handler(struct hdaa_audio_as *as)
{
struct hdaa_pcm_devinfo *pdevinfo = as->pdevinfo;
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_chan *ch = &devinfo->chans[as->chans[0]];
struct hdaa_widget *w;
uint8_t *eld;
int i, total, sub, assume, channels;
uint16_t cpins, upins, tpins;
cpins = upins = 0;
eld = NULL;
for (i = 0; i < 16; i++) {
if (as->pins[i] <= 0)
continue;
w = hdaa_widget_get(devinfo, as->pins[i]);
if (w == NULL)
continue;
if (w->wclass.pin.connected == 1)
cpins |= (1 << i);
else if (w->wclass.pin.connected != 0)
upins |= (1 << i);
if (w->eld != NULL && w->eld_len >= 8)
eld = w->eld;
}
tpins = cpins | upins;
if (as->hpredir >= 0)
tpins &= 0x7fff;
if (tpins == 0)
tpins = as->pinset;
total = sub = assume = channels = 0;
if (eld) {
/* Map CEA speakers to sound(4) channels. */
if (eld[7] & 0x01) /* Front Left/Right */
channels |= SND_CHN_T_MASK_FL | SND_CHN_T_MASK_FR;
if (eld[7] & 0x02) /* Low Frequency Effect */
channels |= SND_CHN_T_MASK_LF;
if (eld[7] & 0x04) /* Front Center */
channels |= SND_CHN_T_MASK_FC;
if (eld[7] & 0x08) { /* Rear Left/Right */
/* If we have both RLR and RLRC, report RLR as side. */
if (eld[7] & 0x40) /* Rear Left/Right Center */
channels |= SND_CHN_T_MASK_SL | SND_CHN_T_MASK_SR;
else
channels |= SND_CHN_T_MASK_BL | SND_CHN_T_MASK_BR;
}
if (eld[7] & 0x10) /* Rear center */
channels |= SND_CHN_T_MASK_BC;
if (eld[7] & 0x20) /* Front Left/Right Center */
channels |= SND_CHN_T_MASK_FLC | SND_CHN_T_MASK_FRC;
if (eld[7] & 0x40) /* Rear Left/Right Center */
channels |= SND_CHN_T_MASK_BL | SND_CHN_T_MASK_BR;
} else if (as->pinset != 0 && (tpins & 0xffe0) == 0) {
/* Map UAA speakers to sound(4) channels. */
if (tpins & 0x0001)
channels |= SND_CHN_T_MASK_FL | SND_CHN_T_MASK_FR;
if (tpins & 0x0002)
channels |= SND_CHN_T_MASK_FC | SND_CHN_T_MASK_LF;
if (tpins & 0x0004)
channels |= SND_CHN_T_MASK_BL | SND_CHN_T_MASK_BR;
if (tpins & 0x0008)
channels |= SND_CHN_T_MASK_FLC | SND_CHN_T_MASK_FRC;
if (tpins & 0x0010) {
/* If there is no back pin, report side as back. */
if ((as->pinset & 0x0004) == 0)
channels |= SND_CHN_T_MASK_BL | SND_CHN_T_MASK_BR;
else
channels |= SND_CHN_T_MASK_SL | SND_CHN_T_MASK_SR;
}
} else if (as->mixed) {
/* Mixed assoc can be only stereo or theoretically mono. */
if (ch->channels == 1)
channels |= SND_CHN_T_MASK_FC;
else
channels |= SND_CHN_T_MASK_FL | SND_CHN_T_MASK_FR;
}
if (channels) { /* We have some usable channels info. */
HDA_BOOTVERBOSE(
device_printf(pdevinfo->dev, "%s channel set is: ",
as->dir == HDAA_CTL_OUT ? "Playback" : "Recording");
for (i = 0; i < SND_CHN_T_MAX; i++)
if (channels & (1 << i))
printf("%s, ", channel_names[i]);
printf("\n");
);
/* Look for maximal fitting matrix. */
for (i = 0; i < sizeof(matrixes) / sizeof(struct matrix); i++) {
if (as->pinset != 0 && matrixes[i].analog == 0)
continue;
if ((matrixes[i].m.mask & ~channels) == 0) {
total = matrixes[i].m.channels;
sub = matrixes[i].m.ext;
}
}
}
if (total == 0) {
assume = 1;
total = ch->channels;
sub = (total == 6 || total == 8) ? 1 : 0;
}
HDA_BOOTVERBOSE(
device_printf(pdevinfo->dev,
"%s channel matrix is: %s%d.%d (%s)\n",
as->dir == HDAA_CTL_OUT ? "Playback" : "Recording",
assume ? "unknown, assuming " : "", total - sub, sub,
cpins != 0 ? "connected" :
(upins != 0 ? "unknown" : "disconnected"));
);
}
/*
* Headphones redirection change handler.
*/
static void
hdaa_hpredir_handler(struct hdaa_widget *w)
{
struct hdaa_devinfo *devinfo = w->devinfo;
struct hdaa_audio_as *as = &devinfo->as[w->bindas];
struct hdaa_widget *w1;
struct hdaa_audio_ctl *ctl;
uint32_t val;
int j, connected = w->wclass.pin.connected;
HDA_BOOTVERBOSE(
device_printf((as->pdevinfo && as->pdevinfo->dev) ?
as->pdevinfo->dev : devinfo->dev,
"Redirect output to: %s\n",
connected ? "headphones": "main");
);
/* (Un)Mute headphone pin. */
ctl = hdaa_audio_ctl_amp_get(devinfo,
w->nid, HDAA_CTL_IN, -1, 1);
if (ctl != NULL && ctl->mute) {
/* If pin has muter - use it. */
val = connected ? 0 : 1;
if (val != ctl->forcemute) {
ctl->forcemute = val;
hdaa_audio_ctl_amp_set(ctl,
HDAA_AMP_MUTE_DEFAULT,
HDAA_AMP_VOL_DEFAULT, HDAA_AMP_VOL_DEFAULT);
}
} else {
/* If there is no muter - disable pin output. */
if (connected)
val = w->wclass.pin.ctrl |
HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE;
else
val = w->wclass.pin.ctrl &
~HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE;
if (val != w->wclass.pin.ctrl) {
w->wclass.pin.ctrl = val;
hda_command(devinfo->dev,
HDA_CMD_SET_PIN_WIDGET_CTRL(0,
w->nid, w->wclass.pin.ctrl));
}
}
/* (Un)Mute other pins. */
for (j = 0; j < 15; j++) {
if (as->pins[j] <= 0)
continue;
ctl = hdaa_audio_ctl_amp_get(devinfo,
as->pins[j], HDAA_CTL_IN, -1, 1);
if (ctl != NULL && ctl->mute) {
/* If pin has muter - use it. */
val = connected ? 1 : 0;
if (val == ctl->forcemute)
continue;
ctl->forcemute = val;
hdaa_audio_ctl_amp_set(ctl,
HDAA_AMP_MUTE_DEFAULT,
HDAA_AMP_VOL_DEFAULT, HDAA_AMP_VOL_DEFAULT);
continue;
}
/* If there is no muter - disable pin output. */
w1 = hdaa_widget_get(devinfo, as->pins[j]);
if (w1 != NULL) {
if (connected)
val = w1->wclass.pin.ctrl &
~HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE;
else
val = w1->wclass.pin.ctrl |
HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE;
if (val != w1->wclass.pin.ctrl) {
w1->wclass.pin.ctrl = val;
hda_command(devinfo->dev,
HDA_CMD_SET_PIN_WIDGET_CTRL(0,
w1->nid, w1->wclass.pin.ctrl));
}
}
}
}
/*
* Recording source change handler.
*/
static void
hdaa_autorecsrc_handler(struct hdaa_audio_as *as, struct hdaa_widget *w)
{
struct hdaa_pcm_devinfo *pdevinfo = as->pdevinfo;
struct hdaa_devinfo *devinfo;
struct hdaa_widget *w1;
int i, mask, fullmask, prio, bestprio;
char buf[128];
if (!as->mixed || pdevinfo == NULL || pdevinfo->mixer == NULL)
return;
/* Don't touch anything if we asked not to. */
if (pdevinfo->autorecsrc == 0 ||
(pdevinfo->autorecsrc == 1 && w != NULL))
return;
/* Don't touch anything if "mix" or "speaker" selected. */
if (pdevinfo->recsrc & (SOUND_MASK_IMIX | SOUND_MASK_SPEAKER))
return;
/* Don't touch anything if several selected. */
if (ffs(pdevinfo->recsrc) != fls(pdevinfo->recsrc))
return;
devinfo = pdevinfo->devinfo;
mask = fullmask = 0;
bestprio = 0;
for (i = 0; i < 16; i++) {
if (as->pins[i] <= 0)
continue;
w1 = hdaa_widget_get(devinfo, as->pins[i]);
if (w1 == NULL || w1->enable == 0)
continue;
if (w1->wclass.pin.connected == 0)
continue;
prio = (w1->wclass.pin.connected == 1) ? 2 : 1;
if (prio < bestprio)
continue;
if (prio > bestprio) {
mask = 0;
bestprio = prio;
}
mask |= (1 << w1->ossdev);
fullmask |= (1 << w1->ossdev);
}
if (mask == 0)
return;
/* Prefer newly connected input. */
if (w != NULL && (mask & (1 << w->ossdev)))
mask = (1 << w->ossdev);
/* Prefer previously selected input */
if (mask & pdevinfo->recsrc)
mask &= pdevinfo->recsrc;
/* Prefer mic. */
if (mask & SOUND_MASK_MIC)
mask = SOUND_MASK_MIC;
/* Prefer monitor (2nd mic). */
if (mask & SOUND_MASK_MONITOR)
mask = SOUND_MASK_MONITOR;
/* Just take first one. */
mask = (1 << (ffs(mask) - 1));
HDA_BOOTVERBOSE(
hdaa_audio_ctl_ossmixer_mask2allname(mask, buf, sizeof(buf));
device_printf(pdevinfo->dev,
"Automatically set rec source to: %s\n", buf);
);
hdaa_unlock(devinfo);
mix_setrecsrc(pdevinfo->mixer, mask);
hdaa_lock(devinfo);
}
/*
* Jack presence detection event handler.
*/
static void
hdaa_presence_handler(struct hdaa_widget *w)
{
struct hdaa_devinfo *devinfo = w->devinfo;
struct hdaa_audio_as *as;
uint32_t res;
int connected, old;
if (w->enable == 0 || w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
return;
if (HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP(w->wclass.pin.cap) == 0 ||
(HDA_CONFIG_DEFAULTCONF_MISC(w->wclass.pin.config) & 1) != 0)
return;
res = hda_command(devinfo->dev, HDA_CMD_GET_PIN_SENSE(0, w->nid));
connected = (res & HDA_CMD_GET_PIN_SENSE_PRESENCE_DETECT) != 0;
if (devinfo->quirks & HDAA_QUIRK_SENSEINV)
connected = !connected;
old = w->wclass.pin.connected;
if (connected == old)
return;
w->wclass.pin.connected = connected;
HDA_BOOTVERBOSE(
if (connected || old != 2) {
device_printf(devinfo->dev,
"Pin sense: nid=%d sense=0x%08x (%sconnected)\n",
w->nid, res, !connected ? "dis" : "");
}
);
as = &devinfo->as[w->bindas];
if (as->hpredir >= 0 && as->pins[15] == w->nid)
hdaa_hpredir_handler(w);
if (as->dir == HDAA_CTL_IN && old != 2)
hdaa_autorecsrc_handler(as, w);
if (old != 2)
hdaa_channels_handler(as);
}
/*
* Callback for poll based presence detection.
*/
static void
hdaa_jack_poll_callback(void *arg)
{
struct hdaa_devinfo *devinfo = arg;
struct hdaa_widget *w;
int i;
hdaa_lock(devinfo);
if (devinfo->poll_ival == 0) {
hdaa_unlock(devinfo);
return;
}
for (i = 0; i < devinfo->ascnt; i++) {
if (devinfo->as[i].hpredir < 0)
continue;
w = hdaa_widget_get(devinfo, devinfo->as[i].pins[15]);
if (w == NULL || w->enable == 0 || w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
hdaa_presence_handler(w);
}
callout_reset(&devinfo->poll_jack, devinfo->poll_ival,
hdaa_jack_poll_callback, devinfo);
hdaa_unlock(devinfo);
}
static void
hdaa_eld_dump(struct hdaa_widget *w)
{
struct hdaa_devinfo *devinfo = w->devinfo;
device_t dev = devinfo->dev;
uint8_t *sad;
int len, mnl, i, sadc, fmt;
if (w->eld == NULL || w->eld_len < 4)
return;
device_printf(dev,
"ELD nid=%d: ELD_Ver=%u Baseline_ELD_Len=%u\n",
w->nid, w->eld[0] >> 3, w->eld[2]);
if ((w->eld[0] >> 3) != 0x02)
return;
len = min(w->eld_len, (u_int)w->eld[2] * 4);
mnl = w->eld[4] & 0x1f;
device_printf(dev,
"ELD nid=%d: CEA_EDID_Ver=%u MNL=%u\n",
w->nid, w->eld[4] >> 5, mnl);
sadc = w->eld[5] >> 4;
device_printf(dev,
"ELD nid=%d: SAD_Count=%u Conn_Type=%u S_AI=%u HDCP=%u\n",
w->nid, sadc, (w->eld[5] >> 2) & 0x3,
(w->eld[5] >> 1) & 0x1, w->eld[5] & 0x1);
device_printf(dev,
"ELD nid=%d: Aud_Synch_Delay=%ums\n",
w->nid, w->eld[6] * 2);
device_printf(dev,
"ELD nid=%d: Channels=0x%b\n",
w->nid, w->eld[7],
"\020\07RLRC\06FLRC\05RC\04RLR\03FC\02LFE\01FLR");
device_printf(dev,
"ELD nid=%d: Port_ID=0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
w->nid, w->eld[8], w->eld[9], w->eld[10], w->eld[11],
w->eld[12], w->eld[13], w->eld[14], w->eld[15]);
device_printf(dev,
"ELD nid=%d: Manufacturer_Name=0x%02x%02x\n",
w->nid, w->eld[16], w->eld[17]);
device_printf(dev,
"ELD nid=%d: Product_Code=0x%02x%02x\n",
w->nid, w->eld[18], w->eld[19]);
device_printf(dev,
"ELD nid=%d: Monitor_Name_String='%.*s'\n",
w->nid, mnl, &w->eld[20]);
for (i = 0; i < sadc; i++) {
sad = &w->eld[20 + mnl + i * 3];
fmt = (sad[0] >> 3) & 0x0f;
if (fmt == HDA_HDMI_CODING_TYPE_REF_CTX) {
fmt = (sad[2] >> 3) & 0x1f;
if (fmt < 1 || fmt > 3)
fmt = 0;
else
fmt += 14;
}
device_printf(dev,
"ELD nid=%d: %s %dch freqs=0x%b",
w->nid, HDA_HDMI_CODING_TYPES[fmt], (sad[0] & 0x07) + 1,
sad[1], "\020\007192\006176\00596\00488\00348\00244\00132");
switch (fmt) {
case HDA_HDMI_CODING_TYPE_LPCM:
printf(" sizes=0x%b",
sad[2] & 0x07, "\020\00324\00220\00116");
break;
case HDA_HDMI_CODING_TYPE_AC3:
case HDA_HDMI_CODING_TYPE_MPEG1:
case HDA_HDMI_CODING_TYPE_MP3:
case HDA_HDMI_CODING_TYPE_MPEG2:
case HDA_HDMI_CODING_TYPE_AACLC:
case HDA_HDMI_CODING_TYPE_DTS:
case HDA_HDMI_CODING_TYPE_ATRAC:
printf(" max_bitrate=%d", sad[2] * 8000);
break;
case HDA_HDMI_CODING_TYPE_WMAPRO:
printf(" profile=%d", sad[2] & 0x07);
break;
}
printf("\n");
}
}
static void
hdaa_eld_handler(struct hdaa_widget *w)
{
struct hdaa_devinfo *devinfo = w->devinfo;
uint32_t res;
int i;
if (w->enable == 0 || w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
return;
if (HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP(w->wclass.pin.cap) == 0 ||
(HDA_CONFIG_DEFAULTCONF_MISC(w->wclass.pin.config) & 1) != 0)
return;
res = hda_command(devinfo->dev, HDA_CMD_GET_PIN_SENSE(0, w->nid));
if ((w->eld != 0) == ((res & HDA_CMD_GET_PIN_SENSE_ELD_VALID) != 0))
return;
if (w->eld != NULL) {
w->eld_len = 0;
free(w->eld, M_HDAA);
w->eld = NULL;
}
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"Pin sense: nid=%d sense=0x%08x "
"(%sconnected, ELD %svalid)\n",
w->nid, res,
(res & HDA_CMD_GET_PIN_SENSE_PRESENCE_DETECT) ? "" : "dis",
(res & HDA_CMD_GET_PIN_SENSE_ELD_VALID) ? "" : "in");
);
if ((res & HDA_CMD_GET_PIN_SENSE_ELD_VALID) == 0)
return;
res = hda_command(devinfo->dev,
HDA_CMD_GET_HDMI_DIP_SIZE(0, w->nid, 0x08));
if (res == HDA_INVALID)
return;
w->eld_len = res & 0xff;
if (w->eld_len != 0)
w->eld = malloc(w->eld_len, M_HDAA, M_ZERO | M_NOWAIT);
if (w->eld == NULL) {
w->eld_len = 0;
return;
}
for (i = 0; i < w->eld_len; i++) {
res = hda_command(devinfo->dev,
HDA_CMD_GET_HDMI_ELDD(0, w->nid, i));
if (res & 0x80000000)
w->eld[i] = res & 0xff;
}
HDA_BOOTVERBOSE(
hdaa_eld_dump(w);
);
hdaa_channels_handler(&devinfo->as[w->bindas]);
}
/*
* Pin sense initializer.
*/
static void
hdaa_sense_init(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as;
struct hdaa_widget *w;
int i, poll = 0;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0 || w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
if (HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP(w->param.widget_cap)) {
if (w->unsol < 0)
w->unsol = HDAC_UNSOL_ALLOC(
device_get_parent(devinfo->dev),
devinfo->dev, w->nid);
hda_command(devinfo->dev,
HDA_CMD_SET_UNSOLICITED_RESPONSE(0, w->nid,
HDA_CMD_SET_UNSOLICITED_RESPONSE_ENABLE | w->unsol));
}
as = &devinfo->as[w->bindas];
if (as->hpredir >= 0 && as->pins[15] == w->nid) {
if (HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP(w->wclass.pin.cap) == 0 ||
(HDA_CONFIG_DEFAULTCONF_MISC(w->wclass.pin.config) & 1) != 0) {
device_printf(devinfo->dev,
"No presence detection support at nid %d\n",
w->nid);
} else {
if (w->unsol < 0)
poll = 1;
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"Headphones redirection for "
"association %d nid=%d using %s.\n",
w->bindas, w->nid,
(w->unsol < 0) ? "polling" :
"unsolicited responses");
);
};
}
hdaa_presence_handler(w);
if (!HDA_PARAM_PIN_CAP_DP(w->wclass.pin.cap) &&
!HDA_PARAM_PIN_CAP_HDMI(w->wclass.pin.cap))
continue;
hdaa_eld_handler(w);
}
if (poll) {
callout_reset(&devinfo->poll_jack, 1,
hdaa_jack_poll_callback, devinfo);
}
}
static void
hdaa_sense_deinit(struct hdaa_devinfo *devinfo)
{
struct hdaa_widget *w;
int i;
callout_stop(&devinfo->poll_jack);
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0 || w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
if (w->unsol < 0)
continue;
hda_command(devinfo->dev,
HDA_CMD_SET_UNSOLICITED_RESPONSE(0, w->nid, 0));
HDAC_UNSOL_FREE(
device_get_parent(devinfo->dev), devinfo->dev,
w->unsol);
w->unsol = -1;
}
}
uint32_t
hdaa_widget_pin_patch(uint32_t config, const char *str)
{
char buf[256];
char *key, *value, *rest, *bad;
int ival, i;
strlcpy(buf, str, sizeof(buf));
rest = buf;
while ((key = strsep(&rest, "=")) != NULL) {
value = strsep(&rest, " \t");
if (value == NULL)
break;
ival = strtol(value, &bad, 10);
if (strcmp(key, "seq") == 0) {
config &= ~HDA_CONFIG_DEFAULTCONF_SEQUENCE_MASK;
config |= ((ival << HDA_CONFIG_DEFAULTCONF_SEQUENCE_SHIFT) &
HDA_CONFIG_DEFAULTCONF_SEQUENCE_MASK);
} else if (strcmp(key, "as") == 0) {
config &= ~HDA_CONFIG_DEFAULTCONF_ASSOCIATION_MASK;
config |= ((ival << HDA_CONFIG_DEFAULTCONF_ASSOCIATION_SHIFT) &
HDA_CONFIG_DEFAULTCONF_ASSOCIATION_MASK);
} else if (strcmp(key, "misc") == 0) {
config &= ~HDA_CONFIG_DEFAULTCONF_MISC_MASK;
config |= ((ival << HDA_CONFIG_DEFAULTCONF_MISC_SHIFT) &
HDA_CONFIG_DEFAULTCONF_MISC_MASK);
} else if (strcmp(key, "color") == 0) {
config &= ~HDA_CONFIG_DEFAULTCONF_COLOR_MASK;
if (bad[0] == 0) {
config |= ((ival << HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT) &
HDA_CONFIG_DEFAULTCONF_COLOR_MASK);
};
for (i = 0; i < 16; i++) {
if (strcasecmp(HDA_COLORS[i], value) == 0) {
config |= (i << HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT);
break;
}
}
} else if (strcmp(key, "ctype") == 0) {
config &= ~HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_MASK;
if (bad[0] == 0) {
config |= ((ival << HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_SHIFT) &
HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_MASK);
}
for (i = 0; i < 16; i++) {
if (strcasecmp(HDA_CONNECTORS[i], value) == 0) {
config |= (i << HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_SHIFT);
break;
}
}
} else if (strcmp(key, "device") == 0) {
config &= ~HDA_CONFIG_DEFAULTCONF_DEVICE_MASK;
if (bad[0] == 0) {
config |= ((ival << HDA_CONFIG_DEFAULTCONF_DEVICE_SHIFT) &
HDA_CONFIG_DEFAULTCONF_DEVICE_MASK);
continue;
};
for (i = 0; i < 16; i++) {
if (strcasecmp(HDA_DEVS[i], value) == 0) {
config |= (i << HDA_CONFIG_DEFAULTCONF_DEVICE_SHIFT);
break;
}
}
} else if (strcmp(key, "loc") == 0) {
config &= ~HDA_CONFIG_DEFAULTCONF_LOCATION_MASK;
if (bad[0] == 0) {
config |= ((ival << HDA_CONFIG_DEFAULTCONF_LOCATION_SHIFT) &
HDA_CONFIG_DEFAULTCONF_LOCATION_MASK);
continue;
}
for (i = 0; i < 64; i++) {
if (strcasecmp(HDA_LOCS[i], value) == 0) {
config |= (i << HDA_CONFIG_DEFAULTCONF_LOCATION_SHIFT);
break;
}
}
} else if (strcmp(key, "conn") == 0) {
config &= ~HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_MASK;
if (bad[0] == 0) {
config |= ((ival << HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_SHIFT) &
HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_MASK);
continue;
};
for (i = 0; i < 4; i++) {
if (strcasecmp(HDA_CONNS[i], value) == 0) {
config |= (i << HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_SHIFT);
break;
}
}
}
}
return (config);
}
uint32_t
hdaa_gpio_patch(uint32_t gpio, const char *str)
{
char buf[256];
char *key, *value, *rest;
int ikey, i;
strlcpy(buf, str, sizeof(buf));
rest = buf;
while ((key = strsep(&rest, "=")) != NULL) {
value = strsep(&rest, " \t");
if (value == NULL)
break;
ikey = strtol(key, NULL, 10);
if (ikey < 0 || ikey > 7)
continue;
for (i = 0; i < 7; i++) {
if (strcasecmp(HDA_GPIO_ACTIONS[i], value) == 0) {
gpio &= ~HDAA_GPIO_MASK(ikey);
gpio |= i << HDAA_GPIO_SHIFT(ikey);
break;
}
}
}
return (gpio);
}
static void
hdaa_local_patch_pin(struct hdaa_widget *w)
{
device_t dev = w->devinfo->dev;
const char *res = NULL;
uint32_t config, orig;
char buf[32];
config = orig = w->wclass.pin.config;
snprintf(buf, sizeof(buf), "cad%u.nid%u.config",
hda_get_codec_id(dev), w->nid);
if (resource_string_value(device_get_name(
device_get_parent(device_get_parent(dev))),
device_get_unit(device_get_parent(device_get_parent(dev))),
buf, &res) == 0) {
if (strncmp(res, "0x", 2) == 0) {
config = strtol(res + 2, NULL, 16);
} else {
config = hdaa_widget_pin_patch(config, res);
}
}
snprintf(buf, sizeof(buf), "nid%u.config", w->nid);
if (resource_string_value(device_get_name(dev), device_get_unit(dev),
buf, &res) == 0) {
if (strncmp(res, "0x", 2) == 0) {
config = strtol(res + 2, NULL, 16);
} else {
config = hdaa_widget_pin_patch(config, res);
}
}
HDA_BOOTVERBOSE(
if (config != orig)
device_printf(w->devinfo->dev,
"Patching pin config nid=%u 0x%08x -> 0x%08x\n",
w->nid, orig, config);
);
w->wclass.pin.newconf = w->wclass.pin.config = config;
}
static void
hdaa_dump_audio_formats_sb(struct sbuf *sb, uint32_t fcap, uint32_t pcmcap)
{
uint32_t cap;
cap = fcap;
if (cap != 0) {
sbuf_printf(sb, " Stream cap: 0x%08x", cap);
if (HDA_PARAM_SUPP_STREAM_FORMATS_AC3(cap))
sbuf_printf(sb, " AC3");
if (HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32(cap))
sbuf_printf(sb, " FLOAT32");
if (HDA_PARAM_SUPP_STREAM_FORMATS_PCM(cap))
sbuf_printf(sb, " PCM");
sbuf_printf(sb, "\n");
}
cap = pcmcap;
if (cap != 0) {
sbuf_printf(sb, " PCM cap: 0x%08x", cap);
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT(cap))
sbuf_printf(sb, " 8");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT(cap))
sbuf_printf(sb, " 16");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT(cap))
sbuf_printf(sb, " 20");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT(cap))
sbuf_printf(sb, " 24");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT(cap))
sbuf_printf(sb, " 32");
sbuf_printf(sb, " bits,");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ(cap))
sbuf_printf(sb, " 8");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ(cap))
sbuf_printf(sb, " 11");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ(cap))
sbuf_printf(sb, " 16");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ(cap))
sbuf_printf(sb, " 22");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ(cap))
sbuf_printf(sb, " 32");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ(cap))
sbuf_printf(sb, " 44");
sbuf_printf(sb, " 48");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ(cap))
sbuf_printf(sb, " 88");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ(cap))
sbuf_printf(sb, " 96");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ(cap))
sbuf_printf(sb, " 176");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ(cap))
sbuf_printf(sb, " 192");
sbuf_printf(sb, " KHz\n");
}
}
static void
hdaa_dump_pin_sb(struct sbuf *sb, struct hdaa_widget *w)
{
uint32_t pincap, conf;
pincap = w->wclass.pin.cap;
sbuf_printf(sb, " Pin cap: 0x%08x", pincap);
if (HDA_PARAM_PIN_CAP_IMP_SENSE_CAP(pincap))
sbuf_printf(sb, " ISC");
if (HDA_PARAM_PIN_CAP_TRIGGER_REQD(pincap))
sbuf_printf(sb, " TRQD");
if (HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP(pincap))
sbuf_printf(sb, " PDC");
if (HDA_PARAM_PIN_CAP_HEADPHONE_CAP(pincap))
sbuf_printf(sb, " HP");
if (HDA_PARAM_PIN_CAP_OUTPUT_CAP(pincap))
sbuf_printf(sb, " OUT");
if (HDA_PARAM_PIN_CAP_INPUT_CAP(pincap))
sbuf_printf(sb, " IN");
if (HDA_PARAM_PIN_CAP_BALANCED_IO_PINS(pincap))
sbuf_printf(sb, " BAL");
if (HDA_PARAM_PIN_CAP_HDMI(pincap))
sbuf_printf(sb, " HDMI");
if (HDA_PARAM_PIN_CAP_VREF_CTRL(pincap)) {
sbuf_printf(sb, " VREF[");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_50(pincap))
sbuf_printf(sb, " 50");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_80(pincap))
sbuf_printf(sb, " 80");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_100(pincap))
sbuf_printf(sb, " 100");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND(pincap))
sbuf_printf(sb, " GROUND");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ(pincap))
sbuf_printf(sb, " HIZ");
sbuf_printf(sb, " ]");
}
if (HDA_PARAM_PIN_CAP_EAPD_CAP(pincap))
sbuf_printf(sb, " EAPD");
if (HDA_PARAM_PIN_CAP_DP(pincap))
sbuf_printf(sb, " DP");
if (HDA_PARAM_PIN_CAP_HBR(pincap))
sbuf_printf(sb, " HBR");
sbuf_printf(sb, "\n");
conf = w->wclass.pin.config;
sbuf_printf(sb, " Pin config: 0x%08x", conf);
sbuf_printf(sb, " as=%d seq=%d "
"device=%s conn=%s ctype=%s loc=%s color=%s misc=%d\n",
HDA_CONFIG_DEFAULTCONF_ASSOCIATION(conf),
HDA_CONFIG_DEFAULTCONF_SEQUENCE(conf),
HDA_DEVS[HDA_CONFIG_DEFAULTCONF_DEVICE(conf)],
HDA_CONNS[HDA_CONFIG_DEFAULTCONF_CONNECTIVITY(conf)],
HDA_CONNECTORS[HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE(conf)],
HDA_LOCS[HDA_CONFIG_DEFAULTCONF_LOCATION(conf)],
HDA_COLORS[HDA_CONFIG_DEFAULTCONF_COLOR(conf)],
HDA_CONFIG_DEFAULTCONF_MISC(conf));
sbuf_printf(sb, " Pin control: 0x%08x", w->wclass.pin.ctrl);
if (w->wclass.pin.ctrl & HDA_CMD_SET_PIN_WIDGET_CTRL_HPHN_ENABLE)
sbuf_printf(sb, " HP");
if (w->wclass.pin.ctrl & HDA_CMD_SET_PIN_WIDGET_CTRL_IN_ENABLE)
sbuf_printf(sb, " IN");
if (w->wclass.pin.ctrl & HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE)
sbuf_printf(sb, " OUT");
if (HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(w->param.widget_cap)) {
if ((w->wclass.pin.ctrl &
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) == 0x03)
sbuf_printf(sb, " HBR");
else if ((w->wclass.pin.ctrl &
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) != 0)
sbuf_printf(sb, " EPTs");
} else {
if ((w->wclass.pin.ctrl &
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) != 0)
sbuf_printf(sb, " VREFs");
}
sbuf_printf(sb, "\n");
}
static void
hdaa_dump_amp_sb(struct sbuf *sb, uint32_t cap, const char *banner)
{
int offset, size, step;
offset = HDA_PARAM_OUTPUT_AMP_CAP_OFFSET(cap);
size = HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE(cap);
step = HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS(cap);
sbuf_printf(sb, " %s amp: 0x%08x "
"mute=%d step=%d size=%d offset=%d (%+d/%+ddB)\n",
banner, cap,
HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP(cap),
step, size, offset,
((0 - offset) * (size + 1)) / 4,
((step - offset) * (size + 1)) / 4);
}
static int
hdaa_sysctl_caps(SYSCTL_HANDLER_ARGS)
{
struct hdaa_devinfo *devinfo;
struct hdaa_widget *w, *cw;
struct sbuf sb;
char buf[64];
int error, j;
w = (struct hdaa_widget *)oidp->oid_arg1;
devinfo = w->devinfo;
sbuf_new_for_sysctl(&sb, NULL, 256, req);
sbuf_printf(&sb, "%s%s\n", w->name,
(w->enable == 0) ? " [DISABLED]" : "");
sbuf_printf(&sb, " Widget cap: 0x%08x",
w->param.widget_cap);
if (w->param.widget_cap & 0x0ee1) {
if (HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP(w->param.widget_cap))
sbuf_printf(&sb, " LRSWAP");
if (HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL(w->param.widget_cap))
sbuf_printf(&sb, " PWR");
if (HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(w->param.widget_cap))
sbuf_printf(&sb, " DIGITAL");
if (HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP(w->param.widget_cap))
sbuf_printf(&sb, " UNSOL");
if (HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET(w->param.widget_cap))
sbuf_printf(&sb, " PROC");
if (HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE(w->param.widget_cap))
sbuf_printf(&sb, " STRIPE(x%d)",
1 << (fls(w->wclass.conv.stripecap) - 1));
j = HDA_PARAM_AUDIO_WIDGET_CAP_CC(w->param.widget_cap);
if (j == 1)
sbuf_printf(&sb, " STEREO");
else if (j > 1)
sbuf_printf(&sb, " %dCH", j + 1);
}
sbuf_printf(&sb, "\n");
if (w->bindas != -1) {
sbuf_printf(&sb, " Association: %d (0x%04x)\n",
w->bindas, w->bindseqmask);
}
if (w->ossmask != 0 || w->ossdev >= 0) {
sbuf_printf(&sb, " OSS: %s",
hdaa_audio_ctl_ossmixer_mask2allname(w->ossmask, buf, sizeof(buf)));
if (w->ossdev >= 0)
sbuf_printf(&sb, " (%s)", ossnames[w->ossdev]);
sbuf_printf(&sb, "\n");
}
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT ||
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT) {
hdaa_dump_audio_formats_sb(&sb,
w->param.supp_stream_formats,
w->param.supp_pcm_size_rate);
} else if (w->type ==
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX || w->waspin)
hdaa_dump_pin_sb(&sb, w);
if (w->param.eapdbtl != HDA_INVALID) {
sbuf_printf(&sb, " EAPD: 0x%08x%s%s%s\n",
w->param.eapdbtl,
(w->param.eapdbtl & HDA_CMD_SET_EAPD_BTL_ENABLE_LR_SWAP) ?
" LRSWAP" : "",
(w->param.eapdbtl & HDA_CMD_SET_EAPD_BTL_ENABLE_EAPD) ?
" EAPD" : "",
(w->param.eapdbtl & HDA_CMD_SET_EAPD_BTL_ENABLE_BTL) ?
" BTL" : "");
}
if (HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP(w->param.widget_cap) &&
w->param.outamp_cap != 0)
hdaa_dump_amp_sb(&sb, w->param.outamp_cap, "Output");
if (HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP(w->param.widget_cap) &&
w->param.inamp_cap != 0)
hdaa_dump_amp_sb(&sb, w->param.inamp_cap, " Input");
if (w->nconns > 0)
sbuf_printf(&sb, " Connections: %d\n", w->nconns);
for (j = 0; j < w->nconns; j++) {
cw = hdaa_widget_get(devinfo, w->conns[j]);
sbuf_printf(&sb, " + %s<- nid=%d [%s]",
(w->connsenable[j] == 0)?"[DISABLED] ":"",
w->conns[j], (cw == NULL) ? "GHOST!" : cw->name);
if (cw == NULL)
sbuf_printf(&sb, " [UNKNOWN]");
else if (cw->enable == 0)
sbuf_printf(&sb, " [DISABLED]");
if (w->nconns > 1 && w->selconn == j && w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER)
sbuf_printf(&sb, " (selected)");
sbuf_printf(&sb, "\n");
}
error = sbuf_finish(&sb);
sbuf_delete(&sb);
return (error);
}
static int
hdaa_sysctl_config(SYSCTL_HANDLER_ARGS)
{
char buf[256];
int error;
uint32_t conf;
conf = *(uint32_t *)oidp->oid_arg1;
snprintf(buf, sizeof(buf), "0x%08x as=%d seq=%d "
"device=%s conn=%s ctype=%s loc=%s color=%s misc=%d",
conf,
HDA_CONFIG_DEFAULTCONF_ASSOCIATION(conf),
HDA_CONFIG_DEFAULTCONF_SEQUENCE(conf),
HDA_DEVS[HDA_CONFIG_DEFAULTCONF_DEVICE(conf)],
HDA_CONNS[HDA_CONFIG_DEFAULTCONF_CONNECTIVITY(conf)],
HDA_CONNECTORS[HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE(conf)],
HDA_LOCS[HDA_CONFIG_DEFAULTCONF_LOCATION(conf)],
HDA_COLORS[HDA_CONFIG_DEFAULTCONF_COLOR(conf)],
HDA_CONFIG_DEFAULTCONF_MISC(conf));
error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
if (error != 0 || req->newptr == NULL)
return (error);
if (strncmp(buf, "0x", 2) == 0)
conf = strtol(buf + 2, NULL, 16);
else
conf = hdaa_widget_pin_patch(conf, buf);
*(uint32_t *)oidp->oid_arg1 = conf;
return (0);
}
static void
hdaa_config_fetch(const char *str, uint32_t *on, uint32_t *off)
{
int i = 0, j, k, len, inv;
for (;;) {
while (str[i] != '\0' &&
(str[i] == ',' || isspace(str[i]) != 0))
i++;
if (str[i] == '\0')
return;
j = i;
while (str[j] != '\0' &&
!(str[j] == ',' || isspace(str[j]) != 0))
j++;
len = j - i;
if (len > 2 && strncmp(str + i, "no", 2) == 0)
inv = 2;
else
inv = 0;
for (k = 0; len > inv && k < nitems(hdaa_quirks_tab); k++) {
if (strncmp(str + i + inv,
hdaa_quirks_tab[k].key, len - inv) != 0)
continue;
if (len - inv != strlen(hdaa_quirks_tab[k].key))
continue;
if (inv == 0) {
*on |= hdaa_quirks_tab[k].value;
*off &= ~hdaa_quirks_tab[k].value;
} else {
*off |= hdaa_quirks_tab[k].value;
*on &= ~hdaa_quirks_tab[k].value;
}
break;
}
i = j;
}
}
static int
hdaa_sysctl_quirks(SYSCTL_HANDLER_ARGS)
{
char buf[256];
int error, n = 0, i;
uint32_t quirks, quirks_off;
quirks = *(uint32_t *)oidp->oid_arg1;
buf[0] = 0;
for (i = 0; i < nitems(hdaa_quirks_tab); i++) {
if ((quirks & hdaa_quirks_tab[i].value) != 0)
n += snprintf(buf + n, sizeof(buf) - n, "%s%s",
n != 0 ? "," : "", hdaa_quirks_tab[i].key);
}
error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
if (error != 0 || req->newptr == NULL)
return (error);
if (strncmp(buf, "0x", 2) == 0)
quirks = strtol(buf + 2, NULL, 16);
else {
quirks = 0;
hdaa_config_fetch(buf, &quirks, &quirks_off);
}
*(uint32_t *)oidp->oid_arg1 = quirks;
return (0);
}
static void
hdaa_local_patch(struct hdaa_devinfo *devinfo)
{
struct hdaa_widget *w;
const char *res = NULL;
uint32_t quirks_on = 0, quirks_off = 0, x;
int i;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL)
continue;
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
hdaa_local_patch_pin(w);
}
if (resource_string_value(device_get_name(devinfo->dev),
device_get_unit(devinfo->dev), "config", &res) == 0) {
if (res != NULL && strlen(res) > 0)
hdaa_config_fetch(res, &quirks_on, &quirks_off);
devinfo->quirks |= quirks_on;
devinfo->quirks &= ~quirks_off;
}
if (devinfo->newquirks == -1)
devinfo->newquirks = devinfo->quirks;
else
devinfo->quirks = devinfo->newquirks;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
"Config options: 0x%08x\n", devinfo->quirks);
);
if (resource_string_value(device_get_name(devinfo->dev),
device_get_unit(devinfo->dev), "gpio_config", &res) == 0) {
if (strncmp(res, "0x", 2) == 0) {
devinfo->gpio = strtol(res + 2, NULL, 16);
} else {
devinfo->gpio = hdaa_gpio_patch(devinfo->gpio, res);
}
}
if (devinfo->newgpio == -1)
devinfo->newgpio = devinfo->gpio;
else
devinfo->gpio = devinfo->newgpio;
if (devinfo->newgpo == -1)
devinfo->newgpo = devinfo->gpo;
else
devinfo->gpo = devinfo->newgpo;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev, "GPIO config options:");
for (i = 0; i < 7; i++) {
x = (devinfo->gpio & HDAA_GPIO_MASK(i)) >> HDAA_GPIO_SHIFT(i);
if (x != 0)
printf(" %d=%s", i, HDA_GPIO_ACTIONS[x]);
}
printf("\n");
);
}
static void
hdaa_widget_connection_parse(struct hdaa_widget *w)
{
uint32_t res;
int i, j, max, ents, entnum;
nid_t nid = w->nid;
nid_t cnid, addcnid, prevcnid;
w->nconns = 0;
res = hda_command(w->devinfo->dev,
HDA_CMD_GET_PARAMETER(0, nid, HDA_PARAM_CONN_LIST_LENGTH));
ents = HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH(res);
if (ents < 1)
return;
entnum = HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM(res) ? 2 : 4;
max = (sizeof(w->conns) / sizeof(w->conns[0])) - 1;
prevcnid = 0;
#define CONN_RMASK(e) (1 << ((32 / (e)) - 1))
#define CONN_NMASK(e) (CONN_RMASK(e) - 1)
#define CONN_RESVAL(r, e, n) ((r) >> ((32 / (e)) * (n)))
#define CONN_RANGE(r, e, n) (CONN_RESVAL(r, e, n) & CONN_RMASK(e))
#define CONN_CNID(r, e, n) (CONN_RESVAL(r, e, n) & CONN_NMASK(e))
for (i = 0; i < ents; i += entnum) {
res = hda_command(w->devinfo->dev,
HDA_CMD_GET_CONN_LIST_ENTRY(0, nid, i));
for (j = 0; j < entnum; j++) {
cnid = CONN_CNID(res, entnum, j);
if (cnid == 0) {
if (w->nconns < ents)
device_printf(w->devinfo->dev,
"WARNING: nid=%d has zero cnid "
"entnum=%d j=%d index=%d "
"entries=%d found=%d res=0x%08x\n",
nid, entnum, j, i,
ents, w->nconns, res);
else
goto getconns_out;
}
if (cnid < w->devinfo->startnode ||
cnid >= w->devinfo->endnode) {
HDA_BOOTVERBOSE(
device_printf(w->devinfo->dev,
"WARNING: nid=%d has cnid outside "
"of the AFG range j=%d "
"entnum=%d index=%d res=0x%08x\n",
nid, j, entnum, i, res);
);
}
if (CONN_RANGE(res, entnum, j) == 0)
addcnid = cnid;
else if (prevcnid == 0 || prevcnid >= cnid) {
device_printf(w->devinfo->dev,
"WARNING: Invalid child range "
"nid=%d index=%d j=%d entnum=%d "
"prevcnid=%d cnid=%d res=0x%08x\n",
nid, i, j, entnum, prevcnid,
cnid, res);
addcnid = cnid;
} else
addcnid = prevcnid + 1;
while (addcnid <= cnid) {
if (w->nconns > max) {
device_printf(w->devinfo->dev,
"Adding %d (nid=%d): "
"Max connection reached! max=%d\n",
addcnid, nid, max + 1);
goto getconns_out;
}
w->connsenable[w->nconns] = 1;
w->conns[w->nconns++] = addcnid++;
}
prevcnid = cnid;
}
}
getconns_out:
return;
}
static void
hdaa_widget_parse(struct hdaa_widget *w)
{
device_t dev = w->devinfo->dev;
uint32_t wcap, cap;
nid_t nid = w->nid;
char buf[64];
w->param.widget_cap = wcap = hda_command(dev,
HDA_CMD_GET_PARAMETER(0, nid, HDA_PARAM_AUDIO_WIDGET_CAP));
w->type = HDA_PARAM_AUDIO_WIDGET_CAP_TYPE(wcap);
hdaa_widget_connection_parse(w);
if (HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP(wcap)) {
if (HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR(wcap))
w->param.outamp_cap =
hda_command(dev,
HDA_CMD_GET_PARAMETER(0, nid,
HDA_PARAM_OUTPUT_AMP_CAP));
else
w->param.outamp_cap =
w->devinfo->outamp_cap;
} else
w->param.outamp_cap = 0;
if (HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP(wcap)) {
if (HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR(wcap))
w->param.inamp_cap =
hda_command(dev,
HDA_CMD_GET_PARAMETER(0, nid,
HDA_PARAM_INPUT_AMP_CAP));
else
w->param.inamp_cap =
w->devinfo->inamp_cap;
} else
w->param.inamp_cap = 0;
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT ||
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT) {
if (HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR(wcap)) {
cap = hda_command(dev,
HDA_CMD_GET_PARAMETER(0, nid,
HDA_PARAM_SUPP_STREAM_FORMATS));
w->param.supp_stream_formats = (cap != 0) ? cap :
w->devinfo->supp_stream_formats;
cap = hda_command(dev,
HDA_CMD_GET_PARAMETER(0, nid,
HDA_PARAM_SUPP_PCM_SIZE_RATE));
w->param.supp_pcm_size_rate = (cap != 0) ? cap :
w->devinfo->supp_pcm_size_rate;
} else {
w->param.supp_stream_formats =
w->devinfo->supp_stream_formats;
w->param.supp_pcm_size_rate =
w->devinfo->supp_pcm_size_rate;
}
if (HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE(w->param.widget_cap)) {
w->wclass.conv.stripecap = hda_command(dev,
HDA_CMD_GET_STRIPE_CONTROL(0, w->nid)) >> 20;
} else
w->wclass.conv.stripecap = 1;
} else {
w->param.supp_stream_formats = 0;
w->param.supp_pcm_size_rate = 0;
}
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX) {
w->wclass.pin.original = w->wclass.pin.newconf =
w->wclass.pin.config = hda_command(dev,
HDA_CMD_GET_CONFIGURATION_DEFAULT(0, w->nid));
w->wclass.pin.cap = hda_command(dev,
HDA_CMD_GET_PARAMETER(0, w->nid, HDA_PARAM_PIN_CAP));
w->wclass.pin.ctrl = hda_command(dev,
HDA_CMD_GET_PIN_WIDGET_CTRL(0, nid));
w->wclass.pin.connected = 2;
if (HDA_PARAM_PIN_CAP_EAPD_CAP(w->wclass.pin.cap)) {
w->param.eapdbtl = hda_command(dev,
HDA_CMD_GET_EAPD_BTL_ENABLE(0, nid));
w->param.eapdbtl &= 0x7;
w->param.eapdbtl |= HDA_CMD_SET_EAPD_BTL_ENABLE_EAPD;
} else
w->param.eapdbtl = HDA_INVALID;
}
w->unsol = -1;
hdaa_unlock(w->devinfo);
snprintf(buf, sizeof(buf), "nid%d", w->nid);
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
buf, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
w, sizeof(w), hdaa_sysctl_caps, "A", "Node capabilities");
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX) {
snprintf(buf, sizeof(buf), "nid%d_config", w->nid);
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
buf, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
&w->wclass.pin.newconf, sizeof(&w->wclass.pin.newconf),
hdaa_sysctl_config, "A", "Current pin configuration");
snprintf(buf, sizeof(buf), "nid%d_original", w->nid);
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
buf, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
&w->wclass.pin.original, sizeof(&w->wclass.pin.original),
hdaa_sysctl_config, "A", "Original pin configuration");
}
hdaa_lock(w->devinfo);
}
static void
hdaa_widget_postprocess(struct hdaa_widget *w)
{
const char *typestr;
w->type = HDA_PARAM_AUDIO_WIDGET_CAP_TYPE(w->param.widget_cap);
switch (w->type) {
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT:
typestr = "audio output";
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT:
typestr = "audio input";
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER:
typestr = "audio mixer";
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR:
typestr = "audio selector";
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX:
typestr = "pin";
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_POWER_WIDGET:
typestr = "power widget";
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_VOLUME_WIDGET:
typestr = "volume widget";
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_BEEP_WIDGET:
typestr = "beep widget";
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_VENDOR_WIDGET:
typestr = "vendor widget";
break;
default:
typestr = "unknown type";
break;
}
strlcpy(w->name, typestr, sizeof(w->name));
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX) {
uint32_t config;
const char *devstr;
int conn, color;
config = w->wclass.pin.config;
devstr = HDA_DEVS[(config & HDA_CONFIG_DEFAULTCONF_DEVICE_MASK) >>
HDA_CONFIG_DEFAULTCONF_DEVICE_SHIFT];
conn = (config & HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_MASK) >>
HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_SHIFT;
color = (config & HDA_CONFIG_DEFAULTCONF_COLOR_MASK) >>
HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT;
strlcat(w->name, ": ", sizeof(w->name));
strlcat(w->name, devstr, sizeof(w->name));
strlcat(w->name, " (", sizeof(w->name));
if (conn == 0 && color != 0 && color != 15) {
strlcat(w->name, HDA_COLORS[color], sizeof(w->name));
strlcat(w->name, " ", sizeof(w->name));
}
strlcat(w->name, HDA_CONNS[conn], sizeof(w->name));
strlcat(w->name, ")", sizeof(w->name));
}
}
struct hdaa_widget *
hdaa_widget_get(struct hdaa_devinfo *devinfo, nid_t nid)
{
if (devinfo == NULL || devinfo->widget == NULL ||
nid < devinfo->startnode || nid >= devinfo->endnode)
return (NULL);
return (&devinfo->widget[nid - devinfo->startnode]);
}
static void
hdaa_audio_ctl_amp_set_internal(struct hdaa_devinfo *devinfo, nid_t nid,
int index, int lmute, int rmute,
int left, int right, int dir)
{
uint16_t v = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
"Setting amplifier nid=%d index=%d %s mute=%d/%d vol=%d/%d\n",
nid,index,dir ? "in" : "out",lmute,rmute,left,right);
);
if (left != right || lmute != rmute) {
v = (1 << (15 - dir)) | (1 << 13) | (index << 8) |
(lmute << 7) | left;
hda_command(devinfo->dev,
HDA_CMD_SET_AMP_GAIN_MUTE(0, nid, v));
v = (1 << (15 - dir)) | (1 << 12) | (index << 8) |
(rmute << 7) | right;
} else
v = (1 << (15 - dir)) | (3 << 12) | (index << 8) |
(lmute << 7) | left;
hda_command(devinfo->dev,
HDA_CMD_SET_AMP_GAIN_MUTE(0, nid, v));
}
static void
hdaa_audio_ctl_amp_set(struct hdaa_audio_ctl *ctl, uint32_t mute,
int left, int right)
{
nid_t nid;
int lmute, rmute;
nid = ctl->widget->nid;
/* Save new values if valid. */
if (mute != HDAA_AMP_MUTE_DEFAULT)
ctl->muted = mute;
if (left != HDAA_AMP_VOL_DEFAULT)
ctl->left = left;
if (right != HDAA_AMP_VOL_DEFAULT)
ctl->right = right;
/* Prepare effective values */
if (ctl->forcemute) {
lmute = 1;
rmute = 1;
left = 0;
right = 0;
} else {
lmute = HDAA_AMP_LEFT_MUTED(ctl->muted);
rmute = HDAA_AMP_RIGHT_MUTED(ctl->muted);
left = ctl->left;
right = ctl->right;
}
/* Apply effective values */
if (ctl->dir & HDAA_CTL_OUT)
hdaa_audio_ctl_amp_set_internal(ctl->widget->devinfo, nid, ctl->index,
lmute, rmute, left, right, 0);
if (ctl->dir & HDAA_CTL_IN)
hdaa_audio_ctl_amp_set_internal(ctl->widget->devinfo, nid, ctl->index,
lmute, rmute, left, right, 1);
}
static void
hdaa_widget_connection_select(struct hdaa_widget *w, uint8_t index)
{
if (w == NULL || w->nconns < 1 || index > (w->nconns - 1))
return;
HDA_BOOTHVERBOSE(
device_printf(w->devinfo->dev,
"Setting selector nid=%d index=%d\n", w->nid, index);
);
hda_command(w->devinfo->dev,
HDA_CMD_SET_CONNECTION_SELECT_CONTROL(0, w->nid, index));
w->selconn = index;
}
/****************************************************************************
* Device Methods
****************************************************************************/
static void *
hdaa_channel_init(kobj_t obj, void *data, struct snd_dbuf *b,
struct pcm_channel *c, int dir)
{
struct hdaa_chan *ch = data;
struct hdaa_pcm_devinfo *pdevinfo = ch->pdevinfo;
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
hdaa_lock(devinfo);
if (devinfo->quirks & HDAA_QUIRK_FIXEDRATE) {
ch->caps.minspeed = ch->caps.maxspeed = 48000;
ch->pcmrates[0] = 48000;
ch->pcmrates[1] = 0;
}
ch->dir = dir;
ch->b = b;
ch->c = c;
ch->blksz = pdevinfo->chan_size / pdevinfo->chan_blkcnt;
ch->blkcnt = pdevinfo->chan_blkcnt;
hdaa_unlock(devinfo);
if (sndbuf_alloc(ch->b, bus_get_dma_tag(devinfo->dev),
hda_get_dma_nocache(devinfo->dev) ? BUS_DMA_NOCACHE : 0,
pdevinfo->chan_size) != 0)
return (NULL);
return (ch);
}
static int
hdaa_channel_setformat(kobj_t obj, void *data, uint32_t format)
{
struct hdaa_chan *ch = data;
int i;
for (i = 0; ch->caps.fmtlist[i] != 0; i++) {
if (format == ch->caps.fmtlist[i]) {
ch->fmt = format;
return (0);
}
}
return (EINVAL);
}
static uint32_t
hdaa_channel_setspeed(kobj_t obj, void *data, uint32_t speed)
{
struct hdaa_chan *ch = data;
uint32_t spd = 0, threshold;
int i;
/* First look for equal or multiple frequency. */
for (i = 0; ch->pcmrates[i] != 0; i++) {
spd = ch->pcmrates[i];
if (speed != 0 && spd / speed * speed == spd) {
ch->spd = spd;
return (spd);
}
}
/* If no match, just find nearest. */
for (i = 0; ch->pcmrates[i] != 0; i++) {
spd = ch->pcmrates[i];
threshold = spd + ((ch->pcmrates[i + 1] != 0) ?
((ch->pcmrates[i + 1] - spd) >> 1) : 0);
if (speed < threshold)
break;
}
ch->spd = spd;
return (spd);
}
static uint16_t
hdaa_stream_format(struct hdaa_chan *ch)
{
int i;
uint16_t fmt;
fmt = 0;
if (ch->fmt & AFMT_S16_LE)
fmt |= ch->bit16 << 4;
else if (ch->fmt & AFMT_S32_LE)
fmt |= ch->bit32 << 4;
else
fmt |= 1 << 4;
for (i = 0; i < HDA_RATE_TAB_LEN; i++) {
if (hda_rate_tab[i].valid && ch->spd == hda_rate_tab[i].rate) {
fmt |= hda_rate_tab[i].base;
fmt |= hda_rate_tab[i].mul;
fmt |= hda_rate_tab[i].div;
break;
}
}
fmt |= (AFMT_CHANNEL(ch->fmt) - 1);
return (fmt);
}
static int
hdaa_allowed_stripes(uint16_t fmt)
{
static const int bits[8] = { 8, 16, 20, 24, 32, 32, 32, 32 };
int size;
size = bits[(fmt >> 4) & 0x03];
size *= (fmt & 0x0f) + 1;
size *= ((fmt >> 11) & 0x07) + 1;
return (0xffffffffU >> (32 - fls(size / 8)));
}
static void
hdaa_audio_setup(struct hdaa_chan *ch)
{
struct hdaa_audio_as *as = &ch->devinfo->as[ch->as];
struct hdaa_widget *w, *wp;
int i, j, k, chn, cchn, totalchn, totalextchn, c;
uint16_t fmt, dfmt;
/* Mapping channel pairs to codec pins/converters. */
const static uint16_t convmap[2][5] =
/* 1.0 2.0 4.0 5.1 7.1 */
{{ 0x0010, 0x0001, 0x0201, 0x0231, 0x4231 }, /* no dup. */
{ 0x0010, 0x0001, 0x2201, 0x2231, 0x4231 }}; /* side dup. */
/* Mapping formats to HDMI channel allocations. */
const static uint8_t hdmica[2][8] =
/* 1 2 3 4 5 6 7 8 */
{{ 0x02, 0x00, 0x04, 0x08, 0x0a, 0x0e, 0x12, 0x12 }, /* x.0 */
{ 0x01, 0x03, 0x01, 0x03, 0x09, 0x0b, 0x0f, 0x13 }}; /* x.1 */
/* Mapping formats to HDMI channels order. */
const static uint32_t hdmich[2][8] =
/* 1 / 5 2 / 6 3 / 7 4 / 8 */
{{ 0xFFFF0F00, 0xFFFFFF10, 0xFFF2FF10, 0xFF32FF10,
0xFF324F10, 0xF5324F10, 0x54326F10, 0x54326F10 }, /* x.0 */
{ 0xFFFFF000, 0xFFFF0100, 0xFFFFF210, 0xFFFF2310,
0xFF32F410, 0xFF324510, 0xF6324510, 0x76325410 }}; /* x.1 */
int convmapid = -1;
nid_t nid;
uint8_t csum;
totalchn = AFMT_CHANNEL(ch->fmt);
totalextchn = AFMT_EXTCHANNEL(ch->fmt);
HDA_BOOTHVERBOSE(
device_printf(ch->pdevinfo->dev,
"PCMDIR_%s: Stream setup fmt=%08x (%d.%d) speed=%d\n",
(ch->dir == PCMDIR_PLAY) ? "PLAY" : "REC",
ch->fmt, totalchn - totalextchn, totalextchn, ch->spd);
);
fmt = hdaa_stream_format(ch);
/* Set channels to I/O converters mapping for known speaker setups. */
if ((as->pinset == 0x0007 || as->pinset == 0x0013) || /* Standard 5.1 */
(as->pinset == 0x0017)) /* Standard 7.1 */
convmapid = (ch->dir == PCMDIR_PLAY);
dfmt = HDA_CMD_SET_DIGITAL_CONV_FMT1_DIGEN;
if (ch->fmt & AFMT_AC3)
dfmt |= HDA_CMD_SET_DIGITAL_CONV_FMT1_NAUDIO;
chn = 0;
for (i = 0; ch->io[i] != -1; i++) {
w = hdaa_widget_get(ch->devinfo, ch->io[i]);
if (w == NULL)
continue;
/* If HP redirection is enabled, but failed to use same
DAC, make last DAC to duplicate first one. */
if (as->fakeredir && i == (as->pincnt - 1)) {
c = (ch->sid << 4);
} else {
/* Map channels to I/O converters, if set. */
if (convmapid >= 0)
chn = (((convmap[convmapid][totalchn / 2]
>> i * 4) & 0xf) - 1) * 2;
if (chn < 0 || chn >= totalchn) {
c = 0;
} else {
c = (ch->sid << 4) | chn;
}
}
hda_command(ch->devinfo->dev,
HDA_CMD_SET_CONV_FMT(0, ch->io[i], fmt));
if (HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(w->param.widget_cap)) {
hda_command(ch->devinfo->dev,
HDA_CMD_SET_DIGITAL_CONV_FMT1(0, ch->io[i], dfmt));
}
hda_command(ch->devinfo->dev,
HDA_CMD_SET_CONV_STREAM_CHAN(0, ch->io[i], c));
if (HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE(w->param.widget_cap)) {
hda_command(ch->devinfo->dev,
HDA_CMD_SET_STRIPE_CONTROL(0, w->nid, ch->stripectl));
}
cchn = HDA_PARAM_AUDIO_WIDGET_CAP_CC(w->param.widget_cap);
if (cchn > 1 && chn < totalchn) {
cchn = min(cchn, totalchn - chn - 1);
hda_command(ch->devinfo->dev,
HDA_CMD_SET_CONV_CHAN_COUNT(0, ch->io[i], cchn));
}
HDA_BOOTHVERBOSE(
device_printf(ch->pdevinfo->dev,
"PCMDIR_%s: Stream setup nid=%d: "
"fmt=0x%04x, dfmt=0x%04x, chan=0x%04x, "
"chan_count=0x%02x, stripe=%d\n",
(ch->dir == PCMDIR_PLAY) ? "PLAY" : "REC",
ch->io[i], fmt, dfmt, c, cchn, ch->stripectl);
);
for (j = 0; j < 16; j++) {
if (as->dacs[ch->asindex][j] != ch->io[i])
continue;
nid = as->pins[j];
wp = hdaa_widget_get(ch->devinfo, nid);
if (wp == NULL)
continue;
if (!HDA_PARAM_PIN_CAP_DP(wp->wclass.pin.cap) &&
!HDA_PARAM_PIN_CAP_HDMI(wp->wclass.pin.cap))
continue;
/* Set channel mapping. */
for (k = 0; k < 8; k++) {
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_CHAN_SLOT(0, nid,
(((hdmich[totalextchn == 0 ? 0 : 1][totalchn - 1]
>> (k * 4)) & 0xf) << 4) | k));
}
/*
* Enable High Bit Rate (HBR) Encoded Packet Type
* (EPT), if supported and needed (8ch data).
*/
if (HDA_PARAM_PIN_CAP_HDMI(wp->wclass.pin.cap) &&
HDA_PARAM_PIN_CAP_HBR(wp->wclass.pin.cap)) {
wp->wclass.pin.ctrl &=
~HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK;
if ((ch->fmt & AFMT_AC3) && (cchn == 7))
wp->wclass.pin.ctrl |= 0x03;
hda_command(ch->devinfo->dev,
HDA_CMD_SET_PIN_WIDGET_CTRL(0, nid,
wp->wclass.pin.ctrl));
}
/* Stop audio infoframe transmission. */
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_INDEX(0, nid, 0x00));
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_XMIT(0, nid, 0x00));
/* Clear audio infoframe buffer. */
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_INDEX(0, nid, 0x00));
for (k = 0; k < 32; k++)
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, 0x00));
/* Write HDMI/DisplayPort audio infoframe. */
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_INDEX(0, nid, 0x00));
if (w->eld != NULL && w->eld_len >= 6 &&
((w->eld[5] >> 2) & 0x3) == 1) { /* DisplayPort */
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, 0x84));
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, 0x1b));
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, 0x44));
} else { /* HDMI */
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, 0x84));
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, 0x01));
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, 0x0a));
csum = 0;
csum -= 0x84 + 0x01 + 0x0a + (totalchn - 1) +
hdmica[totalextchn == 0 ? 0 : 1][totalchn - 1];
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, csum));
}
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, totalchn - 1));
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, 0x00));
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid, 0x00));
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_DATA(0, nid,
hdmica[totalextchn == 0 ? 0 : 1][totalchn - 1]));
/* Start audio infoframe transmission. */
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_INDEX(0, nid, 0x00));
hda_command(ch->devinfo->dev,
HDA_CMD_SET_HDMI_DIP_XMIT(0, nid, 0xc0));
}
chn += cchn + 1;
}
}
/*
* Greatest Common Divisor.
*/
static unsigned
gcd(unsigned a, unsigned b)
{
u_int c;
while (b != 0) {
c = a;
a = b;
b = (c % b);
}
return (a);
}
/*
* Least Common Multiple.
*/
static unsigned
lcm(unsigned a, unsigned b)
{
return ((a * b) / gcd(a, b));
}
static int
hdaa_channel_setfragments(kobj_t obj, void *data,
uint32_t blksz, uint32_t blkcnt)
{
struct hdaa_chan *ch = data;
blksz -= blksz % lcm(HDA_DMA_ALIGNMENT, sndbuf_getalign(ch->b));
if (blksz > (sndbuf_getmaxsize(ch->b) / HDA_BDL_MIN))
blksz = sndbuf_getmaxsize(ch->b) / HDA_BDL_MIN;
if (blksz < HDA_BLK_MIN)
blksz = HDA_BLK_MIN;
if (blkcnt > HDA_BDL_MAX)
blkcnt = HDA_BDL_MAX;
if (blkcnt < HDA_BDL_MIN)
blkcnt = HDA_BDL_MIN;
while ((blksz * blkcnt) > sndbuf_getmaxsize(ch->b)) {
if ((blkcnt >> 1) >= HDA_BDL_MIN)
blkcnt >>= 1;
else if ((blksz >> 1) >= HDA_BLK_MIN)
blksz >>= 1;
else
break;
}
if ((sndbuf_getblksz(ch->b) != blksz ||
sndbuf_getblkcnt(ch->b) != blkcnt) &&
sndbuf_resize(ch->b, blkcnt, blksz) != 0)
device_printf(ch->devinfo->dev, "%s: failed blksz=%u blkcnt=%u\n",
__func__, blksz, blkcnt);
ch->blksz = sndbuf_getblksz(ch->b);
ch->blkcnt = sndbuf_getblkcnt(ch->b);
return (0);
}
static uint32_t
hdaa_channel_setblocksize(kobj_t obj, void *data, uint32_t blksz)
{
struct hdaa_chan *ch = data;
hdaa_channel_setfragments(obj, data, blksz, ch->pdevinfo->chan_blkcnt);
return (ch->blksz);
}
static void
hdaa_channel_stop(struct hdaa_chan *ch)
{
struct hdaa_devinfo *devinfo = ch->devinfo;
struct hdaa_widget *w;
int i;
if ((ch->flags & HDAA_CHN_RUNNING) == 0)
return;
ch->flags &= ~HDAA_CHN_RUNNING;
HDAC_STREAM_STOP(device_get_parent(devinfo->dev), devinfo->dev,
ch->dir == PCMDIR_PLAY ? 1 : 0, ch->sid);
for (i = 0; ch->io[i] != -1; i++) {
w = hdaa_widget_get(ch->devinfo, ch->io[i]);
if (w == NULL)
continue;
if (HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(w->param.widget_cap)) {
hda_command(devinfo->dev,
HDA_CMD_SET_DIGITAL_CONV_FMT1(0, ch->io[i], 0));
}
hda_command(devinfo->dev,
HDA_CMD_SET_CONV_STREAM_CHAN(0, ch->io[i],
0));
}
HDAC_STREAM_FREE(device_get_parent(devinfo->dev), devinfo->dev,
ch->dir == PCMDIR_PLAY ? 1 : 0, ch->sid);
}
static int
hdaa_channel_start(struct hdaa_chan *ch)
{
struct hdaa_devinfo *devinfo = ch->devinfo;
uint32_t fmt;
fmt = hdaa_stream_format(ch);
ch->stripectl = fls(ch->stripecap & hdaa_allowed_stripes(fmt)) - 1;
ch->sid = HDAC_STREAM_ALLOC(device_get_parent(devinfo->dev), devinfo->dev,
ch->dir == PCMDIR_PLAY ? 1 : 0, fmt, ch->stripectl, &ch->dmapos);
if (ch->sid <= 0)
return (EBUSY);
hdaa_audio_setup(ch);
HDAC_STREAM_RESET(device_get_parent(devinfo->dev), devinfo->dev,
ch->dir == PCMDIR_PLAY ? 1 : 0, ch->sid);
HDAC_STREAM_START(device_get_parent(devinfo->dev), devinfo->dev,
ch->dir == PCMDIR_PLAY ? 1 : 0, ch->sid,
sndbuf_getbufaddr(ch->b), ch->blksz, ch->blkcnt);
ch->flags |= HDAA_CHN_RUNNING;
return (0);
}
static int
hdaa_channel_trigger(kobj_t obj, void *data, int go)
{
struct hdaa_chan *ch = data;
int error = 0;
if (!PCMTRIG_COMMON(go))
return (0);
hdaa_lock(ch->devinfo);
switch (go) {
case PCMTRIG_START:
error = hdaa_channel_start(ch);
break;
case PCMTRIG_STOP:
case PCMTRIG_ABORT:
hdaa_channel_stop(ch);
break;
default:
break;
}
hdaa_unlock(ch->devinfo);
return (error);
}
static uint32_t
hdaa_channel_getptr(kobj_t obj, void *data)
{
struct hdaa_chan *ch = data;
struct hdaa_devinfo *devinfo = ch->devinfo;
uint32_t ptr;
hdaa_lock(devinfo);
if (ch->dmapos != NULL) {
ptr = *(ch->dmapos);
} else {
ptr = HDAC_STREAM_GETPTR(
device_get_parent(devinfo->dev), devinfo->dev,
ch->dir == PCMDIR_PLAY ? 1 : 0, ch->sid);
}
hdaa_unlock(devinfo);
/*
* Round to available space and force 128 bytes aligment.
*/
ptr %= ch->blksz * ch->blkcnt;
ptr &= HDA_BLK_ALIGN;
return (ptr);
}
static struct pcmchan_caps *
hdaa_channel_getcaps(kobj_t obj, void *data)
{
return (&((struct hdaa_chan *)data)->caps);
}
static kobj_method_t hdaa_channel_methods[] = {
KOBJMETHOD(channel_init, hdaa_channel_init),
KOBJMETHOD(channel_setformat, hdaa_channel_setformat),
KOBJMETHOD(channel_setspeed, hdaa_channel_setspeed),
KOBJMETHOD(channel_setblocksize, hdaa_channel_setblocksize),
KOBJMETHOD(channel_setfragments, hdaa_channel_setfragments),
KOBJMETHOD(channel_trigger, hdaa_channel_trigger),
KOBJMETHOD(channel_getptr, hdaa_channel_getptr),
KOBJMETHOD(channel_getcaps, hdaa_channel_getcaps),
KOBJMETHOD_END
};
CHANNEL_DECLARE(hdaa_channel);
static int
hdaa_audio_ctl_ossmixer_init(struct snd_mixer *m)
{
struct hdaa_pcm_devinfo *pdevinfo = mix_getdevinfo(m);
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_widget *w, *cw;
uint32_t mask, recmask;
int i, j;
hdaa_lock(devinfo);
pdevinfo->mixer = m;
/* Make sure that in case of soft volume it won't stay muted. */
for (i = 0; i < SOUND_MIXER_NRDEVICES; i++) {
pdevinfo->left[i] = 100;
pdevinfo->right[i] = 100;
}
/* Declare volume controls assigned to this association. */
mask = pdevinfo->ossmask;
if (pdevinfo->playas >= 0) {
/* Declate EAPD as ogain control. */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX ||
w->param.eapdbtl == HDA_INVALID ||
w->bindas != pdevinfo->playas)
continue;
mask |= SOUND_MASK_OGAIN;
break;
}
/* Declare soft PCM volume if needed. */
if ((mask & SOUND_MASK_PCM) == 0 ||
(devinfo->quirks & HDAA_QUIRK_SOFTPCMVOL) ||
pdevinfo->minamp[SOUND_MIXER_PCM] ==
pdevinfo->maxamp[SOUND_MIXER_PCM]) {
mask |= SOUND_MASK_PCM;
pcm_setflags(pdevinfo->dev, pcm_getflags(pdevinfo->dev) | SD_F_SOFTPCMVOL);
HDA_BOOTHVERBOSE(
device_printf(pdevinfo->dev,
"Forcing Soft PCM volume\n");
);
}
/* Declare master volume if needed. */
if ((mask & SOUND_MASK_VOLUME) == 0) {
mask |= SOUND_MASK_VOLUME;
mix_setparentchild(m, SOUND_MIXER_VOLUME,
SOUND_MASK_PCM);
mix_setrealdev(m, SOUND_MIXER_VOLUME,
SOUND_MIXER_NONE);
HDA_BOOTHVERBOSE(
device_printf(pdevinfo->dev,
"Forcing master volume with PCM\n");
);
}
}
/* Declare record sources available to this association. */
recmask = 0;
if (pdevinfo->recas >= 0) {
for (i = 0; i < 16; i++) {
if (devinfo->as[pdevinfo->recas].dacs[0][i] < 0)
continue;
w = hdaa_widget_get(devinfo,
devinfo->as[pdevinfo->recas].dacs[0][i]);
if (w == NULL || w->enable == 0)
continue;
for (j = 0; j < w->nconns; j++) {
if (w->connsenable[j] == 0)
continue;
cw = hdaa_widget_get(devinfo, w->conns[j]);
if (cw == NULL || cw->enable == 0)
continue;
if (cw->bindas != pdevinfo->recas &&
cw->bindas != -2)
continue;
recmask |= cw->ossmask;
}
}
}
recmask &= (1 << SOUND_MIXER_NRDEVICES) - 1;
mask &= (1 << SOUND_MIXER_NRDEVICES) - 1;
pdevinfo->ossmask = mask;
mix_setrecdevs(m, recmask);
mix_setdevs(m, mask);
hdaa_unlock(devinfo);
return (0);
}
/*
* Update amplification per pdevinfo per ossdev, calculate summary coefficient
* and write it to codec, update *left and *right to reflect remaining error.
*/
static void
hdaa_audio_ctl_dev_set(struct hdaa_audio_ctl *ctl, int ossdev,
int mute, int *left, int *right)
{
int i, zleft, zright, sleft, sright, smute, lval, rval;
ctl->devleft[ossdev] = *left;
ctl->devright[ossdev] = *right;
ctl->devmute[ossdev] = mute;
smute = sleft = sright = zleft = zright = 0;
for (i = 0; i < SOUND_MIXER_NRDEVICES; i++) {
sleft += ctl->devleft[i];
sright += ctl->devright[i];
smute |= ctl->devmute[i];
if (i == ossdev)
continue;
zleft += ctl->devleft[i];
zright += ctl->devright[i];
}
lval = QDB2VAL(ctl, sleft);
rval = QDB2VAL(ctl, sright);
hdaa_audio_ctl_amp_set(ctl, smute, lval, rval);
*left -= VAL2QDB(ctl, lval) - VAL2QDB(ctl, QDB2VAL(ctl, zleft));
*right -= VAL2QDB(ctl, rval) - VAL2QDB(ctl, QDB2VAL(ctl, zright));
}
/*
* Trace signal from source, setting volumes on the way.
*/
static void
hdaa_audio_ctl_source_volume(struct hdaa_pcm_devinfo *pdevinfo,
int ossdev, nid_t nid, int index, int mute, int left, int right, int depth)
{
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_widget *w, *wc;
struct hdaa_audio_ctl *ctl;
int i, j, conns = 0;
if (depth > HDA_PARSE_MAXDEPTH)
return;
w = hdaa_widget_get(devinfo, nid);
if (w == NULL || w->enable == 0)
return;
/* Count number of active inputs. */
if (depth > 0) {
for (j = 0; j < w->nconns; j++) {
if (!w->connsenable[j])
continue;
conns++;
}
}
/* If this is not a first step - use input mixer.
Pins have common input ctl so care must be taken. */
if (depth > 0 && (conns == 1 ||
w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)) {
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid, HDAA_CTL_IN,
index, 1);
if (ctl)
hdaa_audio_ctl_dev_set(ctl, ossdev, mute, &left, &right);
}
/* If widget has own ossdev - not traverse it.
It will be traversed on it's own. */
if (w->ossdev >= 0 && depth > 0)
return;
/* We must not traverse pin */
if ((w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT ||
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX) &&
depth > 0)
return;
/*
* If signals mixed, we can't assign controls farther.
* Ignore this on depth zero. Caller must knows why.
*/
if (conns > 1 &&
(w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER ||
w->selconn != index))
return;
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid, HDAA_CTL_OUT, -1, 1);
if (ctl)
hdaa_audio_ctl_dev_set(ctl, ossdev, mute, &left, &right);
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
wc = hdaa_widget_get(devinfo, i);
if (wc == NULL || wc->enable == 0)
continue;
for (j = 0; j < wc->nconns; j++) {
if (wc->connsenable[j] && wc->conns[j] == nid) {
hdaa_audio_ctl_source_volume(pdevinfo, ossdev,
wc->nid, j, mute, left, right, depth + 1);
}
}
}
return;
}
/*
* Trace signal from destination, setting volumes on the way.
*/
static void
hdaa_audio_ctl_dest_volume(struct hdaa_pcm_devinfo *pdevinfo,
int ossdev, nid_t nid, int index, int mute, int left, int right, int depth)
{
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_audio_as *as = devinfo->as;
struct hdaa_widget *w, *wc;
struct hdaa_audio_ctl *ctl;
int i, j, consumers, cleft, cright;
if (depth > HDA_PARSE_MAXDEPTH)
return;
w = hdaa_widget_get(devinfo, nid);
if (w == NULL || w->enable == 0)
return;
if (depth > 0) {
/* If this node produce output for several consumers,
we can't touch it. */
consumers = 0;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
wc = hdaa_widget_get(devinfo, i);
if (wc == NULL || wc->enable == 0)
continue;
for (j = 0; j < wc->nconns; j++) {
if (wc->connsenable[j] && wc->conns[j] == nid)
consumers++;
}
}
/* The only exception is if real HP redirection is configured
and this is a duplication point.
XXX: Actually exception is not completely correct.
XXX: Duplication point check is not perfect. */
if ((consumers == 2 && (w->bindas < 0 ||
as[w->bindas].hpredir < 0 || as[w->bindas].fakeredir ||
(w->bindseqmask & (1 << 15)) == 0)) ||
consumers > 2)
return;
/* Else use it's output mixer. */
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid,
HDAA_CTL_OUT, -1, 1);
if (ctl)
hdaa_audio_ctl_dev_set(ctl, ossdev, mute, &left, &right);
}
/* We must not traverse pin */
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX &&
depth > 0)
return;
for (i = 0; i < w->nconns; i++) {
if (w->connsenable[i] == 0)
continue;
if (index >= 0 && i != index)
continue;
cleft = left;
cright = right;
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid,
HDAA_CTL_IN, i, 1);
if (ctl)
hdaa_audio_ctl_dev_set(ctl, ossdev, mute, &cleft, &cright);
hdaa_audio_ctl_dest_volume(pdevinfo, ossdev, w->conns[i], -1,
mute, cleft, cright, depth + 1);
}
}
/*
* Set volumes for the specified pdevinfo and ossdev.
*/
static void
hdaa_audio_ctl_dev_volume(struct hdaa_pcm_devinfo *pdevinfo, unsigned dev)
{
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_widget *w, *cw;
uint32_t mute;
int lvol, rvol;
int i, j;
mute = 0;
if (pdevinfo->left[dev] == 0) {
mute |= HDAA_AMP_MUTE_LEFT;
lvol = -4000;
} else
lvol = ((pdevinfo->maxamp[dev] - pdevinfo->minamp[dev]) *
pdevinfo->left[dev] + 50) / 100 + pdevinfo->minamp[dev];
if (pdevinfo->right[dev] == 0) {
mute |= HDAA_AMP_MUTE_RIGHT;
rvol = -4000;
} else
rvol = ((pdevinfo->maxamp[dev] - pdevinfo->minamp[dev]) *
pdevinfo->right[dev] + 50) / 100 + pdevinfo->minamp[dev];
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->bindas < 0) {
if (pdevinfo->index != 0)
continue;
} else {
if (w->bindas != pdevinfo->playas &&
w->bindas != pdevinfo->recas)
continue;
}
if (dev == SOUND_MIXER_RECLEV &&
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT) {
hdaa_audio_ctl_dest_volume(pdevinfo, dev,
w->nid, -1, mute, lvol, rvol, 0);
continue;
}
if (dev == SOUND_MIXER_VOLUME &&
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX &&
devinfo->as[w->bindas].dir == HDAA_CTL_OUT) {
hdaa_audio_ctl_dest_volume(pdevinfo, dev,
w->nid, -1, mute, lvol, rvol, 0);
continue;
}
if (dev == SOUND_MIXER_IGAIN &&
w->pflags & HDAA_ADC_MONITOR) {
for (j = 0; j < w->nconns; j++) {
if (!w->connsenable[j])
continue;
cw = hdaa_widget_get(devinfo, w->conns[j]);
if (cw == NULL || cw->enable == 0)
continue;
if (cw->bindas == -1)
continue;
if (cw->bindas >= 0 &&
devinfo->as[cw->bindas].dir != HDAA_CTL_IN)
continue;
hdaa_audio_ctl_dest_volume(pdevinfo, dev,
w->nid, j, mute, lvol, rvol, 0);
}
continue;
}
if (w->ossdev != dev)
continue;
hdaa_audio_ctl_source_volume(pdevinfo, dev,
w->nid, -1, mute, lvol, rvol, 0);
if (dev == SOUND_MIXER_IMIX && (w->pflags & HDAA_IMIX_AS_DST))
hdaa_audio_ctl_dest_volume(pdevinfo, dev,
w->nid, -1, mute, lvol, rvol, 0);
}
}
/*
* OSS Mixer set method.
*/
static int
hdaa_audio_ctl_ossmixer_set(struct snd_mixer *m, unsigned dev,
unsigned left, unsigned right)
{
struct hdaa_pcm_devinfo *pdevinfo = mix_getdevinfo(m);
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_widget *w;
int i;
hdaa_lock(devinfo);
/* Save new values. */
pdevinfo->left[dev] = left;
pdevinfo->right[dev] = right;
/* 'ogain' is the special case implemented with EAPD. */
if (dev == SOUND_MIXER_OGAIN) {
uint32_t orig;
w = NULL;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX ||
w->param.eapdbtl == HDA_INVALID)
continue;
break;
}
if (i >= devinfo->endnode) {
hdaa_unlock(devinfo);
return (-1);
}
orig = w->param.eapdbtl;
if (left == 0)
w->param.eapdbtl &= ~HDA_CMD_SET_EAPD_BTL_ENABLE_EAPD;
else
w->param.eapdbtl |= HDA_CMD_SET_EAPD_BTL_ENABLE_EAPD;
if (orig != w->param.eapdbtl) {
uint32_t val;
val = w->param.eapdbtl;
if (devinfo->quirks & HDAA_QUIRK_EAPDINV)
val ^= HDA_CMD_SET_EAPD_BTL_ENABLE_EAPD;
hda_command(devinfo->dev,
HDA_CMD_SET_EAPD_BTL_ENABLE(0, w->nid, val));
}
hdaa_unlock(devinfo);
return (left | (left << 8));
}
/* Recalculate all controls related to this OSS device. */
hdaa_audio_ctl_dev_volume(pdevinfo, dev);
hdaa_unlock(devinfo);
return (left | (right << 8));
}
/*
* Set mixer settings to our own default values:
* +20dB for mics, -10dB for analog vol, mute for igain, 0dB for others.
*/
static void
hdaa_audio_ctl_set_defaults(struct hdaa_pcm_devinfo *pdevinfo)
{
int amp, vol, dev;
for (dev = 0; dev < SOUND_MIXER_NRDEVICES; dev++) {
if ((pdevinfo->ossmask & (1 << dev)) == 0)
continue;
/* If the value was overriden, leave it as is. */
if (resource_int_value(device_get_name(pdevinfo->dev),
device_get_unit(pdevinfo->dev), ossnames[dev], &vol) == 0)
continue;
vol = -1;
if (dev == SOUND_MIXER_OGAIN)
vol = 100;
else if (dev == SOUND_MIXER_IGAIN)
vol = 0;
else if (dev == SOUND_MIXER_MIC ||
dev == SOUND_MIXER_MONITOR)
amp = 20 * 4; /* +20dB */
else if (dev == SOUND_MIXER_VOLUME && !pdevinfo->digital)
amp = -10 * 4; /* -10dB */
else
amp = 0;
if (vol < 0 &&
(pdevinfo->maxamp[dev] - pdevinfo->minamp[dev]) <= 0) {
vol = 100;
} else if (vol < 0) {
vol = ((amp - pdevinfo->minamp[dev]) * 100 +
(pdevinfo->maxamp[dev] - pdevinfo->minamp[dev]) / 2) /
(pdevinfo->maxamp[dev] - pdevinfo->minamp[dev]);
vol = imin(imax(vol, 1), 100);
}
mix_set(pdevinfo->mixer, dev, vol, vol);
}
}
/*
* Recursively commutate specified record source.
*/
static uint32_t
hdaa_audio_ctl_recsel_comm(struct hdaa_pcm_devinfo *pdevinfo, uint32_t src, nid_t nid, int depth)
{
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_widget *w, *cw;
struct hdaa_audio_ctl *ctl;
char buf[64];
int i, muted;
uint32_t res = 0;
if (depth > HDA_PARSE_MAXDEPTH)
return (0);
w = hdaa_widget_get(devinfo, nid);
if (w == NULL || w->enable == 0)
return (0);
for (i = 0; i < w->nconns; i++) {
if (w->connsenable[i] == 0)
continue;
cw = hdaa_widget_get(devinfo, w->conns[i]);
if (cw == NULL || cw->enable == 0 || cw->bindas == -1)
continue;
/* Call recursively to trace signal to it's source if needed. */
if ((src & cw->ossmask) != 0) {
if (cw->ossdev < 0) {
res |= hdaa_audio_ctl_recsel_comm(pdevinfo, src,
w->conns[i], depth + 1);
} else {
res |= cw->ossmask;
}
}
/* We have two special cases: mixers and others (selectors). */
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER) {
ctl = hdaa_audio_ctl_amp_get(devinfo,
w->nid, HDAA_CTL_IN, i, 1);
if (ctl == NULL)
continue;
/* If we have input control on this node mute them
* according to requested sources. */
muted = (src & cw->ossmask) ? 0 : 1;
if (muted != ctl->forcemute) {
ctl->forcemute = muted;
hdaa_audio_ctl_amp_set(ctl,
HDAA_AMP_MUTE_DEFAULT,
HDAA_AMP_VOL_DEFAULT, HDAA_AMP_VOL_DEFAULT);
}
HDA_BOOTHVERBOSE(
device_printf(pdevinfo->dev,
"Recsel (%s): nid %d source %d %s\n",
hdaa_audio_ctl_ossmixer_mask2allname(
src, buf, sizeof(buf)),
nid, i, muted?"mute":"unmute");
);
} else {
if (w->nconns == 1)
break;
if ((src & cw->ossmask) == 0)
continue;
/* If we found requested source - select it and exit. */
hdaa_widget_connection_select(w, i);
HDA_BOOTHVERBOSE(
device_printf(pdevinfo->dev,
"Recsel (%s): nid %d source %d select\n",
hdaa_audio_ctl_ossmixer_mask2allname(
src, buf, sizeof(buf)),
nid, i);
);
break;
}
}
return (res);
}
static uint32_t
hdaa_audio_ctl_ossmixer_setrecsrc(struct snd_mixer *m, uint32_t src)
{
struct hdaa_pcm_devinfo *pdevinfo = mix_getdevinfo(m);
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_widget *w;
struct hdaa_audio_as *as;
struct hdaa_audio_ctl *ctl;
struct hdaa_chan *ch;
int i, j;
uint32_t ret = 0xffffffff;
hdaa_lock(devinfo);
if (pdevinfo->recas < 0) {
hdaa_unlock(devinfo);
return (0);
}
as = &devinfo->as[pdevinfo->recas];
/* For non-mixed associations we always recording everything. */
if (!as->mixed) {
hdaa_unlock(devinfo);
return (mix_getrecdevs(m));
}
/* Commutate requested recsrc for each ADC. */
for (j = 0; j < as->num_chans; j++) {
ch = &devinfo->chans[as->chans[j]];
for (i = 0; ch->io[i] >= 0; i++) {
w = hdaa_widget_get(devinfo, ch->io[i]);
if (w == NULL || w->enable == 0)
continue;
ret &= hdaa_audio_ctl_recsel_comm(pdevinfo, src,
ch->io[i], 0);
}
}
if (ret == 0xffffffff)
ret = 0;
/*
* Some controls could be shared. Reset volumes for controls
* related to previously chosen devices, as they may no longer
* affect the signal.
*/
i = 0;
while ((ctl = hdaa_audio_ctl_each(devinfo, &i)) != NULL) {
if (ctl->enable == 0 ||
!(ctl->ossmask & pdevinfo->recsrc))
continue;
if (!((pdevinfo->playas >= 0 &&
ctl->widget->bindas == pdevinfo->playas) ||
(pdevinfo->recas >= 0 &&
ctl->widget->bindas == pdevinfo->recas) ||
(pdevinfo->index == 0 &&
ctl->widget->bindas == -2)))
continue;
for (j = 0; j < SOUND_MIXER_NRDEVICES; j++) {
if (pdevinfo->recsrc & (1 << j)) {
ctl->devleft[j] = 0;
ctl->devright[j] = 0;
ctl->devmute[j] = 0;
}
}
}
/*
* Some controls could be shared. Set volumes for controls
* related to devices selected both previously and now.
*/
for (j = 0; j < SOUND_MIXER_NRDEVICES; j++) {
if ((ret | pdevinfo->recsrc) & (1 << j))
hdaa_audio_ctl_dev_volume(pdevinfo, j);
}
pdevinfo->recsrc = ret;
hdaa_unlock(devinfo);
return (ret);
}
static kobj_method_t hdaa_audio_ctl_ossmixer_methods[] = {
KOBJMETHOD(mixer_init, hdaa_audio_ctl_ossmixer_init),
KOBJMETHOD(mixer_set, hdaa_audio_ctl_ossmixer_set),
KOBJMETHOD(mixer_setrecsrc, hdaa_audio_ctl_ossmixer_setrecsrc),
KOBJMETHOD_END
};
MIXER_DECLARE(hdaa_audio_ctl_ossmixer);
static void
hdaa_dump_gpi(struct hdaa_devinfo *devinfo)
{
device_t dev = devinfo->dev;
int i;
uint32_t data, wake, unsol, sticky;
if (HDA_PARAM_GPIO_COUNT_NUM_GPI(devinfo->gpio_cap) > 0) {
data = hda_command(dev,
HDA_CMD_GET_GPI_DATA(0, devinfo->nid));
wake = hda_command(dev,
HDA_CMD_GET_GPI_WAKE_ENABLE_MASK(0, devinfo->nid));
unsol = hda_command(dev,
HDA_CMD_GET_GPI_UNSOLICITED_ENABLE_MASK(0, devinfo->nid));
sticky = hda_command(dev,
HDA_CMD_GET_GPI_STICKY_MASK(0, devinfo->nid));
for (i = 0; i < HDA_PARAM_GPIO_COUNT_NUM_GPI(devinfo->gpio_cap); i++) {
device_printf(dev, " GPI%d:%s%s%s state=%d", i,
(sticky & (1 << i)) ? " sticky" : "",
(unsol & (1 << i)) ? " unsol" : "",
(wake & (1 << i)) ? " wake" : "",
(data >> i) & 1);
}
}
}
static void
hdaa_dump_gpio(struct hdaa_devinfo *devinfo)
{
device_t dev = devinfo->dev;
int i;
uint32_t data, dir, enable, wake, unsol, sticky;
if (HDA_PARAM_GPIO_COUNT_NUM_GPIO(devinfo->gpio_cap) > 0) {
data = hda_command(dev,
HDA_CMD_GET_GPIO_DATA(0, devinfo->nid));
enable = hda_command(dev,
HDA_CMD_GET_GPIO_ENABLE_MASK(0, devinfo->nid));
dir = hda_command(dev,
HDA_CMD_GET_GPIO_DIRECTION(0, devinfo->nid));
wake = hda_command(dev,
HDA_CMD_GET_GPIO_WAKE_ENABLE_MASK(0, devinfo->nid));
unsol = hda_command(dev,
HDA_CMD_GET_GPIO_UNSOLICITED_ENABLE_MASK(0, devinfo->nid));
sticky = hda_command(dev,
HDA_CMD_GET_GPIO_STICKY_MASK(0, devinfo->nid));
for (i = 0; i < HDA_PARAM_GPIO_COUNT_NUM_GPIO(devinfo->gpio_cap); i++) {
device_printf(dev, " GPIO%d: ", i);
if ((enable & (1 << i)) == 0) {
printf("disabled\n");
continue;
}
if ((dir & (1 << i)) == 0) {
printf("input%s%s%s",
(sticky & (1 << i)) ? " sticky" : "",
(unsol & (1 << i)) ? " unsol" : "",
(wake & (1 << i)) ? " wake" : "");
} else
printf("output");
printf(" state=%d\n", (data >> i) & 1);
}
}
}
static void
hdaa_dump_gpo(struct hdaa_devinfo *devinfo)
{
device_t dev = devinfo->dev;
int i;
uint32_t data;
if (HDA_PARAM_GPIO_COUNT_NUM_GPO(devinfo->gpio_cap) > 0) {
data = hda_command(dev,
HDA_CMD_GET_GPO_DATA(0, devinfo->nid));
for (i = 0; i < HDA_PARAM_GPIO_COUNT_NUM_GPO(devinfo->gpio_cap); i++) {
device_printf(dev, " GPO%d: state=%d", i,
(data >> i) & 1);
}
}
}
static void
hdaa_audio_parse(struct hdaa_devinfo *devinfo)
{
struct hdaa_widget *w;
uint32_t res;
int i;
nid_t nid;
nid = devinfo->nid;
res = hda_command(devinfo->dev,
HDA_CMD_GET_PARAMETER(0, nid, HDA_PARAM_GPIO_COUNT));
devinfo->gpio_cap = res;
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"NumGPIO=%d NumGPO=%d "
"NumGPI=%d GPIWake=%d GPIUnsol=%d\n",
HDA_PARAM_GPIO_COUNT_NUM_GPIO(devinfo->gpio_cap),
HDA_PARAM_GPIO_COUNT_NUM_GPO(devinfo->gpio_cap),
HDA_PARAM_GPIO_COUNT_NUM_GPI(devinfo->gpio_cap),
HDA_PARAM_GPIO_COUNT_GPI_WAKE(devinfo->gpio_cap),
HDA_PARAM_GPIO_COUNT_GPI_UNSOL(devinfo->gpio_cap));
hdaa_dump_gpi(devinfo);
hdaa_dump_gpio(devinfo);
hdaa_dump_gpo(devinfo);
);
res = hda_command(devinfo->dev,
HDA_CMD_GET_PARAMETER(0, nid, HDA_PARAM_SUPP_STREAM_FORMATS));
devinfo->supp_stream_formats = res;
res = hda_command(devinfo->dev,
HDA_CMD_GET_PARAMETER(0, nid, HDA_PARAM_SUPP_PCM_SIZE_RATE));
devinfo->supp_pcm_size_rate = res;
res = hda_command(devinfo->dev,
HDA_CMD_GET_PARAMETER(0, nid, HDA_PARAM_OUTPUT_AMP_CAP));
devinfo->outamp_cap = res;
res = hda_command(devinfo->dev,
HDA_CMD_GET_PARAMETER(0, nid, HDA_PARAM_INPUT_AMP_CAP));
devinfo->inamp_cap = res;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL)
device_printf(devinfo->dev, "Ghost widget! nid=%d!\n", i);
else {
w->devinfo = devinfo;
w->nid = i;
w->enable = 1;
w->selconn = -1;
w->pflags = 0;
w->ossdev = -1;
w->bindas = -1;
w->param.eapdbtl = HDA_INVALID;
hdaa_widget_parse(w);
}
}
}
static void
hdaa_audio_postprocess(struct hdaa_devinfo *devinfo)
{
struct hdaa_widget *w;
int i;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL)
continue;
hdaa_widget_postprocess(w);
}
}
static void
hdaa_audio_ctl_parse(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_ctl *ctls;
struct hdaa_widget *w, *cw;
int i, j, cnt, max, ocap, icap;
int mute, offset, step, size;
/* XXX This is redundant */
max = 0;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->param.outamp_cap != 0)
max++;
if (w->param.inamp_cap != 0) {
switch (w->type) {
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR:
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER:
for (j = 0; j < w->nconns; j++) {
cw = hdaa_widget_get(devinfo,
w->conns[j]);
if (cw == NULL || cw->enable == 0)
continue;
max++;
}
break;
default:
max++;
break;
}
}
}
devinfo->ctlcnt = max;
if (max < 1)
return;
ctls = (struct hdaa_audio_ctl *)malloc(
sizeof(*ctls) * max, M_HDAA, M_ZERO | M_NOWAIT);
if (ctls == NULL) {
/* Blekh! */
device_printf(devinfo->dev, "unable to allocate ctls!\n");
devinfo->ctlcnt = 0;
return;
}
cnt = 0;
for (i = devinfo->startnode; cnt < max && i < devinfo->endnode; i++) {
if (cnt >= max) {
device_printf(devinfo->dev, "%s: Ctl overflow!\n",
__func__);
break;
}
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
ocap = w->param.outamp_cap;
icap = w->param.inamp_cap;
if (ocap != 0) {
mute = HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP(ocap);
step = HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS(ocap);
size = HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE(ocap);
offset = HDA_PARAM_OUTPUT_AMP_CAP_OFFSET(ocap);
/*if (offset > step) {
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"BUGGY outamp: nid=%d "
"[offset=%d > step=%d]\n",
w->nid, offset, step);
);
offset = step;
}*/
ctls[cnt].enable = 1;
ctls[cnt].widget = w;
ctls[cnt].mute = mute;
ctls[cnt].step = step;
ctls[cnt].size = size;
ctls[cnt].offset = offset;
ctls[cnt].left = offset;
ctls[cnt].right = offset;
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX ||
w->waspin)
ctls[cnt].ndir = HDAA_CTL_IN;
else
ctls[cnt].ndir = HDAA_CTL_OUT;
ctls[cnt++].dir = HDAA_CTL_OUT;
}
if (icap != 0) {
mute = HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP(icap);
step = HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS(icap);
size = HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE(icap);
offset = HDA_PARAM_OUTPUT_AMP_CAP_OFFSET(icap);
/*if (offset > step) {
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"BUGGY inamp: nid=%d "
"[offset=%d > step=%d]\n",
w->nid, offset, step);
);
offset = step;
}*/
switch (w->type) {
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR:
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER:
for (j = 0; j < w->nconns; j++) {
if (cnt >= max) {
device_printf(devinfo->dev,
"%s: Ctl overflow!\n",
__func__);
break;
}
cw = hdaa_widget_get(devinfo,
w->conns[j]);
if (cw == NULL || cw->enable == 0)
continue;
ctls[cnt].enable = 1;
ctls[cnt].widget = w;
ctls[cnt].childwidget = cw;
ctls[cnt].index = j;
ctls[cnt].mute = mute;
ctls[cnt].step = step;
ctls[cnt].size = size;
ctls[cnt].offset = offset;
ctls[cnt].left = offset;
ctls[cnt].right = offset;
ctls[cnt].ndir = HDAA_CTL_IN;
ctls[cnt++].dir = HDAA_CTL_IN;
}
break;
default:
if (cnt >= max) {
device_printf(devinfo->dev,
"%s: Ctl overflow!\n",
__func__);
break;
}
ctls[cnt].enable = 1;
ctls[cnt].widget = w;
ctls[cnt].mute = mute;
ctls[cnt].step = step;
ctls[cnt].size = size;
ctls[cnt].offset = offset;
ctls[cnt].left = offset;
ctls[cnt].right = offset;
if (w->type ==
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
ctls[cnt].ndir = HDAA_CTL_OUT;
else
ctls[cnt].ndir = HDAA_CTL_IN;
ctls[cnt++].dir = HDAA_CTL_IN;
break;
}
}
}
devinfo->ctl = ctls;
}
static void
hdaa_audio_as_parse(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as;
struct hdaa_widget *w;
int i, j, cnt, max, type, dir, assoc, seq, first, hpredir;
/* Count present associations */
max = 0;
for (j = 1; j < 16; j++) {
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
if (HDA_CONFIG_DEFAULTCONF_ASSOCIATION(w->wclass.pin.config)
!= j)
continue;
max++;
if (j != 15) /* There could be many 1-pin assocs #15 */
break;
}
}
devinfo->ascnt = max;
if (max < 1)
return;
as = (struct hdaa_audio_as *)malloc(
sizeof(*as) * max, M_HDAA, M_ZERO | M_NOWAIT);
if (as == NULL) {
/* Blekh! */
device_printf(devinfo->dev, "unable to allocate assocs!\n");
devinfo->ascnt = 0;
return;
}
for (i = 0; i < max; i++) {
as[i].hpredir = -1;
as[i].digital = 0;
as[i].num_chans = 1;
as[i].location = -1;
}
/* Scan associations skipping as=0. */
cnt = 0;
for (j = 1; j < 16 && cnt < max; j++) {
first = 16;
hpredir = 0;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
assoc = HDA_CONFIG_DEFAULTCONF_ASSOCIATION(w->wclass.pin.config);
seq = HDA_CONFIG_DEFAULTCONF_SEQUENCE(w->wclass.pin.config);
if (assoc != j) {
continue;
}
KASSERT(cnt < max,
("%s: Associations owerflow (%d of %d)",
__func__, cnt, max));
type = w->wclass.pin.config &
HDA_CONFIG_DEFAULTCONF_DEVICE_MASK;
/* Get pin direction. */
if (type == HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_OUT ||
type == HDA_CONFIG_DEFAULTCONF_DEVICE_SPEAKER ||
type == HDA_CONFIG_DEFAULTCONF_DEVICE_HP_OUT ||
type == HDA_CONFIG_DEFAULTCONF_DEVICE_SPDIF_OUT ||
type == HDA_CONFIG_DEFAULTCONF_DEVICE_DIGITAL_OTHER_OUT)
dir = HDAA_CTL_OUT;
else
dir = HDAA_CTL_IN;
/* If this is a first pin - create new association. */
if (as[cnt].pincnt == 0) {
as[cnt].enable = 1;
as[cnt].index = j;
as[cnt].dir = dir;
}
if (seq < first)
first = seq;
/* Check association correctness. */
if (as[cnt].pins[seq] != 0) {
device_printf(devinfo->dev, "%s: Duplicate pin %d (%d) "
"in association %d! Disabling association.\n",
__func__, seq, w->nid, j);
as[cnt].enable = 0;
}
if (dir != as[cnt].dir) {
device_printf(devinfo->dev, "%s: Pin %d has wrong "
"direction for association %d! Disabling "
"association.\n",
__func__, w->nid, j);
as[cnt].enable = 0;
}
if (HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(w->param.widget_cap)) {
as[cnt].digital |= 0x1;
if (HDA_PARAM_PIN_CAP_HDMI(w->wclass.pin.cap))
as[cnt].digital |= 0x2;
if (HDA_PARAM_PIN_CAP_DP(w->wclass.pin.cap))
as[cnt].digital |= 0x4;
}
if (as[cnt].location == -1) {
as[cnt].location =
HDA_CONFIG_DEFAULTCONF_LOCATION(w->wclass.pin.config);
} else if (as[cnt].location !=
HDA_CONFIG_DEFAULTCONF_LOCATION(w->wclass.pin.config)) {
as[cnt].location = -2;
}
/* Headphones with seq=15 may mean redirection. */
if (type == HDA_CONFIG_DEFAULTCONF_DEVICE_HP_OUT &&
seq == 15)
hpredir = 1;
as[cnt].pins[seq] = w->nid;
as[cnt].pincnt++;
/* Association 15 is a multiple unassociated pins. */
if (j == 15)
cnt++;
}
if (j != 15 && as[cnt].pincnt > 0) {
if (hpredir && as[cnt].pincnt > 1)
as[cnt].hpredir = first;
cnt++;
}
}
for (i = 0; i < max; i++) {
if (as[i].dir == HDAA_CTL_IN && (as[i].pincnt == 1 ||
as[i].pins[14] > 0 || as[i].pins[15] > 0))
as[i].mixed = 1;
}
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"%d associations found:\n", max);
for (i = 0; i < max; i++) {
device_printf(devinfo->dev,
"Association %d (%d) %s%s:\n",
i, as[i].index, (as[i].dir == HDAA_CTL_IN)?"in":"out",
as[i].enable?"":" (disabled)");
for (j = 0; j < 16; j++) {
if (as[i].pins[j] == 0)
continue;
device_printf(devinfo->dev,
" Pin nid=%d seq=%d\n",
as[i].pins[j], j);
}
}
);
devinfo->as = as;
}
/*
* Trace path from DAC to pin.
*/
static nid_t
hdaa_audio_trace_dac(struct hdaa_devinfo *devinfo, int as, int seq, nid_t nid,
int dupseq, int min, int only, int depth)
{
struct hdaa_widget *w;
int i, im = -1;
nid_t m = 0, ret;
if (depth > HDA_PARSE_MAXDEPTH)
return (0);
w = hdaa_widget_get(devinfo, nid);
if (w == NULL || w->enable == 0)
return (0);
HDA_BOOTHVERBOSE(
if (!only) {
device_printf(devinfo->dev,
" %*stracing via nid %d\n",
depth + 1, "", w->nid);
}
);
/* Use only unused widgets */
if (w->bindas >= 0 && w->bindas != as) {
HDA_BOOTHVERBOSE(
if (!only) {
device_printf(devinfo->dev,
" %*snid %d busy by association %d\n",
depth + 1, "", w->nid, w->bindas);
}
);
return (0);
}
if (dupseq < 0) {
if (w->bindseqmask != 0) {
HDA_BOOTHVERBOSE(
if (!only) {
device_printf(devinfo->dev,
" %*snid %d busy by seqmask %x\n",
depth + 1, "", w->nid, w->bindseqmask);
}
);
return (0);
}
} else {
/* If this is headphones - allow duplicate first pin. */
if (w->bindseqmask != 0 &&
(w->bindseqmask & (1 << dupseq)) == 0) {
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" %*snid %d busy by seqmask %x\n",
depth + 1, "", w->nid, w->bindseqmask);
);
return (0);
}
}
switch (w->type) {
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT:
/* Do not traverse input. AD1988 has digital monitor
for which we are not ready. */
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT:
/* If we are tracing HP take only dac of first pin. */
if ((only == 0 || only == w->nid) &&
(w->nid >= min) && (dupseq < 0 || w->nid ==
devinfo->as[as].dacs[0][dupseq]))
m = w->nid;
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX:
if (depth > 0)
break;
/* Fall */
default:
/* Find reachable DACs with smallest nid respecting constraints. */
for (i = 0; i < w->nconns; i++) {
if (w->connsenable[i] == 0)
continue;
if (w->selconn != -1 && w->selconn != i)
continue;
if ((ret = hdaa_audio_trace_dac(devinfo, as, seq,
w->conns[i], dupseq, min, only, depth + 1)) != 0) {
if (m == 0 || ret < m) {
m = ret;
im = i;
}
if (only || dupseq >= 0)
break;
}
}
if (im >= 0 && only && ((w->nconns > 1 &&
w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER) ||
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR))
w->selconn = im;
break;
}
if (m && only) {
w->bindas = as;
w->bindseqmask |= (1 << seq);
}
HDA_BOOTHVERBOSE(
if (!only) {
device_printf(devinfo->dev,
" %*snid %d returned %d\n",
depth + 1, "", w->nid, m);
}
);
return (m);
}
/*
* Trace path from widget to ADC.
*/
static nid_t
hdaa_audio_trace_adc(struct hdaa_devinfo *devinfo, int as, int seq, nid_t nid,
int mixed, int min, int only, int depth, int *length, int onlylength)
{
struct hdaa_widget *w, *wc;
int i, j, im, lm = HDA_PARSE_MAXDEPTH;
nid_t m = 0, ret;
if (depth > HDA_PARSE_MAXDEPTH)
return (0);
w = hdaa_widget_get(devinfo, nid);
if (w == NULL || w->enable == 0)
return (0);
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" %*stracing via nid %d\n",
depth + 1, "", w->nid);
);
/* Use only unused widgets */
if (w->bindas >= 0 && w->bindas != as) {
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" %*snid %d busy by association %d\n",
depth + 1, "", w->nid, w->bindas);
);
return (0);
}
if (!mixed && w->bindseqmask != 0) {
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" %*snid %d busy by seqmask %x\n",
depth + 1, "", w->nid, w->bindseqmask);
);
return (0);
}
switch (w->type) {
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT:
if ((only == 0 || only == w->nid) && (w->nid >= min) &&
(onlylength == 0 || onlylength == depth)) {
m = w->nid;
*length = depth;
}
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX:
if (depth > 0)
break;
/* Fall */
default:
/* Try to find reachable ADCs with specified nid. */
for (j = devinfo->startnode; j < devinfo->endnode; j++) {
wc = hdaa_widget_get(devinfo, j);
if (wc == NULL || wc->enable == 0)
continue;
im = -1;
for (i = 0; i < wc->nconns; i++) {
if (wc->connsenable[i] == 0)
continue;
if (wc->conns[i] != nid)
continue;
if ((ret = hdaa_audio_trace_adc(devinfo, as, seq,
j, mixed, min, only, depth + 1,
length, onlylength)) != 0) {
if (m == 0 || ret < m ||
(ret == m && *length < lm)) {
m = ret;
im = i;
lm = *length;
} else
*length = lm;
if (only)
break;
}
}
if (im >= 0 && only && ((wc->nconns > 1 &&
wc->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER) ||
wc->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR))
wc->selconn = im;
}
break;
}
if (m && only) {
w->bindas = as;
w->bindseqmask |= (1 << seq);
}
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" %*snid %d returned %d\n",
depth + 1, "", w->nid, m);
);
return (m);
}
/*
* Erase trace path of the specified association.
*/
static void
hdaa_audio_undo_trace(struct hdaa_devinfo *devinfo, int as, int seq)
{
struct hdaa_widget *w;
int i;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->bindas == as) {
if (seq >= 0) {
w->bindseqmask &= ~(1 << seq);
if (w->bindseqmask == 0) {
w->bindas = -1;
w->selconn = -1;
}
} else {
w->bindas = -1;
w->bindseqmask = 0;
w->selconn = -1;
}
}
}
}
/*
* Trace association path from DAC to output
*/
static int
hdaa_audio_trace_as_out(struct hdaa_devinfo *devinfo, int as, int seq)
{
struct hdaa_audio_as *ases = devinfo->as;
int i, hpredir;
nid_t min, res;
/* Find next pin */
for (i = seq; i < 16 && ases[as].pins[i] == 0; i++)
;
/* Check if there is no any left. If so - we succeeded. */
if (i == 16)
return (1);
hpredir = (i == 15 && ases[as].fakeredir == 0)?ases[as].hpredir:-1;
min = 0;
do {
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Tracing pin %d with min nid %d",
ases[as].pins[i], min);
if (hpredir >= 0)
printf(" and hpredir %d", hpredir);
printf("\n");
);
/* Trace this pin taking min nid into account. */
res = hdaa_audio_trace_dac(devinfo, as, i,
ases[as].pins[i], hpredir, min, 0, 0);
if (res == 0) {
/* If we failed - return to previous and redo it. */
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" Unable to trace pin %d seq %d with min "
"nid %d",
ases[as].pins[i], i, min);
if (hpredir >= 0)
printf(" and hpredir %d", hpredir);
printf("\n");
);
return (0);
}
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" Pin %d traced to DAC %d",
ases[as].pins[i], res);
if (hpredir >= 0)
printf(" and hpredir %d", hpredir);
if (ases[as].fakeredir)
printf(" with fake redirection");
printf("\n");
);
/* Trace again to mark the path */
hdaa_audio_trace_dac(devinfo, as, i,
ases[as].pins[i], hpredir, min, res, 0);
ases[as].dacs[0][i] = res;
/* We succeeded, so call next. */
if (hdaa_audio_trace_as_out(devinfo, as, i + 1))
return (1);
/* If next failed, we should retry with next min */
hdaa_audio_undo_trace(devinfo, as, i);
ases[as].dacs[0][i] = 0;
min = res + 1;
} while (1);
}
/*
* Check equivalency of two DACs.
*/
static int
hdaa_audio_dacs_equal(struct hdaa_widget *w1, struct hdaa_widget *w2)
{
struct hdaa_devinfo *devinfo = w1->devinfo;
struct hdaa_widget *w3;
int i, j, c1, c2;
if (memcmp(&w1->param, &w2->param, sizeof(w1->param)))
return (0);
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w3 = hdaa_widget_get(devinfo, i);
if (w3 == NULL || w3->enable == 0)
continue;
if (w3->bindas != w1->bindas)
continue;
if (w3->nconns == 0)
continue;
c1 = c2 = -1;
for (j = 0; j < w3->nconns; j++) {
if (w3->connsenable[j] == 0)
continue;
if (w3->conns[j] == w1->nid)
c1 = j;
if (w3->conns[j] == w2->nid)
c2 = j;
}
if (c1 < 0)
continue;
if (c2 < 0)
return (0);
if (w3->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER)
return (0);
}
return (1);
}
/*
* Check equivalency of two ADCs.
*/
static int
hdaa_audio_adcs_equal(struct hdaa_widget *w1, struct hdaa_widget *w2)
{
struct hdaa_devinfo *devinfo = w1->devinfo;
struct hdaa_widget *w3, *w4;
int i;
if (memcmp(&w1->param, &w2->param, sizeof(w1->param)))
return (0);
if (w1->nconns != 1 || w2->nconns != 1)
return (0);
if (w1->conns[0] == w2->conns[0])
return (1);
w3 = hdaa_widget_get(devinfo, w1->conns[0]);
if (w3 == NULL || w3->enable == 0)
return (0);
w4 = hdaa_widget_get(devinfo, w2->conns[0]);
if (w4 == NULL || w4->enable == 0)
return (0);
if (w3->bindas == w4->bindas && w3->bindseqmask == w4->bindseqmask)
return (1);
if (w4->bindas >= 0)
return (0);
if (w3->type != w4->type)
return (0);
if (memcmp(&w3->param, &w4->param, sizeof(w3->param)))
return (0);
if (w3->nconns != w4->nconns)
return (0);
for (i = 0; i < w3->nconns; i++) {
if (w3->conns[i] != w4->conns[i])
return (0);
}
return (1);
}
/*
* Look for equivalent DAC/ADC to implement second channel.
*/
static void
hdaa_audio_adddac(struct hdaa_devinfo *devinfo, int asid)
{
struct hdaa_audio_as *as = &devinfo->as[asid];
struct hdaa_widget *w1, *w2;
int i, pos;
nid_t nid1, nid2;
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"Looking for additional %sC "
"for association %d (%d)\n",
(as->dir == HDAA_CTL_OUT) ? "DA" : "AD",
asid, as->index);
);
/* Find the exisitng DAC position and return if found more the one. */
pos = -1;
for (i = 0; i < 16; i++) {
if (as->dacs[0][i] <= 0)
continue;
if (pos >= 0 && as->dacs[0][i] != as->dacs[0][pos])
return;
pos = i;
}
nid1 = as->dacs[0][pos];
w1 = hdaa_widget_get(devinfo, nid1);
w2 = NULL;
for (nid2 = devinfo->startnode; nid2 < devinfo->endnode; nid2++) {
w2 = hdaa_widget_get(devinfo, nid2);
if (w2 == NULL || w2->enable == 0)
continue;
if (w2->bindas >= 0)
continue;
if (w1->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT) {
if (w2->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT)
continue;
if (hdaa_audio_dacs_equal(w1, w2))
break;
} else {
if (w2->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT)
continue;
if (hdaa_audio_adcs_equal(w1, w2))
break;
}
}
if (nid2 >= devinfo->endnode)
return;
w2->bindas = w1->bindas;
w2->bindseqmask = w1->bindseqmask;
if (w1->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT) {
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" ADC %d considered equal to ADC %d\n", nid2, nid1);
);
w1 = hdaa_widget_get(devinfo, w1->conns[0]);
w2 = hdaa_widget_get(devinfo, w2->conns[0]);
w2->bindas = w1->bindas;
w2->bindseqmask = w1->bindseqmask;
} else {
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" DAC %d considered equal to DAC %d\n", nid2, nid1);
);
}
for (i = 0; i < 16; i++) {
if (as->dacs[0][i] <= 0)
continue;
as->dacs[as->num_chans][i] = nid2;
}
as->num_chans++;
}
/*
* Trace association path from input to ADC
*/
static int
hdaa_audio_trace_as_in(struct hdaa_devinfo *devinfo, int as)
{
struct hdaa_audio_as *ases = devinfo->as;
struct hdaa_widget *w;
int i, j, k, length;
for (j = devinfo->startnode; j < devinfo->endnode; j++) {
w = hdaa_widget_get(devinfo, j);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT)
continue;
if (w->bindas >= 0 && w->bindas != as)
continue;
/* Find next pin */
for (i = 0; i < 16; i++) {
if (ases[as].pins[i] == 0)
continue;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Tracing pin %d to ADC %d\n",
ases[as].pins[i], j);
);
/* Trace this pin taking goal into account. */
if (hdaa_audio_trace_adc(devinfo, as, i,
ases[as].pins[i], 1, 0, j, 0, &length, 0) == 0) {
/* If we failed - return to previous and redo it. */
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" Unable to trace pin %d to ADC %d, undo traces\n",
ases[as].pins[i], j);
);
hdaa_audio_undo_trace(devinfo, as, -1);
for (k = 0; k < 16; k++)
ases[as].dacs[0][k] = 0;
break;
}
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" Pin %d traced to ADC %d\n",
ases[as].pins[i], j);
);
ases[as].dacs[0][i] = j;
}
if (i == 16)
return (1);
}
return (0);
}
/*
* Trace association path from input to multiple ADCs
*/
static int
hdaa_audio_trace_as_in_mch(struct hdaa_devinfo *devinfo, int as, int seq)
{
struct hdaa_audio_as *ases = devinfo->as;
int i, length;
nid_t min, res;
/* Find next pin */
for (i = seq; i < 16 && ases[as].pins[i] == 0; i++)
;
/* Check if there is no any left. If so - we succeeded. */
if (i == 16)
return (1);
min = 0;
do {
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Tracing pin %d with min nid %d",
ases[as].pins[i], min);
printf("\n");
);
/* Trace this pin taking min nid into account. */
res = hdaa_audio_trace_adc(devinfo, as, i,
ases[as].pins[i], 0, min, 0, 0, &length, 0);
if (res == 0) {
/* If we failed - return to previous and redo it. */
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" Unable to trace pin %d seq %d with min "
"nid %d",
ases[as].pins[i], i, min);
printf("\n");
);
return (0);
}
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" Pin %d traced to ADC %d\n",
ases[as].pins[i], res);
);
/* Trace again to mark the path */
hdaa_audio_trace_adc(devinfo, as, i,
ases[as].pins[i], 0, min, res, 0, &length, length);
ases[as].dacs[0][i] = res;
/* We succeeded, so call next. */
if (hdaa_audio_trace_as_in_mch(devinfo, as, i + 1))
return (1);
/* If next failed, we should retry with next min */
hdaa_audio_undo_trace(devinfo, as, i);
ases[as].dacs[0][i] = 0;
min = res + 1;
} while (1);
}
/*
* Trace input monitor path from mixer to output association.
*/
static int
hdaa_audio_trace_to_out(struct hdaa_devinfo *devinfo, nid_t nid, int depth)
{
struct hdaa_audio_as *ases = devinfo->as;
struct hdaa_widget *w, *wc;
int i, j;
nid_t res = 0;
if (depth > HDA_PARSE_MAXDEPTH)
return (0);
w = hdaa_widget_get(devinfo, nid);
if (w == NULL || w->enable == 0)
return (0);
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" %*stracing via nid %d\n",
depth + 1, "", w->nid);
);
/* Use only unused widgets */
if (depth > 0 && w->bindas != -1) {
if (w->bindas < 0 || ases[w->bindas].dir == HDAA_CTL_OUT) {
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" %*snid %d found output association %d\n",
depth + 1, "", w->nid, w->bindas);
);
if (w->bindas >= 0)
w->pflags |= HDAA_ADC_MONITOR;
return (1);
} else {
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" %*snid %d busy by input association %d\n",
depth + 1, "", w->nid, w->bindas);
);
return (0);
}
}
switch (w->type) {
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT:
/* Do not traverse input. AD1988 has digital monitor
for which we are not ready. */
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX:
if (depth > 0)
break;
/* Fall */
default:
/* Try to find reachable ADCs with specified nid. */
for (j = devinfo->startnode; j < devinfo->endnode; j++) {
wc = hdaa_widget_get(devinfo, j);
if (wc == NULL || wc->enable == 0)
continue;
for (i = 0; i < wc->nconns; i++) {
if (wc->connsenable[i] == 0)
continue;
if (wc->conns[i] != nid)
continue;
if (hdaa_audio_trace_to_out(devinfo,
j, depth + 1) != 0) {
res = 1;
if (wc->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR &&
wc->selconn == -1)
wc->selconn = i;
}
}
}
break;
}
if (res && w->bindas == -1)
w->bindas = -2;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" %*snid %d returned %d\n",
depth + 1, "", w->nid, res);
);
return (res);
}
/*
* Trace extra associations (beeper, monitor)
*/
static void
hdaa_audio_trace_as_extra(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as = devinfo->as;
struct hdaa_widget *w;
int j;
/* Input monitor */
/* Find mixer associated with input, but supplying signal
for output associations. Hope it will be input monitor. */
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"Tracing input monitor\n");
);
for (j = devinfo->startnode; j < devinfo->endnode; j++) {
w = hdaa_widget_get(devinfo, j);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER)
continue;
if (w->bindas < 0 || as[w->bindas].dir != HDAA_CTL_IN)
continue;
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" Tracing nid %d to out\n",
j);
);
if (hdaa_audio_trace_to_out(devinfo, w->nid, 0)) {
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" nid %d is input monitor\n",
w->nid);
);
w->ossdev = SOUND_MIXER_IMIX;
}
}
/* Other inputs monitor */
/* Find input pins supplying signal for output associations.
Hope it will be input monitoring. */
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"Tracing other input monitors\n");
);
for (j = devinfo->startnode; j < devinfo->endnode; j++) {
w = hdaa_widget_get(devinfo, j);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
if (w->bindas < 0 || as[w->bindas].dir != HDAA_CTL_IN)
continue;
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" Tracing nid %d to out\n",
j);
);
if (hdaa_audio_trace_to_out(devinfo, w->nid, 0)) {
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" nid %d is input monitor\n",
w->nid);
);
}
}
/* Beeper */
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"Tracing beeper\n");
);
for (j = devinfo->startnode; j < devinfo->endnode; j++) {
w = hdaa_widget_get(devinfo, j);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_BEEP_WIDGET)
continue;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Tracing nid %d to out\n",
j);
);
if (hdaa_audio_trace_to_out(devinfo, w->nid, 0)) {
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
" nid %d traced to out\n",
j);
);
}
w->bindas = -2;
}
}
/*
* Bind assotiations to PCM channels
*/
static void
hdaa_audio_bind_as(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as = devinfo->as;
int i, j, cnt = 0, free;
for (j = 0; j < devinfo->ascnt; j++) {
if (as[j].enable)
cnt += as[j].num_chans;
}
if (devinfo->num_chans == 0) {
devinfo->chans = (struct hdaa_chan *)malloc(
sizeof(struct hdaa_chan) * cnt,
M_HDAA, M_ZERO | M_NOWAIT);
if (devinfo->chans == NULL) {
device_printf(devinfo->dev,
"Channels memory allocation failed!\n");
return;
}
} else {
devinfo->chans = (struct hdaa_chan *)realloc(devinfo->chans,
sizeof(struct hdaa_chan) * (devinfo->num_chans + cnt),
M_HDAA, M_ZERO | M_NOWAIT);
if (devinfo->chans == NULL) {
devinfo->num_chans = 0;
device_printf(devinfo->dev,
"Channels memory allocation failed!\n");
return;
}
/* Fixup relative pointers after realloc */
for (j = 0; j < devinfo->num_chans; j++)
devinfo->chans[j].caps.fmtlist = devinfo->chans[j].fmtlist;
}
free = devinfo->num_chans;
devinfo->num_chans += cnt;
for (j = free; j < free + cnt; j++) {
devinfo->chans[j].devinfo = devinfo;
devinfo->chans[j].as = -1;
}
/* Assign associations in order of their numbers, */
for (j = 0; j < devinfo->ascnt; j++) {
if (as[j].enable == 0)
continue;
for (i = 0; i < as[j].num_chans; i++) {
devinfo->chans[free].as = j;
devinfo->chans[free].asindex = i;
devinfo->chans[free].dir =
(as[j].dir == HDAA_CTL_IN) ? PCMDIR_REC : PCMDIR_PLAY;
hdaa_pcmchannel_setup(&devinfo->chans[free]);
as[j].chans[i] = free;
free++;
}
}
}
static void
hdaa_audio_disable_nonaudio(struct hdaa_devinfo *devinfo)
{
struct hdaa_widget *w;
int i;
/* Disable power and volume widgets. */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_POWER_WIDGET ||
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_VOLUME_WIDGET) {
w->enable = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling nid %d due to it's"
" non-audio type.\n",
w->nid);
);
}
}
}
static void
hdaa_audio_disable_useless(struct hdaa_devinfo *devinfo)
{
struct hdaa_widget *w, *cw;
struct hdaa_audio_ctl *ctl;
int done, found, i, j, k;
/* Disable useless pins. */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX) {
if ((w->wclass.pin.config &
HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_MASK) ==
HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_NONE) {
w->enable = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling pin nid %d due"
" to None connectivity.\n",
w->nid);
);
} else if ((w->wclass.pin.config &
HDA_CONFIG_DEFAULTCONF_ASSOCIATION_MASK) == 0) {
w->enable = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling unassociated"
" pin nid %d.\n",
w->nid);
);
}
}
}
do {
done = 1;
/* Disable and mute controls for disabled widgets. */
i = 0;
while ((ctl = hdaa_audio_ctl_each(devinfo, &i)) != NULL) {
if (ctl->enable == 0)
continue;
if (ctl->widget->enable == 0 ||
(ctl->childwidget != NULL &&
ctl->childwidget->enable == 0)) {
ctl->forcemute = 1;
ctl->muted = HDAA_AMP_MUTE_ALL;
ctl->left = 0;
ctl->right = 0;
ctl->enable = 0;
if (ctl->ndir == HDAA_CTL_IN)
ctl->widget->connsenable[ctl->index] = 0;
done = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling ctl %d nid %d cnid %d due"
" to disabled widget.\n", i,
ctl->widget->nid,
(ctl->childwidget != NULL)?
ctl->childwidget->nid:-1);
);
}
}
/* Disable useless widgets. */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
/* Disable inputs with disabled child widgets. */
for (j = 0; j < w->nconns; j++) {
if (w->connsenable[j]) {
cw = hdaa_widget_get(devinfo, w->conns[j]);
if (cw == NULL || cw->enable == 0) {
w->connsenable[j] = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling nid %d connection %d due"
" to disabled child widget.\n",
i, j);
);
}
}
}
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR &&
w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER)
continue;
/* Disable mixers and selectors without inputs. */
found = 0;
for (j = 0; j < w->nconns; j++) {
if (w->connsenable[j]) {
found = 1;
break;
}
}
if (found == 0) {
w->enable = 0;
done = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling nid %d due to all it's"
" inputs disabled.\n", w->nid);
);
}
/* Disable nodes without consumers. */
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR &&
w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER)
continue;
found = 0;
for (k = devinfo->startnode; k < devinfo->endnode; k++) {
cw = hdaa_widget_get(devinfo, k);
if (cw == NULL || cw->enable == 0)
continue;
for (j = 0; j < cw->nconns; j++) {
if (cw->connsenable[j] && cw->conns[j] == i) {
found = 1;
break;
}
}
}
if (found == 0) {
w->enable = 0;
done = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling nid %d due to all it's"
" consumers disabled.\n", w->nid);
);
}
}
} while (done == 0);
}
static void
hdaa_audio_disable_unas(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as = devinfo->as;
struct hdaa_widget *w, *cw;
struct hdaa_audio_ctl *ctl;
int i, j, k;
/* Disable unassosiated widgets. */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->bindas == -1) {
w->enable = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling unassociated nid %d.\n",
w->nid);
);
}
}
/* Disable input connections on input pin and
* output on output. */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
if (w->bindas < 0)
continue;
if (as[w->bindas].dir == HDAA_CTL_IN) {
for (j = 0; j < w->nconns; j++) {
if (w->connsenable[j] == 0)
continue;
w->connsenable[j] = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling connection to input pin "
"nid %d conn %d.\n",
i, j);
);
}
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid,
HDAA_CTL_IN, -1, 1);
if (ctl && ctl->enable) {
ctl->forcemute = 1;
ctl->muted = HDAA_AMP_MUTE_ALL;
ctl->left = 0;
ctl->right = 0;
ctl->enable = 0;
}
} else {
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid,
HDAA_CTL_OUT, -1, 1);
if (ctl && ctl->enable) {
ctl->forcemute = 1;
ctl->muted = HDAA_AMP_MUTE_ALL;
ctl->left = 0;
ctl->right = 0;
ctl->enable = 0;
}
for (k = devinfo->startnode; k < devinfo->endnode; k++) {
cw = hdaa_widget_get(devinfo, k);
if (cw == NULL || cw->enable == 0)
continue;
for (j = 0; j < cw->nconns; j++) {
if (cw->connsenable[j] && cw->conns[j] == i) {
cw->connsenable[j] = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling connection from output pin "
"nid %d conn %d cnid %d.\n",
k, j, i);
);
if (cw->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX &&
cw->nconns > 1)
continue;
ctl = hdaa_audio_ctl_amp_get(devinfo, k,
HDAA_CTL_IN, j, 1);
if (ctl && ctl->enable) {
ctl->forcemute = 1;
ctl->muted = HDAA_AMP_MUTE_ALL;
ctl->left = 0;
ctl->right = 0;
ctl->enable = 0;
}
}
}
}
}
}
}
static void
hdaa_audio_disable_notselected(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as = devinfo->as;
struct hdaa_widget *w;
int i, j;
/* On playback path we can safely disable all unseleted inputs. */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->nconns <= 1)
continue;
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER)
continue;
if (w->bindas < 0 || as[w->bindas].dir == HDAA_CTL_IN)
continue;
for (j = 0; j < w->nconns; j++) {
if (w->connsenable[j] == 0)
continue;
if (w->selconn < 0 || w->selconn == j)
continue;
w->connsenable[j] = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling unselected connection "
"nid %d conn %d.\n",
i, j);
);
}
}
}
static void
hdaa_audio_disable_crossas(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *ases = devinfo->as;
struct hdaa_widget *w, *cw;
struct hdaa_audio_ctl *ctl;
int i, j;
/* Disable crossassociatement and unwanted crosschannel connections. */
/* ... using selectors */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->nconns <= 1)
continue;
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER)
continue;
/* Allow any -> mix */
if (w->bindas == -2)
continue;
for (j = 0; j < w->nconns; j++) {
if (w->connsenable[j] == 0)
continue;
cw = hdaa_widget_get(devinfo, w->conns[j]);
if (cw == NULL || w->enable == 0)
continue;
/* Allow mix -> out. */
if (cw->bindas == -2 && w->bindas >= 0 &&
ases[w->bindas].dir == HDAA_CTL_OUT)
continue;
/* Allow mix -> mixed-in. */
if (cw->bindas == -2 && w->bindas >= 0 &&
ases[w->bindas].mixed)
continue;
/* Allow in -> mix. */
if ((w->pflags & HDAA_ADC_MONITOR) &&
cw->bindas >= 0 &&
ases[cw->bindas].dir == HDAA_CTL_IN)
continue;
/* Allow if have common as/seqs. */
if (w->bindas == cw->bindas &&
(w->bindseqmask & cw->bindseqmask) != 0)
continue;
w->connsenable[j] = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling crossassociatement connection "
"nid %d conn %d cnid %d.\n",
i, j, cw->nid);
);
}
}
/* ... using controls */
i = 0;
while ((ctl = hdaa_audio_ctl_each(devinfo, &i)) != NULL) {
if (ctl->enable == 0 || ctl->childwidget == NULL)
continue;
/* Allow any -> mix */
if (ctl->widget->bindas == -2)
continue;
/* Allow mix -> out. */
if (ctl->childwidget->bindas == -2 &&
ctl->widget->bindas >= 0 &&
ases[ctl->widget->bindas].dir == HDAA_CTL_OUT)
continue;
/* Allow mix -> mixed-in. */
if (ctl->childwidget->bindas == -2 &&
ctl->widget->bindas >= 0 &&
ases[ctl->widget->bindas].mixed)
continue;
/* Allow in -> mix. */
if ((ctl->widget->pflags & HDAA_ADC_MONITOR) &&
ctl->childwidget->bindas >= 0 &&
ases[ctl->childwidget->bindas].dir == HDAA_CTL_IN)
continue;
/* Allow if have common as/seqs. */
if (ctl->widget->bindas == ctl->childwidget->bindas &&
(ctl->widget->bindseqmask & ctl->childwidget->bindseqmask) != 0)
continue;
ctl->forcemute = 1;
ctl->muted = HDAA_AMP_MUTE_ALL;
ctl->left = 0;
ctl->right = 0;
ctl->enable = 0;
if (ctl->ndir == HDAA_CTL_IN)
ctl->widget->connsenable[ctl->index] = 0;
HDA_BOOTHVERBOSE(
device_printf(devinfo->dev,
" Disabling crossassociatement connection "
"ctl %d nid %d cnid %d.\n", i,
ctl->widget->nid,
ctl->childwidget->nid);
);
}
}
/*
* Find controls to control amplification for source and calculate possible
* amplification range.
*/
static int
hdaa_audio_ctl_source_amp(struct hdaa_devinfo *devinfo, nid_t nid, int index,
int ossdev, int ctlable, int depth, int *minamp, int *maxamp)
{
struct hdaa_widget *w, *wc;
struct hdaa_audio_ctl *ctl;
int i, j, conns = 0, tminamp, tmaxamp, cminamp, cmaxamp, found = 0;
if (depth > HDA_PARSE_MAXDEPTH)
return (found);
w = hdaa_widget_get(devinfo, nid);
if (w == NULL || w->enable == 0)
return (found);
/* Count number of active inputs. */
if (depth > 0) {
for (j = 0; j < w->nconns; j++) {
if (!w->connsenable[j])
continue;
conns++;
}
}
/* If this is not a first step - use input mixer.
Pins have common input ctl so care must be taken. */
if (depth > 0 && ctlable && (conns == 1 ||
w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)) {
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid, HDAA_CTL_IN,
index, 1);
if (ctl) {
ctl->ossmask |= (1 << ossdev);
found++;
if (*minamp == *maxamp) {
*minamp += MINQDB(ctl);
*maxamp += MAXQDB(ctl);
}
}
}
/* If widget has own ossdev - not traverse it.
It will be traversed on it's own. */
if (w->ossdev >= 0 && depth > 0)
return (found);
/* We must not traverse pin */
if ((w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT ||
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX) &&
depth > 0)
return (found);
/* record that this widget exports such signal, */
w->ossmask |= (1 << ossdev);
/*
* If signals mixed, we can't assign controls farther.
* Ignore this on depth zero. Caller must knows why.
*/
if (conns > 1 &&
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER)
ctlable = 0;
if (ctlable) {
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid, HDAA_CTL_OUT, -1, 1);
if (ctl) {
ctl->ossmask |= (1 << ossdev);
found++;
if (*minamp == *maxamp) {
*minamp += MINQDB(ctl);
*maxamp += MAXQDB(ctl);
}
}
}
cminamp = cmaxamp = 0;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
wc = hdaa_widget_get(devinfo, i);
if (wc == NULL || wc->enable == 0)
continue;
for (j = 0; j < wc->nconns; j++) {
if (wc->connsenable[j] && wc->conns[j] == nid) {
tminamp = tmaxamp = 0;
found += hdaa_audio_ctl_source_amp(devinfo,
wc->nid, j, ossdev, ctlable, depth + 1,
&tminamp, &tmaxamp);
if (cminamp == 0 && cmaxamp == 0) {
cminamp = tminamp;
cmaxamp = tmaxamp;
} else if (tminamp != tmaxamp) {
cminamp = imax(cminamp, tminamp);
cmaxamp = imin(cmaxamp, tmaxamp);
}
}
}
}
if (*minamp == *maxamp && cminamp < cmaxamp) {
*minamp += cminamp;
*maxamp += cmaxamp;
}
return (found);
}
/*
* Find controls to control amplification for destination and calculate
* possible amplification range.
*/
static int
hdaa_audio_ctl_dest_amp(struct hdaa_devinfo *devinfo, nid_t nid, int index,
int ossdev, int depth, int *minamp, int *maxamp)
{
struct hdaa_audio_as *as = devinfo->as;
struct hdaa_widget *w, *wc;
struct hdaa_audio_ctl *ctl;
int i, j, consumers, tminamp, tmaxamp, cminamp, cmaxamp, found = 0;
if (depth > HDA_PARSE_MAXDEPTH)
return (found);
w = hdaa_widget_get(devinfo, nid);
if (w == NULL || w->enable == 0)
return (found);
if (depth > 0) {
/* If this node produce output for several consumers,
we can't touch it. */
consumers = 0;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
wc = hdaa_widget_get(devinfo, i);
if (wc == NULL || wc->enable == 0)
continue;
for (j = 0; j < wc->nconns; j++) {
if (wc->connsenable[j] && wc->conns[j] == nid)
consumers++;
}
}
/* The only exception is if real HP redirection is configured
and this is a duplication point.
XXX: Actually exception is not completely correct.
XXX: Duplication point check is not perfect. */
if ((consumers == 2 && (w->bindas < 0 ||
as[w->bindas].hpredir < 0 || as[w->bindas].fakeredir ||
(w->bindseqmask & (1 << 15)) == 0)) ||
consumers > 2)
return (found);
/* Else use it's output mixer. */
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid,
HDAA_CTL_OUT, -1, 1);
if (ctl) {
ctl->ossmask |= (1 << ossdev);
found++;
if (*minamp == *maxamp) {
*minamp += MINQDB(ctl);
*maxamp += MAXQDB(ctl);
}
}
}
/* We must not traverse pin */
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX &&
depth > 0)
return (found);
cminamp = cmaxamp = 0;
for (i = 0; i < w->nconns; i++) {
if (w->connsenable[i] == 0)
continue;
if (index >= 0 && i != index)
continue;
tminamp = tmaxamp = 0;
ctl = hdaa_audio_ctl_amp_get(devinfo, w->nid,
HDAA_CTL_IN, i, 1);
if (ctl) {
ctl->ossmask |= (1 << ossdev);
found++;
if (*minamp == *maxamp) {
tminamp += MINQDB(ctl);
tmaxamp += MAXQDB(ctl);
}
}
found += hdaa_audio_ctl_dest_amp(devinfo, w->conns[i], -1, ossdev,
depth + 1, &tminamp, &tmaxamp);
if (cminamp == 0 && cmaxamp == 0) {
cminamp = tminamp;
cmaxamp = tmaxamp;
} else if (tminamp != tmaxamp) {
cminamp = imax(cminamp, tminamp);
cmaxamp = imin(cmaxamp, tmaxamp);
}
}
if (*minamp == *maxamp && cminamp < cmaxamp) {
*minamp += cminamp;
*maxamp += cmaxamp;
}
return (found);
}
/*
* Assign OSS names to sound sources
*/
static void
hdaa_audio_assign_names(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as = devinfo->as;
struct hdaa_widget *w;
int i, j;
int type = -1, use, used = 0;
static const int types[7][13] = {
{ SOUND_MIXER_LINE, SOUND_MIXER_LINE1, SOUND_MIXER_LINE2,
SOUND_MIXER_LINE3, -1 }, /* line */
{ SOUND_MIXER_MONITOR, SOUND_MIXER_MIC, -1 }, /* int mic */
{ SOUND_MIXER_MIC, SOUND_MIXER_MONITOR, -1 }, /* ext mic */
{ SOUND_MIXER_CD, -1 }, /* cd */
{ SOUND_MIXER_SPEAKER, -1 }, /* speaker */
{ SOUND_MIXER_DIGITAL1, SOUND_MIXER_DIGITAL2, SOUND_MIXER_DIGITAL3,
-1 }, /* digital */
{ SOUND_MIXER_LINE, SOUND_MIXER_LINE1, SOUND_MIXER_LINE2,
SOUND_MIXER_LINE3, SOUND_MIXER_PHONEIN, SOUND_MIXER_PHONEOUT,
SOUND_MIXER_VIDEO, SOUND_MIXER_RADIO, SOUND_MIXER_DIGITAL1,
SOUND_MIXER_DIGITAL2, SOUND_MIXER_DIGITAL3, SOUND_MIXER_MONITOR,
-1 } /* others */
};
/* Surely known names */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->bindas == -1)
continue;
use = -1;
switch (w->type) {
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX:
if (as[w->bindas].dir == HDAA_CTL_OUT)
break;
type = -1;
switch (w->wclass.pin.config & HDA_CONFIG_DEFAULTCONF_DEVICE_MASK) {
case HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_IN:
type = 0;
break;
case HDA_CONFIG_DEFAULTCONF_DEVICE_MIC_IN:
if ((w->wclass.pin.config & HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_MASK)
== HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_JACK)
break;
type = 1;
break;
case HDA_CONFIG_DEFAULTCONF_DEVICE_CD:
type = 3;
break;
case HDA_CONFIG_DEFAULTCONF_DEVICE_SPEAKER:
type = 4;
break;
case HDA_CONFIG_DEFAULTCONF_DEVICE_SPDIF_IN:
case HDA_CONFIG_DEFAULTCONF_DEVICE_DIGITAL_OTHER_IN:
type = 5;
break;
}
if (type == -1)
break;
j = 0;
while (types[type][j] >= 0 &&
(used & (1 << types[type][j])) != 0) {
j++;
}
if (types[type][j] >= 0)
use = types[type][j];
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT:
use = SOUND_MIXER_PCM;
break;
case HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_BEEP_WIDGET:
use = SOUND_MIXER_SPEAKER;
break;
default:
break;
}
if (use >= 0) {
w->ossdev = use;
used |= (1 << use);
}
}
/* Semi-known names */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->ossdev >= 0)
continue;
if (w->bindas == -1)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
if (as[w->bindas].dir == HDAA_CTL_OUT)
continue;
type = -1;
switch (w->wclass.pin.config & HDA_CONFIG_DEFAULTCONF_DEVICE_MASK) {
case HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_OUT:
case HDA_CONFIG_DEFAULTCONF_DEVICE_SPEAKER:
case HDA_CONFIG_DEFAULTCONF_DEVICE_HP_OUT:
case HDA_CONFIG_DEFAULTCONF_DEVICE_AUX:
type = 0;
break;
case HDA_CONFIG_DEFAULTCONF_DEVICE_MIC_IN:
type = 2;
break;
case HDA_CONFIG_DEFAULTCONF_DEVICE_SPDIF_OUT:
case HDA_CONFIG_DEFAULTCONF_DEVICE_DIGITAL_OTHER_OUT:
type = 5;
break;
}
if (type == -1)
break;
j = 0;
while (types[type][j] >= 0 &&
(used & (1 << types[type][j])) != 0) {
j++;
}
if (types[type][j] >= 0) {
w->ossdev = types[type][j];
used |= (1 << types[type][j]);
}
}
/* Others */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->ossdev >= 0)
continue;
if (w->bindas == -1)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
if (as[w->bindas].dir == HDAA_CTL_OUT)
continue;
j = 0;
while (types[6][j] >= 0 &&
(used & (1 << types[6][j])) != 0) {
j++;
}
if (types[6][j] >= 0) {
w->ossdev = types[6][j];
used |= (1 << types[6][j]);
}
}
}
static void
hdaa_audio_build_tree(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as = devinfo->as;
int j, res;
/* Trace all associations in order of their numbers. */
for (j = 0; j < devinfo->ascnt; j++) {
if (as[j].enable == 0)
continue;
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"Tracing association %d (%d)\n", j, as[j].index);
);
if (as[j].dir == HDAA_CTL_OUT) {
retry:
res = hdaa_audio_trace_as_out(devinfo, j, 0);
if (res == 0 && as[j].hpredir >= 0 &&
as[j].fakeredir == 0) {
/* If CODEC can't do analog HP redirection
try to make it using one more DAC. */
as[j].fakeredir = 1;
goto retry;
}
} else if (as[j].mixed)
res = hdaa_audio_trace_as_in(devinfo, j);
else
res = hdaa_audio_trace_as_in_mch(devinfo, j, 0);
if (res) {
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"Association %d (%d) trace succeeded\n",
j, as[j].index);
);
} else {
HDA_BOOTVERBOSE(
device_printf(devinfo->dev,
"Association %d (%d) trace failed\n",
j, as[j].index);
);
as[j].enable = 0;
}
}
/* Look for additional DACs/ADCs. */
for (j = 0; j < devinfo->ascnt; j++) {
if (as[j].enable == 0)
continue;
hdaa_audio_adddac(devinfo, j);
}
/* Trace mixer and beeper pseudo associations. */
hdaa_audio_trace_as_extra(devinfo);
}
/*
* Store in pdevinfo new data about whether and how we can control signal
* for OSS device to/from specified widget.
*/
static void
hdaa_adjust_amp(struct hdaa_widget *w, int ossdev,
int found, int minamp, int maxamp)
{
struct hdaa_devinfo *devinfo = w->devinfo;
struct hdaa_pcm_devinfo *pdevinfo;
if (w->bindas >= 0)
pdevinfo = devinfo->as[w->bindas].pdevinfo;
else
pdevinfo = &devinfo->devs[0];
if (found)
pdevinfo->ossmask |= (1 << ossdev);
if (minamp == 0 && maxamp == 0)
return;
if (pdevinfo->minamp[ossdev] == 0 && pdevinfo->maxamp[ossdev] == 0) {
pdevinfo->minamp[ossdev] = minamp;
pdevinfo->maxamp[ossdev] = maxamp;
} else {
pdevinfo->minamp[ossdev] = imax(pdevinfo->minamp[ossdev], minamp);
pdevinfo->maxamp[ossdev] = imin(pdevinfo->maxamp[ossdev], maxamp);
}
}
/*
* Trace signals from/to all possible sources/destionstions to find possible
* recording sources, OSS device control ranges and to assign controls.
*/
static void
hdaa_audio_assign_mixers(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as = devinfo->as;
struct hdaa_widget *w, *cw;
int i, j, minamp, maxamp, found;
/* Assign mixers to the tree. */
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
minamp = maxamp = 0;
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT ||
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_BEEP_WIDGET ||
(w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX &&
as[w->bindas].dir == HDAA_CTL_IN)) {
if (w->ossdev < 0)
continue;
found = hdaa_audio_ctl_source_amp(devinfo, w->nid, -1,
w->ossdev, 1, 0, &minamp, &maxamp);
hdaa_adjust_amp(w, w->ossdev, found, minamp, maxamp);
} else if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT) {
found = hdaa_audio_ctl_dest_amp(devinfo, w->nid, -1,
SOUND_MIXER_RECLEV, 0, &minamp, &maxamp);
hdaa_adjust_amp(w, SOUND_MIXER_RECLEV, found, minamp, maxamp);
} else if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX &&
as[w->bindas].dir == HDAA_CTL_OUT) {
found = hdaa_audio_ctl_dest_amp(devinfo, w->nid, -1,
SOUND_MIXER_VOLUME, 0, &minamp, &maxamp);
hdaa_adjust_amp(w, SOUND_MIXER_VOLUME, found, minamp, maxamp);
}
if (w->ossdev == SOUND_MIXER_IMIX) {
minamp = maxamp = 0;
found = hdaa_audio_ctl_source_amp(devinfo, w->nid, -1,
w->ossdev, 1, 0, &minamp, &maxamp);
if (minamp == maxamp) {
/* If we are unable to control input monitor
as source - try to control it as destination. */
found += hdaa_audio_ctl_dest_amp(devinfo, w->nid, -1,
w->ossdev, 0, &minamp, &maxamp);
w->pflags |= HDAA_IMIX_AS_DST;
}
hdaa_adjust_amp(w, w->ossdev, found, minamp, maxamp);
}
if (w->pflags & HDAA_ADC_MONITOR) {
for (j = 0; j < w->nconns; j++) {
if (!w->connsenable[j])
continue;
cw = hdaa_widget_get(devinfo, w->conns[j]);
if (cw == NULL || cw->enable == 0)
continue;
if (cw->bindas == -1)
continue;
if (cw->bindas >= 0 &&
as[cw->bindas].dir != HDAA_CTL_IN)
continue;
minamp = maxamp = 0;
found = hdaa_audio_ctl_dest_amp(devinfo,
w->nid, j, SOUND_MIXER_IGAIN, 0,
&minamp, &maxamp);
hdaa_adjust_amp(w, SOUND_MIXER_IGAIN,
found, minamp, maxamp);
}
}
}
}
static void
hdaa_audio_prepare_pin_ctrl(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as = devinfo->as;
struct hdaa_widget *w;
uint32_t pincap;
int i;
for (i = 0; i < devinfo->nodecnt; i++) {
w = &devinfo->widget[i];
if (w == NULL)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX &&
w->waspin == 0)
continue;
pincap = w->wclass.pin.cap;
/* Disable everything. */
w->wclass.pin.ctrl &= ~(
HDA_CMD_SET_PIN_WIDGET_CTRL_HPHN_ENABLE |
HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE |
HDA_CMD_SET_PIN_WIDGET_CTRL_IN_ENABLE |
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK);
if (w->enable == 0) {
/* Pin is unused so left it disabled. */
continue;
} else if (w->waspin) {
/* Enable input for beeper input. */
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_IN_ENABLE;
} else if (w->bindas < 0 || as[w->bindas].enable == 0) {
/* Pin is unused so left it disabled. */
continue;
} else if (as[w->bindas].dir == HDAA_CTL_IN) {
/* Input pin, configure for input. */
if (HDA_PARAM_PIN_CAP_INPUT_CAP(pincap))
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_IN_ENABLE;
if ((devinfo->quirks & HDAA_QUIRK_IVREF100) &&
HDA_PARAM_PIN_CAP_VREF_CTRL_100(pincap))
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE(
HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_100);
else if ((devinfo->quirks & HDAA_QUIRK_IVREF80) &&
HDA_PARAM_PIN_CAP_VREF_CTRL_80(pincap))
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE(
HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_80);
else if ((devinfo->quirks & HDAA_QUIRK_IVREF50) &&
HDA_PARAM_PIN_CAP_VREF_CTRL_50(pincap))
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE(
HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_50);
} else {
/* Output pin, configure for output. */
if (HDA_PARAM_PIN_CAP_OUTPUT_CAP(pincap))
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE;
if (HDA_PARAM_PIN_CAP_HEADPHONE_CAP(pincap) &&
(w->wclass.pin.config &
HDA_CONFIG_DEFAULTCONF_DEVICE_MASK) ==
HDA_CONFIG_DEFAULTCONF_DEVICE_HP_OUT)
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_HPHN_ENABLE;
if ((devinfo->quirks & HDAA_QUIRK_OVREF100) &&
HDA_PARAM_PIN_CAP_VREF_CTRL_100(pincap))
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE(
HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_100);
else if ((devinfo->quirks & HDAA_QUIRK_OVREF80) &&
HDA_PARAM_PIN_CAP_VREF_CTRL_80(pincap))
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE(
HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_80);
else if ((devinfo->quirks & HDAA_QUIRK_OVREF50) &&
HDA_PARAM_PIN_CAP_VREF_CTRL_50(pincap))
w->wclass.pin.ctrl |=
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE(
HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_50);
}
}
}
static void
hdaa_audio_ctl_commit(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_ctl *ctl;
int i, z;
i = 0;
while ((ctl = hdaa_audio_ctl_each(devinfo, &i)) != NULL) {
if (ctl->enable == 0 || ctl->ossmask != 0) {
/* Mute disabled and mixer controllable controls.
* Last will be initialized by mixer_init().
* This expected to reduce click on startup. */
hdaa_audio_ctl_amp_set(ctl, HDAA_AMP_MUTE_ALL, 0, 0);
continue;
}
/* Init fixed controls to 0dB amplification. */
z = ctl->offset;
if (z > ctl->step)
z = ctl->step;
hdaa_audio_ctl_amp_set(ctl, HDAA_AMP_MUTE_NONE, z, z);
}
}
static void
hdaa_gpio_commit(struct hdaa_devinfo *devinfo)
{
uint32_t gdata, gmask, gdir;
int i, numgpio;
numgpio = HDA_PARAM_GPIO_COUNT_NUM_GPIO(devinfo->gpio_cap);
if (devinfo->gpio != 0 && numgpio != 0) {
gdata = hda_command(devinfo->dev,
HDA_CMD_GET_GPIO_DATA(0, devinfo->nid));
gmask = hda_command(devinfo->dev,
HDA_CMD_GET_GPIO_ENABLE_MASK(0, devinfo->nid));
gdir = hda_command(devinfo->dev,
HDA_CMD_GET_GPIO_DIRECTION(0, devinfo->nid));
for (i = 0; i < numgpio; i++) {
if ((devinfo->gpio & HDAA_GPIO_MASK(i)) ==
HDAA_GPIO_SET(i)) {
gdata |= (1 << i);
gmask |= (1 << i);
gdir |= (1 << i);
} else if ((devinfo->gpio & HDAA_GPIO_MASK(i)) ==
HDAA_GPIO_CLEAR(i)) {
gdata &= ~(1 << i);
gmask |= (1 << i);
gdir |= (1 << i);
} else if ((devinfo->gpio & HDAA_GPIO_MASK(i)) ==
HDAA_GPIO_DISABLE(i)) {
gmask &= ~(1 << i);
} else if ((devinfo->gpio & HDAA_GPIO_MASK(i)) ==
HDAA_GPIO_INPUT(i)) {
gmask |= (1 << i);
gdir &= ~(1 << i);
}
}
HDA_BOOTVERBOSE(
device_printf(devinfo->dev, "GPIO commit\n");
);
hda_command(devinfo->dev,
HDA_CMD_SET_GPIO_ENABLE_MASK(0, devinfo->nid, gmask));
hda_command(devinfo->dev,
HDA_CMD_SET_GPIO_DIRECTION(0, devinfo->nid, gdir));
hda_command(devinfo->dev,
HDA_CMD_SET_GPIO_DATA(0, devinfo->nid, gdata));
HDA_BOOTVERBOSE(
hdaa_dump_gpio(devinfo);
);
}
}
static void
hdaa_gpo_commit(struct hdaa_devinfo *devinfo)
{
uint32_t gdata;
int i, numgpo;
numgpo = HDA_PARAM_GPIO_COUNT_NUM_GPO(devinfo->gpio_cap);
if (devinfo->gpo != 0 && numgpo != 0) {
gdata = hda_command(devinfo->dev,
HDA_CMD_GET_GPO_DATA(0, devinfo->nid));
for (i = 0; i < numgpo; i++) {
if ((devinfo->gpio & HDAA_GPIO_MASK(i)) ==
HDAA_GPIO_SET(i)) {
gdata |= (1 << i);
} else if ((devinfo->gpio & HDAA_GPIO_MASK(i)) ==
HDAA_GPIO_CLEAR(i)) {
gdata &= ~(1 << i);
}
}
HDA_BOOTVERBOSE(
device_printf(devinfo->dev, "GPO commit\n");
);
hda_command(devinfo->dev,
HDA_CMD_SET_GPO_DATA(0, devinfo->nid, gdata));
HDA_BOOTVERBOSE(
hdaa_dump_gpo(devinfo);
);
}
}
static void
hdaa_audio_commit(struct hdaa_devinfo *devinfo)
{
struct hdaa_widget *w;
int i;
/* Commit controls. */
hdaa_audio_ctl_commit(devinfo);
/* Commit selectors, pins and EAPD. */
for (i = 0; i < devinfo->nodecnt; i++) {
w = &devinfo->widget[i];
if (w == NULL)
continue;
if (w->selconn == -1)
w->selconn = 0;
if (w->nconns > 0)
hdaa_widget_connection_select(w, w->selconn);
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX ||
w->waspin) {
hda_command(devinfo->dev,
HDA_CMD_SET_PIN_WIDGET_CTRL(0, w->nid,
w->wclass.pin.ctrl));
}
if (w->param.eapdbtl != HDA_INVALID) {
uint32_t val;
val = w->param.eapdbtl;
if (devinfo->quirks &
HDAA_QUIRK_EAPDINV)
val ^= HDA_CMD_SET_EAPD_BTL_ENABLE_EAPD;
hda_command(devinfo->dev,
HDA_CMD_SET_EAPD_BTL_ENABLE(0, w->nid,
val));
}
}
hdaa_gpio_commit(devinfo);
hdaa_gpo_commit(devinfo);
}
static void
hdaa_powerup(struct hdaa_devinfo *devinfo)
{
int i;
hda_command(devinfo->dev,
HDA_CMD_SET_POWER_STATE(0,
devinfo->nid, HDA_CMD_POWER_STATE_D0));
DELAY(100);
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
hda_command(devinfo->dev,
HDA_CMD_SET_POWER_STATE(0,
i, HDA_CMD_POWER_STATE_D0));
}
DELAY(1000);
}
static int
hdaa_pcmchannel_setup(struct hdaa_chan *ch)
{
struct hdaa_devinfo *devinfo = ch->devinfo;
struct hdaa_audio_as *as = devinfo->as;
struct hdaa_widget *w;
uint32_t cap, fmtcap, pcmcap;
int i, j, ret, channels, onlystereo;
uint16_t pinset;
ch->caps = hdaa_caps;
ch->caps.fmtlist = ch->fmtlist;
ch->bit16 = 1;
ch->bit32 = 0;
ch->pcmrates[0] = 48000;
ch->pcmrates[1] = 0;
ch->stripecap = 0xff;
ret = 0;
channels = 0;
onlystereo = 1;
pinset = 0;
fmtcap = devinfo->supp_stream_formats;
pcmcap = devinfo->supp_pcm_size_rate;
for (i = 0; i < 16; i++) {
/* Check as is correct */
if (ch->as < 0)
break;
/* Cound only present DACs */
if (as[ch->as].dacs[ch->asindex][i] <= 0)
continue;
/* Ignore duplicates */
for (j = 0; j < ret; j++) {
if (ch->io[j] == as[ch->as].dacs[ch->asindex][i])
break;
}
if (j < ret)
continue;
w = hdaa_widget_get(devinfo, as[ch->as].dacs[ch->asindex][i]);
if (w == NULL || w->enable == 0)
continue;
cap = w->param.supp_stream_formats;
if (!HDA_PARAM_SUPP_STREAM_FORMATS_PCM(cap) &&
!HDA_PARAM_SUPP_STREAM_FORMATS_AC3(cap))
continue;
/* Many CODECs does not declare AC3 support on SPDIF.
I don't beleave that they doesn't support it! */
if (HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(w->param.widget_cap))
cap |= HDA_PARAM_SUPP_STREAM_FORMATS_AC3_MASK;
if (ret == 0) {
fmtcap = cap;
pcmcap = w->param.supp_pcm_size_rate;
} else {
fmtcap &= cap;
pcmcap &= w->param.supp_pcm_size_rate;
}
ch->io[ret++] = as[ch->as].dacs[ch->asindex][i];
ch->stripecap &= w->wclass.conv.stripecap;
/* Do not count redirection pin/dac channels. */
if (i == 15 && as[ch->as].hpredir >= 0)
continue;
channels += HDA_PARAM_AUDIO_WIDGET_CAP_CC(w->param.widget_cap) + 1;
if (HDA_PARAM_AUDIO_WIDGET_CAP_CC(w->param.widget_cap) != 1)
onlystereo = 0;
pinset |= (1 << i);
}
ch->io[ret] = -1;
ch->channels = channels;
if (as[ch->as].fakeredir)
ret--;
/* Standard speaks only about stereo pins and playback, ... */
if ((!onlystereo) || as[ch->as].mixed)
pinset = 0;
/* ..., but there it gives us info about speakers layout. */
as[ch->as].pinset = pinset;
ch->supp_stream_formats = fmtcap;
ch->supp_pcm_size_rate = pcmcap;
/*
* 8bit = 0
* 16bit = 1
* 20bit = 2
* 24bit = 3
* 32bit = 4
*/
if (ret > 0) {
i = 0;
if (HDA_PARAM_SUPP_STREAM_FORMATS_PCM(fmtcap)) {
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT(pcmcap))
ch->bit16 = 1;
else if (HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT(pcmcap))
ch->bit16 = 0;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT(pcmcap))
ch->bit32 = 3;
else if (HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT(pcmcap))
ch->bit32 = 2;
else if (HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT(pcmcap))
ch->bit32 = 4;
if (!(devinfo->quirks & HDAA_QUIRK_FORCESTEREO)) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 1, 0);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 1, 0);
}
if (channels >= 2) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 2, 0);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 2, 0);
}
if (channels >= 3 && !onlystereo) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 3, 0);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 3, 0);
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 3, 1);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 3, 1);
}
if (channels >= 4) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 4, 0);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 4, 0);
if (!onlystereo) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 4, 1);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 4, 1);
}
}
if (channels >= 5 && !onlystereo) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 5, 0);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 5, 0);
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 5, 1);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 5, 1);
}
if (channels >= 6) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 6, 1);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 6, 1);
if (!onlystereo) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 6, 0);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 6, 0);
}
}
if (channels >= 7 && !onlystereo) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 7, 0);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 7, 0);
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 7, 1);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 7, 1);
}
if (channels >= 8) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_S16_LE, 8, 1);
if (ch->bit32)
ch->fmtlist[i++] = SND_FORMAT(AFMT_S32_LE, 8, 1);
}
}
if (HDA_PARAM_SUPP_STREAM_FORMATS_AC3(fmtcap)) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_AC3, 2, 0);
if (channels >= 8) {
ch->fmtlist[i++] = SND_FORMAT(AFMT_AC3, 8, 0);
ch->fmtlist[i++] = SND_FORMAT(AFMT_AC3, 8, 1);
}
}
ch->fmtlist[i] = 0;
i = 0;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ(pcmcap))
ch->pcmrates[i++] = 8000;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ(pcmcap))
ch->pcmrates[i++] = 11025;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ(pcmcap))
ch->pcmrates[i++] = 16000;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ(pcmcap))
ch->pcmrates[i++] = 22050;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ(pcmcap))
ch->pcmrates[i++] = 32000;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ(pcmcap))
ch->pcmrates[i++] = 44100;
/* if (HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ(pcmcap)) */
ch->pcmrates[i++] = 48000;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ(pcmcap))
ch->pcmrates[i++] = 88200;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ(pcmcap))
ch->pcmrates[i++] = 96000;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ(pcmcap))
ch->pcmrates[i++] = 176400;
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ(pcmcap))
ch->pcmrates[i++] = 192000;
/* if (HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ(pcmcap)) */
ch->pcmrates[i] = 0;
if (i > 0) {
ch->caps.minspeed = ch->pcmrates[0];
ch->caps.maxspeed = ch->pcmrates[i - 1];
}
}
return (ret);
}
static void
hdaa_prepare_pcms(struct hdaa_devinfo *devinfo)
{
struct hdaa_audio_as *as = devinfo->as;
int i, j, k, apdev = 0, ardev = 0, dpdev = 0, drdev = 0;
for (i = 0; i < devinfo->ascnt; i++) {
if (as[i].enable == 0)
continue;
if (as[i].dir == HDAA_CTL_IN) {
if (as[i].digital)
drdev++;
else
ardev++;
} else {
if (as[i].digital)
dpdev++;
else
apdev++;
}
}
devinfo->num_devs =
max(ardev, apdev) + max(drdev, dpdev);
devinfo->devs =
(struct hdaa_pcm_devinfo *)malloc(
devinfo->num_devs * sizeof(struct hdaa_pcm_devinfo),
M_HDAA, M_ZERO | M_NOWAIT);
if (devinfo->devs == NULL) {
device_printf(devinfo->dev,
"Unable to allocate memory for devices\n");
return;
}
for (i = 0; i < devinfo->num_devs; i++) {
devinfo->devs[i].index = i;
devinfo->devs[i].devinfo = devinfo;
devinfo->devs[i].playas = -1;
devinfo->devs[i].recas = -1;
devinfo->devs[i].digital = 255;
}
for (i = 0; i < devinfo->ascnt; i++) {
if (as[i].enable == 0)
continue;
for (j = 0; j < devinfo->num_devs; j++) {
if (devinfo->devs[j].digital != 255 &&
(!devinfo->devs[j].digital) !=
(!as[i].digital))
continue;
if (as[i].dir == HDAA_CTL_IN) {
if (devinfo->devs[j].recas >= 0)
continue;
devinfo->devs[j].recas = i;
} else {
if (devinfo->devs[j].playas >= 0)
continue;
devinfo->devs[j].playas = i;
}
as[i].pdevinfo = &devinfo->devs[j];
for (k = 0; k < as[i].num_chans; k++) {
devinfo->chans[as[i].chans[k]].pdevinfo =
&devinfo->devs[j];
}
devinfo->devs[j].digital = as[i].digital;
break;
}
}
}
static void
hdaa_create_pcms(struct hdaa_devinfo *devinfo)
{
int i;
for (i = 0; i < devinfo->num_devs; i++) {
struct hdaa_pcm_devinfo *pdevinfo = &devinfo->devs[i];
pdevinfo->dev = device_add_child(devinfo->dev, "pcm", -1);
device_set_ivars(pdevinfo->dev, (void *)pdevinfo);
}
}
static void
hdaa_dump_ctls(struct hdaa_pcm_devinfo *pdevinfo, const char *banner, uint32_t flag)
{
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_audio_ctl *ctl;
char buf[64];
int i, j, printed = 0;
if (flag == 0) {
flag = ~(SOUND_MASK_VOLUME | SOUND_MASK_PCM |
SOUND_MASK_CD | SOUND_MASK_LINE | SOUND_MASK_RECLEV |
SOUND_MASK_MIC | SOUND_MASK_SPEAKER | SOUND_MASK_IGAIN |
SOUND_MASK_OGAIN | SOUND_MASK_IMIX | SOUND_MASK_MONITOR);
}
for (j = 0; j < SOUND_MIXER_NRDEVICES; j++) {
if ((flag & (1 << j)) == 0)
continue;
i = 0;
printed = 0;
while ((ctl = hdaa_audio_ctl_each(devinfo, &i)) != NULL) {
if (ctl->enable == 0 ||
ctl->widget->enable == 0)
continue;
if (!((pdevinfo->playas >= 0 &&
ctl->widget->bindas == pdevinfo->playas) ||
(pdevinfo->recas >= 0 &&
ctl->widget->bindas == pdevinfo->recas) ||
(ctl->widget->bindas == -2 && pdevinfo->index == 0)))
continue;
if ((ctl->ossmask & (1 << j)) == 0)
continue;
if (printed == 0) {
if (banner != NULL) {
device_printf(pdevinfo->dev, "%s", banner);
} else {
device_printf(pdevinfo->dev, "Unknown Ctl");
}
printf(" (OSS: %s)",
hdaa_audio_ctl_ossmixer_mask2allname(1 << j,
buf, sizeof(buf)));
if (pdevinfo->ossmask & (1 << j)) {
printf(": %+d/%+ddB\n",
pdevinfo->minamp[j] / 4,
pdevinfo->maxamp[j] / 4);
} else
printf("\n");
printed = 1;
}
device_printf(pdevinfo->dev, " +- ctl %2d (nid %3d %s", i,
ctl->widget->nid,
(ctl->ndir == HDAA_CTL_IN)?"in ":"out");
if (ctl->ndir == HDAA_CTL_IN && ctl->ndir == ctl->dir)
printf(" %2d): ", ctl->index);
else
printf("): ");
if (ctl->step > 0) {
printf("%+d/%+ddB (%d steps)%s\n",
MINQDB(ctl) / 4,
MAXQDB(ctl) / 4,
ctl->step + 1,
ctl->mute?" + mute":"");
} else
printf("%s\n", ctl->mute?"mute":"");
}
}
if (printed)
device_printf(pdevinfo->dev, "\n");
}
static void
hdaa_dump_audio_formats(device_t dev, uint32_t fcap, uint32_t pcmcap)
{
uint32_t cap;
cap = fcap;
if (cap != 0) {
device_printf(dev, " Stream cap: 0x%08x", cap);
if (HDA_PARAM_SUPP_STREAM_FORMATS_AC3(cap))
printf(" AC3");
if (HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32(cap))
printf(" FLOAT32");
if (HDA_PARAM_SUPP_STREAM_FORMATS_PCM(cap))
printf(" PCM");
printf("\n");
}
cap = pcmcap;
if (cap != 0) {
device_printf(dev, " PCM cap: 0x%08x", cap);
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT(cap))
printf(" 8");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT(cap))
printf(" 16");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT(cap))
printf(" 20");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT(cap))
printf(" 24");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT(cap))
printf(" 32");
printf(" bits,");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ(cap))
printf(" 8");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ(cap))
printf(" 11");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ(cap))
printf(" 16");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ(cap))
printf(" 22");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ(cap))
printf(" 32");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ(cap))
printf(" 44");
printf(" 48");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ(cap))
printf(" 88");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ(cap))
printf(" 96");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ(cap))
printf(" 176");
if (HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ(cap))
printf(" 192");
printf(" KHz\n");
}
}
static void
hdaa_dump_pin(struct hdaa_widget *w)
{
uint32_t pincap;
pincap = w->wclass.pin.cap;
device_printf(w->devinfo->dev, " Pin cap: 0x%08x", pincap);
if (HDA_PARAM_PIN_CAP_IMP_SENSE_CAP(pincap))
printf(" ISC");
if (HDA_PARAM_PIN_CAP_TRIGGER_REQD(pincap))
printf(" TRQD");
if (HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP(pincap))
printf(" PDC");
if (HDA_PARAM_PIN_CAP_HEADPHONE_CAP(pincap))
printf(" HP");
if (HDA_PARAM_PIN_CAP_OUTPUT_CAP(pincap))
printf(" OUT");
if (HDA_PARAM_PIN_CAP_INPUT_CAP(pincap))
printf(" IN");
if (HDA_PARAM_PIN_CAP_BALANCED_IO_PINS(pincap))
printf(" BAL");
if (HDA_PARAM_PIN_CAP_HDMI(pincap))
printf(" HDMI");
if (HDA_PARAM_PIN_CAP_VREF_CTRL(pincap)) {
printf(" VREF[");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_50(pincap))
printf(" 50");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_80(pincap))
printf(" 80");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_100(pincap))
printf(" 100");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND(pincap))
printf(" GROUND");
if (HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ(pincap))
printf(" HIZ");
printf(" ]");
}
if (HDA_PARAM_PIN_CAP_EAPD_CAP(pincap))
printf(" EAPD");
if (HDA_PARAM_PIN_CAP_DP(pincap))
printf(" DP");
if (HDA_PARAM_PIN_CAP_HBR(pincap))
printf(" HBR");
printf("\n");
device_printf(w->devinfo->dev, " Pin config: 0x%08x\n",
w->wclass.pin.config);
device_printf(w->devinfo->dev, " Pin control: 0x%08x", w->wclass.pin.ctrl);
if (w->wclass.pin.ctrl & HDA_CMD_SET_PIN_WIDGET_CTRL_HPHN_ENABLE)
printf(" HP");
if (w->wclass.pin.ctrl & HDA_CMD_SET_PIN_WIDGET_CTRL_IN_ENABLE)
printf(" IN");
if (w->wclass.pin.ctrl & HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE)
printf(" OUT");
if (HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(w->param.widget_cap)) {
if ((w->wclass.pin.ctrl &
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) == 0x03)
printf(" HBR");
else if ((w->wclass.pin.ctrl &
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) != 0)
printf(" EPTs");
} else {
if ((w->wclass.pin.ctrl &
HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) != 0)
printf(" VREFs");
}
printf("\n");
}
static void
hdaa_dump_pin_config(struct hdaa_widget *w, uint32_t conf)
{
device_printf(w->devinfo->dev, "%2d %08x %-2d %-2d "
"%-13s %-5s %-7s %-10s %-7s %d%s\n",
w->nid, conf,
HDA_CONFIG_DEFAULTCONF_ASSOCIATION(conf),
HDA_CONFIG_DEFAULTCONF_SEQUENCE(conf),
HDA_DEVS[HDA_CONFIG_DEFAULTCONF_DEVICE(conf)],
HDA_CONNS[HDA_CONFIG_DEFAULTCONF_CONNECTIVITY(conf)],
HDA_CONNECTORS[HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE(conf)],
HDA_LOCS[HDA_CONFIG_DEFAULTCONF_LOCATION(conf)],
HDA_COLORS[HDA_CONFIG_DEFAULTCONF_COLOR(conf)],
HDA_CONFIG_DEFAULTCONF_MISC(conf),
(w->enable == 0)?" DISA":"");
}
static void
hdaa_dump_pin_configs(struct hdaa_devinfo *devinfo)
{
struct hdaa_widget *w;
int i;
device_printf(devinfo->dev, "nid 0x as seq "
"device conn jack loc color misc\n");
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
hdaa_dump_pin_config(w, w->wclass.pin.config);
}
}
static void
hdaa_dump_amp(device_t dev, uint32_t cap, const char *banner)
{
int offset, size, step;
offset = HDA_PARAM_OUTPUT_AMP_CAP_OFFSET(cap);
size = HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE(cap);
step = HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS(cap);
device_printf(dev, " %s amp: 0x%08x "
"mute=%d step=%d size=%d offset=%d (%+d/%+ddB)\n",
banner, cap,
HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP(cap),
step, size, offset,
((0 - offset) * (size + 1)) / 4,
((step - offset) * (size + 1)) / 4);
}
static void
hdaa_dump_nodes(struct hdaa_devinfo *devinfo)
{
struct hdaa_widget *w, *cw;
char buf[64];
int i, j;
device_printf(devinfo->dev, "\n");
device_printf(devinfo->dev, "Default parameters:\n");
hdaa_dump_audio_formats(devinfo->dev,
devinfo->supp_stream_formats,
devinfo->supp_pcm_size_rate);
hdaa_dump_amp(devinfo->dev, devinfo->inamp_cap, " Input");
hdaa_dump_amp(devinfo->dev, devinfo->outamp_cap, "Output");
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL) {
device_printf(devinfo->dev, "Ghost widget nid=%d\n", i);
continue;
}
device_printf(devinfo->dev, "\n");
device_printf(devinfo->dev, " nid: %d%s\n", w->nid,
(w->enable == 0) ? " [DISABLED]" : "");
device_printf(devinfo->dev, " Name: %s\n", w->name);
device_printf(devinfo->dev, " Widget cap: 0x%08x",
w->param.widget_cap);
if (w->param.widget_cap & 0x0ee1) {
if (HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP(w->param.widget_cap))
printf(" LRSWAP");
if (HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL(w->param.widget_cap))
printf(" PWR");
if (HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(w->param.widget_cap))
printf(" DIGITAL");
if (HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP(w->param.widget_cap))
printf(" UNSOL");
if (HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET(w->param.widget_cap))
printf(" PROC");
if (HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE(w->param.widget_cap))
printf(" STRIPE(x%d)",
1 << (fls(w->wclass.conv.stripecap) - 1));
j = HDA_PARAM_AUDIO_WIDGET_CAP_CC(w->param.widget_cap);
if (j == 1)
printf(" STEREO");
else if (j > 1)
printf(" %dCH", j + 1);
}
printf("\n");
if (w->bindas != -1) {
device_printf(devinfo->dev, " Association: %d (0x%04x)\n",
w->bindas, w->bindseqmask);
}
if (w->ossmask != 0 || w->ossdev >= 0) {
device_printf(devinfo->dev, " OSS: %s",
hdaa_audio_ctl_ossmixer_mask2allname(w->ossmask, buf, sizeof(buf)));
if (w->ossdev >= 0)
printf(" (%s)", ossnames[w->ossdev]);
printf("\n");
}
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT ||
w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT) {
hdaa_dump_audio_formats(devinfo->dev,
w->param.supp_stream_formats,
w->param.supp_pcm_size_rate);
} else if (w->type ==
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX || w->waspin)
hdaa_dump_pin(w);
if (w->param.eapdbtl != HDA_INVALID)
device_printf(devinfo->dev, " EAPD: 0x%08x\n",
w->param.eapdbtl);
if (HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP(w->param.widget_cap) &&
w->param.outamp_cap != 0)
hdaa_dump_amp(devinfo->dev, w->param.outamp_cap, "Output");
if (HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP(w->param.widget_cap) &&
w->param.inamp_cap != 0)
hdaa_dump_amp(devinfo->dev, w->param.inamp_cap, " Input");
if (w->nconns > 0)
device_printf(devinfo->dev, " Connections: %d\n", w->nconns);
for (j = 0; j < w->nconns; j++) {
cw = hdaa_widget_get(devinfo, w->conns[j]);
device_printf(devinfo->dev, " + %s<- nid=%d [%s]",
(w->connsenable[j] == 0)?"[DISABLED] ":"",
w->conns[j], (cw == NULL) ? "GHOST!" : cw->name);
if (cw == NULL)
printf(" [UNKNOWN]");
else if (cw->enable == 0)
printf(" [DISABLED]");
if (w->nconns > 1 && w->selconn == j && w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER)
printf(" (selected)");
printf("\n");
}
}
}
static void
hdaa_dump_dst_nid(struct hdaa_pcm_devinfo *pdevinfo, nid_t nid, int depth)
{
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_widget *w, *cw;
char buf[64];
int i;
if (depth > HDA_PARSE_MAXDEPTH)
return;
w = hdaa_widget_get(devinfo, nid);
if (w == NULL || w->enable == 0)
return;
if (depth == 0)
device_printf(pdevinfo->dev, "%*s", 4, "");
else
device_printf(pdevinfo->dev, "%*s + <- ", 4 + (depth - 1) * 7, "");
printf("nid=%d [%s]", w->nid, w->name);
if (depth > 0) {
if (w->ossmask == 0) {
printf("\n");
return;
}
printf(" [src: %s]",
hdaa_audio_ctl_ossmixer_mask2allname(
w->ossmask, buf, sizeof(buf)));
if (w->ossdev >= 0) {
printf("\n");
return;
}
}
printf("\n");
for (i = 0; i < w->nconns; i++) {
if (w->connsenable[i] == 0)
continue;
cw = hdaa_widget_get(devinfo, w->conns[i]);
if (cw == NULL || cw->enable == 0 || cw->bindas == -1)
continue;
hdaa_dump_dst_nid(pdevinfo, w->conns[i], depth + 1);
}
}
static void
hdaa_dump_dac(struct hdaa_pcm_devinfo *pdevinfo)
{
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_audio_as *as;
struct hdaa_widget *w;
nid_t *nids;
int chid, i;
if (pdevinfo->playas < 0)
return;
device_printf(pdevinfo->dev, "Playback:\n");
chid = devinfo->as[pdevinfo->playas].chans[0];
hdaa_dump_audio_formats(pdevinfo->dev,
devinfo->chans[chid].supp_stream_formats,
devinfo->chans[chid].supp_pcm_size_rate);
for (i = 0; i < devinfo->as[pdevinfo->playas].num_chans; i++) {
chid = devinfo->as[pdevinfo->playas].chans[i];
device_printf(pdevinfo->dev, " DAC:");
for (nids = devinfo->chans[chid].io; *nids != -1; nids++)
printf(" %d", *nids);
printf("\n");
}
as = &devinfo->as[pdevinfo->playas];
for (i = 0; i < 16; i++) {
if (as->pins[i] <= 0)
continue;
w = hdaa_widget_get(devinfo, as->pins[i]);
if (w == NULL || w->enable == 0)
continue;
device_printf(pdevinfo->dev, "\n");
hdaa_dump_dst_nid(pdevinfo, as->pins[i], 0);
}
device_printf(pdevinfo->dev, "\n");
}
static void
hdaa_dump_adc(struct hdaa_pcm_devinfo *pdevinfo)
{
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_widget *w;
nid_t *nids;
int chid, i;
if (pdevinfo->recas < 0)
return;
device_printf(pdevinfo->dev, "Record:\n");
chid = devinfo->as[pdevinfo->recas].chans[0];
hdaa_dump_audio_formats(pdevinfo->dev,
devinfo->chans[chid].supp_stream_formats,
devinfo->chans[chid].supp_pcm_size_rate);
for (i = 0; i < devinfo->as[pdevinfo->recas].num_chans; i++) {
chid = devinfo->as[pdevinfo->recas].chans[i];
device_printf(pdevinfo->dev, " ADC:");
for (nids = devinfo->chans[chid].io; *nids != -1; nids++)
printf(" %d", *nids);
printf("\n");
}
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->type != HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT)
continue;
if (w->bindas != pdevinfo->recas)
continue;
device_printf(pdevinfo->dev, "\n");
hdaa_dump_dst_nid(pdevinfo, i, 0);
}
device_printf(pdevinfo->dev, "\n");
}
static void
hdaa_dump_mix(struct hdaa_pcm_devinfo *pdevinfo)
{
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_widget *w;
int i;
int printed = 0;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0)
continue;
if (w->ossdev != SOUND_MIXER_IMIX)
continue;
if (w->bindas != pdevinfo->recas)
continue;
if (printed == 0) {
printed = 1;
device_printf(pdevinfo->dev, "Input Mix:\n");
}
device_printf(pdevinfo->dev, "\n");
hdaa_dump_dst_nid(pdevinfo, i, 0);
}
if (printed)
device_printf(pdevinfo->dev, "\n");
}
static void
hdaa_pindump(device_t dev)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
struct hdaa_widget *w;
uint32_t res, pincap, delay;
int i;
device_printf(dev, "Dumping AFG pins:\n");
device_printf(dev, "nid 0x as seq "
"device conn jack loc color misc\n");
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
hdaa_dump_pin_config(w, w->wclass.pin.config);
pincap = w->wclass.pin.cap;
device_printf(dev, " Caps: %2s %3s %2s %4s %4s",
HDA_PARAM_PIN_CAP_INPUT_CAP(pincap)?"IN":"",
HDA_PARAM_PIN_CAP_OUTPUT_CAP(pincap)?"OUT":"",
HDA_PARAM_PIN_CAP_HEADPHONE_CAP(pincap)?"HP":"",
HDA_PARAM_PIN_CAP_EAPD_CAP(pincap)?"EAPD":"",
HDA_PARAM_PIN_CAP_VREF_CTRL(pincap)?"VREF":"");
if (HDA_PARAM_PIN_CAP_IMP_SENSE_CAP(pincap) ||
HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP(pincap)) {
if (HDA_PARAM_PIN_CAP_TRIGGER_REQD(pincap)) {
delay = 0;
hda_command(dev,
HDA_CMD_SET_PIN_SENSE(0, w->nid, 0));
do {
res = hda_command(dev,
HDA_CMD_GET_PIN_SENSE(0, w->nid));
if (res != 0x7fffffff && res != 0xffffffff)
break;
DELAY(10);
} while (++delay < 10000);
} else {
delay = 0;
res = hda_command(dev, HDA_CMD_GET_PIN_SENSE(0,
w->nid));
}
printf(" Sense: 0x%08x (%sconnected%s)", res,
(res & HDA_CMD_GET_PIN_SENSE_PRESENCE_DETECT) ?
"" : "dis",
(HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(w->param.widget_cap) &&
(res & HDA_CMD_GET_PIN_SENSE_ELD_VALID)) ?
", ELD valid" : "");
if (delay > 0)
printf(" delay %dus", delay * 10);
}
printf("\n");
}
device_printf(dev,
"NumGPIO=%d NumGPO=%d NumGPI=%d GPIWake=%d GPIUnsol=%d\n",
HDA_PARAM_GPIO_COUNT_NUM_GPIO(devinfo->gpio_cap),
HDA_PARAM_GPIO_COUNT_NUM_GPO(devinfo->gpio_cap),
HDA_PARAM_GPIO_COUNT_NUM_GPI(devinfo->gpio_cap),
HDA_PARAM_GPIO_COUNT_GPI_WAKE(devinfo->gpio_cap),
HDA_PARAM_GPIO_COUNT_GPI_UNSOL(devinfo->gpio_cap));
hdaa_dump_gpi(devinfo);
hdaa_dump_gpio(devinfo);
hdaa_dump_gpo(devinfo);
}
static void
hdaa_configure(device_t dev)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
struct hdaa_audio_ctl *ctl;
int i;
HDA_BOOTHVERBOSE(
device_printf(dev, "Applying built-in patches...\n");
);
hdaa_patch(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Applying local patches...\n");
);
hdaa_local_patch(devinfo);
hdaa_audio_postprocess(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Parsing Ctls...\n");
);
hdaa_audio_ctl_parse(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Disabling nonaudio...\n");
);
hdaa_audio_disable_nonaudio(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Disabling useless...\n");
);
hdaa_audio_disable_useless(devinfo);
HDA_BOOTVERBOSE(
device_printf(dev, "Patched pins configuration:\n");
hdaa_dump_pin_configs(devinfo);
);
HDA_BOOTHVERBOSE(
device_printf(dev, "Parsing pin associations...\n");
);
hdaa_audio_as_parse(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Building AFG tree...\n");
);
hdaa_audio_build_tree(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Disabling unassociated "
"widgets...\n");
);
hdaa_audio_disable_unas(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Disabling nonselected "
"inputs...\n");
);
hdaa_audio_disable_notselected(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Disabling useless...\n");
);
hdaa_audio_disable_useless(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Disabling "
"crossassociatement connections...\n");
);
hdaa_audio_disable_crossas(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Disabling useless...\n");
);
hdaa_audio_disable_useless(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Binding associations to channels...\n");
);
hdaa_audio_bind_as(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Assigning names to signal sources...\n");
);
hdaa_audio_assign_names(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Preparing PCM devices...\n");
);
hdaa_prepare_pcms(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Assigning mixers to the tree...\n");
);
hdaa_audio_assign_mixers(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Preparing pin controls...\n");
);
hdaa_audio_prepare_pin_ctrl(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "AFG commit...\n");
);
hdaa_audio_commit(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Applying direct built-in patches...\n");
);
hdaa_patch_direct(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Pin sense init...\n");
);
hdaa_sense_init(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Creating PCM devices...\n");
);
hdaa_create_pcms(devinfo);
HDA_BOOTVERBOSE(
if (devinfo->quirks != 0) {
device_printf(dev, "FG config/quirks:");
for (i = 0; i < nitems(hdaa_quirks_tab); i++) {
if ((devinfo->quirks &
hdaa_quirks_tab[i].value) ==
hdaa_quirks_tab[i].value)
printf(" %s", hdaa_quirks_tab[i].key);
}
printf("\n");
}
);
HDA_BOOTHVERBOSE(
device_printf(dev, "\n");
device_printf(dev, "+-----------+\n");
device_printf(dev, "| HDA NODES |\n");
device_printf(dev, "+-----------+\n");
hdaa_dump_nodes(devinfo);
device_printf(dev, "\n");
device_printf(dev, "+----------------+\n");
device_printf(dev, "| HDA AMPLIFIERS |\n");
device_printf(dev, "+----------------+\n");
device_printf(dev, "\n");
i = 0;
while ((ctl = hdaa_audio_ctl_each(devinfo, &i)) != NULL) {
device_printf(dev, "%3d: nid %3d %s (%s) index %d", i,
(ctl->widget != NULL) ? ctl->widget->nid : -1,
(ctl->ndir == HDAA_CTL_IN)?"in ":"out",
(ctl->dir == HDAA_CTL_IN)?"in ":"out",
ctl->index);
if (ctl->childwidget != NULL)
printf(" cnid %3d", ctl->childwidget->nid);
else
printf(" ");
printf(" ossmask=0x%08x\n",
ctl->ossmask);
device_printf(dev,
" mute: %d step: %3d size: %3d off: %3d%s\n",
ctl->mute, ctl->step, ctl->size, ctl->offset,
(ctl->enable == 0) ? " [DISABLED]" :
((ctl->ossmask == 0) ? " [UNUSED]" : ""));
}
device_printf(dev, "\n");
);
}
static void
hdaa_unconfigure(device_t dev)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
struct hdaa_widget *w;
int i, j;
HDA_BOOTHVERBOSE(
device_printf(dev, "Pin sense deinit...\n");
);
hdaa_sense_deinit(devinfo);
free(devinfo->ctl, M_HDAA);
devinfo->ctl = NULL;
devinfo->ctlcnt = 0;
free(devinfo->as, M_HDAA);
devinfo->as = NULL;
devinfo->ascnt = 0;
free(devinfo->devs, M_HDAA);
devinfo->devs = NULL;
devinfo->num_devs = 0;
free(devinfo->chans, M_HDAA);
devinfo->chans = NULL;
devinfo->num_chans = 0;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL)
continue;
w->enable = 1;
w->selconn = -1;
w->pflags = 0;
w->bindas = -1;
w->bindseqmask = 0;
w->ossdev = -1;
w->ossmask = 0;
for (j = 0; j < w->nconns; j++)
w->connsenable[j] = 1;
if (w->type == HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
w->wclass.pin.config = w->wclass.pin.newconf;
if (w->eld != NULL) {
w->eld_len = 0;
free(w->eld, M_HDAA);
w->eld = NULL;
}
}
}
static int
hdaa_sysctl_gpi_state(SYSCTL_HANDLER_ARGS)
{
struct hdaa_devinfo *devinfo = oidp->oid_arg1;
device_t dev = devinfo->dev;
char buf[256];
int n = 0, i, numgpi;
uint32_t data = 0;
buf[0] = 0;
hdaa_lock(devinfo);
numgpi = HDA_PARAM_GPIO_COUNT_NUM_GPI(devinfo->gpio_cap);
if (numgpi > 0) {
data = hda_command(dev,
HDA_CMD_GET_GPI_DATA(0, devinfo->nid));
}
hdaa_unlock(devinfo);
for (i = 0; i < numgpi; i++) {
n += snprintf(buf + n, sizeof(buf) - n, "%s%d=%d",
n != 0 ? " " : "", i, ((data >> i) & 1));
}
return (sysctl_handle_string(oidp, buf, sizeof(buf), req));
}
static int
hdaa_sysctl_gpio_state(SYSCTL_HANDLER_ARGS)
{
struct hdaa_devinfo *devinfo = oidp->oid_arg1;
device_t dev = devinfo->dev;
char buf[256];
int n = 0, i, numgpio;
uint32_t data = 0, enable = 0, dir = 0;
buf[0] = 0;
hdaa_lock(devinfo);
numgpio = HDA_PARAM_GPIO_COUNT_NUM_GPIO(devinfo->gpio_cap);
if (numgpio > 0) {
data = hda_command(dev,
HDA_CMD_GET_GPIO_DATA(0, devinfo->nid));
enable = hda_command(dev,
HDA_CMD_GET_GPIO_ENABLE_MASK(0, devinfo->nid));
dir = hda_command(dev,
HDA_CMD_GET_GPIO_DIRECTION(0, devinfo->nid));
}
hdaa_unlock(devinfo);
for (i = 0; i < numgpio; i++) {
n += snprintf(buf + n, sizeof(buf) - n, "%s%d=",
n != 0 ? " " : "", i);
if ((enable & (1 << i)) == 0) {
n += snprintf(buf + n, sizeof(buf) - n, "disabled");
continue;
}
n += snprintf(buf + n, sizeof(buf) - n, "%sput(%d)",
((dir >> i) & 1) ? "out" : "in", ((data >> i) & 1));
}
return (sysctl_handle_string(oidp, buf, sizeof(buf), req));
}
static int
hdaa_sysctl_gpio_config(SYSCTL_HANDLER_ARGS)
{
struct hdaa_devinfo *devinfo = oidp->oid_arg1;
char buf[256];
int error, n = 0, i, numgpio;
uint32_t gpio, x;
gpio = devinfo->newgpio;
numgpio = HDA_PARAM_GPIO_COUNT_NUM_GPIO(devinfo->gpio_cap);
buf[0] = 0;
for (i = 0; i < numgpio; i++) {
x = (gpio & HDAA_GPIO_MASK(i)) >> HDAA_GPIO_SHIFT(i);
n += snprintf(buf + n, sizeof(buf) - n, "%s%d=%s",
n != 0 ? " " : "", i, HDA_GPIO_ACTIONS[x]);
}
error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
if (error != 0 || req->newptr == NULL)
return (error);
if (strncmp(buf, "0x", 2) == 0)
gpio = strtol(buf + 2, NULL, 16);
else
gpio = hdaa_gpio_patch(gpio, buf);
hdaa_lock(devinfo);
devinfo->newgpio = devinfo->gpio = gpio;
hdaa_gpio_commit(devinfo);
hdaa_unlock(devinfo);
return (0);
}
static int
hdaa_sysctl_gpo_state(SYSCTL_HANDLER_ARGS)
{
struct hdaa_devinfo *devinfo = oidp->oid_arg1;
device_t dev = devinfo->dev;
char buf[256];
int n = 0, i, numgpo;
uint32_t data = 0;
buf[0] = 0;
hdaa_lock(devinfo);
numgpo = HDA_PARAM_GPIO_COUNT_NUM_GPO(devinfo->gpio_cap);
if (numgpo > 0) {
data = hda_command(dev,
HDA_CMD_GET_GPO_DATA(0, devinfo->nid));
}
hdaa_unlock(devinfo);
for (i = 0; i < numgpo; i++) {
n += snprintf(buf + n, sizeof(buf) - n, "%s%d=%d",
n != 0 ? " " : "", i, ((data >> i) & 1));
}
return (sysctl_handle_string(oidp, buf, sizeof(buf), req));
}
static int
hdaa_sysctl_gpo_config(SYSCTL_HANDLER_ARGS)
{
struct hdaa_devinfo *devinfo = oidp->oid_arg1;
char buf[256];
int error, n = 0, i, numgpo;
uint32_t gpo, x;
gpo = devinfo->newgpo;
numgpo = HDA_PARAM_GPIO_COUNT_NUM_GPO(devinfo->gpio_cap);
buf[0] = 0;
for (i = 0; i < numgpo; i++) {
x = (gpo & HDAA_GPIO_MASK(i)) >> HDAA_GPIO_SHIFT(i);
n += snprintf(buf + n, sizeof(buf) - n, "%s%d=%s",
n != 0 ? " " : "", i, HDA_GPIO_ACTIONS[x]);
}
error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
if (error != 0 || req->newptr == NULL)
return (error);
if (strncmp(buf, "0x", 2) == 0)
gpo = strtol(buf + 2, NULL, 16);
else
gpo = hdaa_gpio_patch(gpo, buf);
hdaa_lock(devinfo);
devinfo->newgpo = devinfo->gpo = gpo;
hdaa_gpo_commit(devinfo);
hdaa_unlock(devinfo);
return (0);
}
static int
hdaa_sysctl_reconfig(SYSCTL_HANDLER_ARGS)
{
device_t dev;
struct hdaa_devinfo *devinfo;
int error, val;
dev = oidp->oid_arg1;
devinfo = device_get_softc(dev);
if (devinfo == NULL)
return (EINVAL);
val = 0;
error = sysctl_handle_int(oidp, &val, 0, req);
if (error != 0 || req->newptr == NULL || val == 0)
return (error);
HDA_BOOTHVERBOSE(
device_printf(dev, "Reconfiguration...\n");
);
if ((error = device_delete_children(dev)) != 0)
return (error);
hdaa_lock(devinfo);
hdaa_unconfigure(dev);
hdaa_configure(dev);
hdaa_unlock(devinfo);
bus_generic_attach(dev);
HDA_BOOTHVERBOSE(
device_printf(dev, "Reconfiguration done\n");
);
return (0);
}
static int
hdaa_suspend(device_t dev)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
int i;
HDA_BOOTHVERBOSE(
device_printf(dev, "Suspend...\n");
);
hdaa_lock(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Stop streams...\n");
);
for (i = 0; i < devinfo->num_chans; i++) {
if (devinfo->chans[i].flags & HDAA_CHN_RUNNING) {
devinfo->chans[i].flags |= HDAA_CHN_SUSPEND;
hdaa_channel_stop(&devinfo->chans[i]);
}
}
HDA_BOOTHVERBOSE(
device_printf(dev, "Power down FG"
" nid=%d to the D3 state...\n",
devinfo->nid);
);
hda_command(devinfo->dev,
HDA_CMD_SET_POWER_STATE(0,
devinfo->nid, HDA_CMD_POWER_STATE_D3));
callout_stop(&devinfo->poll_jack);
hdaa_unlock(devinfo);
callout_drain(&devinfo->poll_jack);
HDA_BOOTHVERBOSE(
device_printf(dev, "Suspend done\n");
);
return (0);
}
static int
hdaa_resume(device_t dev)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
int i;
HDA_BOOTHVERBOSE(
device_printf(dev, "Resume...\n");
);
hdaa_lock(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Power up audio FG nid=%d...\n",
devinfo->nid);
);
hdaa_powerup(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "AFG commit...\n");
);
hdaa_audio_commit(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Applying direct built-in patches...\n");
);
hdaa_patch_direct(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Pin sense init...\n");
);
hdaa_sense_init(devinfo);
hdaa_unlock(devinfo);
for (i = 0; i < devinfo->num_devs; i++) {
struct hdaa_pcm_devinfo *pdevinfo = &devinfo->devs[i];
HDA_BOOTHVERBOSE(
device_printf(pdevinfo->dev,
"OSS mixer reinitialization...\n");
);
if (mixer_reinit(pdevinfo->dev) == -1)
device_printf(pdevinfo->dev,
"unable to reinitialize the mixer\n");
}
hdaa_lock(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Start streams...\n");
);
for (i = 0; i < devinfo->num_chans; i++) {
if (devinfo->chans[i].flags & HDAA_CHN_SUSPEND) {
devinfo->chans[i].flags &= ~HDAA_CHN_SUSPEND;
hdaa_channel_start(&devinfo->chans[i]);
}
}
hdaa_unlock(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Resume done\n");
);
return (0);
}
static int
hdaa_probe(device_t dev)
{
const char *pdesc;
char buf[128];
if (hda_get_node_type(dev) != HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_AUDIO)
return (ENXIO);
pdesc = device_get_desc(device_get_parent(dev));
snprintf(buf, sizeof(buf), "%.*s Audio Function Group",
(int)(strlen(pdesc) - 10), pdesc);
device_set_desc_copy(dev, buf);
return (BUS_PROBE_DEFAULT);
}
static int
hdaa_attach(device_t dev)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
uint32_t res;
nid_t nid = hda_get_node_id(dev);
devinfo->dev = dev;
devinfo->lock = HDAC_GET_MTX(device_get_parent(dev), dev);
devinfo->nid = nid;
devinfo->newquirks = -1;
devinfo->newgpio = -1;
devinfo->newgpo = -1;
- callout_init(&devinfo->poll_jack, CALLOUT_MPSAFE);
+ callout_init(&devinfo->poll_jack, 1);
devinfo->poll_ival = hz;
hdaa_lock(devinfo);
res = hda_command(dev,
HDA_CMD_GET_PARAMETER(0 , nid, HDA_PARAM_SUB_NODE_COUNT));
hdaa_unlock(devinfo);
devinfo->nodecnt = HDA_PARAM_SUB_NODE_COUNT_TOTAL(res);
devinfo->startnode = HDA_PARAM_SUB_NODE_COUNT_START(res);
devinfo->endnode = devinfo->startnode + devinfo->nodecnt;
HDA_BOOTVERBOSE(
device_printf(dev, "Subsystem ID: 0x%08x\n",
hda_get_subsystem_id(dev));
);
HDA_BOOTHVERBOSE(
device_printf(dev,
"Audio Function Group at nid=%d: %d subnodes %d-%d\n",
nid, devinfo->nodecnt,
devinfo->startnode, devinfo->endnode - 1);
);
if (devinfo->nodecnt > 0)
devinfo->widget = (struct hdaa_widget *)malloc(
sizeof(*(devinfo->widget)) * devinfo->nodecnt, M_HDAA,
M_WAITOK | M_ZERO);
else
devinfo->widget = NULL;
hdaa_lock(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Powering up...\n");
);
hdaa_powerup(devinfo);
HDA_BOOTHVERBOSE(
device_printf(dev, "Parsing audio FG...\n");
);
hdaa_audio_parse(devinfo);
HDA_BOOTVERBOSE(
device_printf(dev, "Original pins configuration:\n");
hdaa_dump_pin_configs(devinfo);
);
hdaa_configure(dev);
hdaa_unlock(devinfo);
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"config", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
&devinfo->newquirks, sizeof(&devinfo->newquirks),
hdaa_sysctl_quirks, "A", "Configuration options");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"gpi_state", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
devinfo, sizeof(devinfo),
hdaa_sysctl_gpi_state, "A", "GPI state");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"gpio_state", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
devinfo, sizeof(devinfo),
hdaa_sysctl_gpio_state, "A", "GPIO state");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"gpio_config", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
devinfo, sizeof(devinfo),
hdaa_sysctl_gpio_config, "A", "GPIO configuration");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"gpo_state", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
devinfo, sizeof(devinfo),
hdaa_sysctl_gpo_state, "A", "GPO state");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"gpo_config", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
devinfo, sizeof(devinfo),
hdaa_sysctl_gpo_config, "A", "GPO configuration");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"reconfig", CTLTYPE_INT | CTLFLAG_RW,
dev, sizeof(dev),
hdaa_sysctl_reconfig, "I", "Reprocess configuration");
bus_generic_attach(dev);
return (0);
}
static int
hdaa_detach(device_t dev)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
int error;
if ((error = device_delete_children(dev)) != 0)
return (error);
hdaa_lock(devinfo);
hdaa_unconfigure(dev);
devinfo->poll_ival = 0;
callout_stop(&devinfo->poll_jack);
hdaa_unlock(devinfo);
callout_drain(&devinfo->poll_jack);
free(devinfo->widget, M_HDAA);
return (0);
}
static int
hdaa_print_child(device_t dev, device_t child)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
struct hdaa_pcm_devinfo *pdevinfo =
(struct hdaa_pcm_devinfo *)device_get_ivars(child);
struct hdaa_audio_as *as;
int retval, first = 1, i;
retval = bus_print_child_header(dev, child);
retval += printf(" at nid ");
if (pdevinfo->playas >= 0) {
as = &devinfo->as[pdevinfo->playas];
for (i = 0; i < 16; i++) {
if (as->pins[i] <= 0)
continue;
retval += printf("%s%d", first ? "" : ",", as->pins[i]);
first = 0;
}
}
if (pdevinfo->recas >= 0) {
if (pdevinfo->playas >= 0) {
retval += printf(" and ");
first = 1;
}
as = &devinfo->as[pdevinfo->recas];
for (i = 0; i < 16; i++) {
if (as->pins[i] <= 0)
continue;
retval += printf("%s%d", first ? "" : ",", as->pins[i]);
first = 0;
}
}
retval += bus_print_child_footer(dev, child);
return (retval);
}
static int
hdaa_child_location_str(device_t dev, device_t child, char *buf,
size_t buflen)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
struct hdaa_pcm_devinfo *pdevinfo =
(struct hdaa_pcm_devinfo *)device_get_ivars(child);
struct hdaa_audio_as *as;
int first = 1, i, len = 0;
len += snprintf(buf + len, buflen - len, "nid=");
if (pdevinfo->playas >= 0) {
as = &devinfo->as[pdevinfo->playas];
for (i = 0; i < 16; i++) {
if (as->pins[i] <= 0)
continue;
len += snprintf(buf + len, buflen - len,
"%s%d", first ? "" : ",", as->pins[i]);
first = 0;
}
}
if (pdevinfo->recas >= 0) {
as = &devinfo->as[pdevinfo->recas];
for (i = 0; i < 16; i++) {
if (as->pins[i] <= 0)
continue;
len += snprintf(buf + len, buflen - len,
"%s%d", first ? "" : ",", as->pins[i]);
first = 0;
}
}
return (0);
}
static void
hdaa_stream_intr(device_t dev, int dir, int stream)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
struct hdaa_chan *ch;
int i;
for (i = 0; i < devinfo->num_chans; i++) {
ch = &devinfo->chans[i];
if (!(ch->flags & HDAA_CHN_RUNNING))
continue;
if (ch->dir == ((dir == 1) ? PCMDIR_PLAY : PCMDIR_REC) &&
ch->sid == stream) {
hdaa_unlock(devinfo);
chn_intr(ch->c);
hdaa_lock(devinfo);
}
}
}
static void
hdaa_unsol_intr(device_t dev, uint32_t resp)
{
struct hdaa_devinfo *devinfo = device_get_softc(dev);
struct hdaa_widget *w;
int i, tag, flags;
HDA_BOOTHVERBOSE(
device_printf(dev, "Unsolicited response %08x\n", resp);
);
tag = resp >> 26;
for (i = devinfo->startnode; i < devinfo->endnode; i++) {
w = hdaa_widget_get(devinfo, i);
if (w == NULL || w->enable == 0 || w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
if (w->unsol != tag)
continue;
if (HDA_PARAM_PIN_CAP_DP(w->wclass.pin.cap) ||
HDA_PARAM_PIN_CAP_HDMI(w->wclass.pin.cap))
flags = resp & 0x03;
else
flags = 0x01;
if (flags & 0x01)
hdaa_presence_handler(w);
if (flags & 0x02)
hdaa_eld_handler(w);
}
}
static device_method_t hdaa_methods[] = {
/* device interface */
DEVMETHOD(device_probe, hdaa_probe),
DEVMETHOD(device_attach, hdaa_attach),
DEVMETHOD(device_detach, hdaa_detach),
DEVMETHOD(device_suspend, hdaa_suspend),
DEVMETHOD(device_resume, hdaa_resume),
/* Bus interface */
DEVMETHOD(bus_print_child, hdaa_print_child),
DEVMETHOD(bus_child_location_str, hdaa_child_location_str),
DEVMETHOD(hdac_stream_intr, hdaa_stream_intr),
DEVMETHOD(hdac_unsol_intr, hdaa_unsol_intr),
DEVMETHOD(hdac_pindump, hdaa_pindump),
DEVMETHOD_END
};
static driver_t hdaa_driver = {
"hdaa",
hdaa_methods,
sizeof(struct hdaa_devinfo),
};
static devclass_t hdaa_devclass;
DRIVER_MODULE(snd_hda, hdacc, hdaa_driver, hdaa_devclass, NULL, NULL);
static void
hdaa_chan_formula(struct hdaa_devinfo *devinfo, int asid,
char *buf, int buflen)
{
struct hdaa_audio_as *as;
int c;
as = &devinfo->as[asid];
c = devinfo->chans[as->chans[0]].channels;
if (c == 1)
snprintf(buf, buflen, "mono");
else if (c == 2) {
if (as->hpredir < 0)
buf[0] = 0;
else
snprintf(buf, buflen, "2.0");
} else if (as->pinset == 0x0003)
snprintf(buf, buflen, "3.1");
else if (as->pinset == 0x0005 || as->pinset == 0x0011)
snprintf(buf, buflen, "4.0");
else if (as->pinset == 0x0007 || as->pinset == 0x0013)
snprintf(buf, buflen, "5.1");
else if (as->pinset == 0x0017)
snprintf(buf, buflen, "7.1");
else
snprintf(buf, buflen, "%dch", c);
if (as->hpredir >= 0)
strlcat(buf, "+HP", buflen);
}
static int
hdaa_chan_type(struct hdaa_devinfo *devinfo, int asid)
{
struct hdaa_audio_as *as;
struct hdaa_widget *w;
int i, t = -1, t1;
as = &devinfo->as[asid];
for (i = 0; i < 16; i++) {
w = hdaa_widget_get(devinfo, as->pins[i]);
if (w == NULL || w->enable == 0 || w->type !=
HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX)
continue;
t1 = HDA_CONFIG_DEFAULTCONF_DEVICE(w->wclass.pin.config);
if (t == -1)
t = t1;
else if (t != t1) {
t = -2;
break;
}
}
return (t);
}
static int
hdaa_sysctl_32bit(SYSCTL_HANDLER_ARGS)
{
struct hdaa_audio_as *as = (struct hdaa_audio_as *)oidp->oid_arg1;
struct hdaa_pcm_devinfo *pdevinfo = as->pdevinfo;
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_chan *ch;
int error, val, i;
uint32_t pcmcap;
ch = &devinfo->chans[as->chans[0]];
val = (ch->bit32 == 4) ? 32 : ((ch->bit32 == 3) ? 24 :
((ch->bit32 == 2) ? 20 : 0));
error = sysctl_handle_int(oidp, &val, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
pcmcap = ch->supp_pcm_size_rate;
if (val == 32 && HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT(pcmcap))
ch->bit32 = 4;
else if (val == 24 && HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT(pcmcap))
ch->bit32 = 3;
else if (val == 20 && HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT(pcmcap))
ch->bit32 = 2;
else
return (EINVAL);
for (i = 1; i < as->num_chans; i++)
devinfo->chans[as->chans[i]].bit32 = ch->bit32;
return (0);
}
static int
hdaa_pcm_probe(device_t dev)
{
struct hdaa_pcm_devinfo *pdevinfo =
(struct hdaa_pcm_devinfo *)device_get_ivars(dev);
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
const char *pdesc;
char chans1[8], chans2[8];
char buf[128];
int loc1, loc2, t1, t2;
if (pdevinfo->playas >= 0)
loc1 = devinfo->as[pdevinfo->playas].location;
else
loc1 = devinfo->as[pdevinfo->recas].location;
if (pdevinfo->recas >= 0)
loc2 = devinfo->as[pdevinfo->recas].location;
else
loc2 = loc1;
if (loc1 != loc2)
loc1 = -2;
if (loc1 >= 0 && HDA_LOCS[loc1][0] == '0')
loc1 = -2;
chans1[0] = 0;
chans2[0] = 0;
t1 = t2 = -1;
if (pdevinfo->playas >= 0) {
hdaa_chan_formula(devinfo, pdevinfo->playas,
chans1, sizeof(chans1));
t1 = hdaa_chan_type(devinfo, pdevinfo->playas);
}
if (pdevinfo->recas >= 0) {
hdaa_chan_formula(devinfo, pdevinfo->recas,
chans2, sizeof(chans2));
t2 = hdaa_chan_type(devinfo, pdevinfo->recas);
}
if (chans1[0] != 0 || chans2[0] != 0) {
if (chans1[0] == 0 && pdevinfo->playas >= 0)
snprintf(chans1, sizeof(chans1), "2.0");
else if (chans2[0] == 0 && pdevinfo->recas >= 0)
snprintf(chans2, sizeof(chans2), "2.0");
if (strcmp(chans1, chans2) == 0)
chans2[0] = 0;
}
if (t1 == -1)
t1 = t2;
else if (t2 == -1)
t2 = t1;
if (t1 != t2)
t1 = -2;
if (pdevinfo->digital)
t1 = -2;
pdesc = device_get_desc(device_get_parent(dev));
snprintf(buf, sizeof(buf), "%.*s (%s%s%s%s%s%s%s%s%s)",
(int)(strlen(pdesc) - 21), pdesc,
loc1 >= 0 ? HDA_LOCS[loc1] : "", loc1 >= 0 ? " " : "",
(pdevinfo->digital == 0x7)?"HDMI/DP":
((pdevinfo->digital == 0x5)?"DisplayPort":
((pdevinfo->digital == 0x3)?"HDMI":
((pdevinfo->digital)?"Digital":"Analog"))),
chans1[0] ? " " : "", chans1,
chans2[0] ? "/" : "", chans2,
t1 >= 0 ? " " : "", t1 >= 0 ? HDA_DEVS[t1] : "");
device_set_desc_copy(dev, buf);
return (BUS_PROBE_SPECIFIC);
}
static int
hdaa_pcm_attach(device_t dev)
{
struct hdaa_pcm_devinfo *pdevinfo =
(struct hdaa_pcm_devinfo *)device_get_ivars(dev);
struct hdaa_devinfo *devinfo = pdevinfo->devinfo;
struct hdaa_audio_as *as;
struct snddev_info *d;
char status[SND_STATUSLEN];
int i;
pdevinfo->chan_size = pcm_getbuffersize(dev,
HDA_BUFSZ_MIN, HDA_BUFSZ_DEFAULT, HDA_BUFSZ_MAX);
HDA_BOOTVERBOSE(
hdaa_dump_dac(pdevinfo);
hdaa_dump_adc(pdevinfo);
hdaa_dump_mix(pdevinfo);
hdaa_dump_ctls(pdevinfo, "Master Volume", SOUND_MASK_VOLUME);
hdaa_dump_ctls(pdevinfo, "PCM Volume", SOUND_MASK_PCM);
hdaa_dump_ctls(pdevinfo, "CD Volume", SOUND_MASK_CD);
hdaa_dump_ctls(pdevinfo, "Microphone Volume", SOUND_MASK_MIC);
hdaa_dump_ctls(pdevinfo, "Microphone2 Volume", SOUND_MASK_MONITOR);
hdaa_dump_ctls(pdevinfo, "Line-in Volume", SOUND_MASK_LINE);
hdaa_dump_ctls(pdevinfo, "Speaker/Beep Volume", SOUND_MASK_SPEAKER);
hdaa_dump_ctls(pdevinfo, "Recording Level", SOUND_MASK_RECLEV);
hdaa_dump_ctls(pdevinfo, "Input Mix Level", SOUND_MASK_IMIX);
hdaa_dump_ctls(pdevinfo, "Input Monitoring Level", SOUND_MASK_IGAIN);
hdaa_dump_ctls(pdevinfo, NULL, 0);
);
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "blocksize", &i) == 0 && i > 0) {
i &= HDA_BLK_ALIGN;
if (i < HDA_BLK_MIN)
i = HDA_BLK_MIN;
pdevinfo->chan_blkcnt = pdevinfo->chan_size / i;
i = 0;
while (pdevinfo->chan_blkcnt >> i)
i++;
pdevinfo->chan_blkcnt = 1 << (i - 1);
if (pdevinfo->chan_blkcnt < HDA_BDL_MIN)
pdevinfo->chan_blkcnt = HDA_BDL_MIN;
else if (pdevinfo->chan_blkcnt > HDA_BDL_MAX)
pdevinfo->chan_blkcnt = HDA_BDL_MAX;
} else
pdevinfo->chan_blkcnt = HDA_BDL_DEFAULT;
/*
* We don't register interrupt handler with snd_setup_intr
* in pcm device. Mark pcm device as MPSAFE manually.
*/
pcm_setflags(dev, pcm_getflags(dev) | SD_F_MPSAFE);
HDA_BOOTHVERBOSE(
device_printf(dev, "OSS mixer initialization...\n");
);
if (mixer_init(dev, &hdaa_audio_ctl_ossmixer_class, pdevinfo) != 0)
device_printf(dev, "Can't register mixer\n");
HDA_BOOTHVERBOSE(
device_printf(dev, "Registering PCM channels...\n");
);
if (pcm_register(dev, pdevinfo, (pdevinfo->playas >= 0)?1:0,
(pdevinfo->recas >= 0)?1:0) != 0)
device_printf(dev, "Can't register PCM\n");
pdevinfo->registered++;
d = device_get_softc(dev);
if (pdevinfo->playas >= 0) {
as = &devinfo->as[pdevinfo->playas];
for (i = 0; i < as->num_chans; i++)
pcm_addchan(dev, PCMDIR_PLAY, &hdaa_channel_class,
&devinfo->chans[as->chans[i]]);
SYSCTL_ADD_PROC(&d->play_sysctl_ctx,
SYSCTL_CHILDREN(d->play_sysctl_tree), OID_AUTO,
"32bit", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
as, sizeof(as), hdaa_sysctl_32bit, "I",
"Resolution of 32bit samples (20/24/32bit)");
}
if (pdevinfo->recas >= 0) {
as = &devinfo->as[pdevinfo->recas];
for (i = 0; i < as->num_chans; i++)
pcm_addchan(dev, PCMDIR_REC, &hdaa_channel_class,
&devinfo->chans[as->chans[i]]);
SYSCTL_ADD_PROC(&d->rec_sysctl_ctx,
SYSCTL_CHILDREN(d->rec_sysctl_tree), OID_AUTO,
"32bit", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
as, sizeof(as), hdaa_sysctl_32bit, "I",
"Resolution of 32bit samples (20/24/32bit)");
pdevinfo->autorecsrc = 2;
resource_int_value(device_get_name(dev), device_get_unit(dev),
"rec.autosrc", &pdevinfo->autorecsrc);
SYSCTL_ADD_INT(&d->rec_sysctl_ctx,
SYSCTL_CHILDREN(d->rec_sysctl_tree), OID_AUTO,
"autosrc", CTLFLAG_RW,
&pdevinfo->autorecsrc, 0,
"Automatic recording source selection");
}
if (pdevinfo->mixer != NULL) {
hdaa_audio_ctl_set_defaults(pdevinfo);
hdaa_lock(devinfo);
if (pdevinfo->playas >= 0) {
as = &devinfo->as[pdevinfo->playas];
hdaa_channels_handler(as);
}
if (pdevinfo->recas >= 0) {
as = &devinfo->as[pdevinfo->recas];
hdaa_autorecsrc_handler(as, NULL);
hdaa_channels_handler(as);
}
hdaa_unlock(devinfo);
}
snprintf(status, SND_STATUSLEN, "on %s %s",
device_get_nameunit(device_get_parent(dev)),
PCM_KLDSTRING(snd_hda));
pcm_setstatus(dev, status);
return (0);
}
static int
hdaa_pcm_detach(device_t dev)
{
struct hdaa_pcm_devinfo *pdevinfo =
(struct hdaa_pcm_devinfo *)device_get_ivars(dev);
int err;
if (pdevinfo->registered > 0) {
err = pcm_unregister(dev);
if (err != 0)
return (err);
}
return (0);
}
static device_method_t hdaa_pcm_methods[] = {
/* device interface */
DEVMETHOD(device_probe, hdaa_pcm_probe),
DEVMETHOD(device_attach, hdaa_pcm_attach),
DEVMETHOD(device_detach, hdaa_pcm_detach),
DEVMETHOD_END
};
static driver_t hdaa_pcm_driver = {
"pcm",
hdaa_pcm_methods,
PCM_SOFTC_SIZE,
};
DRIVER_MODULE(snd_hda_pcm, hdaa, hdaa_pcm_driver, pcm_devclass, NULL, NULL);
MODULE_DEPEND(snd_hda, sound, SOUND_MINVER, SOUND_PREFVER, SOUND_MAXVER);
MODULE_VERSION(snd_hda, 1);
Index: head/sys/dev/sound/pci/hda/hdac.c
===================================================================
--- head/sys/dev/sound/pci/hda/hdac.c (revision 283290)
+++ head/sys/dev/sound/pci/hda/hdac.c (revision 283291)
@@ -1,2090 +1,2090 @@
/*-
* Copyright (c) 2006 Stephane E. Potvin <sepotvin@videotron.ca>
* Copyright (c) 2006 Ariff Abdullah <ariff@FreeBSD.org>
* Copyright (c) 2008-2012 Alexander Motin <mav@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Intel High Definition Audio (Controller) driver for FreeBSD.
*/
#ifdef HAVE_KERNEL_OPTION_HEADERS
#include "opt_snd.h"
#endif
#include <dev/sound/pcm/sound.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <sys/ctype.h>
#include <sys/taskqueue.h>
#include <dev/sound/pci/hda/hdac_private.h>
#include <dev/sound/pci/hda/hdac_reg.h>
#include <dev/sound/pci/hda/hda_reg.h>
#include <dev/sound/pci/hda/hdac.h>
#define HDA_DRV_TEST_REV "20120126_0002"
SND_DECLARE_FILE("$FreeBSD$");
#define hdac_lock(sc) snd_mtxlock((sc)->lock)
#define hdac_unlock(sc) snd_mtxunlock((sc)->lock)
#define hdac_lockassert(sc) snd_mtxassert((sc)->lock)
#define hdac_lockowned(sc) mtx_owned((sc)->lock)
#define HDAC_QUIRK_64BIT (1 << 0)
#define HDAC_QUIRK_DMAPOS (1 << 1)
#define HDAC_QUIRK_MSI (1 << 2)
static const struct {
const char *key;
uint32_t value;
} hdac_quirks_tab[] = {
{ "64bit", HDAC_QUIRK_DMAPOS },
{ "dmapos", HDAC_QUIRK_DMAPOS },
{ "msi", HDAC_QUIRK_MSI },
};
MALLOC_DEFINE(M_HDAC, "hdac", "HDA Controller");
static const struct {
uint32_t model;
const char *desc;
char quirks_on;
char quirks_off;
} hdac_devices[] = {
{ HDA_INTEL_OAK, "Intel Oaktrail", 0, 0 },
{ HDA_INTEL_BAY, "Intel BayTrail", 0, 0 },
{ HDA_INTEL_HSW1, "Intel Haswell", 0, 0 },
{ HDA_INTEL_HSW2, "Intel Haswell", 0, 0 },
{ HDA_INTEL_HSW3, "Intel Haswell", 0, 0 },
{ HDA_INTEL_BDW1, "Intel Broadwell", 0, 0 },
{ HDA_INTEL_BDW2, "Intel Broadwell", 0, 0 },
{ HDA_INTEL_CPT, "Intel Cougar Point", 0, 0 },
{ HDA_INTEL_PATSBURG,"Intel Patsburg", 0, 0 },
{ HDA_INTEL_PPT1, "Intel Panther Point", 0, 0 },
{ HDA_INTEL_LPT1, "Intel Lynx Point", 0, 0 },
{ HDA_INTEL_LPT2, "Intel Lynx Point", 0, 0 },
{ HDA_INTEL_WCPT, "Intel Wildcat Point", 0, 0 },
{ HDA_INTEL_WELLS1, "Intel Wellsburg", 0, 0 },
{ HDA_INTEL_WELLS2, "Intel Wellsburg", 0, 0 },
{ HDA_INTEL_LPTLP1, "Intel Lynx Point-LP", 0, 0 },
{ HDA_INTEL_LPTLP2, "Intel Lynx Point-LP", 0, 0 },
{ HDA_INTEL_82801F, "Intel 82801F", 0, 0 },
{ HDA_INTEL_63XXESB, "Intel 631x/632xESB", 0, 0 },
{ HDA_INTEL_82801G, "Intel 82801G", 0, 0 },
{ HDA_INTEL_82801H, "Intel 82801H", 0, 0 },
{ HDA_INTEL_82801I, "Intel 82801I", 0, 0 },
{ HDA_INTEL_82801JI, "Intel 82801JI", 0, 0 },
{ HDA_INTEL_82801JD, "Intel 82801JD", 0, 0 },
{ HDA_INTEL_PCH, "Intel 5 Series/3400 Series", 0, 0 },
{ HDA_INTEL_PCH2, "Intel 5 Series/3400 Series", 0, 0 },
{ HDA_INTEL_SCH, "Intel SCH", 0, 0 },
{ HDA_NVIDIA_MCP51, "NVIDIA MCP51", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_MCP55, "NVIDIA MCP55", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_MCP61_1, "NVIDIA MCP61", 0, 0 },
{ HDA_NVIDIA_MCP61_2, "NVIDIA MCP61", 0, 0 },
{ HDA_NVIDIA_MCP65_1, "NVIDIA MCP65", 0, 0 },
{ HDA_NVIDIA_MCP65_2, "NVIDIA MCP65", 0, 0 },
{ HDA_NVIDIA_MCP67_1, "NVIDIA MCP67", 0, 0 },
{ HDA_NVIDIA_MCP67_2, "NVIDIA MCP67", 0, 0 },
{ HDA_NVIDIA_MCP73_1, "NVIDIA MCP73", 0, 0 },
{ HDA_NVIDIA_MCP73_2, "NVIDIA MCP73", 0, 0 },
{ HDA_NVIDIA_MCP78_1, "NVIDIA MCP78", 0, HDAC_QUIRK_64BIT },
{ HDA_NVIDIA_MCP78_2, "NVIDIA MCP78", 0, HDAC_QUIRK_64BIT },
{ HDA_NVIDIA_MCP78_3, "NVIDIA MCP78", 0, HDAC_QUIRK_64BIT },
{ HDA_NVIDIA_MCP78_4, "NVIDIA MCP78", 0, HDAC_QUIRK_64BIT },
{ HDA_NVIDIA_MCP79_1, "NVIDIA MCP79", 0, 0 },
{ HDA_NVIDIA_MCP79_2, "NVIDIA MCP79", 0, 0 },
{ HDA_NVIDIA_MCP79_3, "NVIDIA MCP79", 0, 0 },
{ HDA_NVIDIA_MCP79_4, "NVIDIA MCP79", 0, 0 },
{ HDA_NVIDIA_MCP89_1, "NVIDIA MCP89", 0, 0 },
{ HDA_NVIDIA_MCP89_2, "NVIDIA MCP89", 0, 0 },
{ HDA_NVIDIA_MCP89_3, "NVIDIA MCP89", 0, 0 },
{ HDA_NVIDIA_MCP89_4, "NVIDIA MCP89", 0, 0 },
{ HDA_NVIDIA_0BE2, "NVIDIA (0x0be2)", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_0BE3, "NVIDIA (0x0be3)", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_0BE4, "NVIDIA (0x0be4)", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_GT100, "NVIDIA GT100", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_GT104, "NVIDIA GT104", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_GT106, "NVIDIA GT106", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_GT108, "NVIDIA GT108", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_GT116, "NVIDIA GT116", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_GF119, "NVIDIA GF119", 0, 0 },
{ HDA_NVIDIA_GF110_1, "NVIDIA GF110", 0, HDAC_QUIRK_MSI },
{ HDA_NVIDIA_GF110_2, "NVIDIA GF110", 0, HDAC_QUIRK_MSI },
{ HDA_ATI_SB450, "ATI SB450", 0, 0 },
{ HDA_ATI_SB600, "ATI SB600", 0, 0 },
{ HDA_ATI_RS600, "ATI RS600", 0, 0 },
{ HDA_ATI_RS690, "ATI RS690", 0, 0 },
{ HDA_ATI_RS780, "ATI RS780", 0, 0 },
{ HDA_ATI_R600, "ATI R600", 0, 0 },
{ HDA_ATI_RV610, "ATI RV610", 0, 0 },
{ HDA_ATI_RV620, "ATI RV620", 0, 0 },
{ HDA_ATI_RV630, "ATI RV630", 0, 0 },
{ HDA_ATI_RV635, "ATI RV635", 0, 0 },
{ HDA_ATI_RV710, "ATI RV710", 0, 0 },
{ HDA_ATI_RV730, "ATI RV730", 0, 0 },
{ HDA_ATI_RV740, "ATI RV740", 0, 0 },
{ HDA_ATI_RV770, "ATI RV770", 0, 0 },
{ HDA_ATI_RV810, "ATI RV810", 0, 0 },
{ HDA_ATI_RV830, "ATI RV830", 0, 0 },
{ HDA_ATI_RV840, "ATI RV840", 0, 0 },
{ HDA_ATI_RV870, "ATI RV870", 0, 0 },
{ HDA_ATI_RV910, "ATI RV910", 0, 0 },
{ HDA_ATI_RV930, "ATI RV930", 0, 0 },
{ HDA_ATI_RV940, "ATI RV940", 0, 0 },
{ HDA_ATI_RV970, "ATI RV970", 0, 0 },
{ HDA_ATI_R1000, "ATI R1000", 0, 0 },
{ HDA_RDC_M3010, "RDC M3010", 0, 0 },
{ HDA_VIA_VT82XX, "VIA VT8251/8237A",0, 0 },
{ HDA_SIS_966, "SiS 966", 0, 0 },
{ HDA_ULI_M5461, "ULI M5461", 0, 0 },
/* Unknown */
{ HDA_INTEL_ALL, "Intel", 0, 0 },
{ HDA_NVIDIA_ALL, "NVIDIA", 0, 0 },
{ HDA_ATI_ALL, "ATI", 0, 0 },
{ HDA_VIA_ALL, "VIA", 0, 0 },
{ HDA_SIS_ALL, "SiS", 0, 0 },
{ HDA_ULI_ALL, "ULI", 0, 0 },
};
static const struct {
uint16_t vendor;
uint8_t reg;
uint8_t mask;
uint8_t enable;
} hdac_pcie_snoop[] = {
{ INTEL_VENDORID, 0x00, 0x00, 0x00 },
{ ATI_VENDORID, 0x42, 0xf8, 0x02 },
{ NVIDIA_VENDORID, 0x4e, 0xf0, 0x0f },
};
/****************************************************************************
* Function prototypes
****************************************************************************/
static void hdac_intr_handler(void *);
static int hdac_reset(struct hdac_softc *, int);
static int hdac_get_capabilities(struct hdac_softc *);
static void hdac_dma_cb(void *, bus_dma_segment_t *, int, int);
static int hdac_dma_alloc(struct hdac_softc *,
struct hdac_dma *, bus_size_t);
static void hdac_dma_free(struct hdac_softc *, struct hdac_dma *);
static int hdac_mem_alloc(struct hdac_softc *);
static void hdac_mem_free(struct hdac_softc *);
static int hdac_irq_alloc(struct hdac_softc *);
static void hdac_irq_free(struct hdac_softc *);
static void hdac_corb_init(struct hdac_softc *);
static void hdac_rirb_init(struct hdac_softc *);
static void hdac_corb_start(struct hdac_softc *);
static void hdac_rirb_start(struct hdac_softc *);
static void hdac_attach2(void *);
static uint32_t hdac_send_command(struct hdac_softc *, nid_t, uint32_t);
static int hdac_probe(device_t);
static int hdac_attach(device_t);
static int hdac_detach(device_t);
static int hdac_suspend(device_t);
static int hdac_resume(device_t);
static int hdac_rirb_flush(struct hdac_softc *sc);
static int hdac_unsolq_flush(struct hdac_softc *sc);
#define hdac_command(a1, a2, a3) \
hdac_send_command(a1, a3, a2)
/* This function surely going to make its way into upper level someday. */
static void
hdac_config_fetch(struct hdac_softc *sc, uint32_t *on, uint32_t *off)
{
const char *res = NULL;
int i = 0, j, k, len, inv;
if (resource_string_value(device_get_name(sc->dev),
device_get_unit(sc->dev), "config", &res) != 0)
return;
if (!(res != NULL && strlen(res) > 0))
return;
HDA_BOOTVERBOSE(
device_printf(sc->dev, "Config options:");
);
for (;;) {
while (res[i] != '\0' &&
(res[i] == ',' || isspace(res[i]) != 0))
i++;
if (res[i] == '\0') {
HDA_BOOTVERBOSE(
printf("\n");
);
return;
}
j = i;
while (res[j] != '\0' &&
!(res[j] == ',' || isspace(res[j]) != 0))
j++;
len = j - i;
if (len > 2 && strncmp(res + i, "no", 2) == 0)
inv = 2;
else
inv = 0;
for (k = 0; len > inv && k < nitems(hdac_quirks_tab); k++) {
if (strncmp(res + i + inv,
hdac_quirks_tab[k].key, len - inv) != 0)
continue;
if (len - inv != strlen(hdac_quirks_tab[k].key))
continue;
HDA_BOOTVERBOSE(
printf(" %s%s", (inv != 0) ? "no" : "",
hdac_quirks_tab[k].key);
);
if (inv == 0) {
*on |= hdac_quirks_tab[k].value;
*on &= ~hdac_quirks_tab[k].value;
} else if (inv != 0) {
*off |= hdac_quirks_tab[k].value;
*off &= ~hdac_quirks_tab[k].value;
}
break;
}
i = j;
}
}
/****************************************************************************
* void hdac_intr_handler(void *)
*
* Interrupt handler. Processes interrupts received from the hdac.
****************************************************************************/
static void
hdac_intr_handler(void *context)
{
struct hdac_softc *sc;
device_t dev;
uint32_t intsts;
uint8_t rirbsts;
int i;
sc = (struct hdac_softc *)context;
hdac_lock(sc);
/* Do we have anything to do? */
intsts = HDAC_READ_4(&sc->mem, HDAC_INTSTS);
if ((intsts & HDAC_INTSTS_GIS) == 0) {
hdac_unlock(sc);
return;
}
/* Was this a controller interrupt? */
if (intsts & HDAC_INTSTS_CIS) {
rirbsts = HDAC_READ_1(&sc->mem, HDAC_RIRBSTS);
/* Get as many responses that we can */
while (rirbsts & HDAC_RIRBSTS_RINTFL) {
HDAC_WRITE_1(&sc->mem,
HDAC_RIRBSTS, HDAC_RIRBSTS_RINTFL);
hdac_rirb_flush(sc);
rirbsts = HDAC_READ_1(&sc->mem, HDAC_RIRBSTS);
}
if (sc->unsolq_rp != sc->unsolq_wp)
taskqueue_enqueue(taskqueue_thread, &sc->unsolq_task);
}
if (intsts & HDAC_INTSTS_SIS_MASK) {
for (i = 0; i < sc->num_ss; i++) {
if ((intsts & (1 << i)) == 0)
continue;
HDAC_WRITE_1(&sc->mem, (i << 5) + HDAC_SDSTS,
HDAC_SDSTS_DESE | HDAC_SDSTS_FIFOE | HDAC_SDSTS_BCIS );
if ((dev = sc->streams[i].dev) != NULL) {
HDAC_STREAM_INTR(dev,
sc->streams[i].dir, sc->streams[i].stream);
}
}
}
HDAC_WRITE_4(&sc->mem, HDAC_INTSTS, intsts);
hdac_unlock(sc);
}
static void
hdac_poll_callback(void *arg)
{
struct hdac_softc *sc = arg;
if (sc == NULL)
return;
hdac_lock(sc);
if (sc->polling == 0) {
hdac_unlock(sc);
return;
}
callout_reset(&sc->poll_callout, sc->poll_ival,
hdac_poll_callback, sc);
hdac_unlock(sc);
hdac_intr_handler(sc);
}
/****************************************************************************
* int hdac_reset(hdac_softc *, int)
*
* Reset the hdac to a quiescent and known state.
****************************************************************************/
static int
hdac_reset(struct hdac_softc *sc, int wakeup)
{
uint32_t gctl;
int count, i;
/*
* Stop all Streams DMA engine
*/
for (i = 0; i < sc->num_iss; i++)
HDAC_WRITE_4(&sc->mem, HDAC_ISDCTL(sc, i), 0x0);
for (i = 0; i < sc->num_oss; i++)
HDAC_WRITE_4(&sc->mem, HDAC_OSDCTL(sc, i), 0x0);
for (i = 0; i < sc->num_bss; i++)
HDAC_WRITE_4(&sc->mem, HDAC_BSDCTL(sc, i), 0x0);
/*
* Stop Control DMA engines.
*/
HDAC_WRITE_1(&sc->mem, HDAC_CORBCTL, 0x0);
HDAC_WRITE_1(&sc->mem, HDAC_RIRBCTL, 0x0);
/*
* Reset DMA position buffer.
*/
HDAC_WRITE_4(&sc->mem, HDAC_DPIBLBASE, 0x0);
HDAC_WRITE_4(&sc->mem, HDAC_DPIBUBASE, 0x0);
/*
* Reset the controller. The reset must remain asserted for
* a minimum of 100us.
*/
gctl = HDAC_READ_4(&sc->mem, HDAC_GCTL);
HDAC_WRITE_4(&sc->mem, HDAC_GCTL, gctl & ~HDAC_GCTL_CRST);
count = 10000;
do {
gctl = HDAC_READ_4(&sc->mem, HDAC_GCTL);
if (!(gctl & HDAC_GCTL_CRST))
break;
DELAY(10);
} while (--count);
if (gctl & HDAC_GCTL_CRST) {
device_printf(sc->dev, "Unable to put hdac in reset\n");
return (ENXIO);
}
/* If wakeup is not requested - leave the controller in reset state. */
if (!wakeup)
return (0);
DELAY(100);
gctl = HDAC_READ_4(&sc->mem, HDAC_GCTL);
HDAC_WRITE_4(&sc->mem, HDAC_GCTL, gctl | HDAC_GCTL_CRST);
count = 10000;
do {
gctl = HDAC_READ_4(&sc->mem, HDAC_GCTL);
if (gctl & HDAC_GCTL_CRST)
break;
DELAY(10);
} while (--count);
if (!(gctl & HDAC_GCTL_CRST)) {
device_printf(sc->dev, "Device stuck in reset\n");
return (ENXIO);
}
/*
* Wait for codecs to finish their own reset sequence. The delay here
* should be of 250us but for some reasons, on it's not enough on my
* computer. Let's use twice as much as necessary to make sure that
* it's reset properly.
*/
DELAY(1000);
return (0);
}
/****************************************************************************
* int hdac_get_capabilities(struct hdac_softc *);
*
* Retreive the general capabilities of the hdac;
* Number of Input Streams
* Number of Output Streams
* Number of bidirectional Streams
* 64bit ready
* CORB and RIRB sizes
****************************************************************************/
static int
hdac_get_capabilities(struct hdac_softc *sc)
{
uint16_t gcap;
uint8_t corbsize, rirbsize;
gcap = HDAC_READ_2(&sc->mem, HDAC_GCAP);
sc->num_iss = HDAC_GCAP_ISS(gcap);
sc->num_oss = HDAC_GCAP_OSS(gcap);
sc->num_bss = HDAC_GCAP_BSS(gcap);
sc->num_ss = sc->num_iss + sc->num_oss + sc->num_bss;
sc->num_sdo = HDAC_GCAP_NSDO(gcap);
sc->support_64bit = (gcap & HDAC_GCAP_64OK) != 0;
if (sc->quirks_on & HDAC_QUIRK_64BIT)
sc->support_64bit = 1;
else if (sc->quirks_off & HDAC_QUIRK_64BIT)
sc->support_64bit = 0;
corbsize = HDAC_READ_1(&sc->mem, HDAC_CORBSIZE);
if ((corbsize & HDAC_CORBSIZE_CORBSZCAP_256) ==
HDAC_CORBSIZE_CORBSZCAP_256)
sc->corb_size = 256;
else if ((corbsize & HDAC_CORBSIZE_CORBSZCAP_16) ==
HDAC_CORBSIZE_CORBSZCAP_16)
sc->corb_size = 16;
else if ((corbsize & HDAC_CORBSIZE_CORBSZCAP_2) ==
HDAC_CORBSIZE_CORBSZCAP_2)
sc->corb_size = 2;
else {
device_printf(sc->dev, "%s: Invalid corb size (%x)\n",
__func__, corbsize);
return (ENXIO);
}
rirbsize = HDAC_READ_1(&sc->mem, HDAC_RIRBSIZE);
if ((rirbsize & HDAC_RIRBSIZE_RIRBSZCAP_256) ==
HDAC_RIRBSIZE_RIRBSZCAP_256)
sc->rirb_size = 256;
else if ((rirbsize & HDAC_RIRBSIZE_RIRBSZCAP_16) ==
HDAC_RIRBSIZE_RIRBSZCAP_16)
sc->rirb_size = 16;
else if ((rirbsize & HDAC_RIRBSIZE_RIRBSZCAP_2) ==
HDAC_RIRBSIZE_RIRBSZCAP_2)
sc->rirb_size = 2;
else {
device_printf(sc->dev, "%s: Invalid rirb size (%x)\n",
__func__, rirbsize);
return (ENXIO);
}
HDA_BOOTVERBOSE(
device_printf(sc->dev, "Caps: OSS %d, ISS %d, BSS %d, "
"NSDO %d%s, CORB %d, RIRB %d\n",
sc->num_oss, sc->num_iss, sc->num_bss, 1 << sc->num_sdo,
sc->support_64bit ? ", 64bit" : "",
sc->corb_size, sc->rirb_size);
);
return (0);
}
/****************************************************************************
* void hdac_dma_cb
*
* This function is called by bus_dmamap_load when the mapping has been
* established. We just record the physical address of the mapping into
* the struct hdac_dma passed in.
****************************************************************************/
static void
hdac_dma_cb(void *callback_arg, bus_dma_segment_t *segs, int nseg, int error)
{
struct hdac_dma *dma;
if (error == 0) {
dma = (struct hdac_dma *)callback_arg;
dma->dma_paddr = segs[0].ds_addr;
}
}
/****************************************************************************
* int hdac_dma_alloc
*
* This function allocate and setup a dma region (struct hdac_dma).
* It must be freed by a corresponding hdac_dma_free.
****************************************************************************/
static int
hdac_dma_alloc(struct hdac_softc *sc, struct hdac_dma *dma, bus_size_t size)
{
bus_size_t roundsz;
int result;
roundsz = roundup2(size, HDA_DMA_ALIGNMENT);
bzero(dma, sizeof(*dma));
/*
* Create a DMA tag
*/
result = bus_dma_tag_create(
bus_get_dma_tag(sc->dev), /* parent */
HDA_DMA_ALIGNMENT, /* alignment */
0, /* boundary */
(sc->support_64bit) ? BUS_SPACE_MAXADDR :
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, /* filtfunc */
NULL, /* fistfuncarg */
roundsz, /* maxsize */
1, /* nsegments */
roundsz, /* maxsegsz */
0, /* flags */
NULL, /* lockfunc */
NULL, /* lockfuncarg */
&dma->dma_tag); /* dmat */
if (result != 0) {
device_printf(sc->dev, "%s: bus_dma_tag_create failed (%x)\n",
__func__, result);
goto hdac_dma_alloc_fail;
}
/*
* Allocate DMA memory
*/
result = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
BUS_DMA_NOWAIT | BUS_DMA_ZERO |
((sc->flags & HDAC_F_DMA_NOCACHE) ? BUS_DMA_NOCACHE : 0),
&dma->dma_map);
if (result != 0) {
device_printf(sc->dev, "%s: bus_dmamem_alloc failed (%x)\n",
__func__, result);
goto hdac_dma_alloc_fail;
}
dma->dma_size = roundsz;
/*
* Map the memory
*/
result = bus_dmamap_load(dma->dma_tag, dma->dma_map,
(void *)dma->dma_vaddr, roundsz, hdac_dma_cb, (void *)dma, 0);
if (result != 0 || dma->dma_paddr == 0) {
if (result == 0)
result = ENOMEM;
device_printf(sc->dev, "%s: bus_dmamem_load failed (%x)\n",
__func__, result);
goto hdac_dma_alloc_fail;
}
HDA_BOOTHVERBOSE(
device_printf(sc->dev, "%s: size=%ju -> roundsz=%ju\n",
__func__, (uintmax_t)size, (uintmax_t)roundsz);
);
return (0);
hdac_dma_alloc_fail:
hdac_dma_free(sc, dma);
return (result);
}
/****************************************************************************
* void hdac_dma_free(struct hdac_softc *, struct hdac_dma *)
*
* Free a struct dhac_dma that has been previously allocated via the
* hdac_dma_alloc function.
****************************************************************************/
static void
hdac_dma_free(struct hdac_softc *sc, struct hdac_dma *dma)
{
if (dma->dma_paddr != 0) {
#if 0
/* Flush caches */
bus_dmamap_sync(dma->dma_tag, dma->dma_map,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
#endif
bus_dmamap_unload(dma->dma_tag, dma->dma_map);
dma->dma_paddr = 0;
}
if (dma->dma_vaddr != NULL) {
bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
dma->dma_vaddr = NULL;
}
if (dma->dma_tag != NULL) {
bus_dma_tag_destroy(dma->dma_tag);
dma->dma_tag = NULL;
}
dma->dma_size = 0;
}
/****************************************************************************
* int hdac_mem_alloc(struct hdac_softc *)
*
* Allocate all the bus resources necessary to speak with the physical
* controller.
****************************************************************************/
static int
hdac_mem_alloc(struct hdac_softc *sc)
{
struct hdac_mem *mem;
mem = &sc->mem;
mem->mem_rid = PCIR_BAR(0);
mem->mem_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
&mem->mem_rid, RF_ACTIVE);
if (mem->mem_res == NULL) {
device_printf(sc->dev,
"%s: Unable to allocate memory resource\n", __func__);
return (ENOMEM);
}
mem->mem_tag = rman_get_bustag(mem->mem_res);
mem->mem_handle = rman_get_bushandle(mem->mem_res);
return (0);
}
/****************************************************************************
* void hdac_mem_free(struct hdac_softc *)
*
* Free up resources previously allocated by hdac_mem_alloc.
****************************************************************************/
static void
hdac_mem_free(struct hdac_softc *sc)
{
struct hdac_mem *mem;
mem = &sc->mem;
if (mem->mem_res != NULL)
bus_release_resource(sc->dev, SYS_RES_MEMORY, mem->mem_rid,
mem->mem_res);
mem->mem_res = NULL;
}
/****************************************************************************
* int hdac_irq_alloc(struct hdac_softc *)
*
* Allocate and setup the resources necessary for interrupt handling.
****************************************************************************/
static int
hdac_irq_alloc(struct hdac_softc *sc)
{
struct hdac_irq *irq;
int result;
irq = &sc->irq;
irq->irq_rid = 0x0;
if ((sc->quirks_off & HDAC_QUIRK_MSI) == 0 &&
(result = pci_msi_count(sc->dev)) == 1 &&
pci_alloc_msi(sc->dev, &result) == 0)
irq->irq_rid = 0x1;
irq->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
&irq->irq_rid, RF_SHAREABLE | RF_ACTIVE);
if (irq->irq_res == NULL) {
device_printf(sc->dev, "%s: Unable to allocate irq\n",
__func__);
goto hdac_irq_alloc_fail;
}
result = bus_setup_intr(sc->dev, irq->irq_res, INTR_MPSAFE | INTR_TYPE_AV,
NULL, hdac_intr_handler, sc, &irq->irq_handle);
if (result != 0) {
device_printf(sc->dev,
"%s: Unable to setup interrupt handler (%x)\n",
__func__, result);
goto hdac_irq_alloc_fail;
}
return (0);
hdac_irq_alloc_fail:
hdac_irq_free(sc);
return (ENXIO);
}
/****************************************************************************
* void hdac_irq_free(struct hdac_softc *)
*
* Free up resources previously allocated by hdac_irq_alloc.
****************************************************************************/
static void
hdac_irq_free(struct hdac_softc *sc)
{
struct hdac_irq *irq;
irq = &sc->irq;
if (irq->irq_res != NULL && irq->irq_handle != NULL)
bus_teardown_intr(sc->dev, irq->irq_res, irq->irq_handle);
if (irq->irq_res != NULL)
bus_release_resource(sc->dev, SYS_RES_IRQ, irq->irq_rid,
irq->irq_res);
if (irq->irq_rid == 0x1)
pci_release_msi(sc->dev);
irq->irq_handle = NULL;
irq->irq_res = NULL;
irq->irq_rid = 0x0;
}
/****************************************************************************
* void hdac_corb_init(struct hdac_softc *)
*
* Initialize the corb registers for operations but do not start it up yet.
* The CORB engine must not be running when this function is called.
****************************************************************************/
static void
hdac_corb_init(struct hdac_softc *sc)
{
uint8_t corbsize;
uint64_t corbpaddr;
/* Setup the CORB size. */
switch (sc->corb_size) {
case 256:
corbsize = HDAC_CORBSIZE_CORBSIZE(HDAC_CORBSIZE_CORBSIZE_256);
break;
case 16:
corbsize = HDAC_CORBSIZE_CORBSIZE(HDAC_CORBSIZE_CORBSIZE_16);
break;
case 2:
corbsize = HDAC_CORBSIZE_CORBSIZE(HDAC_CORBSIZE_CORBSIZE_2);
break;
default:
panic("%s: Invalid CORB size (%x)\n", __func__, sc->corb_size);
}
HDAC_WRITE_1(&sc->mem, HDAC_CORBSIZE, corbsize);
/* Setup the CORB Address in the hdac */
corbpaddr = (uint64_t)sc->corb_dma.dma_paddr;
HDAC_WRITE_4(&sc->mem, HDAC_CORBLBASE, (uint32_t)corbpaddr);
HDAC_WRITE_4(&sc->mem, HDAC_CORBUBASE, (uint32_t)(corbpaddr >> 32));
/* Set the WP and RP */
sc->corb_wp = 0;
HDAC_WRITE_2(&sc->mem, HDAC_CORBWP, sc->corb_wp);
HDAC_WRITE_2(&sc->mem, HDAC_CORBRP, HDAC_CORBRP_CORBRPRST);
/*
* The HDA specification indicates that the CORBRPRST bit will always
* read as zero. Unfortunately, it seems that at least the 82801G
* doesn't reset the bit to zero, which stalls the corb engine.
* manually reset the bit to zero before continuing.
*/
HDAC_WRITE_2(&sc->mem, HDAC_CORBRP, 0x0);
/* Enable CORB error reporting */
#if 0
HDAC_WRITE_1(&sc->mem, HDAC_CORBCTL, HDAC_CORBCTL_CMEIE);
#endif
}
/****************************************************************************
* void hdac_rirb_init(struct hdac_softc *)
*
* Initialize the rirb registers for operations but do not start it up yet.
* The RIRB engine must not be running when this function is called.
****************************************************************************/
static void
hdac_rirb_init(struct hdac_softc *sc)
{
uint8_t rirbsize;
uint64_t rirbpaddr;
/* Setup the RIRB size. */
switch (sc->rirb_size) {
case 256:
rirbsize = HDAC_RIRBSIZE_RIRBSIZE(HDAC_RIRBSIZE_RIRBSIZE_256);
break;
case 16:
rirbsize = HDAC_RIRBSIZE_RIRBSIZE(HDAC_RIRBSIZE_RIRBSIZE_16);
break;
case 2:
rirbsize = HDAC_RIRBSIZE_RIRBSIZE(HDAC_RIRBSIZE_RIRBSIZE_2);
break;
default:
panic("%s: Invalid RIRB size (%x)\n", __func__, sc->rirb_size);
}
HDAC_WRITE_1(&sc->mem, HDAC_RIRBSIZE, rirbsize);
/* Setup the RIRB Address in the hdac */
rirbpaddr = (uint64_t)sc->rirb_dma.dma_paddr;
HDAC_WRITE_4(&sc->mem, HDAC_RIRBLBASE, (uint32_t)rirbpaddr);
HDAC_WRITE_4(&sc->mem, HDAC_RIRBUBASE, (uint32_t)(rirbpaddr >> 32));
/* Setup the WP and RP */
sc->rirb_rp = 0;
HDAC_WRITE_2(&sc->mem, HDAC_RIRBWP, HDAC_RIRBWP_RIRBWPRST);
/* Setup the interrupt threshold */
HDAC_WRITE_2(&sc->mem, HDAC_RINTCNT, sc->rirb_size / 2);
/* Enable Overrun and response received reporting */
#if 0
HDAC_WRITE_1(&sc->mem, HDAC_RIRBCTL,
HDAC_RIRBCTL_RIRBOIC | HDAC_RIRBCTL_RINTCTL);
#else
HDAC_WRITE_1(&sc->mem, HDAC_RIRBCTL, HDAC_RIRBCTL_RINTCTL);
#endif
#if 0
/*
* Make sure that the Host CPU cache doesn't contain any dirty
* cache lines that falls in the rirb. If I understood correctly, it
* should be sufficient to do this only once as the rirb is purely
* read-only from now on.
*/
bus_dmamap_sync(sc->rirb_dma.dma_tag, sc->rirb_dma.dma_map,
BUS_DMASYNC_PREREAD);
#endif
}
/****************************************************************************
* void hdac_corb_start(hdac_softc *)
*
* Startup the corb DMA engine
****************************************************************************/
static void
hdac_corb_start(struct hdac_softc *sc)
{
uint32_t corbctl;
corbctl = HDAC_READ_1(&sc->mem, HDAC_CORBCTL);
corbctl |= HDAC_CORBCTL_CORBRUN;
HDAC_WRITE_1(&sc->mem, HDAC_CORBCTL, corbctl);
}
/****************************************************************************
* void hdac_rirb_start(hdac_softc *)
*
* Startup the rirb DMA engine
****************************************************************************/
static void
hdac_rirb_start(struct hdac_softc *sc)
{
uint32_t rirbctl;
rirbctl = HDAC_READ_1(&sc->mem, HDAC_RIRBCTL);
rirbctl |= HDAC_RIRBCTL_RIRBDMAEN;
HDAC_WRITE_1(&sc->mem, HDAC_RIRBCTL, rirbctl);
}
static int
hdac_rirb_flush(struct hdac_softc *sc)
{
struct hdac_rirb *rirb_base, *rirb;
nid_t cad;
uint32_t resp;
uint8_t rirbwp;
int ret;
rirb_base = (struct hdac_rirb *)sc->rirb_dma.dma_vaddr;
rirbwp = HDAC_READ_1(&sc->mem, HDAC_RIRBWP);
#if 0
bus_dmamap_sync(sc->rirb_dma.dma_tag, sc->rirb_dma.dma_map,
BUS_DMASYNC_POSTREAD);
#endif
ret = 0;
while (sc->rirb_rp != rirbwp) {
sc->rirb_rp++;
sc->rirb_rp %= sc->rirb_size;
rirb = &rirb_base[sc->rirb_rp];
cad = HDAC_RIRB_RESPONSE_EX_SDATA_IN(rirb->response_ex);
resp = rirb->response;
if (rirb->response_ex & HDAC_RIRB_RESPONSE_EX_UNSOLICITED) {
sc->unsolq[sc->unsolq_wp++] = resp;
sc->unsolq_wp %= HDAC_UNSOLQ_MAX;
sc->unsolq[sc->unsolq_wp++] = cad;
sc->unsolq_wp %= HDAC_UNSOLQ_MAX;
} else if (sc->codecs[cad].pending <= 0) {
device_printf(sc->dev, "Unexpected unsolicited "
"response from address %d: %08x\n", cad, resp);
} else {
sc->codecs[cad].response = resp;
sc->codecs[cad].pending--;
}
ret++;
}
return (ret);
}
static int
hdac_unsolq_flush(struct hdac_softc *sc)
{
device_t child;
nid_t cad;
uint32_t resp;
int ret = 0;
if (sc->unsolq_st == HDAC_UNSOLQ_READY) {
sc->unsolq_st = HDAC_UNSOLQ_BUSY;
while (sc->unsolq_rp != sc->unsolq_wp) {
resp = sc->unsolq[sc->unsolq_rp++];
sc->unsolq_rp %= HDAC_UNSOLQ_MAX;
cad = sc->unsolq[sc->unsolq_rp++];
sc->unsolq_rp %= HDAC_UNSOLQ_MAX;
if ((child = sc->codecs[cad].dev) != NULL)
HDAC_UNSOL_INTR(child, resp);
ret++;
}
sc->unsolq_st = HDAC_UNSOLQ_READY;
}
return (ret);
}
/****************************************************************************
* uint32_t hdac_command_sendone_internal
*
* Wrapper function that sends only one command to a given codec
****************************************************************************/
static uint32_t
hdac_send_command(struct hdac_softc *sc, nid_t cad, uint32_t verb)
{
int timeout;
uint32_t *corb;
if (!hdac_lockowned(sc))
device_printf(sc->dev, "WARNING!!!! mtx not owned!!!!\n");
verb &= ~HDA_CMD_CAD_MASK;
verb |= ((uint32_t)cad) << HDA_CMD_CAD_SHIFT;
sc->codecs[cad].response = HDA_INVALID;
sc->codecs[cad].pending++;
sc->corb_wp++;
sc->corb_wp %= sc->corb_size;
corb = (uint32_t *)sc->corb_dma.dma_vaddr;
#if 0
bus_dmamap_sync(sc->corb_dma.dma_tag,
sc->corb_dma.dma_map, BUS_DMASYNC_PREWRITE);
#endif
corb[sc->corb_wp] = verb;
#if 0
bus_dmamap_sync(sc->corb_dma.dma_tag,
sc->corb_dma.dma_map, BUS_DMASYNC_POSTWRITE);
#endif
HDAC_WRITE_2(&sc->mem, HDAC_CORBWP, sc->corb_wp);
timeout = 10000;
do {
if (hdac_rirb_flush(sc) == 0)
DELAY(10);
} while (sc->codecs[cad].pending != 0 && --timeout);
if (sc->codecs[cad].pending != 0) {
device_printf(sc->dev, "Command timeout on address %d\n", cad);
sc->codecs[cad].pending = 0;
}
if (sc->unsolq_rp != sc->unsolq_wp)
taskqueue_enqueue(taskqueue_thread, &sc->unsolq_task);
return (sc->codecs[cad].response);
}
/****************************************************************************
* Device Methods
****************************************************************************/
/****************************************************************************
* int hdac_probe(device_t)
*
* Probe for the presence of an hdac. If none is found, check for a generic
* match using the subclass of the device.
****************************************************************************/
static int
hdac_probe(device_t dev)
{
int i, result;
uint32_t model;
uint16_t class, subclass;
char desc[64];
model = (uint32_t)pci_get_device(dev) << 16;
model |= (uint32_t)pci_get_vendor(dev) & 0x0000ffff;
class = pci_get_class(dev);
subclass = pci_get_subclass(dev);
bzero(desc, sizeof(desc));
result = ENXIO;
for (i = 0; i < nitems(hdac_devices); i++) {
if (hdac_devices[i].model == model) {
strlcpy(desc, hdac_devices[i].desc, sizeof(desc));
result = BUS_PROBE_DEFAULT;
break;
}
if (HDA_DEV_MATCH(hdac_devices[i].model, model) &&
class == PCIC_MULTIMEDIA &&
subclass == PCIS_MULTIMEDIA_HDA) {
snprintf(desc, sizeof(desc),
"%s (0x%04x)",
hdac_devices[i].desc, pci_get_device(dev));
result = BUS_PROBE_GENERIC;
break;
}
}
if (result == ENXIO && class == PCIC_MULTIMEDIA &&
subclass == PCIS_MULTIMEDIA_HDA) {
snprintf(desc, sizeof(desc), "Generic (0x%08x)", model);
result = BUS_PROBE_GENERIC;
}
if (result != ENXIO) {
strlcat(desc, " HDA Controller", sizeof(desc));
device_set_desc_copy(dev, desc);
}
return (result);
}
static void
hdac_unsolq_task(void *context, int pending)
{
struct hdac_softc *sc;
sc = (struct hdac_softc *)context;
hdac_lock(sc);
hdac_unsolq_flush(sc);
hdac_unlock(sc);
}
/****************************************************************************
* int hdac_attach(device_t)
*
* Attach the device into the kernel. Interrupts usually won't be enabled
* when this function is called. Setup everything that doesn't require
* interrupts and defer probing of codecs until interrupts are enabled.
****************************************************************************/
static int
hdac_attach(device_t dev)
{
struct hdac_softc *sc;
int result;
int i, devid = -1;
uint32_t model;
uint16_t class, subclass;
uint16_t vendor;
uint8_t v;
sc = device_get_softc(dev);
HDA_BOOTVERBOSE(
device_printf(dev, "PCI card vendor: 0x%04x, device: 0x%04x\n",
pci_get_subvendor(dev), pci_get_subdevice(dev));
device_printf(dev, "HDA Driver Revision: %s\n",
HDA_DRV_TEST_REV);
);
model = (uint32_t)pci_get_device(dev) << 16;
model |= (uint32_t)pci_get_vendor(dev) & 0x0000ffff;
class = pci_get_class(dev);
subclass = pci_get_subclass(dev);
for (i = 0; i < nitems(hdac_devices); i++) {
if (hdac_devices[i].model == model) {
devid = i;
break;
}
if (HDA_DEV_MATCH(hdac_devices[i].model, model) &&
class == PCIC_MULTIMEDIA &&
subclass == PCIS_MULTIMEDIA_HDA) {
devid = i;
break;
}
}
sc->lock = snd_mtxcreate(device_get_nameunit(dev), "HDA driver mutex");
sc->dev = dev;
TASK_INIT(&sc->unsolq_task, 0, hdac_unsolq_task, sc);
- callout_init(&sc->poll_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->poll_callout, 1);
for (i = 0; i < HDAC_CODEC_MAX; i++)
sc->codecs[i].dev = NULL;
if (devid >= 0) {
sc->quirks_on = hdac_devices[devid].quirks_on;
sc->quirks_off = hdac_devices[devid].quirks_off;
} else {
sc->quirks_on = 0;
sc->quirks_off = 0;
}
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "msi", &i) == 0) {
if (i == 0)
sc->quirks_off |= HDAC_QUIRK_MSI;
else {
sc->quirks_on |= HDAC_QUIRK_MSI;
sc->quirks_off |= ~HDAC_QUIRK_MSI;
}
}
hdac_config_fetch(sc, &sc->quirks_on, &sc->quirks_off);
HDA_BOOTVERBOSE(
device_printf(sc->dev,
"Config options: on=0x%08x off=0x%08x\n",
sc->quirks_on, sc->quirks_off);
);
sc->poll_ival = hz;
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "polling", &i) == 0 && i != 0)
sc->polling = 1;
else
sc->polling = 0;
pci_enable_busmaster(dev);
vendor = pci_get_vendor(dev);
if (vendor == INTEL_VENDORID) {
/* TCSEL -> TC0 */
v = pci_read_config(dev, 0x44, 1);
pci_write_config(dev, 0x44, v & 0xf8, 1);
HDA_BOOTHVERBOSE(
device_printf(dev, "TCSEL: 0x%02d -> 0x%02d\n", v,
pci_read_config(dev, 0x44, 1));
);
}
#if defined(__i386__) || defined(__amd64__)
sc->flags |= HDAC_F_DMA_NOCACHE;
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "snoop", &i) == 0 && i != 0) {
#else
sc->flags &= ~HDAC_F_DMA_NOCACHE;
#endif
/*
* Try to enable PCIe snoop to avoid messing around with
* uncacheable DMA attribute. Since PCIe snoop register
* config is pretty much vendor specific, there are no
* general solutions on how to enable it, forcing us (even
* Microsoft) to enable uncacheable or write combined DMA
* by default.
*
* http://msdn2.microsoft.com/en-us/library/ms790324.aspx
*/
for (i = 0; i < nitems(hdac_pcie_snoop); i++) {
if (hdac_pcie_snoop[i].vendor != vendor)
continue;
sc->flags &= ~HDAC_F_DMA_NOCACHE;
if (hdac_pcie_snoop[i].reg == 0x00)
break;
v = pci_read_config(dev, hdac_pcie_snoop[i].reg, 1);
if ((v & hdac_pcie_snoop[i].enable) ==
hdac_pcie_snoop[i].enable)
break;
v &= hdac_pcie_snoop[i].mask;
v |= hdac_pcie_snoop[i].enable;
pci_write_config(dev, hdac_pcie_snoop[i].reg, v, 1);
v = pci_read_config(dev, hdac_pcie_snoop[i].reg, 1);
if ((v & hdac_pcie_snoop[i].enable) !=
hdac_pcie_snoop[i].enable) {
HDA_BOOTVERBOSE(
device_printf(dev,
"WARNING: Failed to enable PCIe "
"snoop!\n");
);
#if defined(__i386__) || defined(__amd64__)
sc->flags |= HDAC_F_DMA_NOCACHE;
#endif
}
break;
}
#if defined(__i386__) || defined(__amd64__)
}
#endif
HDA_BOOTHVERBOSE(
device_printf(dev, "DMA Coherency: %s / vendor=0x%04x\n",
(sc->flags & HDAC_F_DMA_NOCACHE) ?
"Uncacheable" : "PCIe snoop", vendor);
);
/* Allocate resources */
result = hdac_mem_alloc(sc);
if (result != 0)
goto hdac_attach_fail;
result = hdac_irq_alloc(sc);
if (result != 0)
goto hdac_attach_fail;
/* Get Capabilities */
result = hdac_get_capabilities(sc);
if (result != 0)
goto hdac_attach_fail;
/* Allocate CORB, RIRB, POS and BDLs dma memory */
result = hdac_dma_alloc(sc, &sc->corb_dma,
sc->corb_size * sizeof(uint32_t));
if (result != 0)
goto hdac_attach_fail;
result = hdac_dma_alloc(sc, &sc->rirb_dma,
sc->rirb_size * sizeof(struct hdac_rirb));
if (result != 0)
goto hdac_attach_fail;
sc->streams = malloc(sizeof(struct hdac_stream) * sc->num_ss,
M_HDAC, M_ZERO | M_WAITOK);
for (i = 0; i < sc->num_ss; i++) {
result = hdac_dma_alloc(sc, &sc->streams[i].bdl,
sizeof(struct hdac_bdle) * HDA_BDL_MAX);
if (result != 0)
goto hdac_attach_fail;
}
if (sc->quirks_on & HDAC_QUIRK_DMAPOS) {
if (hdac_dma_alloc(sc, &sc->pos_dma, (sc->num_ss) * 8) != 0) {
HDA_BOOTVERBOSE(
device_printf(dev, "Failed to "
"allocate DMA pos buffer "
"(non-fatal)\n");
);
} else {
uint64_t addr = sc->pos_dma.dma_paddr;
HDAC_WRITE_4(&sc->mem, HDAC_DPIBUBASE, addr >> 32);
HDAC_WRITE_4(&sc->mem, HDAC_DPIBLBASE,
(addr & HDAC_DPLBASE_DPLBASE_MASK) |
HDAC_DPLBASE_DPLBASE_DMAPBE);
}
}
result = bus_dma_tag_create(
bus_get_dma_tag(sc->dev), /* parent */
HDA_DMA_ALIGNMENT, /* alignment */
0, /* boundary */
(sc->support_64bit) ? BUS_SPACE_MAXADDR :
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, /* filtfunc */
NULL, /* fistfuncarg */
HDA_BUFSZ_MAX, /* maxsize */
1, /* nsegments */
HDA_BUFSZ_MAX, /* maxsegsz */
0, /* flags */
NULL, /* lockfunc */
NULL, /* lockfuncarg */
&sc->chan_dmat); /* dmat */
if (result != 0) {
device_printf(dev, "%s: bus_dma_tag_create failed (%x)\n",
__func__, result);
goto hdac_attach_fail;
}
/* Quiesce everything */
HDA_BOOTHVERBOSE(
device_printf(dev, "Reset controller...\n");
);
hdac_reset(sc, 1);
/* Initialize the CORB and RIRB */
hdac_corb_init(sc);
hdac_rirb_init(sc);
/* Defer remaining of initialization until interrupts are enabled */
sc->intrhook.ich_func = hdac_attach2;
sc->intrhook.ich_arg = (void *)sc;
if (cold == 0 || config_intrhook_establish(&sc->intrhook) != 0) {
sc->intrhook.ich_func = NULL;
hdac_attach2((void *)sc);
}
return (0);
hdac_attach_fail:
hdac_irq_free(sc);
for (i = 0; i < sc->num_ss; i++)
hdac_dma_free(sc, &sc->streams[i].bdl);
free(sc->streams, M_HDAC);
hdac_dma_free(sc, &sc->rirb_dma);
hdac_dma_free(sc, &sc->corb_dma);
hdac_mem_free(sc);
snd_mtxfree(sc->lock);
return (ENXIO);
}
static int
sysctl_hdac_pindump(SYSCTL_HANDLER_ARGS)
{
struct hdac_softc *sc;
device_t *devlist;
device_t dev;
int devcount, i, err, val;
dev = oidp->oid_arg1;
sc = device_get_softc(dev);
if (sc == NULL)
return (EINVAL);
val = 0;
err = sysctl_handle_int(oidp, &val, 0, req);
if (err != 0 || req->newptr == NULL || val == 0)
return (err);
/* XXX: Temporary. For debugging. */
if (val == 100) {
hdac_suspend(dev);
return (0);
} else if (val == 101) {
hdac_resume(dev);
return (0);
}
if ((err = device_get_children(dev, &devlist, &devcount)) != 0)
return (err);
hdac_lock(sc);
for (i = 0; i < devcount; i++)
HDAC_PINDUMP(devlist[i]);
hdac_unlock(sc);
free(devlist, M_TEMP);
return (0);
}
static int
hdac_mdata_rate(uint16_t fmt)
{
static const int mbits[8] = { 8, 16, 32, 32, 32, 32, 32, 32 };
int rate, bits;
if (fmt & (1 << 14))
rate = 44100;
else
rate = 48000;
rate *= ((fmt >> 11) & 0x07) + 1;
rate /= ((fmt >> 8) & 0x07) + 1;
bits = mbits[(fmt >> 4) & 0x03];
bits *= (fmt & 0x0f) + 1;
return (rate * bits);
}
static int
hdac_bdata_rate(uint16_t fmt, int output)
{
static const int bbits[8] = { 8, 16, 20, 24, 32, 32, 32, 32 };
int rate, bits;
rate = 48000;
rate *= ((fmt >> 11) & 0x07) + 1;
bits = bbits[(fmt >> 4) & 0x03];
bits *= (fmt & 0x0f) + 1;
if (!output)
bits = ((bits + 7) & ~0x07) + 10;
return (rate * bits);
}
static void
hdac_poll_reinit(struct hdac_softc *sc)
{
int i, pollticks, min = 1000000;
struct hdac_stream *s;
if (sc->polling == 0)
return;
if (sc->unsol_registered > 0)
min = hz / 2;
for (i = 0; i < sc->num_ss; i++) {
s = &sc->streams[i];
if (s->running == 0)
continue;
pollticks = ((uint64_t)hz * s->blksz) /
(hdac_mdata_rate(s->format) / 8);
pollticks >>= 1;
if (pollticks > hz)
pollticks = hz;
if (pollticks < 1) {
HDA_BOOTVERBOSE(
device_printf(sc->dev,
"poll interval < 1 tick !\n");
);
pollticks = 1;
}
if (min > pollticks)
min = pollticks;
}
HDA_BOOTVERBOSE(
device_printf(sc->dev,
"poll interval %d -> %d ticks\n",
sc->poll_ival, min);
);
sc->poll_ival = min;
if (min == 1000000)
callout_stop(&sc->poll_callout);
else
callout_reset(&sc->poll_callout, 1, hdac_poll_callback, sc);
}
static int
sysctl_hdac_polling(SYSCTL_HANDLER_ARGS)
{
struct hdac_softc *sc;
device_t dev;
uint32_t ctl;
int err, val;
dev = oidp->oid_arg1;
sc = device_get_softc(dev);
if (sc == NULL)
return (EINVAL);
hdac_lock(sc);
val = sc->polling;
hdac_unlock(sc);
err = sysctl_handle_int(oidp, &val, 0, req);
if (err != 0 || req->newptr == NULL)
return (err);
if (val < 0 || val > 1)
return (EINVAL);
hdac_lock(sc);
if (val != sc->polling) {
if (val == 0) {
callout_stop(&sc->poll_callout);
hdac_unlock(sc);
callout_drain(&sc->poll_callout);
hdac_lock(sc);
sc->polling = 0;
ctl = HDAC_READ_4(&sc->mem, HDAC_INTCTL);
ctl |= HDAC_INTCTL_GIE;
HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, ctl);
} else {
ctl = HDAC_READ_4(&sc->mem, HDAC_INTCTL);
ctl &= ~HDAC_INTCTL_GIE;
HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, ctl);
sc->polling = 1;
hdac_poll_reinit(sc);
}
}
hdac_unlock(sc);
return (err);
}
static void
hdac_attach2(void *arg)
{
struct hdac_softc *sc;
device_t child;
uint32_t vendorid, revisionid;
int i;
uint16_t statests;
sc = (struct hdac_softc *)arg;
hdac_lock(sc);
/* Remove ourselves from the config hooks */
if (sc->intrhook.ich_func != NULL) {
config_intrhook_disestablish(&sc->intrhook);
sc->intrhook.ich_func = NULL;
}
HDA_BOOTHVERBOSE(
device_printf(sc->dev, "Starting CORB Engine...\n");
);
hdac_corb_start(sc);
HDA_BOOTHVERBOSE(
device_printf(sc->dev, "Starting RIRB Engine...\n");
);
hdac_rirb_start(sc);
HDA_BOOTHVERBOSE(
device_printf(sc->dev,
"Enabling controller interrupt...\n");
);
HDAC_WRITE_4(&sc->mem, HDAC_GCTL, HDAC_READ_4(&sc->mem, HDAC_GCTL) |
HDAC_GCTL_UNSOL);
if (sc->polling == 0) {
HDAC_WRITE_4(&sc->mem, HDAC_INTCTL,
HDAC_INTCTL_CIE | HDAC_INTCTL_GIE);
}
DELAY(1000);
HDA_BOOTHVERBOSE(
device_printf(sc->dev, "Scanning HDA codecs ...\n");
);
statests = HDAC_READ_2(&sc->mem, HDAC_STATESTS);
hdac_unlock(sc);
for (i = 0; i < HDAC_CODEC_MAX; i++) {
if (HDAC_STATESTS_SDIWAKE(statests, i)) {
HDA_BOOTHVERBOSE(
device_printf(sc->dev,
"Found CODEC at address %d\n", i);
);
hdac_lock(sc);
vendorid = hdac_send_command(sc, i,
HDA_CMD_GET_PARAMETER(0, 0x0, HDA_PARAM_VENDOR_ID));
revisionid = hdac_send_command(sc, i,
HDA_CMD_GET_PARAMETER(0, 0x0, HDA_PARAM_REVISION_ID));
hdac_unlock(sc);
if (vendorid == HDA_INVALID &&
revisionid == HDA_INVALID) {
device_printf(sc->dev,
"CODEC is not responding!\n");
continue;
}
sc->codecs[i].vendor_id =
HDA_PARAM_VENDOR_ID_VENDOR_ID(vendorid);
sc->codecs[i].device_id =
HDA_PARAM_VENDOR_ID_DEVICE_ID(vendorid);
sc->codecs[i].revision_id =
HDA_PARAM_REVISION_ID_REVISION_ID(revisionid);
sc->codecs[i].stepping_id =
HDA_PARAM_REVISION_ID_STEPPING_ID(revisionid);
child = device_add_child(sc->dev, "hdacc", -1);
if (child == NULL) {
device_printf(sc->dev,
"Failed to add CODEC device\n");
continue;
}
device_set_ivars(child, (void *)(intptr_t)i);
sc->codecs[i].dev = child;
}
}
bus_generic_attach(sc->dev);
SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO,
"pindump", CTLTYPE_INT | CTLFLAG_RW, sc->dev, sizeof(sc->dev),
sysctl_hdac_pindump, "I", "Dump pin states/data");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO,
"polling", CTLTYPE_INT | CTLFLAG_RW, sc->dev, sizeof(sc->dev),
sysctl_hdac_polling, "I", "Enable polling mode");
}
/****************************************************************************
* int hdac_suspend(device_t)
*
* Suspend and power down HDA bus and codecs.
****************************************************************************/
static int
hdac_suspend(device_t dev)
{
struct hdac_softc *sc = device_get_softc(dev);
HDA_BOOTHVERBOSE(
device_printf(dev, "Suspend...\n");
);
bus_generic_suspend(dev);
hdac_lock(sc);
HDA_BOOTHVERBOSE(
device_printf(dev, "Reset controller...\n");
);
callout_stop(&sc->poll_callout);
hdac_reset(sc, 0);
hdac_unlock(sc);
callout_drain(&sc->poll_callout);
taskqueue_drain(taskqueue_thread, &sc->unsolq_task);
HDA_BOOTHVERBOSE(
device_printf(dev, "Suspend done\n");
);
return (0);
}
/****************************************************************************
* int hdac_resume(device_t)
*
* Powerup and restore HDA bus and codecs state.
****************************************************************************/
static int
hdac_resume(device_t dev)
{
struct hdac_softc *sc = device_get_softc(dev);
int error;
HDA_BOOTHVERBOSE(
device_printf(dev, "Resume...\n");
);
hdac_lock(sc);
/* Quiesce everything */
HDA_BOOTHVERBOSE(
device_printf(dev, "Reset controller...\n");
);
hdac_reset(sc, 1);
/* Initialize the CORB and RIRB */
hdac_corb_init(sc);
hdac_rirb_init(sc);
HDA_BOOTHVERBOSE(
device_printf(dev, "Starting CORB Engine...\n");
);
hdac_corb_start(sc);
HDA_BOOTHVERBOSE(
device_printf(dev, "Starting RIRB Engine...\n");
);
hdac_rirb_start(sc);
HDA_BOOTHVERBOSE(
device_printf(dev, "Enabling controller interrupt...\n");
);
HDAC_WRITE_4(&sc->mem, HDAC_GCTL, HDAC_READ_4(&sc->mem, HDAC_GCTL) |
HDAC_GCTL_UNSOL);
HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, HDAC_INTCTL_CIE | HDAC_INTCTL_GIE);
DELAY(1000);
hdac_poll_reinit(sc);
hdac_unlock(sc);
error = bus_generic_resume(dev);
HDA_BOOTHVERBOSE(
device_printf(dev, "Resume done\n");
);
return (error);
}
/****************************************************************************
* int hdac_detach(device_t)
*
* Detach and free up resources utilized by the hdac device.
****************************************************************************/
static int
hdac_detach(device_t dev)
{
struct hdac_softc *sc = device_get_softc(dev);
device_t *devlist;
int cad, i, devcount, error;
if ((error = device_get_children(dev, &devlist, &devcount)) != 0)
return (error);
for (i = 0; i < devcount; i++) {
cad = (intptr_t)device_get_ivars(devlist[i]);
if ((error = device_delete_child(dev, devlist[i])) != 0) {
free(devlist, M_TEMP);
return (error);
}
sc->codecs[cad].dev = NULL;
}
free(devlist, M_TEMP);
hdac_lock(sc);
hdac_reset(sc, 0);
hdac_unlock(sc);
taskqueue_drain(taskqueue_thread, &sc->unsolq_task);
hdac_irq_free(sc);
for (i = 0; i < sc->num_ss; i++)
hdac_dma_free(sc, &sc->streams[i].bdl);
free(sc->streams, M_HDAC);
hdac_dma_free(sc, &sc->pos_dma);
hdac_dma_free(sc, &sc->rirb_dma);
hdac_dma_free(sc, &sc->corb_dma);
if (sc->chan_dmat != NULL) {
bus_dma_tag_destroy(sc->chan_dmat);
sc->chan_dmat = NULL;
}
hdac_mem_free(sc);
snd_mtxfree(sc->lock);
return (0);
}
static bus_dma_tag_t
hdac_get_dma_tag(device_t dev, device_t child)
{
struct hdac_softc *sc = device_get_softc(dev);
return (sc->chan_dmat);
}
static int
hdac_print_child(device_t dev, device_t child)
{
int retval;
retval = bus_print_child_header(dev, child);
retval += printf(" at cad %d",
(int)(intptr_t)device_get_ivars(child));
retval += bus_print_child_footer(dev, child);
return (retval);
}
static int
hdac_child_location_str(device_t dev, device_t child, char *buf,
size_t buflen)
{
snprintf(buf, buflen, "cad=%d",
(int)(intptr_t)device_get_ivars(child));
return (0);
}
static int
hdac_child_pnpinfo_str_method(device_t dev, device_t child, char *buf,
size_t buflen)
{
struct hdac_softc *sc = device_get_softc(dev);
nid_t cad = (uintptr_t)device_get_ivars(child);
snprintf(buf, buflen, "vendor=0x%04x device=0x%04x revision=0x%02x "
"stepping=0x%02x",
sc->codecs[cad].vendor_id, sc->codecs[cad].device_id,
sc->codecs[cad].revision_id, sc->codecs[cad].stepping_id);
return (0);
}
static int
hdac_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
{
struct hdac_softc *sc = device_get_softc(dev);
nid_t cad = (uintptr_t)device_get_ivars(child);
switch (which) {
case HDA_IVAR_CODEC_ID:
*result = cad;
break;
case HDA_IVAR_VENDOR_ID:
*result = sc->codecs[cad].vendor_id;
break;
case HDA_IVAR_DEVICE_ID:
*result = sc->codecs[cad].device_id;
break;
case HDA_IVAR_REVISION_ID:
*result = sc->codecs[cad].revision_id;
break;
case HDA_IVAR_STEPPING_ID:
*result = sc->codecs[cad].stepping_id;
break;
case HDA_IVAR_SUBVENDOR_ID:
*result = pci_get_subvendor(dev);
break;
case HDA_IVAR_SUBDEVICE_ID:
*result = pci_get_subdevice(dev);
break;
case HDA_IVAR_DMA_NOCACHE:
*result = (sc->flags & HDAC_F_DMA_NOCACHE) != 0;
break;
default:
return (ENOENT);
}
return (0);
}
static struct mtx *
hdac_get_mtx(device_t dev, device_t child)
{
struct hdac_softc *sc = device_get_softc(dev);
return (sc->lock);
}
static uint32_t
hdac_codec_command(device_t dev, device_t child, uint32_t verb)
{
return (hdac_send_command(device_get_softc(dev),
(intptr_t)device_get_ivars(child), verb));
}
static int
hdac_find_stream(struct hdac_softc *sc, int dir, int stream)
{
int i, ss;
ss = -1;
/* Allocate ISS/BSS first. */
if (dir == 0) {
for (i = 0; i < sc->num_iss; i++) {
if (sc->streams[i].stream == stream) {
ss = i;
break;
}
}
} else {
for (i = 0; i < sc->num_oss; i++) {
if (sc->streams[i + sc->num_iss].stream == stream) {
ss = i + sc->num_iss;
break;
}
}
}
/* Fallback to BSS. */
if (ss == -1) {
for (i = 0; i < sc->num_bss; i++) {
if (sc->streams[i + sc->num_iss + sc->num_oss].stream
== stream) {
ss = i + sc->num_iss + sc->num_oss;
break;
}
}
}
return (ss);
}
static int
hdac_stream_alloc(device_t dev, device_t child, int dir, int format, int stripe,
uint32_t **dmapos)
{
struct hdac_softc *sc = device_get_softc(dev);
nid_t cad = (uintptr_t)device_get_ivars(child);
int stream, ss, bw, maxbw, prevbw;
/* Look for empty stream. */
ss = hdac_find_stream(sc, dir, 0);
/* Return if found nothing. */
if (ss < 0)
return (0);
/* Check bus bandwidth. */
bw = hdac_bdata_rate(format, dir);
if (dir == 1) {
bw *= 1 << (sc->num_sdo - stripe);
prevbw = sc->sdo_bw_used;
maxbw = 48000 * 960 * (1 << sc->num_sdo);
} else {
prevbw = sc->codecs[cad].sdi_bw_used;
maxbw = 48000 * 464;
}
HDA_BOOTHVERBOSE(
device_printf(dev, "%dKbps of %dKbps bandwidth used%s\n",
(bw + prevbw) / 1000, maxbw / 1000,
bw + prevbw > maxbw ? " -- OVERFLOW!" : "");
);
if (bw + prevbw > maxbw)
return (0);
if (dir == 1)
sc->sdo_bw_used += bw;
else
sc->codecs[cad].sdi_bw_used += bw;
/* Allocate stream number */
if (ss >= sc->num_iss + sc->num_oss)
stream = 15 - (ss - sc->num_iss + sc->num_oss);
else if (ss >= sc->num_iss)
stream = ss - sc->num_iss + 1;
else
stream = ss + 1;
sc->streams[ss].dev = child;
sc->streams[ss].dir = dir;
sc->streams[ss].stream = stream;
sc->streams[ss].bw = bw;
sc->streams[ss].format = format;
sc->streams[ss].stripe = stripe;
if (dmapos != NULL) {
if (sc->pos_dma.dma_vaddr != NULL)
*dmapos = (uint32_t *)(sc->pos_dma.dma_vaddr + ss * 8);
else
*dmapos = NULL;
}
return (stream);
}
static void
hdac_stream_free(device_t dev, device_t child, int dir, int stream)
{
struct hdac_softc *sc = device_get_softc(dev);
nid_t cad = (uintptr_t)device_get_ivars(child);
int ss;
ss = hdac_find_stream(sc, dir, stream);
KASSERT(ss >= 0,
("Free for not allocated stream (%d/%d)\n", dir, stream));
if (dir == 1)
sc->sdo_bw_used -= sc->streams[ss].bw;
else
sc->codecs[cad].sdi_bw_used -= sc->streams[ss].bw;
sc->streams[ss].stream = 0;
sc->streams[ss].dev = NULL;
}
static int
hdac_stream_start(device_t dev, device_t child,
int dir, int stream, bus_addr_t buf, int blksz, int blkcnt)
{
struct hdac_softc *sc = device_get_softc(dev);
struct hdac_bdle *bdle;
uint64_t addr;
int i, ss, off;
uint32_t ctl;
ss = hdac_find_stream(sc, dir, stream);
KASSERT(ss >= 0,
("Start for not allocated stream (%d/%d)\n", dir, stream));
addr = (uint64_t)buf;
bdle = (struct hdac_bdle *)sc->streams[ss].bdl.dma_vaddr;
for (i = 0; i < blkcnt; i++, bdle++) {
bdle->addrl = (uint32_t)addr;
bdle->addrh = (uint32_t)(addr >> 32);
bdle->len = blksz;
bdle->ioc = 1;
addr += blksz;
}
off = ss << 5;
HDAC_WRITE_4(&sc->mem, off + HDAC_SDCBL, blksz * blkcnt);
HDAC_WRITE_2(&sc->mem, off + HDAC_SDLVI, blkcnt - 1);
addr = sc->streams[ss].bdl.dma_paddr;
HDAC_WRITE_4(&sc->mem, off + HDAC_SDBDPL, (uint32_t)addr);
HDAC_WRITE_4(&sc->mem, off + HDAC_SDBDPU, (uint32_t)(addr >> 32));
ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL2);
if (dir)
ctl |= HDAC_SDCTL2_DIR;
else
ctl &= ~HDAC_SDCTL2_DIR;
ctl &= ~HDAC_SDCTL2_STRM_MASK;
ctl |= stream << HDAC_SDCTL2_STRM_SHIFT;
ctl &= ~HDAC_SDCTL2_STRIPE_MASK;
ctl |= sc->streams[ss].stripe << HDAC_SDCTL2_STRIPE_SHIFT;
HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL2, ctl);
HDAC_WRITE_2(&sc->mem, off + HDAC_SDFMT, sc->streams[ss].format);
ctl = HDAC_READ_4(&sc->mem, HDAC_INTCTL);
ctl |= 1 << ss;
HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, ctl);
HDAC_WRITE_1(&sc->mem, off + HDAC_SDSTS,
HDAC_SDSTS_DESE | HDAC_SDSTS_FIFOE | HDAC_SDSTS_BCIS);
ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
ctl |= HDAC_SDCTL_IOCE | HDAC_SDCTL_FEIE | HDAC_SDCTL_DEIE |
HDAC_SDCTL_RUN;
HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL0, ctl);
sc->streams[ss].blksz = blksz;
sc->streams[ss].running = 1;
hdac_poll_reinit(sc);
return (0);
}
static void
hdac_stream_stop(device_t dev, device_t child, int dir, int stream)
{
struct hdac_softc *sc = device_get_softc(dev);
int ss, off;
uint32_t ctl;
ss = hdac_find_stream(sc, dir, stream);
KASSERT(ss >= 0,
("Stop for not allocated stream (%d/%d)\n", dir, stream));
off = ss << 5;
ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
ctl &= ~(HDAC_SDCTL_IOCE | HDAC_SDCTL_FEIE | HDAC_SDCTL_DEIE |
HDAC_SDCTL_RUN);
HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL0, ctl);
ctl = HDAC_READ_4(&sc->mem, HDAC_INTCTL);
ctl &= ~(1 << ss);
HDAC_WRITE_4(&sc->mem, HDAC_INTCTL, ctl);
sc->streams[ss].running = 0;
hdac_poll_reinit(sc);
}
static void
hdac_stream_reset(device_t dev, device_t child, int dir, int stream)
{
struct hdac_softc *sc = device_get_softc(dev);
int timeout = 1000;
int to = timeout;
int ss, off;
uint32_t ctl;
ss = hdac_find_stream(sc, dir, stream);
KASSERT(ss >= 0,
("Reset for not allocated stream (%d/%d)\n", dir, stream));
off = ss << 5;
ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
ctl |= HDAC_SDCTL_SRST;
HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL0, ctl);
do {
ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
if (ctl & HDAC_SDCTL_SRST)
break;
DELAY(10);
} while (--to);
if (!(ctl & HDAC_SDCTL_SRST))
device_printf(dev, "Reset setting timeout\n");
ctl &= ~HDAC_SDCTL_SRST;
HDAC_WRITE_1(&sc->mem, off + HDAC_SDCTL0, ctl);
to = timeout;
do {
ctl = HDAC_READ_1(&sc->mem, off + HDAC_SDCTL0);
if (!(ctl & HDAC_SDCTL_SRST))
break;
DELAY(10);
} while (--to);
if (ctl & HDAC_SDCTL_SRST)
device_printf(dev, "Reset timeout!\n");
}
static uint32_t
hdac_stream_getptr(device_t dev, device_t child, int dir, int stream)
{
struct hdac_softc *sc = device_get_softc(dev);
int ss, off;
ss = hdac_find_stream(sc, dir, stream);
KASSERT(ss >= 0,
("Reset for not allocated stream (%d/%d)\n", dir, stream));
off = ss << 5;
return (HDAC_READ_4(&sc->mem, off + HDAC_SDLPIB));
}
static int
hdac_unsol_alloc(device_t dev, device_t child, int tag)
{
struct hdac_softc *sc = device_get_softc(dev);
sc->unsol_registered++;
hdac_poll_reinit(sc);
return (tag);
}
static void
hdac_unsol_free(device_t dev, device_t child, int tag)
{
struct hdac_softc *sc = device_get_softc(dev);
sc->unsol_registered--;
hdac_poll_reinit(sc);
}
static device_method_t hdac_methods[] = {
/* device interface */
DEVMETHOD(device_probe, hdac_probe),
DEVMETHOD(device_attach, hdac_attach),
DEVMETHOD(device_detach, hdac_detach),
DEVMETHOD(device_suspend, hdac_suspend),
DEVMETHOD(device_resume, hdac_resume),
/* Bus interface */
DEVMETHOD(bus_get_dma_tag, hdac_get_dma_tag),
DEVMETHOD(bus_print_child, hdac_print_child),
DEVMETHOD(bus_child_location_str, hdac_child_location_str),
DEVMETHOD(bus_child_pnpinfo_str, hdac_child_pnpinfo_str_method),
DEVMETHOD(bus_read_ivar, hdac_read_ivar),
DEVMETHOD(hdac_get_mtx, hdac_get_mtx),
DEVMETHOD(hdac_codec_command, hdac_codec_command),
DEVMETHOD(hdac_stream_alloc, hdac_stream_alloc),
DEVMETHOD(hdac_stream_free, hdac_stream_free),
DEVMETHOD(hdac_stream_start, hdac_stream_start),
DEVMETHOD(hdac_stream_stop, hdac_stream_stop),
DEVMETHOD(hdac_stream_reset, hdac_stream_reset),
DEVMETHOD(hdac_stream_getptr, hdac_stream_getptr),
DEVMETHOD(hdac_unsol_alloc, hdac_unsol_alloc),
DEVMETHOD(hdac_unsol_free, hdac_unsol_free),
DEVMETHOD_END
};
static driver_t hdac_driver = {
"hdac",
hdac_methods,
sizeof(struct hdac_softc),
};
static devclass_t hdac_devclass;
DRIVER_MODULE(snd_hda, pci, hdac_driver, hdac_devclass, NULL, NULL);
Index: head/sys/dev/sound/pci/via8233.c
===================================================================
--- head/sys/dev/sound/pci/via8233.c (revision 283290)
+++ head/sys/dev/sound/pci/via8233.c (revision 283291)
@@ -1,1445 +1,1445 @@
/*-
* Copyright (c) 2002 Orion Hodson <orion@freebsd.org>
* Portions of this code derived from via82c686.c:
* Copyright (c) 2000 David Jones <dej@ox.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Credits due to:
*
* Grzybowski Rafal, Russell Davies, Mark Handley, Daniel O'Connor for
* comments, machine time, testing patches, and patience. VIA for
* providing specs. ALSA for helpful comments and some register poke
* ordering.
*/
#ifdef HAVE_KERNEL_OPTION_HEADERS
#include "opt_snd.h"
#endif
#include <dev/sound/pcm/sound.h>
#include <dev/sound/pcm/ac97.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <sys/sysctl.h>
#include <dev/sound/pci/via8233.h>
SND_DECLARE_FILE("$FreeBSD$");
#define VIA8233_PCI_ID 0x30591106
#define VIA8233_REV_ID_8233PRE 0x10
#define VIA8233_REV_ID_8233C 0x20
#define VIA8233_REV_ID_8233 0x30
#define VIA8233_REV_ID_8233A 0x40
#define VIA8233_REV_ID_8235 0x50
#define VIA8233_REV_ID_8237 0x60
#define VIA8233_REV_ID_8251 0x70
#define SEGS_PER_CHAN 2 /* Segments per channel */
#define NDXSCHANS 4 /* No of DXS channels */
#define NMSGDCHANS 1 /* No of multichannel SGD */
#define NWRCHANS 1 /* No of write channels */
#define NCHANS (NWRCHANS + NDXSCHANS + NMSGDCHANS)
#define NSEGS NCHANS * SEGS_PER_CHAN /* Segments in SGD table */
#define VIA_SEGS_MIN 2
#define VIA_SEGS_MAX 64
#define VIA_SEGS_DEFAULT 2
#define VIA_BLK_MIN 32
#define VIA_BLK_ALIGN (~(VIA_BLK_MIN - 1))
#define VIA_DEFAULT_BUFSZ 0x1000
/* we rely on this struct being packed to 64 bits */
struct via_dma_op {
volatile uint32_t ptr;
volatile uint32_t flags;
#define VIA_DMAOP_EOL 0x80000000
#define VIA_DMAOP_FLAG 0x40000000
#define VIA_DMAOP_STOP 0x20000000
#define VIA_DMAOP_COUNT(x) ((x)&0x00FFFFFF)
};
struct via_info;
struct via_chinfo {
struct via_info *parent;
struct pcm_channel *channel;
struct snd_dbuf *buffer;
struct via_dma_op *sgd_table;
bus_addr_t sgd_addr;
int dir, rbase, active;
unsigned int blksz, blkcnt;
unsigned int ptr, prevptr;
};
struct via_info {
device_t dev;
bus_space_tag_t st;
bus_space_handle_t sh;
bus_dma_tag_t parent_dmat;
bus_dma_tag_t sgd_dmat;
bus_dmamap_t sgd_dmamap;
bus_addr_t sgd_addr;
struct resource *reg, *irq;
int regid, irqid;
void *ih;
struct ac97_info *codec;
unsigned int bufsz, blkcnt;
int dxs_src, dma_eol_wake;
struct via_chinfo pch[NDXSCHANS + NMSGDCHANS];
struct via_chinfo rch[NWRCHANS];
struct via_dma_op *sgd_table;
uint16_t codec_caps;
uint16_t n_dxs_registered;
int play_num, rec_num;
struct mtx *lock;
struct callout poll_timer;
int poll_ticks, polling;
};
static uint32_t via_fmt[] = {
SND_FORMAT(AFMT_U8, 1, 0),
SND_FORMAT(AFMT_U8, 2, 0),
SND_FORMAT(AFMT_S16_LE, 1, 0),
SND_FORMAT(AFMT_S16_LE, 2, 0),
0
};
static struct pcmchan_caps via_vracaps = { 4000, 48000, via_fmt, 0 };
static struct pcmchan_caps via_caps = { 48000, 48000, via_fmt, 0 };
static __inline int
via_chan_active(struct via_info *via)
{
int i, ret = 0;
if (via == NULL)
return (0);
for (i = 0; i < NDXSCHANS + NMSGDCHANS; i++)
ret += via->pch[i].active;
for (i = 0; i < NWRCHANS; i++)
ret += via->rch[i].active;
return (ret);
}
static int
sysctl_via8233_spdif_enable(SYSCTL_HANDLER_ARGS)
{
struct via_info *via;
device_t dev;
uint32_t r;
int err, new_en;
dev = oidp->oid_arg1;
via = pcm_getdevinfo(dev);
snd_mtxlock(via->lock);
r = pci_read_config(dev, VIA_PCI_SPDIF, 1);
snd_mtxunlock(via->lock);
new_en = (r & VIA_SPDIF_EN) ? 1 : 0;
err = sysctl_handle_int(oidp, &new_en, 0, req);
if (err || req->newptr == NULL)
return (err);
if (new_en < 0 || new_en > 1)
return (EINVAL);
if (new_en)
r |= VIA_SPDIF_EN;
else
r &= ~VIA_SPDIF_EN;
snd_mtxlock(via->lock);
pci_write_config(dev, VIA_PCI_SPDIF, r, 1);
snd_mtxunlock(via->lock);
return (0);
}
static int
sysctl_via8233_dxs_src(SYSCTL_HANDLER_ARGS)
{
struct via_info *via;
device_t dev;
int err, val;
dev = oidp->oid_arg1;
via = pcm_getdevinfo(dev);
snd_mtxlock(via->lock);
val = via->dxs_src;
snd_mtxunlock(via->lock);
err = sysctl_handle_int(oidp, &val, 0, req);
if (err || req->newptr == NULL)
return (err);
if (val < 0 || val > 1)
return (EINVAL);
snd_mtxlock(via->lock);
via->dxs_src = val;
snd_mtxunlock(via->lock);
return (0);
}
static int
sysctl_via_polling(SYSCTL_HANDLER_ARGS)
{
struct via_info *via;
device_t dev;
int err, val;
dev = oidp->oid_arg1;
via = pcm_getdevinfo(dev);
if (via == NULL)
return (EINVAL);
snd_mtxlock(via->lock);
val = via->polling;
snd_mtxunlock(via->lock);
err = sysctl_handle_int(oidp, &val, 0, req);
if (err || req->newptr == NULL)
return (err);
if (val < 0 || val > 1)
return (EINVAL);
snd_mtxlock(via->lock);
if (val != via->polling) {
if (via_chan_active(via) != 0)
err = EBUSY;
else if (val == 0)
via->polling = 0;
else
via->polling = 1;
}
snd_mtxunlock(via->lock);
return (err);
}
static void
via_init_sysctls(device_t dev)
{
/* XXX: an user should be able to set this with a control tool,
if not done before 7.0-RELEASE, this needs to be converted to
a device specific sysctl "dev.pcm.X.yyy" via device_get_sysctl_*()
as discussed on multimedia@ in msg-id <861wujij2q.fsf@xps.des.no> */
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"spdif_enabled", CTLTYPE_INT | CTLFLAG_RW, dev, sizeof(dev),
sysctl_via8233_spdif_enable, "I",
"Enable S/PDIF output on primary playback channel");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"dxs_src", CTLTYPE_INT | CTLFLAG_RW, dev, sizeof(dev),
sysctl_via8233_dxs_src, "I",
"Enable VIA DXS Sample Rate Converter");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,
"polling", CTLTYPE_INT | CTLFLAG_RW, dev, sizeof(dev),
sysctl_via_polling, "I",
"Enable polling mode");
}
static __inline uint32_t
via_rd(struct via_info *via, int regno, int size)
{
switch (size) {
case 1:
return (bus_space_read_1(via->st, via->sh, regno));
case 2:
return (bus_space_read_2(via->st, via->sh, regno));
case 4:
return (bus_space_read_4(via->st, via->sh, regno));
default:
return (0xFFFFFFFF);
}
}
static __inline void
via_wr(struct via_info *via, int regno, uint32_t data, int size)
{
switch (size) {
case 1:
bus_space_write_1(via->st, via->sh, regno, data);
break;
case 2:
bus_space_write_2(via->st, via->sh, regno, data);
break;
case 4:
bus_space_write_4(via->st, via->sh, regno, data);
break;
}
}
/* -------------------------------------------------------------------- */
/* Codec interface */
static int
via_waitready_codec(struct via_info *via)
{
int i;
/* poll until codec not busy */
for (i = 0; i < 1000; i++) {
if ((via_rd(via, VIA_AC97_CONTROL, 4) & VIA_AC97_BUSY) == 0)
return (0);
DELAY(1);
}
device_printf(via->dev, "%s: codec busy\n", __func__);
return (1);
}
static int
via_waitvalid_codec(struct via_info *via)
{
int i;
/* poll until codec valid */
for (i = 0; i < 1000; i++) {
if (via_rd(via, VIA_AC97_CONTROL, 4) & VIA_AC97_CODEC00_VALID)
return (0);
DELAY(1);
}
device_printf(via->dev, "%s: codec invalid\n", __func__);
return (1);
}
static int
via_write_codec(kobj_t obj, void *addr, int reg, uint32_t val)
{
struct via_info *via = addr;
if (via_waitready_codec(via))
return (-1);
via_wr(via, VIA_AC97_CONTROL,
VIA_AC97_CODEC00_VALID | VIA_AC97_INDEX(reg) |
VIA_AC97_DATA(val), 4);
return (0);
}
static int
via_read_codec(kobj_t obj, void *addr, int reg)
{
struct via_info *via = addr;
if (via_waitready_codec(via))
return (-1);
via_wr(via, VIA_AC97_CONTROL, VIA_AC97_CODEC00_VALID |
VIA_AC97_READ | VIA_AC97_INDEX(reg), 4);
if (via_waitready_codec(via))
return (-1);
if (via_waitvalid_codec(via))
return (-1);
return (via_rd(via, VIA_AC97_CONTROL, 2));
}
static kobj_method_t via_ac97_methods[] = {
KOBJMETHOD(ac97_read, via_read_codec),
KOBJMETHOD(ac97_write, via_write_codec),
KOBJMETHOD_END
};
AC97_DECLARE(via_ac97);
/* -------------------------------------------------------------------- */
static int
via_buildsgdt(struct via_chinfo *ch)
{
uint32_t phys_addr, flag;
int i;
phys_addr = sndbuf_getbufaddr(ch->buffer);
for (i = 0; i < ch->blkcnt; i++) {
flag = (i == ch->blkcnt - 1) ? VIA_DMAOP_EOL : VIA_DMAOP_FLAG;
ch->sgd_table[i].ptr = phys_addr + (i * ch->blksz);
ch->sgd_table[i].flags = flag | ch->blksz;
}
return (0);
}
/* -------------------------------------------------------------------- */
/* Format setting functions */
static int
via8233wr_setformat(kobj_t obj, void *data, uint32_t format)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
uint32_t f = WR_FORMAT_STOP_INDEX;
if (AFMT_CHANNEL(format) > 1)
f |= WR_FORMAT_STEREO;
if (format & AFMT_S16_LE)
f |= WR_FORMAT_16BIT;
snd_mtxlock(via->lock);
via_wr(via, VIA_WR0_FORMAT, f, 4);
snd_mtxunlock(via->lock);
return (0);
}
static int
via8233dxs_setformat(kobj_t obj, void *data, uint32_t format)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
uint32_t r, v;
r = ch->rbase + VIA8233_RP_DXS_RATEFMT;
snd_mtxlock(via->lock);
v = via_rd(via, r, 4);
v &= ~(VIA8233_DXS_RATEFMT_STEREO | VIA8233_DXS_RATEFMT_16BIT);
if (AFMT_CHANNEL(format) > 1)
v |= VIA8233_DXS_RATEFMT_STEREO;
if (format & AFMT_16BIT)
v |= VIA8233_DXS_RATEFMT_16BIT;
via_wr(via, r, v, 4);
snd_mtxunlock(via->lock);
return (0);
}
static int
via8233msgd_setformat(kobj_t obj, void *data, uint32_t format)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
uint32_t s = 0xff000000;
uint8_t v = (format & AFMT_S16_LE) ? MC_SGD_16BIT : MC_SGD_8BIT;
if (AFMT_CHANNEL(format) > 1) {
v |= MC_SGD_CHANNELS(2);
s |= SLOT3(1) | SLOT4(2);
} else {
v |= MC_SGD_CHANNELS(1);
s |= SLOT3(1) | SLOT4(1);
}
snd_mtxlock(via->lock);
via_wr(via, VIA_MC_SLOT_SELECT, s, 4);
via_wr(via, VIA_MC_SGD_FORMAT, v, 1);
snd_mtxunlock(via->lock);
return (0);
}
/* -------------------------------------------------------------------- */
/* Speed setting functions */
static uint32_t
via8233wr_setspeed(kobj_t obj, void *data, uint32_t speed)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
if (via->codec_caps & AC97_EXTCAP_VRA)
return (ac97_setrate(via->codec, AC97_REGEXT_LADCRATE, speed));
return (48000);
}
static uint32_t
via8233dxs_setspeed(kobj_t obj, void *data, uint32_t speed)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
uint32_t r, v;
r = ch->rbase + VIA8233_RP_DXS_RATEFMT;
snd_mtxlock(via->lock);
v = via_rd(via, r, 4) & ~VIA8233_DXS_RATEFMT_48K;
/* Careful to avoid overflow (divide by 48 per vt8233c docs) */
v |= VIA8233_DXS_RATEFMT_48K * (speed / 48) / (48000 / 48);
via_wr(via, r, v, 4);
snd_mtxunlock(via->lock);
return (speed);
}
static uint32_t
via8233msgd_setspeed(kobj_t obj, void *data, uint32_t speed)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
if (via->codec_caps & AC97_EXTCAP_VRA)
return (ac97_setrate(via->codec, AC97_REGEXT_FDACRATE, speed));
return (48000);
}
/* -------------------------------------------------------------------- */
/* Format probing functions */
static struct pcmchan_caps *
via8233wr_getcaps(kobj_t obj, void *data)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
/* Controlled by ac97 registers */
if (via->codec_caps & AC97_EXTCAP_VRA)
return (&via_vracaps);
return (&via_caps);
}
static struct pcmchan_caps *
via8233dxs_getcaps(kobj_t obj, void *data)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
/*
* Controlled by onboard registers
*
* Apparently, few boards can do DXS sample rate
* conversion.
*/
if (via->dxs_src)
return (&via_vracaps);
return (&via_caps);
}
static struct pcmchan_caps *
via8233msgd_getcaps(kobj_t obj, void *data)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
/* Controlled by ac97 registers */
if (via->codec_caps & AC97_EXTCAP_VRA)
return (&via_vracaps);
return (&via_caps);
}
/* -------------------------------------------------------------------- */
/* Common functions */
static int
via8233chan_setfragments(kobj_t obj, void *data,
uint32_t blksz, uint32_t blkcnt)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
blksz &= VIA_BLK_ALIGN;
if (blksz > (sndbuf_getmaxsize(ch->buffer) / VIA_SEGS_MIN))
blksz = sndbuf_getmaxsize(ch->buffer) / VIA_SEGS_MIN;
if (blksz < VIA_BLK_MIN)
blksz = VIA_BLK_MIN;
if (blkcnt > VIA_SEGS_MAX)
blkcnt = VIA_SEGS_MAX;
if (blkcnt < VIA_SEGS_MIN)
blkcnt = VIA_SEGS_MIN;
while ((blksz * blkcnt) > sndbuf_getmaxsize(ch->buffer)) {
if ((blkcnt >> 1) >= VIA_SEGS_MIN)
blkcnt >>= 1;
else if ((blksz >> 1) >= VIA_BLK_MIN)
blksz >>= 1;
else
break;
}
if ((sndbuf_getblksz(ch->buffer) != blksz ||
sndbuf_getblkcnt(ch->buffer) != blkcnt) &&
sndbuf_resize(ch->buffer, blkcnt, blksz) != 0)
device_printf(via->dev, "%s: failed blksz=%u blkcnt=%u\n",
__func__, blksz, blkcnt);
ch->blksz = sndbuf_getblksz(ch->buffer);
ch->blkcnt = sndbuf_getblkcnt(ch->buffer);
return (0);
}
static uint32_t
via8233chan_setblocksize(kobj_t obj, void *data, uint32_t blksz)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
via8233chan_setfragments(obj, data, blksz, via->blkcnt);
return (ch->blksz);
}
static uint32_t
via8233chan_getptr(kobj_t obj, void *data)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
uint32_t v, index, count, ptr;
snd_mtxlock(via->lock);
if (via->polling != 0) {
ptr = ch->ptr;
snd_mtxunlock(via->lock);
} else {
v = via_rd(via, ch->rbase + VIA_RP_CURRENT_COUNT, 4);
snd_mtxunlock(via->lock);
index = v >> 24; /* Last completed buffer */
count = v & 0x00ffffff; /* Bytes remaining */
ptr = (index + 1) * ch->blksz - count;
ptr %= ch->blkcnt * ch->blksz; /* Wrap to available space */
}
return (ptr);
}
static void
via8233chan_reset(struct via_info *via, struct via_chinfo *ch)
{
via_wr(via, ch->rbase + VIA_RP_CONTROL, SGD_CONTROL_STOP, 1);
via_wr(via, ch->rbase + VIA_RP_CONTROL, 0x00, 1);
via_wr(via, ch->rbase + VIA_RP_STATUS,
SGD_STATUS_EOL | SGD_STATUS_FLAG, 1);
}
/* -------------------------------------------------------------------- */
/* Channel initialization functions */
static void
via8233chan_sgdinit(struct via_info *via, struct via_chinfo *ch, int chnum)
{
ch->sgd_table = &via->sgd_table[chnum * VIA_SEGS_MAX];
ch->sgd_addr = via->sgd_addr + chnum * VIA_SEGS_MAX *
sizeof(struct via_dma_op);
}
static void*
via8233wr_init(kobj_t obj, void *devinfo, struct snd_dbuf *b,
struct pcm_channel *c, int dir)
{
struct via_info *via = devinfo;
struct via_chinfo *ch;
int num;
snd_mtxlock(via->lock);
num = via->rec_num++;
ch = &via->rch[num];
ch->parent = via;
ch->channel = c;
ch->buffer = b;
ch->dir = dir;
ch->blkcnt = via->blkcnt;
ch->rbase = VIA_WR_BASE(num);
via_wr(via, ch->rbase + VIA_WR_RP_SGD_FORMAT, WR_FIFO_ENABLE, 1);
snd_mtxunlock(via->lock);
if (sndbuf_alloc(ch->buffer, via->parent_dmat, 0, via->bufsz) != 0)
return (NULL);
snd_mtxlock(via->lock);
via8233chan_sgdinit(via, ch, num);
via8233chan_reset(via, ch);
snd_mtxunlock(via->lock);
return (ch);
}
static void*
via8233dxs_init(kobj_t obj, void *devinfo, struct snd_dbuf *b,
struct pcm_channel *c, int dir)
{
struct via_info *via = devinfo;
struct via_chinfo *ch;
int num;
snd_mtxlock(via->lock);
num = via->play_num++;
ch = &via->pch[num];
ch->parent = via;
ch->channel = c;
ch->buffer = b;
ch->dir = dir;
ch->blkcnt = via->blkcnt;
/*
* All cards apparently support DXS3, but not other DXS
* channels. We therefore want to align first DXS channel to
* DXS3.
*/
ch->rbase = VIA_DXS_BASE(NDXSCHANS - 1 - via->n_dxs_registered);
via->n_dxs_registered++;
snd_mtxunlock(via->lock);
if (sndbuf_alloc(ch->buffer, via->parent_dmat, 0, via->bufsz) != 0)
return (NULL);
snd_mtxlock(via->lock);
via8233chan_sgdinit(via, ch, NWRCHANS + num);
via8233chan_reset(via, ch);
snd_mtxunlock(via->lock);
return (ch);
}
static void*
via8233msgd_init(kobj_t obj, void *devinfo, struct snd_dbuf *b,
struct pcm_channel *c, int dir)
{
struct via_info *via = devinfo;
struct via_chinfo *ch;
int num;
snd_mtxlock(via->lock);
num = via->play_num++;
ch = &via->pch[num];
ch->parent = via;
ch->channel = c;
ch->buffer = b;
ch->dir = dir;
ch->rbase = VIA_MC_SGD_STATUS;
ch->blkcnt = via->blkcnt;
snd_mtxunlock(via->lock);
if (sndbuf_alloc(ch->buffer, via->parent_dmat, 0, via->bufsz) != 0)
return (NULL);
snd_mtxlock(via->lock);
via8233chan_sgdinit(via, ch, NWRCHANS + num);
via8233chan_reset(via, ch);
snd_mtxunlock(via->lock);
return (ch);
}
static void
via8233chan_mute(struct via_info *via, struct via_chinfo *ch, int muted)
{
if (BASE_IS_VIA_DXS_REG(ch->rbase)) {
int r;
muted = (muted) ? VIA8233_DXS_MUTE : 0;
via_wr(via, ch->rbase + VIA8233_RP_DXS_LVOL, muted, 1);
via_wr(via, ch->rbase + VIA8233_RP_DXS_RVOL, muted, 1);
r = via_rd(via, ch->rbase + VIA8233_RP_DXS_LVOL, 1) &
VIA8233_DXS_MUTE;
if (r != muted)
device_printf(via->dev,
"%s: failed to set dxs volume "
"(dxs base 0x%02x).\n", __func__, ch->rbase);
}
}
static __inline int
via_poll_channel(struct via_chinfo *ch)
{
struct via_info *via;
uint32_t sz, delta;
uint32_t v, index, count;
int ptr;
if (ch == NULL || ch->channel == NULL || ch->active == 0)
return (0);
via = ch->parent;
sz = ch->blksz * ch->blkcnt;
v = via_rd(via, ch->rbase + VIA_RP_CURRENT_COUNT, 4);
index = v >> 24;
count = v & 0x00ffffff;
ptr = ((index + 1) * ch->blksz) - count;
ptr %= sz;
ptr &= ~(ch->blksz - 1);
ch->ptr = ptr;
delta = (sz + ptr - ch->prevptr) % sz;
if (delta < ch->blksz)
return (0);
ch->prevptr = ptr;
return (1);
}
static void
via_poll_callback(void *arg)
{
struct via_info *via = arg;
uint32_t ptrigger = 0, rtrigger = 0;
int i;
if (via == NULL)
return;
snd_mtxlock(via->lock);
if (via->polling == 0 || via_chan_active(via) == 0) {
snd_mtxunlock(via->lock);
return;
}
for (i = 0; i < NDXSCHANS + NMSGDCHANS; i++)
ptrigger |= (via_poll_channel(&via->pch[i]) != 0) ?
(1 << i) : 0;
for (i = 0; i < NWRCHANS; i++)
rtrigger |= (via_poll_channel(&via->rch[i]) != 0) ?
(1 << i) : 0;
/* XXX */
callout_reset(&via->poll_timer, 1/*via->poll_ticks*/,
via_poll_callback, via);
snd_mtxunlock(via->lock);
for (i = 0; i < NDXSCHANS + NMSGDCHANS; i++) {
if (ptrigger & (1 << i))
chn_intr(via->pch[i].channel);
}
for (i = 0; i < NWRCHANS; i++) {
if (rtrigger & (1 << i))
chn_intr(via->rch[i].channel);
}
}
static int
via_poll_ticks(struct via_info *via)
{
struct via_chinfo *ch;
int i;
int ret = hz;
int pollticks;
for (i = 0; i < NDXSCHANS + NMSGDCHANS; i++) {
ch = &via->pch[i];
if (ch->channel == NULL || ch->active == 0)
continue;
pollticks = ((uint64_t)hz * ch->blksz) /
((uint64_t)sndbuf_getalign(ch->buffer) *
sndbuf_getspd(ch->buffer));
pollticks >>= 2;
if (pollticks > hz)
pollticks = hz;
if (pollticks < 1)
pollticks = 1;
if (pollticks < ret)
ret = pollticks;
}
for (i = 0; i < NWRCHANS; i++) {
ch = &via->rch[i];
if (ch->channel == NULL || ch->active == 0)
continue;
pollticks = ((uint64_t)hz * ch->blksz) /
((uint64_t)sndbuf_getalign(ch->buffer) *
sndbuf_getspd(ch->buffer));
pollticks >>= 2;
if (pollticks > hz)
pollticks = hz;
if (pollticks < 1)
pollticks = 1;
if (pollticks < ret)
ret = pollticks;
}
return (ret);
}
static int
via8233chan_trigger(kobj_t obj, void* data, int go)
{
struct via_chinfo *ch = data;
struct via_info *via = ch->parent;
int pollticks;
if (!PCMTRIG_COMMON(go))
return (0);
snd_mtxlock(via->lock);
switch(go) {
case PCMTRIG_START:
via_buildsgdt(ch);
via8233chan_mute(via, ch, 0);
via_wr(via, ch->rbase + VIA_RP_TABLE_PTR, ch->sgd_addr, 4);
if (via->polling != 0) {
ch->ptr = 0;
ch->prevptr = 0;
pollticks = ((uint64_t)hz * ch->blksz) /
((uint64_t)sndbuf_getalign(ch->buffer) *
sndbuf_getspd(ch->buffer));
pollticks >>= 2;
if (pollticks > hz)
pollticks = hz;
if (pollticks < 1)
pollticks = 1;
if (via_chan_active(via) == 0 ||
pollticks < via->poll_ticks) {
if (bootverbose) {
if (via_chan_active(via) == 0)
printf("%s: pollticks=%d\n",
__func__, pollticks);
else
printf("%s: "
"pollticks %d -> %d\n",
__func__, via->poll_ticks,
pollticks);
}
via->poll_ticks = pollticks;
callout_reset(&via->poll_timer, 1,
via_poll_callback, via);
}
}
via_wr(via, ch->rbase + VIA_RP_CONTROL,
SGD_CONTROL_START | SGD_CONTROL_AUTOSTART |
((via->polling == 0) ?
(SGD_CONTROL_I_EOL | SGD_CONTROL_I_FLAG) : 0), 1);
ch->active = 1;
break;
case PCMTRIG_STOP:
case PCMTRIG_ABORT:
via_wr(via, ch->rbase + VIA_RP_CONTROL, SGD_CONTROL_STOP, 1);
via8233chan_mute(via, ch, 1);
via8233chan_reset(via, ch);
ch->active = 0;
if (via->polling != 0) {
if (via_chan_active(via) == 0) {
callout_stop(&via->poll_timer);
via->poll_ticks = 1;
} else {
pollticks = via_poll_ticks(via);
if (pollticks > via->poll_ticks) {
if (bootverbose)
printf("%s: pollticks "
"%d -> %d\n",
__func__, via->poll_ticks,
pollticks);
via->poll_ticks = pollticks;
callout_reset(&via->poll_timer,
1, via_poll_callback,
via);
}
}
}
break;
default:
break;
}
snd_mtxunlock(via->lock);
return (0);
}
static kobj_method_t via8233wr_methods[] = {
KOBJMETHOD(channel_init, via8233wr_init),
KOBJMETHOD(channel_setformat, via8233wr_setformat),
KOBJMETHOD(channel_setspeed, via8233wr_setspeed),
KOBJMETHOD(channel_getcaps, via8233wr_getcaps),
KOBJMETHOD(channel_setblocksize, via8233chan_setblocksize),
KOBJMETHOD(channel_setfragments, via8233chan_setfragments),
KOBJMETHOD(channel_trigger, via8233chan_trigger),
KOBJMETHOD(channel_getptr, via8233chan_getptr),
KOBJMETHOD_END
};
CHANNEL_DECLARE(via8233wr);
static kobj_method_t via8233dxs_methods[] = {
KOBJMETHOD(channel_init, via8233dxs_init),
KOBJMETHOD(channel_setformat, via8233dxs_setformat),
KOBJMETHOD(channel_setspeed, via8233dxs_setspeed),
KOBJMETHOD(channel_getcaps, via8233dxs_getcaps),
KOBJMETHOD(channel_setblocksize, via8233chan_setblocksize),
KOBJMETHOD(channel_setfragments, via8233chan_setfragments),
KOBJMETHOD(channel_trigger, via8233chan_trigger),
KOBJMETHOD(channel_getptr, via8233chan_getptr),
KOBJMETHOD_END
};
CHANNEL_DECLARE(via8233dxs);
static kobj_method_t via8233msgd_methods[] = {
KOBJMETHOD(channel_init, via8233msgd_init),
KOBJMETHOD(channel_setformat, via8233msgd_setformat),
KOBJMETHOD(channel_setspeed, via8233msgd_setspeed),
KOBJMETHOD(channel_getcaps, via8233msgd_getcaps),
KOBJMETHOD(channel_setblocksize, via8233chan_setblocksize),
KOBJMETHOD(channel_setfragments, via8233chan_setfragments),
KOBJMETHOD(channel_trigger, via8233chan_trigger),
KOBJMETHOD(channel_getptr, via8233chan_getptr),
KOBJMETHOD_END
};
CHANNEL_DECLARE(via8233msgd);
/* -------------------------------------------------------------------- */
static void
via_intr(void *p)
{
struct via_info *via = p;
uint32_t ptrigger = 0, rtrigger = 0;
int i, reg, stat;
snd_mtxlock(via->lock);
if (via->polling != 0) {
snd_mtxunlock(via->lock);
return;
}
/* Poll playback channels */
for (i = 0; i < NDXSCHANS + NMSGDCHANS; i++) {
if (via->pch[i].channel == NULL || via->pch[i].active == 0)
continue;
reg = via->pch[i].rbase + VIA_RP_STATUS;
stat = via_rd(via, reg, 1);
if (stat & SGD_STATUS_INTR) {
if (via->dma_eol_wake && ((stat & SGD_STATUS_EOL) ||
!(stat & SGD_STATUS_ACTIVE)))
via_wr(via, via->pch[i].rbase + VIA_RP_CONTROL,
SGD_CONTROL_START | SGD_CONTROL_AUTOSTART |
SGD_CONTROL_I_EOL | SGD_CONTROL_I_FLAG, 1);
via_wr(via, reg, stat, 1);
ptrigger |= 1 << i;
}
}
/* Poll record channels */
for (i = 0; i < NWRCHANS; i++) {
if (via->rch[i].channel == NULL || via->rch[i].active == 0)
continue;
reg = via->rch[i].rbase + VIA_RP_STATUS;
stat = via_rd(via, reg, 1);
if (stat & SGD_STATUS_INTR) {
if (via->dma_eol_wake && ((stat & SGD_STATUS_EOL) ||
!(stat & SGD_STATUS_ACTIVE)))
via_wr(via, via->rch[i].rbase + VIA_RP_CONTROL,
SGD_CONTROL_START | SGD_CONTROL_AUTOSTART |
SGD_CONTROL_I_EOL | SGD_CONTROL_I_FLAG, 1);
via_wr(via, reg, stat, 1);
rtrigger |= 1 << i;
}
}
snd_mtxunlock(via->lock);
for (i = 0; i < NDXSCHANS + NMSGDCHANS; i++) {
if (ptrigger & (1 << i))
chn_intr(via->pch[i].channel);
}
for (i = 0; i < NWRCHANS; i++) {
if (rtrigger & (1 << i))
chn_intr(via->rch[i].channel);
}
}
/*
* Probe and attach the card
*/
static int
via_probe(device_t dev)
{
switch(pci_get_devid(dev)) {
case VIA8233_PCI_ID:
switch(pci_get_revid(dev)) {
case VIA8233_REV_ID_8233PRE:
device_set_desc(dev, "VIA VT8233 (pre)");
return (BUS_PROBE_DEFAULT);
case VIA8233_REV_ID_8233C:
device_set_desc(dev, "VIA VT8233C");
return (BUS_PROBE_DEFAULT);
case VIA8233_REV_ID_8233:
device_set_desc(dev, "VIA VT8233");
return (BUS_PROBE_DEFAULT);
case VIA8233_REV_ID_8233A:
device_set_desc(dev, "VIA VT8233A");
return (BUS_PROBE_DEFAULT);
case VIA8233_REV_ID_8235:
device_set_desc(dev, "VIA VT8235");
return (BUS_PROBE_DEFAULT);
case VIA8233_REV_ID_8237:
device_set_desc(dev, "VIA VT8237");
return (BUS_PROBE_DEFAULT);
case VIA8233_REV_ID_8251:
device_set_desc(dev, "VIA VT8251");
return (BUS_PROBE_DEFAULT);
default:
device_set_desc(dev, "VIA VT8233X"); /* Unknown */
return (BUS_PROBE_DEFAULT);
}
}
return (ENXIO);
}
static void
dma_cb(void *p, bus_dma_segment_t *bds, int a, int b)
{
struct via_info *via = (struct via_info *)p;
via->sgd_addr = bds->ds_addr;
}
static int
via_chip_init(device_t dev)
{
uint32_t data, cnt;
/* Wake up and reset AC97 if necessary */
data = pci_read_config(dev, VIA_PCI_ACLINK_STAT, 1);
if ((data & VIA_PCI_ACLINK_C00_READY) == 0) {
/* Cold reset per ac97r2.3 spec (page 95) */
/* Assert low */
pci_write_config(dev, VIA_PCI_ACLINK_CTRL,
VIA_PCI_ACLINK_EN, 1);
/* Wait T_rst_low */
DELAY(100);
/* Assert high */
pci_write_config(dev, VIA_PCI_ACLINK_CTRL,
VIA_PCI_ACLINK_EN | VIA_PCI_ACLINK_NRST, 1);
/* Wait T_rst2clk */
DELAY(5);
/* Assert low */
pci_write_config(dev, VIA_PCI_ACLINK_CTRL,
VIA_PCI_ACLINK_EN, 1);
} else {
/* Warm reset */
/* Force no sync */
pci_write_config(dev, VIA_PCI_ACLINK_CTRL,
VIA_PCI_ACLINK_EN, 1);
DELAY(100);
/* Sync */
pci_write_config(dev, VIA_PCI_ACLINK_CTRL,
VIA_PCI_ACLINK_EN | VIA_PCI_ACLINK_SYNC, 1);
/* Wait T_sync_high */
DELAY(5);
/* Force no sync */
pci_write_config(dev, VIA_PCI_ACLINK_CTRL,
VIA_PCI_ACLINK_EN, 1);
/* Wait T_sync2clk */
DELAY(5);
}
/* Power everything up */
pci_write_config(dev, VIA_PCI_ACLINK_CTRL, VIA_PCI_ACLINK_DESIRED, 1);
/* Wait for codec to become ready (largest reported delay 310ms) */
for (cnt = 0; cnt < 2000; cnt++) {
data = pci_read_config(dev, VIA_PCI_ACLINK_STAT, 1);
if (data & VIA_PCI_ACLINK_C00_READY)
return (0);
DELAY(5000);
}
device_printf(dev, "primary codec not ready (cnt = 0x%02x)\n", cnt);
return (ENXIO);
}
static int
via_attach(device_t dev)
{
struct via_info *via = 0;
char status[SND_STATUSLEN];
int i, via_dxs_disabled, via_dxs_src, via_dxs_chnum, via_sgd_chnum;
int nsegs;
uint32_t revid;
via = malloc(sizeof *via, M_DEVBUF, M_WAITOK | M_ZERO);
via->lock = snd_mtxcreate(device_get_nameunit(dev),
"snd_via8233 softc");
via->dev = dev;
- callout_init(&via->poll_timer, CALLOUT_MPSAFE);
+ callout_init(&via->poll_timer, 1);
via->poll_ticks = 1;
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "polling", &i) == 0 && i != 0)
via->polling = 1;
else
via->polling = 0;
pci_set_powerstate(dev, PCI_POWERSTATE_D0);
pci_enable_busmaster(dev);
via->regid = PCIR_BAR(0);
via->reg = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &via->regid,
RF_ACTIVE);
if (!via->reg) {
device_printf(dev, "cannot allocate bus resource.");
goto bad;
}
via->st = rman_get_bustag(via->reg);
via->sh = rman_get_bushandle(via->reg);
via->irqid = 0;
via->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &via->irqid,
RF_ACTIVE | RF_SHAREABLE);
if (!via->irq ||
snd_setup_intr(dev, via->irq, INTR_MPSAFE,
via_intr, via, &via->ih)) {
device_printf(dev, "unable to map interrupt\n");
goto bad;
}
via->bufsz = pcm_getbuffersize(dev, 4096, VIA_DEFAULT_BUFSZ, 65536);
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "blocksize", &i) == 0 && i > 0) {
i &= VIA_BLK_ALIGN;
if (i < VIA_BLK_MIN)
i = VIA_BLK_MIN;
via->blkcnt = via->bufsz / i;
i = 0;
while (via->blkcnt >> i)
i++;
via->blkcnt = 1 << (i - 1);
if (via->blkcnt < VIA_SEGS_MIN)
via->blkcnt = VIA_SEGS_MIN;
else if (via->blkcnt > VIA_SEGS_MAX)
via->blkcnt = VIA_SEGS_MAX;
} else
via->blkcnt = VIA_SEGS_DEFAULT;
revid = pci_get_revid(dev);
/*
* VIA8251 lost its interrupt after DMA EOL, and need
* a gentle spank on its face within interrupt handler.
*/
if (revid == VIA8233_REV_ID_8251)
via->dma_eol_wake = 1;
else
via->dma_eol_wake = 0;
/*
* Decide whether DXS had to be disabled or not
*/
if (revid == VIA8233_REV_ID_8233A) {
/*
* DXS channel is disabled. Reports from multiple users
* that it plays at half-speed. Do not see this behaviour
* on available 8233C or when emulating 8233A register set
* on 8233C (either with or without ac97 VRA).
*/
via_dxs_disabled = 1;
} else if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "via_dxs_disabled",
&via_dxs_disabled) == 0)
via_dxs_disabled = (via_dxs_disabled > 0) ? 1 : 0;
else
via_dxs_disabled = 0;
if (via_dxs_disabled) {
via_dxs_chnum = 0;
via_sgd_chnum = 1;
} else {
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "via_dxs_channels",
&via_dxs_chnum) != 0)
via_dxs_chnum = NDXSCHANS;
if (resource_int_value(device_get_name(dev),
device_get_unit(dev), "via_sgd_channels",
&via_sgd_chnum) != 0)
via_sgd_chnum = NMSGDCHANS;
}
if (via_dxs_chnum > NDXSCHANS)
via_dxs_chnum = NDXSCHANS;
else if (via_dxs_chnum < 0)
via_dxs_chnum = 0;
if (via_sgd_chnum > NMSGDCHANS)
via_sgd_chnum = NMSGDCHANS;
else if (via_sgd_chnum < 0)
via_sgd_chnum = 0;
if (via_dxs_chnum + via_sgd_chnum < 1) {
/* Minimalist ? */
via_dxs_chnum = 1;
via_sgd_chnum = 0;
}
if (via_dxs_chnum > 0 && resource_int_value(device_get_name(dev),
device_get_unit(dev), "via_dxs_src", &via_dxs_src) == 0)
via->dxs_src = (via_dxs_src > 0) ? 1 : 0;
else
via->dxs_src = 0;
nsegs = (via_dxs_chnum + via_sgd_chnum + NWRCHANS) * VIA_SEGS_MAX;
/* DMA tag for buffers */
if (bus_dma_tag_create(/*parent*/bus_get_dma_tag(dev), /*alignment*/2,
/*boundary*/0,
/*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
/*highaddr*/BUS_SPACE_MAXADDR,
/*filter*/NULL, /*filterarg*/NULL,
/*maxsize*/via->bufsz, /*nsegments*/1, /*maxsegz*/0x3ffff,
/*flags*/0, /*lockfunc*/NULL,
/*lockarg*/NULL, &via->parent_dmat) != 0) {
device_printf(dev, "unable to create dma tag\n");
goto bad;
}
/*
* DMA tag for SGD table. The 686 uses scatter/gather DMA and
* requires a list in memory of work to do. We need only 16 bytes
* for this list, and it is wasteful to allocate 16K.
*/
if (bus_dma_tag_create(/*parent*/bus_get_dma_tag(dev), /*alignment*/2,
/*boundary*/0,
/*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
/*highaddr*/BUS_SPACE_MAXADDR,
/*filter*/NULL, /*filterarg*/NULL,
/*maxsize*/nsegs * sizeof(struct via_dma_op),
/*nsegments*/1, /*maxsegz*/0x3ffff,
/*flags*/0, /*lockfunc*/NULL,
/*lockarg*/NULL, &via->sgd_dmat) != 0) {
device_printf(dev, "unable to create dma tag\n");
goto bad;
}
if (bus_dmamem_alloc(via->sgd_dmat, (void **)&via->sgd_table,
BUS_DMA_NOWAIT, &via->sgd_dmamap) == -1)
goto bad;
if (bus_dmamap_load(via->sgd_dmat, via->sgd_dmamap, via->sgd_table,
nsegs * sizeof(struct via_dma_op), dma_cb, via, 0))
goto bad;
if (via_chip_init(dev))
goto bad;
via->codec = AC97_CREATE(dev, via, via_ac97);
if (!via->codec)
goto bad;
mixer_init(dev, ac97_getmixerclass(), via->codec);
via->codec_caps = ac97_getextcaps(via->codec);
/* Try to set VRA without generating an error, VRM not reqrd yet */
if (via->codec_caps &
(AC97_EXTCAP_VRA | AC97_EXTCAP_VRM | AC97_EXTCAP_DRA)) {
uint16_t ext = ac97_getextmode(via->codec);
ext |= (via->codec_caps &
(AC97_EXTCAP_VRA | AC97_EXTCAP_VRM));
ext &= ~AC97_EXTCAP_DRA;
ac97_setextmode(via->codec, ext);
}
snprintf(status, SND_STATUSLEN, "at io 0x%lx irq %ld %s",
rman_get_start(via->reg), rman_get_start(via->irq),
PCM_KLDSTRING(snd_via8233));
/* Register */
if (pcm_register(dev, via, via_dxs_chnum + via_sgd_chnum, NWRCHANS))
goto bad;
for (i = 0; i < via_dxs_chnum; i++)
pcm_addchan(dev, PCMDIR_PLAY, &via8233dxs_class, via);
for (i = 0; i < via_sgd_chnum; i++)
pcm_addchan(dev, PCMDIR_PLAY, &via8233msgd_class, via);
for (i = 0; i < NWRCHANS; i++)
pcm_addchan(dev, PCMDIR_REC, &via8233wr_class, via);
if (via_dxs_chnum > 0)
via_init_sysctls(dev);
device_printf(dev, "<VIA DXS %sabled: DXS%s %d / SGD %d / REC %d>\n",
(via_dxs_chnum > 0) ? "En" : "Dis", (via->dxs_src) ? "(SRC)" : "",
via_dxs_chnum, via_sgd_chnum, NWRCHANS);
pcm_setstatus(dev, status);
return (0);
bad:
if (via->codec)
ac97_destroy(via->codec);
if (via->reg)
bus_release_resource(dev, SYS_RES_IOPORT, via->regid, via->reg);
if (via->ih)
bus_teardown_intr(dev, via->irq, via->ih);
if (via->irq)
bus_release_resource(dev, SYS_RES_IRQ, via->irqid, via->irq);
if (via->parent_dmat)
bus_dma_tag_destroy(via->parent_dmat);
if (via->sgd_addr)
bus_dmamap_unload(via->sgd_dmat, via->sgd_dmamap);
if (via->sgd_table)
bus_dmamem_free(via->sgd_dmat, via->sgd_table, via->sgd_dmamap);
if (via->sgd_dmat)
bus_dma_tag_destroy(via->sgd_dmat);
if (via->lock)
snd_mtxfree(via->lock);
if (via)
free(via, M_DEVBUF);
return (ENXIO);
}
static int
via_detach(device_t dev)
{
int r;
struct via_info *via;
r = pcm_unregister(dev);
if (r)
return (r);
via = pcm_getdevinfo(dev);
if (via != NULL && (via->play_num != 0 || via->rec_num != 0)) {
snd_mtxlock(via->lock);
via->polling = 0;
callout_stop(&via->poll_timer);
snd_mtxunlock(via->lock);
callout_drain(&via->poll_timer);
}
bus_release_resource(dev, SYS_RES_IOPORT, via->regid, via->reg);
bus_teardown_intr(dev, via->irq, via->ih);
bus_release_resource(dev, SYS_RES_IRQ, via->irqid, via->irq);
bus_dma_tag_destroy(via->parent_dmat);
bus_dmamap_unload(via->sgd_dmat, via->sgd_dmamap);
bus_dmamem_free(via->sgd_dmat, via->sgd_table, via->sgd_dmamap);
bus_dma_tag_destroy(via->sgd_dmat);
snd_mtxfree(via->lock);
free(via, M_DEVBUF);
return (0);
}
static device_method_t via_methods[] = {
DEVMETHOD(device_probe, via_probe),
DEVMETHOD(device_attach, via_attach),
DEVMETHOD(device_detach, via_detach),
{ 0, 0}
};
static driver_t via_driver = {
"pcm",
via_methods,
PCM_SOFTC_SIZE,
};
DRIVER_MODULE(snd_via8233, pci, via_driver, pcm_devclass, 0, 0);
MODULE_DEPEND(snd_via8233, sound, SOUND_MINVER, SOUND_PREFVER, SOUND_MAXVER);
MODULE_VERSION(snd_via8233, 1);
Index: head/sys/dev/twa/tw_osl_freebsd.c
===================================================================
--- head/sys/dev/twa/tw_osl_freebsd.c (revision 283290)
+++ head/sys/dev/twa/tw_osl_freebsd.c (revision 283291)
@@ -1,1712 +1,1712 @@
/*
* Copyright (c) 2004-07 Applied Micro Circuits Corporation.
* Copyright (c) 2004-05 Vinod Kashyap.
* Copyright (c) 2000 Michael Smith
* Copyright (c) 2000 BSDi
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* AMCC'S 3ware driver for 9000 series storage controllers.
*
* Author: Vinod Kashyap
* Modifications by: Adam Radford
* Modifications by: Manjunath Ranganathaiah
*/
/*
* FreeBSD specific functions not related to CAM, and other
* miscellaneous functions.
*/
#include <dev/twa/tw_osl_includes.h>
#include <dev/twa/tw_cl_fwif.h>
#include <dev/twa/tw_cl_ioctl.h>
#include <dev/twa/tw_osl_ioctl.h>
#ifdef TW_OSL_DEBUG
TW_INT32 TW_DEBUG_LEVEL_FOR_OSL = TW_OSL_DEBUG;
TW_INT32 TW_OSL_DEBUG_LEVEL_FOR_CL = TW_OSL_DEBUG;
#endif /* TW_OSL_DEBUG */
static MALLOC_DEFINE(TW_OSLI_MALLOC_CLASS, "twa_commands", "twa commands");
static d_open_t twa_open;
static d_close_t twa_close;
static d_ioctl_t twa_ioctl;
static struct cdevsw twa_cdevsw = {
.d_version = D_VERSION,
.d_open = twa_open,
.d_close = twa_close,
.d_ioctl = twa_ioctl,
.d_name = "twa",
};
static devclass_t twa_devclass;
/*
* Function name: twa_open
* Description: Called when the controller is opened.
* Simply marks the controller as open.
*
* Input: dev -- control device corresponding to the ctlr
* flags -- mode of open
* fmt -- device type (character/block etc.)
* proc -- current process
* Output: None
* Return value: 0 -- success
* non-zero-- failure
*/
static TW_INT32
twa_open(struct cdev *dev, TW_INT32 flags, TW_INT32 fmt, struct thread *proc)
{
struct twa_softc *sc = (struct twa_softc *)(dev->si_drv1);
tw_osli_dbg_dprintf(5, sc, "entered");
sc->open = TW_CL_TRUE;
return(0);
}
/*
* Function name: twa_close
* Description: Called when the controller is closed.
* Simply marks the controller as not open.
*
* Input: dev -- control device corresponding to the ctlr
* flags -- mode of corresponding open
* fmt -- device type (character/block etc.)
* proc -- current process
* Output: None
* Return value: 0 -- success
* non-zero-- failure
*/
static TW_INT32
twa_close(struct cdev *dev, TW_INT32 flags, TW_INT32 fmt, struct thread *proc)
{
struct twa_softc *sc = (struct twa_softc *)(dev->si_drv1);
tw_osli_dbg_dprintf(5, sc, "entered");
sc->open = TW_CL_FALSE;
return(0);
}
/*
* Function name: twa_ioctl
* Description: Called when an ioctl is posted to the controller.
* Handles any OS Layer specific cmds, passes the rest
* on to the Common Layer.
*
* Input: dev -- control device corresponding to the ctlr
* cmd -- ioctl cmd
* buf -- ptr to buffer in kernel memory, which is
* a copy of the input buffer in user-space
* flags -- mode of corresponding open
* proc -- current process
* Output: buf -- ptr to buffer in kernel memory, which will
* be copied to the output buffer in user-space
* Return value: 0 -- success
* non-zero-- failure
*/
static TW_INT32
twa_ioctl(struct cdev *dev, u_long cmd, caddr_t buf, TW_INT32 flags, struct thread *proc)
{
struct twa_softc *sc = (struct twa_softc *)(dev->si_drv1);
TW_INT32 error;
tw_osli_dbg_dprintf(5, sc, "entered");
switch (cmd) {
case TW_OSL_IOCTL_FIRMWARE_PASS_THROUGH:
tw_osli_dbg_dprintf(6, sc, "ioctl: fw_passthru");
error = tw_osli_fw_passthru(sc, (TW_INT8 *)buf);
break;
case TW_OSL_IOCTL_SCAN_BUS:
/* Request CAM for a bus scan. */
tw_osli_dbg_dprintf(6, sc, "ioctl: scan bus");
error = tw_osli_request_bus_scan(sc);
break;
default:
tw_osli_dbg_dprintf(6, sc, "ioctl: 0x%lx", cmd);
error = tw_cl_ioctl(&sc->ctlr_handle, cmd, buf);
break;
}
return(error);
}
static TW_INT32 twa_probe(device_t dev);
static TW_INT32 twa_attach(device_t dev);
static TW_INT32 twa_detach(device_t dev);
static TW_INT32 twa_shutdown(device_t dev);
static TW_VOID twa_busdma_lock(TW_VOID *lock_arg, bus_dma_lock_op_t op);
static TW_VOID twa_pci_intr(TW_VOID *arg);
static TW_VOID twa_watchdog(TW_VOID *arg);
int twa_setup_intr(struct twa_softc *sc);
int twa_teardown_intr(struct twa_softc *sc);
static TW_INT32 tw_osli_alloc_mem(struct twa_softc *sc);
static TW_VOID tw_osli_free_resources(struct twa_softc *sc);
static TW_VOID twa_map_load_data_callback(TW_VOID *arg,
bus_dma_segment_t *segs, TW_INT32 nsegments, TW_INT32 error);
static TW_VOID twa_map_load_callback(TW_VOID *arg,
bus_dma_segment_t *segs, TW_INT32 nsegments, TW_INT32 error);
static device_method_t twa_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, twa_probe),
DEVMETHOD(device_attach, twa_attach),
DEVMETHOD(device_detach, twa_detach),
DEVMETHOD(device_shutdown, twa_shutdown),
DEVMETHOD_END
};
static driver_t twa_pci_driver = {
"twa",
twa_methods,
sizeof(struct twa_softc)
};
DRIVER_MODULE(twa, pci, twa_pci_driver, twa_devclass, 0, 0);
MODULE_DEPEND(twa, cam, 1, 1, 1);
MODULE_DEPEND(twa, pci, 1, 1, 1);
/*
* Function name: twa_probe
* Description: Called at driver load time. Claims 9000 ctlrs.
*
* Input: dev -- bus device corresponding to the ctlr
* Output: None
* Return value: <= 0 -- success
* > 0 -- failure
*/
static TW_INT32
twa_probe(device_t dev)
{
static TW_UINT8 first_ctlr = 1;
tw_osli_dbg_printf(3, "entered");
if (tw_cl_ctlr_supported(pci_get_vendor(dev), pci_get_device(dev))) {
device_set_desc(dev, TW_OSLI_DEVICE_NAME);
/* Print the driver version only once. */
if (first_ctlr) {
printf("3ware device driver for 9000 series storage "
"controllers, version: %s\n",
TW_OSL_DRIVER_VERSION_STRING);
first_ctlr = 0;
}
return(0);
}
return(ENXIO);
}
int twa_setup_intr(struct twa_softc *sc)
{
int error = 0;
if (!(sc->intr_handle) && (sc->irq_res)) {
error = bus_setup_intr(sc->bus_dev, sc->irq_res,
INTR_TYPE_CAM | INTR_MPSAFE,
NULL, twa_pci_intr,
sc, &sc->intr_handle);
}
return( error );
}
int twa_teardown_intr(struct twa_softc *sc)
{
int error = 0;
if ((sc->intr_handle) && (sc->irq_res)) {
error = bus_teardown_intr(sc->bus_dev,
sc->irq_res, sc->intr_handle);
sc->intr_handle = NULL;
}
return( error );
}
/*
* Function name: twa_attach
* Description: Allocates pci resources; updates sc; adds a node to the
* sysctl tree to expose the driver version; makes calls
* (to the Common Layer) to initialize ctlr, and to
* attach to CAM.
*
* Input: dev -- bus device corresponding to the ctlr
* Output: None
* Return value: 0 -- success
* non-zero-- failure
*/
static TW_INT32
twa_attach(device_t dev)
{
struct twa_softc *sc = device_get_softc(dev);
TW_INT32 bar_num;
TW_INT32 bar0_offset;
TW_INT32 bar_size;
TW_INT32 error;
tw_osli_dbg_dprintf(3, sc, "entered");
sc->ctlr_handle.osl_ctlr_ctxt = sc;
/* Initialize the softc structure. */
sc->bus_dev = dev;
sc->device_id = pci_get_device(dev);
/* Initialize the mutexes right here. */
sc->io_lock = &(sc->io_lock_handle);
mtx_init(sc->io_lock, "tw_osl_io_lock", NULL, MTX_SPIN);
sc->q_lock = &(sc->q_lock_handle);
mtx_init(sc->q_lock, "tw_osl_q_lock", NULL, MTX_SPIN);
sc->sim_lock = &(sc->sim_lock_handle);
mtx_init(sc->sim_lock, "tw_osl_sim_lock", NULL, MTX_DEF | MTX_RECURSE);
sysctl_ctx_init(&sc->sysctl_ctxt);
sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctxt,
SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
device_get_nameunit(dev), CTLFLAG_RD, 0, "");
if (sc->sysctl_tree == NULL) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2000,
"Cannot add sysctl tree node",
ENXIO);
return(ENXIO);
}
SYSCTL_ADD_STRING(&sc->sysctl_ctxt, SYSCTL_CHILDREN(sc->sysctl_tree),
OID_AUTO, "driver_version", CTLFLAG_RD,
TW_OSL_DRIVER_VERSION_STRING, 0, "TWA driver version");
/* Force the busmaster enable bit on, in case the BIOS forgot. */
pci_enable_busmaster(dev);
/* Allocate the PCI register window. */
if ((error = tw_cl_get_pci_bar_info(sc->device_id, TW_CL_BAR_TYPE_MEM,
&bar_num, &bar0_offset, &bar_size))) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x201F,
"Can't get PCI BAR info",
error);
tw_osli_free_resources(sc);
return(error);
}
sc->reg_res_id = PCIR_BARS + bar0_offset;
if ((sc->reg_res = bus_alloc_resource(dev, SYS_RES_MEMORY,
&(sc->reg_res_id), 0, ~0, 1, RF_ACTIVE))
== NULL) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2002,
"Can't allocate register window",
ENXIO);
tw_osli_free_resources(sc);
return(ENXIO);
}
sc->bus_tag = rman_get_bustag(sc->reg_res);
sc->bus_handle = rman_get_bushandle(sc->reg_res);
/* Allocate and register our interrupt. */
sc->irq_res_id = 0;
if ((sc->irq_res = bus_alloc_resource(sc->bus_dev, SYS_RES_IRQ,
&(sc->irq_res_id), 0, ~0, 1,
RF_SHAREABLE | RF_ACTIVE)) == NULL) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2003,
"Can't allocate interrupt",
ENXIO);
tw_osli_free_resources(sc);
return(ENXIO);
}
if ((error = twa_setup_intr(sc))) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2004,
"Can't set up interrupt",
error);
tw_osli_free_resources(sc);
return(error);
}
if ((error = tw_osli_alloc_mem(sc))) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2005,
"Memory allocation failure",
error);
tw_osli_free_resources(sc);
return(error);
}
/* Initialize the Common Layer for this controller. */
if ((error = tw_cl_init_ctlr(&sc->ctlr_handle, sc->flags, sc->device_id,
TW_OSLI_MAX_NUM_REQUESTS, TW_OSLI_MAX_NUM_AENS,
sc->non_dma_mem, sc->dma_mem,
sc->dma_mem_phys
))) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2006,
"Failed to initialize Common Layer/controller",
error);
tw_osli_free_resources(sc);
return(error);
}
/* Create the control device. */
sc->ctrl_dev = make_dev(&twa_cdevsw, device_get_unit(sc->bus_dev),
UID_ROOT, GID_OPERATOR, S_IRUSR | S_IWUSR,
"twa%d", device_get_unit(sc->bus_dev));
sc->ctrl_dev->si_drv1 = sc;
if ((error = tw_osli_cam_attach(sc))) {
tw_osli_free_resources(sc);
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2007,
"Failed to initialize CAM",
error);
return(error);
}
sc->watchdog_index = 0;
- callout_init(&(sc->watchdog_callout[0]), CALLOUT_MPSAFE);
- callout_init(&(sc->watchdog_callout[1]), CALLOUT_MPSAFE);
+ callout_init(&(sc->watchdog_callout[0]), 1);
+ callout_init(&(sc->watchdog_callout[1]), 1);
callout_reset(&(sc->watchdog_callout[0]), 5*hz, twa_watchdog, &sc->ctlr_handle);
return(0);
}
static TW_VOID
twa_watchdog(TW_VOID *arg)
{
struct tw_cl_ctlr_handle *ctlr_handle =
(struct tw_cl_ctlr_handle *)arg;
struct twa_softc *sc = ctlr_handle->osl_ctlr_ctxt;
int i;
int i_need_a_reset = 0;
int driver_is_active = 0;
int my_watchdog_was_pending = 1234;
TW_UINT64 current_time;
struct tw_osli_req_context *my_req;
//==============================================================================
current_time = (TW_UINT64) (tw_osl_get_local_time());
for (i = 0; i < TW_OSLI_MAX_NUM_REQUESTS; i++) {
my_req = &(sc->req_ctx_buf[i]);
if ((my_req->state == TW_OSLI_REQ_STATE_BUSY) &&
(my_req->deadline) &&
(my_req->deadline < current_time)) {
tw_cl_set_reset_needed(ctlr_handle);
#ifdef TW_OSL_DEBUG
device_printf((sc)->bus_dev, "Request %d timed out! d = %llu, c = %llu\n", i, my_req->deadline, current_time);
#else /* TW_OSL_DEBUG */
device_printf((sc)->bus_dev, "Request %d timed out!\n", i);
#endif /* TW_OSL_DEBUG */
break;
}
}
//==============================================================================
i_need_a_reset = tw_cl_is_reset_needed(ctlr_handle);
i = (int) ((sc->watchdog_index++) & 1);
driver_is_active = tw_cl_is_active(ctlr_handle);
if (i_need_a_reset) {
#ifdef TW_OSL_DEBUG
device_printf((sc)->bus_dev, "Watchdog rescheduled in 70 seconds\n");
#endif /* TW_OSL_DEBUG */
my_watchdog_was_pending =
callout_reset(&(sc->watchdog_callout[i]), 70*hz, twa_watchdog, &sc->ctlr_handle);
tw_cl_reset_ctlr(ctlr_handle);
#ifdef TW_OSL_DEBUG
device_printf((sc)->bus_dev, "Watchdog reset completed!\n");
#endif /* TW_OSL_DEBUG */
} else if (driver_is_active) {
my_watchdog_was_pending =
callout_reset(&(sc->watchdog_callout[i]), 5*hz, twa_watchdog, &sc->ctlr_handle);
}
#ifdef TW_OSL_DEBUG
if (i_need_a_reset || my_watchdog_was_pending)
device_printf((sc)->bus_dev, "i_need_a_reset = %d, "
"driver_is_active = %d, my_watchdog_was_pending = %d\n",
i_need_a_reset, driver_is_active, my_watchdog_was_pending);
#endif /* TW_OSL_DEBUG */
}
/*
* Function name: tw_osli_alloc_mem
* Description: Allocates memory needed both by CL and OSL.
*
* Input: sc -- OSL internal controller context
* Output: None
* Return value: 0 -- success
* non-zero-- failure
*/
static TW_INT32
tw_osli_alloc_mem(struct twa_softc *sc)
{
struct tw_osli_req_context *req;
TW_UINT32 max_sg_elements;
TW_UINT32 non_dma_mem_size;
TW_UINT32 dma_mem_size;
TW_INT32 error;
TW_INT32 i;
tw_osli_dbg_dprintf(3, sc, "entered");
sc->flags |= (sizeof(bus_addr_t) == 8) ? TW_CL_64BIT_ADDRESSES : 0;
sc->flags |= (sizeof(bus_size_t) == 8) ? TW_CL_64BIT_SG_LENGTH : 0;
max_sg_elements = (sizeof(bus_addr_t) == 8) ?
TW_CL_MAX_64BIT_SG_ELEMENTS : TW_CL_MAX_32BIT_SG_ELEMENTS;
if ((error = tw_cl_get_mem_requirements(&sc->ctlr_handle, sc->flags,
sc->device_id, TW_OSLI_MAX_NUM_REQUESTS, TW_OSLI_MAX_NUM_AENS,
&(sc->alignment), &(sc->sg_size_factor),
&non_dma_mem_size, &dma_mem_size
))) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2008,
"Can't get Common Layer's memory requirements",
error);
return(error);
}
if ((sc->non_dma_mem = malloc(non_dma_mem_size, TW_OSLI_MALLOC_CLASS,
M_WAITOK)) == NULL) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2009,
"Can't allocate non-dma memory",
ENOMEM);
return(ENOMEM);
}
/* Create the parent dma tag. */
if (bus_dma_tag_create(bus_get_dma_tag(sc->bus_dev), /* parent */
sc->alignment, /* alignment */
0, /* boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
TW_CL_MAX_IO_SIZE, /* maxsize */
max_sg_elements, /* nsegments */
TW_CL_MAX_IO_SIZE, /* maxsegsize */
0, /* flags */
NULL, /* lockfunc */
NULL, /* lockfuncarg */
&sc->parent_tag /* tag */)) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x200A,
"Can't allocate parent DMA tag",
ENOMEM);
return(ENOMEM);
}
/* Create a dma tag for Common Layer's DMA'able memory (dma_mem). */
if (bus_dma_tag_create(sc->parent_tag, /* parent */
sc->alignment, /* alignment */
0, /* boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
dma_mem_size, /* maxsize */
1, /* nsegments */
BUS_SPACE_MAXSIZE, /* maxsegsize */
0, /* flags */
NULL, /* lockfunc */
NULL, /* lockfuncarg */
&sc->cmd_tag /* tag */)) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x200B,
"Can't allocate DMA tag for Common Layer's "
"DMA'able memory",
ENOMEM);
return(ENOMEM);
}
if (bus_dmamem_alloc(sc->cmd_tag, &sc->dma_mem,
BUS_DMA_NOWAIT, &sc->cmd_map)) {
/* Try a second time. */
if (bus_dmamem_alloc(sc->cmd_tag, &sc->dma_mem,
BUS_DMA_NOWAIT, &sc->cmd_map)) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x200C,
"Can't allocate DMA'able memory for the"
"Common Layer",
ENOMEM);
return(ENOMEM);
}
}
bus_dmamap_load(sc->cmd_tag, sc->cmd_map, sc->dma_mem,
dma_mem_size, twa_map_load_callback,
&sc->dma_mem_phys, 0);
/*
* Create a dma tag for data buffers; size will be the maximum
* possible I/O size (128kB).
*/
if (bus_dma_tag_create(sc->parent_tag, /* parent */
sc->alignment, /* alignment */
0, /* boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
TW_CL_MAX_IO_SIZE, /* maxsize */
max_sg_elements, /* nsegments */
TW_CL_MAX_IO_SIZE, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
twa_busdma_lock, /* lockfunc */
sc->io_lock, /* lockfuncarg */
&sc->dma_tag /* tag */)) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x200F,
"Can't allocate DMA tag for data buffers",
ENOMEM);
return(ENOMEM);
}
/*
* Create a dma tag for ioctl data buffers; size will be the maximum
* possible I/O size (128kB).
*/
if (bus_dma_tag_create(sc->parent_tag, /* parent */
sc->alignment, /* alignment */
0, /* boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
TW_CL_MAX_IO_SIZE, /* maxsize */
max_sg_elements, /* nsegments */
TW_CL_MAX_IO_SIZE, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
twa_busdma_lock, /* lockfunc */
sc->io_lock, /* lockfuncarg */
&sc->ioctl_tag /* tag */)) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2010,
"Can't allocate DMA tag for ioctl data buffers",
ENOMEM);
return(ENOMEM);
}
/* Create just one map for all ioctl request data buffers. */
if (bus_dmamap_create(sc->ioctl_tag, 0, &sc->ioctl_map)) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2011,
"Can't create ioctl map",
ENOMEM);
return(ENOMEM);
}
/* Initialize request queues. */
tw_osli_req_q_init(sc, TW_OSLI_FREE_Q);
tw_osli_req_q_init(sc, TW_OSLI_BUSY_Q);
if ((sc->req_ctx_buf = (struct tw_osli_req_context *)
malloc((sizeof(struct tw_osli_req_context) *
TW_OSLI_MAX_NUM_REQUESTS),
TW_OSLI_MALLOC_CLASS, M_WAITOK)) == NULL) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2012,
"Failed to allocate request packets",
ENOMEM);
return(ENOMEM);
}
bzero(sc->req_ctx_buf,
sizeof(struct tw_osli_req_context) * TW_OSLI_MAX_NUM_REQUESTS);
for (i = 0; i < TW_OSLI_MAX_NUM_REQUESTS; i++) {
req = &(sc->req_ctx_buf[i]);
req->ctlr = sc;
if (bus_dmamap_create(sc->dma_tag, 0, &req->dma_map)) {
tw_osli_printf(sc, "request # = %d, error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2013,
"Can't create dma map",
i, ENOMEM);
return(ENOMEM);
}
/* Initialize the ioctl wakeup/ timeout mutex */
req->ioctl_wake_timeout_lock = &(req->ioctl_wake_timeout_lock_handle);
mtx_init(req->ioctl_wake_timeout_lock, "tw_ioctl_wake_timeout_lock", NULL, MTX_DEF);
/* Insert request into the free queue. */
tw_osli_req_q_insert_tail(req, TW_OSLI_FREE_Q);
}
return(0);
}
/*
* Function name: tw_osli_free_resources
* Description: Performs clean-up at the time of going down.
*
* Input: sc -- ptr to OSL internal ctlr context
* Output: None
* Return value: None
*/
static TW_VOID
tw_osli_free_resources(struct twa_softc *sc)
{
struct tw_osli_req_context *req;
TW_INT32 error = 0;
tw_osli_dbg_dprintf(3, sc, "entered");
/* Detach from CAM */
tw_osli_cam_detach(sc);
if (sc->req_ctx_buf)
while ((req = tw_osli_req_q_remove_head(sc, TW_OSLI_FREE_Q)) !=
NULL) {
mtx_destroy(req->ioctl_wake_timeout_lock);
if ((error = bus_dmamap_destroy(sc->dma_tag,
req->dma_map)))
tw_osli_dbg_dprintf(1, sc,
"dmamap_destroy(dma) returned %d",
error);
}
if ((sc->ioctl_tag) && (sc->ioctl_map))
if ((error = bus_dmamap_destroy(sc->ioctl_tag, sc->ioctl_map)))
tw_osli_dbg_dprintf(1, sc,
"dmamap_destroy(ioctl) returned %d", error);
/* Free all memory allocated so far. */
if (sc->req_ctx_buf)
free(sc->req_ctx_buf, TW_OSLI_MALLOC_CLASS);
if (sc->non_dma_mem)
free(sc->non_dma_mem, TW_OSLI_MALLOC_CLASS);
if (sc->dma_mem) {
bus_dmamap_unload(sc->cmd_tag, sc->cmd_map);
bus_dmamem_free(sc->cmd_tag, sc->dma_mem,
sc->cmd_map);
}
if (sc->cmd_tag)
if ((error = bus_dma_tag_destroy(sc->cmd_tag)))
tw_osli_dbg_dprintf(1, sc,
"dma_tag_destroy(cmd) returned %d", error);
if (sc->dma_tag)
if ((error = bus_dma_tag_destroy(sc->dma_tag)))
tw_osli_dbg_dprintf(1, sc,
"dma_tag_destroy(dma) returned %d", error);
if (sc->ioctl_tag)
if ((error = bus_dma_tag_destroy(sc->ioctl_tag)))
tw_osli_dbg_dprintf(1, sc,
"dma_tag_destroy(ioctl) returned %d", error);
if (sc->parent_tag)
if ((error = bus_dma_tag_destroy(sc->parent_tag)))
tw_osli_dbg_dprintf(1, sc,
"dma_tag_destroy(parent) returned %d", error);
/* Disconnect the interrupt handler. */
if ((error = twa_teardown_intr(sc)))
tw_osli_dbg_dprintf(1, sc,
"teardown_intr returned %d", error);
if (sc->irq_res != NULL)
if ((error = bus_release_resource(sc->bus_dev,
SYS_RES_IRQ, sc->irq_res_id, sc->irq_res)))
tw_osli_dbg_dprintf(1, sc,
"release_resource(irq) returned %d", error);
/* Release the register window mapping. */
if (sc->reg_res != NULL)
if ((error = bus_release_resource(sc->bus_dev,
SYS_RES_MEMORY, sc->reg_res_id, sc->reg_res)))
tw_osli_dbg_dprintf(1, sc,
"release_resource(io) returned %d", error);
/* Destroy the control device. */
if (sc->ctrl_dev != (struct cdev *)NULL)
destroy_dev(sc->ctrl_dev);
if ((error = sysctl_ctx_free(&sc->sysctl_ctxt)))
tw_osli_dbg_dprintf(1, sc,
"sysctl_ctx_free returned %d", error);
}
/*
* Function name: twa_detach
* Description: Called when the controller is being detached from
* the pci bus.
*
* Input: dev -- bus device corresponding to the ctlr
* Output: None
* Return value: 0 -- success
* non-zero-- failure
*/
static TW_INT32
twa_detach(device_t dev)
{
struct twa_softc *sc = device_get_softc(dev);
TW_INT32 error;
tw_osli_dbg_dprintf(3, sc, "entered");
error = EBUSY;
if (sc->open) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2014,
"Device open",
error);
goto out;
}
/* Shut the controller down. */
if ((error = twa_shutdown(dev)))
goto out;
/* Free all resources associated with this controller. */
tw_osli_free_resources(sc);
error = 0;
out:
return(error);
}
/*
* Function name: twa_shutdown
* Description: Called at unload/shutdown time. Lets the controller
* know that we are going down.
*
* Input: dev -- bus device corresponding to the ctlr
* Output: None
* Return value: 0 -- success
* non-zero-- failure
*/
static TW_INT32
twa_shutdown(device_t dev)
{
struct twa_softc *sc = device_get_softc(dev);
TW_INT32 error = 0;
tw_osli_dbg_dprintf(3, sc, "entered");
/* Disconnect interrupts. */
error = twa_teardown_intr(sc);
/* Stop watchdog task. */
callout_drain(&(sc->watchdog_callout[0]));
callout_drain(&(sc->watchdog_callout[1]));
/* Disconnect from the controller. */
if ((error = tw_cl_shutdown_ctlr(&(sc->ctlr_handle), 0))) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2015,
"Failed to shutdown Common Layer/controller",
error);
}
return(error);
}
/*
* Function name: twa_busdma_lock
* Description: Function to provide synchronization during busdma_swi.
*
* Input: lock_arg -- lock mutex sent as argument
* op -- operation (lock/unlock) expected of the function
* Output: None
* Return value: None
*/
TW_VOID
twa_busdma_lock(TW_VOID *lock_arg, bus_dma_lock_op_t op)
{
struct mtx *lock;
lock = (struct mtx *)lock_arg;
switch (op) {
case BUS_DMA_LOCK:
mtx_lock_spin(lock);
break;
case BUS_DMA_UNLOCK:
mtx_unlock_spin(lock);
break;
default:
panic("Unknown operation 0x%x for twa_busdma_lock!", op);
}
}
/*
* Function name: twa_pci_intr
* Description: Interrupt handler. Wrapper for twa_interrupt.
*
* Input: arg -- ptr to OSL internal ctlr context
* Output: None
* Return value: None
*/
static TW_VOID
twa_pci_intr(TW_VOID *arg)
{
struct twa_softc *sc = (struct twa_softc *)arg;
tw_osli_dbg_dprintf(10, sc, "entered");
tw_cl_interrupt(&(sc->ctlr_handle));
}
/*
* Function name: tw_osli_fw_passthru
* Description: Builds a fw passthru cmd pkt, and submits it to CL.
*
* Input: sc -- ptr to OSL internal ctlr context
* buf -- ptr to ioctl pkt understood by CL
* Output: None
* Return value: 0 -- success
* non-zero-- failure
*/
TW_INT32
tw_osli_fw_passthru(struct twa_softc *sc, TW_INT8 *buf)
{
struct tw_osli_req_context *req;
struct tw_osli_ioctl_no_data_buf *user_buf =
(struct tw_osli_ioctl_no_data_buf *)buf;
TW_TIME end_time;
TW_UINT32 timeout = 60;
TW_UINT32 data_buf_size_adjusted;
struct tw_cl_req_packet *req_pkt;
struct tw_cl_passthru_req_packet *pt_req;
TW_INT32 error;
tw_osli_dbg_dprintf(5, sc, "ioctl: passthru");
if ((req = tw_osli_get_request(sc)) == NULL)
return(EBUSY);
req->req_handle.osl_req_ctxt = req;
req->orig_req = buf;
req->flags |= TW_OSLI_REQ_FLAGS_PASSTHRU;
req_pkt = &(req->req_pkt);
req_pkt->status = 0;
req_pkt->tw_osl_callback = tw_osl_complete_passthru;
/* Let the Common Layer retry the request on cmd queue full. */
req_pkt->flags |= TW_CL_REQ_RETRY_ON_BUSY;
pt_req = &(req_pkt->gen_req_pkt.pt_req);
/*
* Make sure that the data buffer sent to firmware is a
* 512 byte multiple in size.
*/
data_buf_size_adjusted =
(user_buf->driver_pkt.buffer_length +
(sc->sg_size_factor - 1)) & ~(sc->sg_size_factor - 1);
if ((req->length = data_buf_size_adjusted)) {
if ((req->data = malloc(data_buf_size_adjusted,
TW_OSLI_MALLOC_CLASS, M_WAITOK)) == NULL) {
error = ENOMEM;
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2016,
"Could not alloc mem for "
"fw_passthru data_buf",
error);
goto fw_passthru_err;
}
/* Copy the payload. */
if ((error = copyin((TW_VOID *)(user_buf->pdata),
req->data,
user_buf->driver_pkt.buffer_length)) != 0) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2017,
"Could not copyin fw_passthru data_buf",
error);
goto fw_passthru_err;
}
pt_req->sgl_entries = 1; /* will be updated during mapping */
req->flags |= (TW_OSLI_REQ_FLAGS_DATA_IN |
TW_OSLI_REQ_FLAGS_DATA_OUT);
} else
pt_req->sgl_entries = 0; /* no payload */
pt_req->cmd_pkt = (TW_VOID *)(&(user_buf->cmd_pkt));
pt_req->cmd_pkt_length = sizeof(struct tw_cl_command_packet);
if ((error = tw_osli_map_request(req)))
goto fw_passthru_err;
end_time = tw_osl_get_local_time() + timeout;
while (req->state != TW_OSLI_REQ_STATE_COMPLETE) {
mtx_lock(req->ioctl_wake_timeout_lock);
req->flags |= TW_OSLI_REQ_FLAGS_SLEEPING;
error = mtx_sleep(req, req->ioctl_wake_timeout_lock, 0,
"twa_passthru", timeout*hz);
mtx_unlock(req->ioctl_wake_timeout_lock);
if (!(req->flags & TW_OSLI_REQ_FLAGS_SLEEPING))
error = 0;
req->flags &= ~TW_OSLI_REQ_FLAGS_SLEEPING;
if (! error) {
if (((error = req->error_code)) ||
((error = (req->state !=
TW_OSLI_REQ_STATE_COMPLETE))) ||
((error = req_pkt->status)))
goto fw_passthru_err;
break;
}
if (req_pkt->status) {
error = req_pkt->status;
goto fw_passthru_err;
}
if (error == EWOULDBLOCK) {
/* Time out! */
if ((!(req->error_code)) &&
(req->state == TW_OSLI_REQ_STATE_COMPLETE) &&
(!(req_pkt->status)) ) {
#ifdef TW_OSL_DEBUG
tw_osli_printf(sc, "request = %p",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x7777,
"FALSE Passthru timeout!",
req);
#endif /* TW_OSL_DEBUG */
error = 0; /* False error */
break;
}
if (!(tw_cl_is_reset_needed(&(req->ctlr->ctlr_handle)))) {
#ifdef TW_OSL_DEBUG
tw_osli_printf(sc, "request = %p",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2018,
"Passthru request timed out!",
req);
#else /* TW_OSL_DEBUG */
device_printf((sc)->bus_dev, "Passthru request timed out!\n");
#endif /* TW_OSL_DEBUG */
tw_cl_reset_ctlr(&(req->ctlr->ctlr_handle));
}
error = 0;
end_time = tw_osl_get_local_time() + timeout;
continue;
/*
* Don't touch req after a reset. It (and any
* associated data) will be
* unmapped by the callback.
*/
}
/*
* Either the request got completed, or we were woken up by a
* signal. Calculate the new timeout, in case it was the latter.
*/
timeout = (end_time - tw_osl_get_local_time());
} /* End of while loop */
/* If there was a payload, copy it back. */
if ((!error) && (req->length))
if ((error = copyout(req->data, user_buf->pdata,
user_buf->driver_pkt.buffer_length)))
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2019,
"Could not copyout fw_passthru data_buf",
error);
fw_passthru_err:
if (req_pkt->status == TW_CL_ERR_REQ_BUS_RESET)
error = EBUSY;
user_buf->driver_pkt.os_status = error;
/* Free resources. */
if (req->data)
free(req->data, TW_OSLI_MALLOC_CLASS);
tw_osli_req_q_insert_tail(req, TW_OSLI_FREE_Q);
return(error);
}
/*
* Function name: tw_osl_complete_passthru
* Description: Called to complete passthru requests.
*
* Input: req_handle -- ptr to request handle
* Output: None
* Return value: None
*/
TW_VOID
tw_osl_complete_passthru(struct tw_cl_req_handle *req_handle)
{
struct tw_osli_req_context *req = req_handle->osl_req_ctxt;
struct tw_cl_req_packet *req_pkt =
(struct tw_cl_req_packet *)(&req->req_pkt);
struct twa_softc *sc = req->ctlr;
tw_osli_dbg_dprintf(5, sc, "entered");
if (req->state != TW_OSLI_REQ_STATE_BUSY) {
tw_osli_printf(sc, "request = %p, status = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x201B,
"Unposted command completed!!",
req, req->state);
}
/*
* Remove request from the busy queue. Just mark it complete.
* There's no need to move it into the complete queue as we are
* going to be done with it right now.
*/
req->state = TW_OSLI_REQ_STATE_COMPLETE;
tw_osli_req_q_remove_item(req, TW_OSLI_BUSY_Q);
tw_osli_unmap_request(req);
/*
* Don't do a wake up if there was an error even before the request
* was sent down to the Common Layer, and we hadn't gotten an
* EINPROGRESS. The request originator will then be returned an
* error, and he can do the clean-up.
*/
if ((req->error_code) && (!(req->flags & TW_OSLI_REQ_FLAGS_IN_PROGRESS)))
return;
if (req->flags & TW_OSLI_REQ_FLAGS_PASSTHRU) {
if (req->flags & TW_OSLI_REQ_FLAGS_SLEEPING) {
/* Wake up the sleeping command originator. */
tw_osli_dbg_dprintf(5, sc,
"Waking up originator of request %p", req);
req->flags &= ~TW_OSLI_REQ_FLAGS_SLEEPING;
wakeup_one(req);
} else {
/*
* If the request completed even before mtx_sleep
* was called, simply return.
*/
if (req->flags & TW_OSLI_REQ_FLAGS_MAPPED)
return;
if (req_pkt->status == TW_CL_ERR_REQ_BUS_RESET)
return;
tw_osli_printf(sc, "request = %p",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x201C,
"Passthru callback called, "
"and caller not sleeping",
req);
}
} else {
tw_osli_printf(sc, "request = %p",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x201D,
"Passthru callback called for non-passthru request",
req);
}
}
/*
* Function name: tw_osli_get_request
* Description: Gets a request pkt from the free queue.
*
* Input: sc -- ptr to OSL internal ctlr context
* Output: None
* Return value: ptr to request pkt -- success
* NULL -- failure
*/
struct tw_osli_req_context *
tw_osli_get_request(struct twa_softc *sc)
{
struct tw_osli_req_context *req;
tw_osli_dbg_dprintf(4, sc, "entered");
/* Get a free request packet. */
req = tw_osli_req_q_remove_head(sc, TW_OSLI_FREE_Q);
/* Initialize some fields to their defaults. */
if (req) {
req->req_handle.osl_req_ctxt = NULL;
req->req_handle.cl_req_ctxt = NULL;
req->req_handle.is_io = 0;
req->data = NULL;
req->length = 0;
req->deadline = 0;
req->real_data = NULL;
req->real_length = 0;
req->state = TW_OSLI_REQ_STATE_INIT;/* req being initialized */
req->flags = 0;
req->error_code = 0;
req->orig_req = NULL;
bzero(&(req->req_pkt), sizeof(struct tw_cl_req_packet));
}
return(req);
}
/*
* Function name: twa_map_load_data_callback
* Description: Callback of bus_dmamap_load for the buffer associated
* with data. Updates the cmd pkt (size/sgl_entries
* fields, as applicable) to reflect the number of sg
* elements.
*
* Input: arg -- ptr to OSL internal request context
* segs -- ptr to a list of segment descriptors
* nsegments--# of segments
* error -- 0 if no errors encountered before callback,
* non-zero if errors were encountered
* Output: None
* Return value: None
*/
static TW_VOID
twa_map_load_data_callback(TW_VOID *arg, bus_dma_segment_t *segs,
TW_INT32 nsegments, TW_INT32 error)
{
struct tw_osli_req_context *req =
(struct tw_osli_req_context *)arg;
struct twa_softc *sc = req->ctlr;
struct tw_cl_req_packet *req_pkt = &(req->req_pkt);
tw_osli_dbg_dprintf(10, sc, "entered");
if (error == EINVAL) {
req->error_code = error;
return;
}
/* Mark the request as currently being processed. */
req->state = TW_OSLI_REQ_STATE_BUSY;
/* Move the request into the busy queue. */
tw_osli_req_q_insert_tail(req, TW_OSLI_BUSY_Q);
req->flags |= TW_OSLI_REQ_FLAGS_MAPPED;
if (error == EFBIG) {
req->error_code = error;
goto out;
}
if (req->flags & TW_OSLI_REQ_FLAGS_PASSTHRU) {
struct tw_cl_passthru_req_packet *pt_req;
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_IN)
bus_dmamap_sync(sc->ioctl_tag, sc->ioctl_map,
BUS_DMASYNC_PREREAD);
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_OUT) {
/*
* If we're using an alignment buffer, and we're
* writing data, copy the real data out.
*/
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_COPY_NEEDED)
bcopy(req->real_data, req->data, req->real_length);
bus_dmamap_sync(sc->ioctl_tag, sc->ioctl_map,
BUS_DMASYNC_PREWRITE);
}
pt_req = &(req_pkt->gen_req_pkt.pt_req);
pt_req->sg_list = (TW_UINT8 *)segs;
pt_req->sgl_entries += (nsegments - 1);
error = tw_cl_fw_passthru(&(sc->ctlr_handle), req_pkt,
&(req->req_handle));
} else {
struct tw_cl_scsi_req_packet *scsi_req;
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_IN)
bus_dmamap_sync(sc->dma_tag, req->dma_map,
BUS_DMASYNC_PREREAD);
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_OUT) {
/*
* If we're using an alignment buffer, and we're
* writing data, copy the real data out.
*/
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_COPY_NEEDED)
bcopy(req->real_data, req->data, req->real_length);
bus_dmamap_sync(sc->dma_tag, req->dma_map,
BUS_DMASYNC_PREWRITE);
}
scsi_req = &(req_pkt->gen_req_pkt.scsi_req);
scsi_req->sg_list = (TW_UINT8 *)segs;
scsi_req->sgl_entries += (nsegments - 1);
error = tw_cl_start_io(&(sc->ctlr_handle), req_pkt,
&(req->req_handle));
}
out:
if (error) {
req->error_code = error;
req_pkt->tw_osl_callback(&(req->req_handle));
/*
* If the caller had been returned EINPROGRESS, and he has
* registered a callback for handling completion, the callback
* will never get called because we were unable to submit the
* request. So, free up the request right here.
*/
if (req->flags & TW_OSLI_REQ_FLAGS_IN_PROGRESS)
tw_osli_req_q_insert_tail(req, TW_OSLI_FREE_Q);
}
}
/*
* Function name: twa_map_load_callback
* Description: Callback of bus_dmamap_load for the buffer associated
* with a cmd pkt.
*
* Input: arg -- ptr to variable to hold phys addr
* segs -- ptr to a list of segment descriptors
* nsegments--# of segments
* error -- 0 if no errors encountered before callback,
* non-zero if errors were encountered
* Output: None
* Return value: None
*/
static TW_VOID
twa_map_load_callback(TW_VOID *arg, bus_dma_segment_t *segs,
TW_INT32 nsegments, TW_INT32 error)
{
*((bus_addr_t *)arg) = segs[0].ds_addr;
}
/*
* Function name: tw_osli_map_request
* Description: Maps a cmd pkt and data associated with it, into
* DMA'able memory.
*
* Input: req -- ptr to request pkt
* Output: None
* Return value: 0 -- success
* non-zero-- failure
*/
TW_INT32
tw_osli_map_request(struct tw_osli_req_context *req)
{
struct twa_softc *sc = req->ctlr;
TW_INT32 error = 0;
tw_osli_dbg_dprintf(10, sc, "entered");
/* If the command involves data, map that too. */
if (req->data != NULL) {
/*
* It's sufficient for the data pointer to be 4-byte aligned
* to work with 9000. However, if 4-byte aligned addresses
* are passed to bus_dmamap_load, we can get back sg elements
* that are not 512-byte multiples in size. So, we will let
* only those buffers that are 512-byte aligned to pass
* through, and bounce the rest, so as to make sure that we
* always get back sg elements that are 512-byte multiples
* in size.
*/
if (((vm_offset_t)req->data % sc->sg_size_factor) ||
(req->length % sc->sg_size_factor)) {
req->flags |= TW_OSLI_REQ_FLAGS_DATA_COPY_NEEDED;
/* Save original data pointer and length. */
req->real_data = req->data;
req->real_length = req->length;
req->length = (req->length +
(sc->sg_size_factor - 1)) &
~(sc->sg_size_factor - 1);
req->data = malloc(req->length, TW_OSLI_MALLOC_CLASS,
M_NOWAIT);
if (req->data == NULL) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x201E,
"Failed to allocate memory "
"for bounce buffer",
ENOMEM);
/* Restore original data pointer and length. */
req->data = req->real_data;
req->length = req->real_length;
return(ENOMEM);
}
}
/*
* Map the data buffer into bus space and build the SG list.
*/
if (req->flags & TW_OSLI_REQ_FLAGS_PASSTHRU) {
/* Lock against multiple simultaneous ioctl calls. */
mtx_lock_spin(sc->io_lock);
error = bus_dmamap_load(sc->ioctl_tag, sc->ioctl_map,
req->data, req->length,
twa_map_load_data_callback, req,
BUS_DMA_WAITOK);
mtx_unlock_spin(sc->io_lock);
} else if (req->flags & TW_OSLI_REQ_FLAGS_CCB) {
error = bus_dmamap_load_ccb(sc->dma_tag, req->dma_map,
req->orig_req, twa_map_load_data_callback, req,
BUS_DMA_WAITOK);
} else {
/*
* There's only one CAM I/O thread running at a time.
* So, there's no need to hold the io_lock.
*/
error = bus_dmamap_load(sc->dma_tag, req->dma_map,
req->data, req->length,
twa_map_load_data_callback, req,
BUS_DMA_WAITOK);
}
if (!error)
error = req->error_code;
else {
if (error == EINPROGRESS) {
/*
* Specifying sc->io_lock as the lockfuncarg
* in ...tag_create should protect the access
* of ...FLAGS_MAPPED from the callback.
*/
mtx_lock_spin(sc->io_lock);
if (!(req->flags & TW_OSLI_REQ_FLAGS_MAPPED))
req->flags |= TW_OSLI_REQ_FLAGS_IN_PROGRESS;
tw_osli_disallow_new_requests(sc, &(req->req_handle));
mtx_unlock_spin(sc->io_lock);
error = 0;
} else {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x9999,
"Failed to map DMA memory "
"for I/O request",
error);
req->flags |= TW_OSLI_REQ_FLAGS_FAILED;
/* Free alignment buffer if it was used. */
if (req->flags &
TW_OSLI_REQ_FLAGS_DATA_COPY_NEEDED) {
free(req->data, TW_OSLI_MALLOC_CLASS);
/*
* Restore original data pointer
* and length.
*/
req->data = req->real_data;
req->length = req->real_length;
}
}
}
} else {
/* Mark the request as currently being processed. */
req->state = TW_OSLI_REQ_STATE_BUSY;
/* Move the request into the busy queue. */
tw_osli_req_q_insert_tail(req, TW_OSLI_BUSY_Q);
if (req->flags & TW_OSLI_REQ_FLAGS_PASSTHRU)
error = tw_cl_fw_passthru(&sc->ctlr_handle,
&(req->req_pkt), &(req->req_handle));
else
error = tw_cl_start_io(&sc->ctlr_handle,
&(req->req_pkt), &(req->req_handle));
if (error) {
req->error_code = error;
req->req_pkt.tw_osl_callback(&(req->req_handle));
}
}
return(error);
}
/*
* Function name: tw_osli_unmap_request
* Description: Undoes the mapping done by tw_osli_map_request.
*
* Input: req -- ptr to request pkt
* Output: None
* Return value: None
*/
TW_VOID
tw_osli_unmap_request(struct tw_osli_req_context *req)
{
struct twa_softc *sc = req->ctlr;
tw_osli_dbg_dprintf(10, sc, "entered");
/* If the command involved data, unmap that too. */
if (req->data != NULL) {
if (req->flags & TW_OSLI_REQ_FLAGS_PASSTHRU) {
/* Lock against multiple simultaneous ioctl calls. */
mtx_lock_spin(sc->io_lock);
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_IN) {
bus_dmamap_sync(sc->ioctl_tag,
sc->ioctl_map, BUS_DMASYNC_POSTREAD);
/*
* If we are using a bounce buffer, and we are
* reading data, copy the real data in.
*/
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_COPY_NEEDED)
bcopy(req->data, req->real_data,
req->real_length);
}
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_OUT)
bus_dmamap_sync(sc->ioctl_tag, sc->ioctl_map,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->ioctl_tag, sc->ioctl_map);
mtx_unlock_spin(sc->io_lock);
} else {
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_IN) {
bus_dmamap_sync(sc->dma_tag,
req->dma_map, BUS_DMASYNC_POSTREAD);
/*
* If we are using a bounce buffer, and we are
* reading data, copy the real data in.
*/
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_COPY_NEEDED)
bcopy(req->data, req->real_data,
req->real_length);
}
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_OUT)
bus_dmamap_sync(sc->dma_tag, req->dma_map,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->dma_tag, req->dma_map);
}
}
/* Free alignment buffer if it was used. */
if (req->flags & TW_OSLI_REQ_FLAGS_DATA_COPY_NEEDED) {
free(req->data, TW_OSLI_MALLOC_CLASS);
/* Restore original data pointer and length. */
req->data = req->real_data;
req->length = req->real_length;
}
}
#ifdef TW_OSL_DEBUG
TW_VOID twa_report_stats(TW_VOID);
TW_VOID twa_reset_stats(TW_VOID);
TW_VOID tw_osli_print_ctlr_stats(struct twa_softc *sc);
TW_VOID twa_print_req_info(struct tw_osli_req_context *req);
/*
* Function name: twa_report_stats
* Description: For being called from ddb. Calls functions that print
* OSL and CL internal stats for the controller.
*
* Input: None
* Output: None
* Return value: None
*/
TW_VOID
twa_report_stats(TW_VOID)
{
struct twa_softc *sc;
TW_INT32 i;
for (i = 0; (sc = devclass_get_softc(twa_devclass, i)) != NULL; i++) {
tw_osli_print_ctlr_stats(sc);
tw_cl_print_ctlr_stats(&sc->ctlr_handle);
}
}
/*
* Function name: tw_osli_print_ctlr_stats
* Description: For being called from ddb. Prints OSL controller stats
*
* Input: sc -- ptr to OSL internal controller context
* Output: None
* Return value: None
*/
TW_VOID
tw_osli_print_ctlr_stats(struct twa_softc *sc)
{
twa_printf(sc, "osl_ctlr_ctxt = %p\n", sc);
twa_printf(sc, "OSLq type current max\n");
twa_printf(sc, "free %04d %04d\n",
sc->q_stats[TW_OSLI_FREE_Q].cur_len,
sc->q_stats[TW_OSLI_FREE_Q].max_len);
twa_printf(sc, "busy %04d %04d\n",
sc->q_stats[TW_OSLI_BUSY_Q].cur_len,
sc->q_stats[TW_OSLI_BUSY_Q].max_len);
}
/*
* Function name: twa_print_req_info
* Description: For being called from ddb. Calls functions that print
* OSL and CL internal details for the request.
*
* Input: req -- ptr to OSL internal request context
* Output: None
* Return value: None
*/
TW_VOID
twa_print_req_info(struct tw_osli_req_context *req)
{
struct twa_softc *sc = req->ctlr;
twa_printf(sc, "OSL details for request:\n");
twa_printf(sc, "osl_req_ctxt = %p, cl_req_ctxt = %p\n"
"data = %p, length = 0x%x, real_data = %p, real_length = 0x%x\n"
"state = 0x%x, flags = 0x%x, error = 0x%x, orig_req = %p\n"
"next_req = %p, prev_req = %p, dma_map = %p\n",
req->req_handle.osl_req_ctxt, req->req_handle.cl_req_ctxt,
req->data, req->length, req->real_data, req->real_length,
req->state, req->flags, req->error_code, req->orig_req,
req->link.next, req->link.prev, req->dma_map);
tw_cl_print_req_info(&(req->req_handle));
}
/*
* Function name: twa_reset_stats
* Description: For being called from ddb.
* Resets some OSL controller stats.
*
* Input: None
* Output: None
* Return value: None
*/
TW_VOID
twa_reset_stats(TW_VOID)
{
struct twa_softc *sc;
TW_INT32 i;
for (i = 0; (sc = devclass_get_softc(twa_devclass, i)) != NULL; i++) {
sc->q_stats[TW_OSLI_FREE_Q].max_len = 0;
sc->q_stats[TW_OSLI_BUSY_Q].max_len = 0;
tw_cl_reset_stats(&sc->ctlr_handle);
}
}
#endif /* TW_OSL_DEBUG */
Index: head/sys/dev/tws/tws.c
===================================================================
--- head/sys/dev/tws/tws.c (revision 283290)
+++ head/sys/dev/tws/tws.c (revision 283291)
@@ -1,922 +1,922 @@
/*
* Copyright (c) 2010, LSI Corp.
* All rights reserved.
* Author : Manjunath Ranganathaiah
* Support: freebsdraid@lsi.com
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* 3. Neither the name of the <ORGANIZATION> nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <dev/tws/tws.h>
#include <dev/tws/tws_services.h>
#include <dev/tws/tws_hdm.h>
#include <cam/cam.h>
#include <cam/cam_ccb.h>
MALLOC_DEFINE(M_TWS, "twsbuf", "buffers used by tws driver");
int tws_queue_depth = TWS_MAX_REQS;
int tws_enable_msi = 0;
int tws_enable_msix = 0;
/* externs */
extern int tws_cam_attach(struct tws_softc *sc);
extern void tws_cam_detach(struct tws_softc *sc);
extern int tws_init_ctlr(struct tws_softc *sc);
extern boolean tws_ctlr_ready(struct tws_softc *sc);
extern void tws_turn_off_interrupts(struct tws_softc *sc);
extern void tws_q_insert_tail(struct tws_softc *sc, struct tws_request *req,
u_int8_t q_type );
extern struct tws_request *tws_q_remove_request(struct tws_softc *sc,
struct tws_request *req, u_int8_t q_type );
extern struct tws_request *tws_q_remove_head(struct tws_softc *sc,
u_int8_t q_type );
extern boolean tws_get_response(struct tws_softc *sc, u_int16_t *req_id);
extern boolean tws_ctlr_reset(struct tws_softc *sc);
extern void tws_intr(void *arg);
extern int tws_use_32bit_sgls;
struct tws_request *tws_get_request(struct tws_softc *sc, u_int16_t type);
int tws_init_connect(struct tws_softc *sc, u_int16_t mc);
void tws_send_event(struct tws_softc *sc, u_int8_t event);
uint8_t tws_get_state(struct tws_softc *sc);
void tws_release_request(struct tws_request *req);
/* Function prototypes */
static d_open_t tws_open;
static d_close_t tws_close;
static d_read_t tws_read;
static d_write_t tws_write;
extern d_ioctl_t tws_ioctl;
static int tws_init(struct tws_softc *sc);
static void tws_dmamap_cmds_load_cbfn(void *arg, bus_dma_segment_t *segs,
int nseg, int error);
static int tws_init_reqs(struct tws_softc *sc, u_int32_t dma_mem_size);
static int tws_init_aen_q(struct tws_softc *sc);
static int tws_init_trace_q(struct tws_softc *sc);
static int tws_setup_irq(struct tws_softc *sc);
int tws_setup_intr(struct tws_softc *sc, int irqs);
int tws_teardown_intr(struct tws_softc *sc);
/* Character device entry points */
static struct cdevsw tws_cdevsw = {
.d_version = D_VERSION,
.d_open = tws_open,
.d_close = tws_close,
.d_read = tws_read,
.d_write = tws_write,
.d_ioctl = tws_ioctl,
.d_name = "tws",
};
/*
* In the cdevsw routines, we find our softc by using the si_drv1 member
* of struct cdev. We set this variable to point to our softc in our
* attach routine when we create the /dev entry.
*/
int
tws_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
{
struct tws_softc *sc = dev->si_drv1;
if ( sc )
TWS_TRACE_DEBUG(sc, "entry", dev, oflags);
return (0);
}
int
tws_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
{
struct tws_softc *sc = dev->si_drv1;
if ( sc )
TWS_TRACE_DEBUG(sc, "entry", dev, fflag);
return (0);
}
int
tws_read(struct cdev *dev, struct uio *uio, int ioflag)
{
struct tws_softc *sc = dev->si_drv1;
if ( sc )
TWS_TRACE_DEBUG(sc, "entry", dev, ioflag);
return (0);
}
int
tws_write(struct cdev *dev, struct uio *uio, int ioflag)
{
struct tws_softc *sc = dev->si_drv1;
if ( sc )
TWS_TRACE_DEBUG(sc, "entry", dev, ioflag);
return (0);
}
/* PCI Support Functions */
/*
* Compare the device ID of this device against the IDs that this driver
* supports. If there is a match, set the description and return success.
*/
static int
tws_probe(device_t dev)
{
static u_int8_t first_ctlr = 1;
if ((pci_get_vendor(dev) == TWS_VENDOR_ID) &&
(pci_get_device(dev) == TWS_DEVICE_ID)) {
device_set_desc(dev, "LSI 3ware SAS/SATA Storage Controller");
if (first_ctlr) {
printf("LSI 3ware device driver for SAS/SATA storage "
"controllers, version: %s\n", TWS_DRIVER_VERSION_STRING);
first_ctlr = 0;
}
return(BUS_PROBE_DEFAULT);
}
return (ENXIO);
}
/* Attach function is only called if the probe is successful. */
static int
tws_attach(device_t dev)
{
struct tws_softc *sc = device_get_softc(dev);
u_int32_t bar;
int error=0,i;
/* no tracing yet */
/* Look up our softc and initialize its fields. */
sc->tws_dev = dev;
sc->device_id = pci_get_device(dev);
sc->subvendor_id = pci_get_subvendor(dev);
sc->subdevice_id = pci_get_subdevice(dev);
/* Intialize mutexes */
mtx_init( &sc->q_lock, "tws_q_lock", NULL, MTX_DEF);
mtx_init( &sc->sim_lock, "tws_sim_lock", NULL, MTX_DEF);
mtx_init( &sc->gen_lock, "tws_gen_lock", NULL, MTX_DEF);
mtx_init( &sc->io_lock, "tws_io_lock", NULL, MTX_DEF | MTX_RECURSE);
- callout_init(&sc->stats_timer, CALLOUT_MPSAFE);
+ callout_init(&sc->stats_timer, 1);
if ( tws_init_trace_q(sc) == FAILURE )
printf("trace init failure\n");
/* send init event */
mtx_lock(&sc->gen_lock);
tws_send_event(sc, TWS_INIT_START);
mtx_unlock(&sc->gen_lock);
#if _BYTE_ORDER == _BIG_ENDIAN
TWS_TRACE(sc, "BIG endian", 0, 0);
#endif
/* sysctl context setup */
sysctl_ctx_init(&sc->tws_clist);
sc->tws_oidp = SYSCTL_ADD_NODE(&sc->tws_clist,
SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
device_get_nameunit(dev),
CTLFLAG_RD, 0, "");
if ( sc->tws_oidp == NULL ) {
tws_log(sc, SYSCTL_TREE_NODE_ADD);
goto attach_fail_1;
}
SYSCTL_ADD_STRING(&sc->tws_clist, SYSCTL_CHILDREN(sc->tws_oidp),
OID_AUTO, "driver_version", CTLFLAG_RD,
TWS_DRIVER_VERSION_STRING, 0, "TWS driver version");
pci_enable_busmaster(dev);
bar = pci_read_config(dev, TWS_PCI_BAR0, 4);
TWS_TRACE_DEBUG(sc, "bar0 ", bar, 0);
bar = pci_read_config(dev, TWS_PCI_BAR1, 4);
bar = bar & ~TWS_BIT2;
TWS_TRACE_DEBUG(sc, "bar1 ", bar, 0);
/* MFA base address is BAR2 register used for
* push mode. Firmware will evatualy move to
* pull mode during witch this needs to change
*/
#ifndef TWS_PULL_MODE_ENABLE
sc->mfa_base = (u_int64_t)pci_read_config(dev, TWS_PCI_BAR2, 4);
sc->mfa_base = sc->mfa_base & ~TWS_BIT2;
TWS_TRACE_DEBUG(sc, "bar2 ", sc->mfa_base, 0);
#endif
/* allocate MMIO register space */
sc->reg_res_id = TWS_PCI_BAR1; /* BAR1 offset */
if ((sc->reg_res = bus_alloc_resource(dev, SYS_RES_MEMORY,
&(sc->reg_res_id), 0, ~0, 1, RF_ACTIVE))
== NULL) {
tws_log(sc, ALLOC_MEMORY_RES);
goto attach_fail_1;
}
sc->bus_tag = rman_get_bustag(sc->reg_res);
sc->bus_handle = rman_get_bushandle(sc->reg_res);
#ifndef TWS_PULL_MODE_ENABLE
/* Allocate bus space for inbound mfa */
sc->mfa_res_id = TWS_PCI_BAR2; /* BAR2 offset */
if ((sc->mfa_res = bus_alloc_resource(dev, SYS_RES_MEMORY,
&(sc->mfa_res_id), 0, ~0, 0x100000, RF_ACTIVE))
== NULL) {
tws_log(sc, ALLOC_MEMORY_RES);
goto attach_fail_2;
}
sc->bus_mfa_tag = rman_get_bustag(sc->mfa_res);
sc->bus_mfa_handle = rman_get_bushandle(sc->mfa_res);
#endif
/* Allocate and register our interrupt. */
sc->intr_type = TWS_INTx; /* default */
if ( tws_enable_msi )
sc->intr_type = TWS_MSI;
if ( tws_setup_irq(sc) == FAILURE ) {
tws_log(sc, ALLOC_MEMORY_RES);
goto attach_fail_3;
}
/*
* Create a /dev entry for this device. The kernel will assign us
* a major number automatically. We use the unit number of this
* device as the minor number and name the character device
* "tws<unit>".
*/
sc->tws_cdev = make_dev(&tws_cdevsw, device_get_unit(dev),
UID_ROOT, GID_OPERATOR, S_IRUSR | S_IWUSR, "tws%u",
device_get_unit(dev));
sc->tws_cdev->si_drv1 = sc;
if ( tws_init(sc) == FAILURE ) {
tws_log(sc, TWS_INIT_FAILURE);
goto attach_fail_4;
}
if ( tws_init_ctlr(sc) == FAILURE ) {
tws_log(sc, TWS_CTLR_INIT_FAILURE);
goto attach_fail_4;
}
if ((error = tws_cam_attach(sc))) {
tws_log(sc, TWS_CAM_ATTACH);
goto attach_fail_4;
}
/* send init complete event */
mtx_lock(&sc->gen_lock);
tws_send_event(sc, TWS_INIT_COMPLETE);
mtx_unlock(&sc->gen_lock);
TWS_TRACE_DEBUG(sc, "attached successfully", 0, sc->device_id);
return(0);
attach_fail_4:
tws_teardown_intr(sc);
destroy_dev(sc->tws_cdev);
if (sc->dma_mem_phys)
bus_dmamap_unload(sc->cmd_tag, sc->cmd_map);
if (sc->dma_mem)
bus_dmamem_free(sc->cmd_tag, sc->dma_mem, sc->cmd_map);
if (sc->cmd_tag)
bus_dma_tag_destroy(sc->cmd_tag);
attach_fail_3:
for(i=0;i<sc->irqs;i++) {
if ( sc->irq_res[i] ){
if (bus_release_resource(sc->tws_dev,
SYS_RES_IRQ, sc->irq_res_id[i], sc->irq_res[i]))
TWS_TRACE(sc, "bus irq res", 0, 0);
}
}
#ifndef TWS_PULL_MODE_ENABLE
attach_fail_2:
#endif
if ( sc->mfa_res ){
if (bus_release_resource(sc->tws_dev,
SYS_RES_MEMORY, sc->mfa_res_id, sc->mfa_res))
TWS_TRACE(sc, "bus release ", 0, sc->mfa_res_id);
}
if ( sc->reg_res ){
if (bus_release_resource(sc->tws_dev,
SYS_RES_MEMORY, sc->reg_res_id, sc->reg_res))
TWS_TRACE(sc, "bus release2 ", 0, sc->reg_res_id);
}
attach_fail_1:
mtx_destroy(&sc->q_lock);
mtx_destroy(&sc->sim_lock);
mtx_destroy(&sc->gen_lock);
mtx_destroy(&sc->io_lock);
sysctl_ctx_free(&sc->tws_clist);
return (ENXIO);
}
/* Detach device. */
static int
tws_detach(device_t dev)
{
struct tws_softc *sc = device_get_softc(dev);
int i;
u_int32_t reg;
TWS_TRACE_DEBUG(sc, "entry", 0, 0);
mtx_lock(&sc->gen_lock);
tws_send_event(sc, TWS_UNINIT_START);
mtx_unlock(&sc->gen_lock);
/* needs to disable interrupt before detaching from cam */
tws_turn_off_interrupts(sc);
/* clear door bell */
tws_write_reg(sc, TWS_I2O0_HOBDBC, ~0, 4);
reg = tws_read_reg(sc, TWS_I2O0_HIMASK, 4);
TWS_TRACE_DEBUG(sc, "turn-off-intr", reg, 0);
sc->obfl_q_overrun = false;
tws_init_connect(sc, 1);
/* Teardown the state in our softc created in our attach routine. */
/* Disconnect the interrupt handler. */
tws_teardown_intr(sc);
/* Release irq resource */
for(i=0;i<sc->irqs;i++) {
if ( sc->irq_res[i] ){
if (bus_release_resource(sc->tws_dev,
SYS_RES_IRQ, sc->irq_res_id[i], sc->irq_res[i]))
TWS_TRACE(sc, "bus release irq resource",
i, sc->irq_res_id[i]);
}
}
if ( sc->intr_type == TWS_MSI ) {
pci_release_msi(sc->tws_dev);
}
tws_cam_detach(sc);
if (sc->dma_mem_phys)
bus_dmamap_unload(sc->cmd_tag, sc->cmd_map);
if (sc->dma_mem)
bus_dmamem_free(sc->cmd_tag, sc->dma_mem, sc->cmd_map);
if (sc->cmd_tag)
bus_dma_tag_destroy(sc->cmd_tag);
/* Release memory resource */
if ( sc->mfa_res ){
if (bus_release_resource(sc->tws_dev,
SYS_RES_MEMORY, sc->mfa_res_id, sc->mfa_res))
TWS_TRACE(sc, "bus release mem resource", 0, sc->mfa_res_id);
}
if ( sc->reg_res ){
if (bus_release_resource(sc->tws_dev,
SYS_RES_MEMORY, sc->reg_res_id, sc->reg_res))
TWS_TRACE(sc, "bus release mem resource", 0, sc->reg_res_id);
}
for ( i=0; i< tws_queue_depth; i++) {
if (sc->reqs[i].dma_map)
bus_dmamap_destroy(sc->data_tag, sc->reqs[i].dma_map);
callout_drain(&sc->reqs[i].timeout);
}
callout_drain(&sc->stats_timer);
free(sc->reqs, M_TWS);
free(sc->sense_bufs, M_TWS);
free(sc->scan_ccb, M_TWS);
if (sc->ioctl_data_mem)
bus_dmamem_free(sc->data_tag, sc->ioctl_data_mem, sc->ioctl_data_map);
if (sc->data_tag)
bus_dma_tag_destroy(sc->data_tag);
free(sc->aen_q.q, M_TWS);
free(sc->trace_q.q, M_TWS);
mtx_destroy(&sc->q_lock);
mtx_destroy(&sc->sim_lock);
mtx_destroy(&sc->gen_lock);
mtx_destroy(&sc->io_lock);
destroy_dev(sc->tws_cdev);
sysctl_ctx_free(&sc->tws_clist);
return (0);
}
int
tws_setup_intr(struct tws_softc *sc, int irqs)
{
int i, error;
for(i=0;i<irqs;i++) {
if (!(sc->intr_handle[i])) {
if ((error = bus_setup_intr(sc->tws_dev, sc->irq_res[i],
INTR_TYPE_CAM | INTR_MPSAFE,
#if (__FreeBSD_version >= 700000)
NULL,
#endif
tws_intr, sc, &sc->intr_handle[i]))) {
tws_log(sc, SETUP_INTR_RES);
return(FAILURE);
}
}
}
return(SUCCESS);
}
int
tws_teardown_intr(struct tws_softc *sc)
{
int i, error;
for(i=0;i<sc->irqs;i++) {
if (sc->intr_handle[i]) {
error = bus_teardown_intr(sc->tws_dev,
sc->irq_res[i], sc->intr_handle[i]);
sc->intr_handle[i] = NULL;
}
}
return(SUCCESS);
}
static int
tws_setup_irq(struct tws_softc *sc)
{
int messages;
switch(sc->intr_type) {
case TWS_INTx :
sc->irqs = 1;
sc->irq_res_id[0] = 0;
sc->irq_res[0] = bus_alloc_resource_any(sc->tws_dev, SYS_RES_IRQ,
&sc->irq_res_id[0], RF_SHAREABLE | RF_ACTIVE);
if ( ! sc->irq_res[0] )
return(FAILURE);
if ( tws_setup_intr(sc, sc->irqs) == FAILURE )
return(FAILURE);
device_printf(sc->tws_dev, "Using legacy INTx\n");
break;
case TWS_MSI :
sc->irqs = 1;
sc->irq_res_id[0] = 1;
messages = 1;
if (pci_alloc_msi(sc->tws_dev, &messages) != 0 ) {
TWS_TRACE(sc, "pci alloc msi fail", 0, messages);
return(FAILURE);
}
sc->irq_res[0] = bus_alloc_resource_any(sc->tws_dev, SYS_RES_IRQ,
&sc->irq_res_id[0], RF_SHAREABLE | RF_ACTIVE);
if ( !sc->irq_res[0] )
return(FAILURE);
if ( tws_setup_intr(sc, sc->irqs) == FAILURE )
return(FAILURE);
device_printf(sc->tws_dev, "Using MSI\n");
break;
}
return(SUCCESS);
}
static int
tws_init(struct tws_softc *sc)
{
u_int32_t max_sg_elements;
u_int32_t dma_mem_size;
int error;
u_int32_t reg;
sc->seq_id = 0;
if ( tws_queue_depth > TWS_MAX_REQS )
tws_queue_depth = TWS_MAX_REQS;
if (tws_queue_depth < TWS_RESERVED_REQS+1)
tws_queue_depth = TWS_RESERVED_REQS+1;
sc->is64bit = (sizeof(bus_addr_t) == 8) ? true : false;
max_sg_elements = (sc->is64bit && !tws_use_32bit_sgls) ?
TWS_MAX_64BIT_SG_ELEMENTS :
TWS_MAX_32BIT_SG_ELEMENTS;
dma_mem_size = (sizeof(struct tws_command_packet) * tws_queue_depth) +
(TWS_SECTOR_SIZE) ;
if ( bus_dma_tag_create(bus_get_dma_tag(sc->tws_dev), /* PCI parent */
TWS_ALIGNMENT, /* alignment */
0, /* boundary */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
BUS_SPACE_MAXSIZE, /* maxsize */
max_sg_elements, /* numsegs */
BUS_SPACE_MAXSIZE, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockfuncarg */
&sc->parent_tag /* tag */
)) {
TWS_TRACE_DEBUG(sc, "DMA parent tag Create fail", max_sg_elements,
sc->is64bit);
return(ENOMEM);
}
/* In bound message frame requires 16byte alignment.
* Outbound MF's can live with 4byte alignment - for now just
* use 16 for both.
*/
if ( bus_dma_tag_create(sc->parent_tag, /* parent */
TWS_IN_MF_ALIGNMENT, /* alignment */
0, /* boundary */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
dma_mem_size, /* maxsize */
1, /* numsegs */
BUS_SPACE_MAXSIZE, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockfuncarg */
&sc->cmd_tag /* tag */
)) {
TWS_TRACE_DEBUG(sc, "DMA cmd tag Create fail", max_sg_elements, sc->is64bit);
return(ENOMEM);
}
if (bus_dmamem_alloc(sc->cmd_tag, &sc->dma_mem,
BUS_DMA_NOWAIT, &sc->cmd_map)) {
TWS_TRACE_DEBUG(sc, "DMA mem alloc fail", max_sg_elements, sc->is64bit);
return(ENOMEM);
}
/* if bus_dmamem_alloc succeeds then bus_dmamap_load will succeed */
sc->dma_mem_phys=0;
error = bus_dmamap_load(sc->cmd_tag, sc->cmd_map, sc->dma_mem,
dma_mem_size, tws_dmamap_cmds_load_cbfn,
&sc->dma_mem_phys, 0);
/*
* Create a dma tag for data buffers; size will be the maximum
* possible I/O size (128kB).
*/
if (bus_dma_tag_create(sc->parent_tag, /* parent */
TWS_ALIGNMENT, /* alignment */
0, /* boundary */
BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
TWS_MAX_IO_SIZE, /* maxsize */
max_sg_elements, /* nsegments */
TWS_MAX_IO_SIZE, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
busdma_lock_mutex, /* lockfunc */
&sc->io_lock, /* lockfuncarg */
&sc->data_tag /* tag */)) {
TWS_TRACE_DEBUG(sc, "DMA cmd tag Create fail", max_sg_elements, sc->is64bit);
return(ENOMEM);
}
sc->reqs = malloc(sizeof(struct tws_request) * tws_queue_depth, M_TWS,
M_WAITOK | M_ZERO);
if ( sc->reqs == NULL ) {
TWS_TRACE_DEBUG(sc, "malloc failed", 0, sc->is64bit);
return(ENOMEM);
}
sc->sense_bufs = malloc(sizeof(struct tws_sense) * tws_queue_depth, M_TWS,
M_WAITOK | M_ZERO);
if ( sc->sense_bufs == NULL ) {
TWS_TRACE_DEBUG(sc, "sense malloc failed", 0, sc->is64bit);
return(ENOMEM);
}
sc->scan_ccb = malloc(sizeof(union ccb), M_TWS, M_WAITOK | M_ZERO);
if ( sc->scan_ccb == NULL ) {
TWS_TRACE_DEBUG(sc, "ccb malloc failed", 0, sc->is64bit);
return(ENOMEM);
}
if (bus_dmamem_alloc(sc->data_tag, (void **)&sc->ioctl_data_mem,
(BUS_DMA_NOWAIT | BUS_DMA_ZERO), &sc->ioctl_data_map)) {
device_printf(sc->tws_dev, "Cannot allocate ioctl data mem\n");
return(ENOMEM);
}
if ( !tws_ctlr_ready(sc) )
if( !tws_ctlr_reset(sc) )
return(FAILURE);
bzero(&sc->stats, sizeof(struct tws_stats));
tws_init_qs(sc);
tws_turn_off_interrupts(sc);
/*
* enable pull mode by setting bit1 .
* setting bit0 to 1 will enable interrupt coalesing
* will revisit.
*/
#ifdef TWS_PULL_MODE_ENABLE
reg = tws_read_reg(sc, TWS_I2O0_CTL, 4);
TWS_TRACE_DEBUG(sc, "i20 ctl", reg, TWS_I2O0_CTL);
tws_write_reg(sc, TWS_I2O0_CTL, reg | TWS_BIT1, 4);
#endif
TWS_TRACE_DEBUG(sc, "dma_mem_phys", sc->dma_mem_phys, TWS_I2O0_CTL);
if ( tws_init_reqs(sc, dma_mem_size) == FAILURE )
return(FAILURE);
if ( tws_init_aen_q(sc) == FAILURE )
return(FAILURE);
return(SUCCESS);
}
static int
tws_init_aen_q(struct tws_softc *sc)
{
sc->aen_q.head=0;
sc->aen_q.tail=0;
sc->aen_q.depth=256;
sc->aen_q.overflow=0;
sc->aen_q.q = malloc(sizeof(struct tws_event_packet)*sc->aen_q.depth,
M_TWS, M_WAITOK | M_ZERO);
if ( ! sc->aen_q.q )
return(FAILURE);
return(SUCCESS);
}
static int
tws_init_trace_q(struct tws_softc *sc)
{
sc->trace_q.head=0;
sc->trace_q.tail=0;
sc->trace_q.depth=256;
sc->trace_q.overflow=0;
sc->trace_q.q = malloc(sizeof(struct tws_trace_rec)*sc->trace_q.depth,
M_TWS, M_WAITOK | M_ZERO);
if ( ! sc->trace_q.q )
return(FAILURE);
return(SUCCESS);
}
static int
tws_init_reqs(struct tws_softc *sc, u_int32_t dma_mem_size)
{
struct tws_command_packet *cmd_buf;
cmd_buf = (struct tws_command_packet *)sc->dma_mem;
int i;
bzero(cmd_buf, dma_mem_size);
TWS_TRACE_DEBUG(sc, "phy cmd", sc->dma_mem_phys, 0);
mtx_lock(&sc->q_lock);
for ( i=0; i< tws_queue_depth; i++)
{
if (bus_dmamap_create(sc->data_tag, 0, &sc->reqs[i].dma_map)) {
/* log a ENOMEM failure msg here */
mtx_unlock(&sc->q_lock);
return(FAILURE);
}
sc->reqs[i].cmd_pkt = &cmd_buf[i];
sc->sense_bufs[i].hdr = &cmd_buf[i].hdr ;
sc->sense_bufs[i].hdr_pkt_phy = sc->dma_mem_phys +
(i * sizeof(struct tws_command_packet));
sc->reqs[i].cmd_pkt_phy = sc->dma_mem_phys +
sizeof(struct tws_command_header) +
(i * sizeof(struct tws_command_packet));
sc->reqs[i].request_id = i;
sc->reqs[i].sc = sc;
sc->reqs[i].cmd_pkt->hdr.header_desc.size_header = 128;
- callout_init(&sc->reqs[i].timeout, CALLOUT_MPSAFE);
+ callout_init(&sc->reqs[i].timeout, 1);
sc->reqs[i].state = TWS_REQ_STATE_FREE;
if ( i >= TWS_RESERVED_REQS )
tws_q_insert_tail(sc, &sc->reqs[i], TWS_FREE_Q);
}
mtx_unlock(&sc->q_lock);
return(SUCCESS);
}
static void
tws_dmamap_cmds_load_cbfn(void *arg, bus_dma_segment_t *segs,
int nseg, int error)
{
/* printf("command load done \n"); */
*((bus_addr_t *)arg) = segs[0].ds_addr;
}
void
tws_send_event(struct tws_softc *sc, u_int8_t event)
{
mtx_assert(&sc->gen_lock, MA_OWNED);
TWS_TRACE_DEBUG(sc, "received event ", 0, event);
switch (event) {
case TWS_INIT_START:
sc->tws_state = TWS_INIT;
break;
case TWS_INIT_COMPLETE:
if (sc->tws_state != TWS_INIT) {
device_printf(sc->tws_dev, "invalid state transition %d => TWS_ONLINE\n", sc->tws_state);
} else {
sc->tws_state = TWS_ONLINE;
}
break;
case TWS_RESET_START:
/* We can transition to reset state from any state except reset*/
if (sc->tws_state != TWS_RESET) {
sc->tws_prev_state = sc->tws_state;
sc->tws_state = TWS_RESET;
}
break;
case TWS_RESET_COMPLETE:
if (sc->tws_state != TWS_RESET) {
device_printf(sc->tws_dev, "invalid state transition %d => %d (previous state)\n", sc->tws_state, sc->tws_prev_state);
} else {
sc->tws_state = sc->tws_prev_state;
}
break;
case TWS_SCAN_FAILURE:
if (sc->tws_state != TWS_ONLINE) {
device_printf(sc->tws_dev, "invalid state transition %d => TWS_OFFLINE\n", sc->tws_state);
} else {
sc->tws_state = TWS_OFFLINE;
}
break;
case TWS_UNINIT_START:
if ((sc->tws_state != TWS_ONLINE) && (sc->tws_state != TWS_OFFLINE)) {
device_printf(sc->tws_dev, "invalid state transition %d => TWS_UNINIT\n", sc->tws_state);
} else {
sc->tws_state = TWS_UNINIT;
}
break;
}
}
uint8_t
tws_get_state(struct tws_softc *sc)
{
return((u_int8_t)sc->tws_state);
}
/* Called during system shutdown after sync. */
static int
tws_shutdown(device_t dev)
{
struct tws_softc *sc = device_get_softc(dev);
TWS_TRACE_DEBUG(sc, "entry", 0, 0);
tws_turn_off_interrupts(sc);
tws_init_connect(sc, 1);
return (0);
}
/*
* Device suspend routine.
*/
static int
tws_suspend(device_t dev)
{
struct tws_softc *sc = device_get_softc(dev);
if ( sc )
TWS_TRACE_DEBUG(sc, "entry", 0, 0);
return (0);
}
/*
* Device resume routine.
*/
static int
tws_resume(device_t dev)
{
struct tws_softc *sc = device_get_softc(dev);
if ( sc )
TWS_TRACE_DEBUG(sc, "entry", 0, 0);
return (0);
}
struct tws_request *
tws_get_request(struct tws_softc *sc, u_int16_t type)
{
struct mtx *my_mutex = ((type == TWS_REQ_TYPE_SCSI_IO) ? &sc->q_lock : &sc->gen_lock);
struct tws_request *r = NULL;
mtx_lock(my_mutex);
if (type == TWS_REQ_TYPE_SCSI_IO) {
r = tws_q_remove_head(sc, TWS_FREE_Q);
} else {
if ( sc->reqs[type].state == TWS_REQ_STATE_FREE ) {
r = &sc->reqs[type];
}
}
if ( r ) {
bzero(&r->cmd_pkt->cmd, sizeof(struct tws_command_apache));
r->data = NULL;
r->length = 0;
r->type = type;
r->flags = TWS_DIR_UNKNOWN;
r->error_code = TWS_REQ_RET_INVALID;
r->cb = NULL;
r->ccb_ptr = NULL;
callout_stop(&r->timeout);
r->next = r->prev = NULL;
r->state = ((type == TWS_REQ_TYPE_SCSI_IO) ? TWS_REQ_STATE_TRAN : TWS_REQ_STATE_BUSY);
}
mtx_unlock(my_mutex);
return(r);
}
void
tws_release_request(struct tws_request *req)
{
struct tws_softc *sc = req->sc;
TWS_TRACE_DEBUG(sc, "entry", sc, 0);
mtx_lock(&sc->q_lock);
tws_q_insert_tail(sc, req, TWS_FREE_Q);
mtx_unlock(&sc->q_lock);
}
static device_method_t tws_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, tws_probe),
DEVMETHOD(device_attach, tws_attach),
DEVMETHOD(device_detach, tws_detach),
DEVMETHOD(device_shutdown, tws_shutdown),
DEVMETHOD(device_suspend, tws_suspend),
DEVMETHOD(device_resume, tws_resume),
DEVMETHOD_END
};
static driver_t tws_driver = {
"tws",
tws_methods,
sizeof(struct tws_softc)
};
static devclass_t tws_devclass;
/* DEFINE_CLASS_0(tws, tws_driver, tws_methods, sizeof(struct tws_softc)); */
DRIVER_MODULE(tws, pci, tws_driver, tws_devclass, 0, 0);
MODULE_DEPEND(tws, cam, 1, 1, 1);
MODULE_DEPEND(tws, pci, 1, 1, 1);
TUNABLE_INT("hw.tws.queue_depth", &tws_queue_depth);
TUNABLE_INT("hw.tws.enable_msi", &tws_enable_msi);
Index: head/sys/dev/ubsec/ubsec.c
===================================================================
--- head/sys/dev/ubsec/ubsec.c (revision 283290)
+++ head/sys/dev/ubsec/ubsec.c (revision 283291)
@@ -1,2858 +1,2858 @@
/* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */
/*-
* Copyright (c) 2000 Jason L. Wright (jason@thought.net)
* Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
* Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by Jason L. Wright
* 4. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* Effort sponsored in part by the Defense Advanced Research Projects
* Agency (DARPA) and Air Force Research Laboratory, Air Force
* Materiel Command, USAF, under agreement number F30602-01-2-0537.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* uBsec 5[56]01, 58xx hardware crypto accelerator
*/
#include "opt_ubsec.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/errno.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/mbuf.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <sys/endian.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <crypto/sha1.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/cryptosoft.h>
#include <sys/md5.h>
#include <sys/random.h>
#include <sys/kobj.h>
#include "cryptodev_if.h"
#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
/* grr, #defines for gratuitous incompatibility in queue.h */
#define SIMPLEQ_HEAD STAILQ_HEAD
#define SIMPLEQ_ENTRY STAILQ_ENTRY
#define SIMPLEQ_INIT STAILQ_INIT
#define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL
#define SIMPLEQ_EMPTY STAILQ_EMPTY
#define SIMPLEQ_FIRST STAILQ_FIRST
#define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD
#define SIMPLEQ_FOREACH STAILQ_FOREACH
/* ditto for endian.h */
#define letoh16(x) le16toh(x)
#define letoh32(x) le32toh(x)
#ifdef UBSEC_RNDTEST
#include <dev/rndtest/rndtest.h>
#endif
#include <dev/ubsec/ubsecreg.h>
#include <dev/ubsec/ubsecvar.h>
/*
* Prototypes and count for the pci_device structure
*/
static int ubsec_probe(device_t);
static int ubsec_attach(device_t);
static int ubsec_detach(device_t);
static int ubsec_suspend(device_t);
static int ubsec_resume(device_t);
static int ubsec_shutdown(device_t);
static int ubsec_newsession(device_t, u_int32_t *, struct cryptoini *);
static int ubsec_freesession(device_t, u_int64_t);
static int ubsec_process(device_t, struct cryptop *, int);
static int ubsec_kprocess(device_t, struct cryptkop *, int);
static device_method_t ubsec_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, ubsec_probe),
DEVMETHOD(device_attach, ubsec_attach),
DEVMETHOD(device_detach, ubsec_detach),
DEVMETHOD(device_suspend, ubsec_suspend),
DEVMETHOD(device_resume, ubsec_resume),
DEVMETHOD(device_shutdown, ubsec_shutdown),
/* crypto device methods */
DEVMETHOD(cryptodev_newsession, ubsec_newsession),
DEVMETHOD(cryptodev_freesession,ubsec_freesession),
DEVMETHOD(cryptodev_process, ubsec_process),
DEVMETHOD(cryptodev_kprocess, ubsec_kprocess),
DEVMETHOD_END
};
static driver_t ubsec_driver = {
"ubsec",
ubsec_methods,
sizeof (struct ubsec_softc)
};
static devclass_t ubsec_devclass;
DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
#ifdef UBSEC_RNDTEST
MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
#endif
static void ubsec_intr(void *);
static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
static void ubsec_feed(struct ubsec_softc *);
static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
static int ubsec_feed2(struct ubsec_softc *);
static void ubsec_rng(void *);
static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
struct ubsec_dma_alloc *, int);
#define ubsec_dma_sync(_dma, _flags) \
bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
static int ubsec_dmamap_aligned(struct ubsec_operand *op);
static void ubsec_reset_board(struct ubsec_softc *sc);
static void ubsec_init_board(struct ubsec_softc *sc);
static void ubsec_init_pciregs(device_t dev);
static void ubsec_totalreset(struct ubsec_softc *sc);
static int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
static int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
static int ubsec_ksigbits(struct crparam *);
static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
static SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0,
"Broadcom driver parameters");
#ifdef UBSEC_DEBUG
static void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
static void ubsec_dump_mcr(struct ubsec_mcr *);
static void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
static int ubsec_debug = 0;
SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
0, "control debugging msgs");
#endif
#define READ_REG(sc,r) \
bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
#define WRITE_REG(sc,reg,val) \
bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
#define SWAP32(x) (x) = htole32(ntohl((x)))
#define HTOLE32(x) (x) = htole32(x)
struct ubsec_stats ubsecstats;
SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
ubsec_stats, "driver statistics");
static int
ubsec_probe(device_t dev)
{
if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
(pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
return (BUS_PROBE_DEFAULT);
if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
(pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
return (BUS_PROBE_DEFAULT);
if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
(pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825
))
return (BUS_PROBE_DEFAULT);
return (ENXIO);
}
static const char*
ubsec_partname(struct ubsec_softc *sc)
{
/* XXX sprintf numbers when not decoded */
switch (pci_get_vendor(sc->sc_dev)) {
case PCI_VENDOR_BROADCOM:
switch (pci_get_device(sc->sc_dev)) {
case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801";
case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802";
case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805";
case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820";
case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821";
case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822";
case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823";
case PCI_PRODUCT_BROADCOM_5825: return "Broadcom 5825";
}
return "Broadcom unknown-part";
case PCI_VENDOR_BLUESTEEL:
switch (pci_get_device(sc->sc_dev)) {
case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
}
return "Bluesteel unknown-part";
case PCI_VENDOR_SUN:
switch (pci_get_device(sc->sc_dev)) {
case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
}
return "Sun unknown-part";
}
return "Unknown-vendor unknown-part";
}
static void
default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
{
random_harvest(buf, count, count*NBBY/2, RANDOM_PURE_UBSEC);
}
static int
ubsec_attach(device_t dev)
{
struct ubsec_softc *sc = device_get_softc(dev);
struct ubsec_dma *dmap;
u_int32_t i;
int rid;
bzero(sc, sizeof (*sc));
sc->sc_dev = dev;
SIMPLEQ_INIT(&sc->sc_queue);
SIMPLEQ_INIT(&sc->sc_qchip);
SIMPLEQ_INIT(&sc->sc_queue2);
SIMPLEQ_INIT(&sc->sc_qchip2);
SIMPLEQ_INIT(&sc->sc_q2free);
/* XXX handle power management */
sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
(pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
(pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825)) ||
(pci_get_vendor(dev) == PCI_VENDOR_SUN &&
(pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
/* NB: the 5821/5822 defines some additional status bits */
sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
BS_STAT_MCR2_ALLEMPTY;
sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
}
pci_enable_busmaster(dev);
/*
* Setup memory-mapping of PCI registers.
*/
rid = BS_BAR;
sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (sc->sc_sr == NULL) {
device_printf(dev, "cannot map register space\n");
goto bad;
}
sc->sc_st = rman_get_bustag(sc->sc_sr);
sc->sc_sh = rman_get_bushandle(sc->sc_sr);
/*
* Arrange interrupt line.
*/
rid = 0;
sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_SHAREABLE|RF_ACTIVE);
if (sc->sc_irq == NULL) {
device_printf(dev, "could not map interrupt\n");
goto bad1;
}
/*
* NB: Network code assumes we are blocked with splimp()
* so make sure the IRQ is mapped appropriately.
*/
if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
NULL, ubsec_intr, sc, &sc->sc_ih)) {
device_printf(dev, "could not establish interrupt\n");
goto bad2;
}
sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
if (sc->sc_cid < 0) {
device_printf(dev, "could not get crypto driver id\n");
goto bad3;
}
/*
* Setup DMA descriptor area.
*/
if (bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
0x3ffff, /* maxsize */
UBS_MAX_SCATTER, /* nsegments */
0xffff, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&sc->sc_dmat)) {
device_printf(dev, "cannot allocate DMA tag\n");
goto bad4;
}
SIMPLEQ_INIT(&sc->sc_freequeue);
dmap = sc->sc_dmaa;
for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
struct ubsec_q *q;
q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
M_DEVBUF, M_NOWAIT);
if (q == NULL) {
device_printf(dev, "cannot allocate queue buffers\n");
break;
}
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
&dmap->d_alloc, 0)) {
device_printf(dev, "cannot allocate dma buffers\n");
free(q, M_DEVBUF);
break;
}
dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
q->q_dma = dmap;
sc->sc_queuea[i] = q;
SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
}
mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev),
"mcr1 operations", MTX_DEF);
mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev),
"mcr1 free q", MTX_DEF);
device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
/*
* Reset Broadcom chip
*/
ubsec_reset_board(sc);
/*
* Init Broadcom specific PCI settings
*/
ubsec_init_pciregs(dev);
/*
* Init Broadcom chip
*/
ubsec_init_board(sc);
#ifndef UBSEC_NO_RNG
if (sc->sc_flags & UBS_FLAGS_RNG) {
sc->sc_statmask |= BS_STAT_MCR2_DONE;
#ifdef UBSEC_RNDTEST
sc->sc_rndtest = rndtest_attach(dev);
if (sc->sc_rndtest)
sc->sc_harvest = rndtest_harvest;
else
sc->sc_harvest = default_harvest;
#else
sc->sc_harvest = default_harvest;
#endif
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
&sc->sc_rng.rng_q.q_mcr, 0))
goto skip_rng;
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
&sc->sc_rng.rng_q.q_ctx, 0)) {
ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
goto skip_rng;
}
if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
goto skip_rng;
}
if (hz >= 100)
sc->sc_rnghz = hz / 100;
else
sc->sc_rnghz = 1;
- callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_rngto, 1);
callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
skip_rng:
;
}
#endif /* UBSEC_NO_RNG */
mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev),
"mcr2 operations", MTX_DEF);
if (sc->sc_flags & UBS_FLAGS_KEY) {
sc->sc_statmask |= BS_STAT_MCR2_DONE;
crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
#if 0
crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
#endif
}
return (0);
bad4:
crypto_unregister_all(sc->sc_cid);
bad3:
bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
bad2:
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
bad1:
bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
bad:
return (ENXIO);
}
/*
* Detach a device that successfully probed.
*/
static int
ubsec_detach(device_t dev)
{
struct ubsec_softc *sc = device_get_softc(dev);
/* XXX wait/abort active ops */
/* disable interrupts */
WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
callout_stop(&sc->sc_rngto);
crypto_unregister_all(sc->sc_cid);
#ifdef UBSEC_RNDTEST
if (sc->sc_rndtest)
rndtest_detach(sc->sc_rndtest);
#endif
while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
struct ubsec_q *q;
q = SIMPLEQ_FIRST(&sc->sc_freequeue);
SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
ubsec_dma_free(sc, &q->q_dma->d_alloc);
free(q, M_DEVBUF);
}
mtx_destroy(&sc->sc_mcr1lock);
mtx_destroy(&sc->sc_freeqlock);
#ifndef UBSEC_NO_RNG
if (sc->sc_flags & UBS_FLAGS_RNG) {
ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
}
#endif /* UBSEC_NO_RNG */
mtx_destroy(&sc->sc_mcr2lock);
bus_generic_detach(dev);
bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
bus_dma_tag_destroy(sc->sc_dmat);
bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
return (0);
}
/*
* Stop all chip i/o so that the kernel's probe routines don't
* get confused by errant DMAs when rebooting.
*/
static int
ubsec_shutdown(device_t dev)
{
#ifdef notyet
ubsec_stop(device_get_softc(dev));
#endif
return (0);
}
/*
* Device suspend routine.
*/
static int
ubsec_suspend(device_t dev)
{
struct ubsec_softc *sc = device_get_softc(dev);
#ifdef notyet
/* XXX stop the device and save PCI settings */
#endif
sc->sc_suspended = 1;
return (0);
}
static int
ubsec_resume(device_t dev)
{
struct ubsec_softc *sc = device_get_softc(dev);
#ifdef notyet
/* XXX retore PCI settings and start the device */
#endif
sc->sc_suspended = 0;
return (0);
}
/*
* UBSEC Interrupt routine
*/
static void
ubsec_intr(void *arg)
{
struct ubsec_softc *sc = arg;
volatile u_int32_t stat;
struct ubsec_q *q;
struct ubsec_dma *dmap;
int npkts = 0, i;
stat = READ_REG(sc, BS_STAT);
stat &= sc->sc_statmask;
if (stat == 0)
return;
WRITE_REG(sc, BS_STAT, stat); /* IACK */
/*
* Check to see if we have any packets waiting for us
*/
if ((stat & BS_STAT_MCR1_DONE)) {
mtx_lock(&sc->sc_mcr1lock);
while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
q = SIMPLEQ_FIRST(&sc->sc_qchip);
dmap = q->q_dma;
if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
break;
SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
npkts = q->q_nstacked_mcrs;
sc->sc_nqchip -= 1+npkts;
/*
* search for further sc_qchip ubsec_q's that share
* the same MCR, and complete them too, they must be
* at the top.
*/
for (i = 0; i < npkts; i++) {
if(q->q_stacked_mcr[i]) {
ubsec_callback(sc, q->q_stacked_mcr[i]);
} else {
break;
}
}
ubsec_callback(sc, q);
}
/*
* Don't send any more packet to chip if there has been
* a DMAERR.
*/
if (!(stat & BS_STAT_DMAERR))
ubsec_feed(sc);
mtx_unlock(&sc->sc_mcr1lock);
}
/*
* Check to see if we have any key setups/rng's waiting for us
*/
if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
(stat & BS_STAT_MCR2_DONE)) {
struct ubsec_q2 *q2;
struct ubsec_mcr *mcr;
mtx_lock(&sc->sc_mcr2lock);
while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
ubsec_dma_sync(&q2->q_mcr,
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
ubsec_dma_sync(&q2->q_mcr,
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
break;
}
SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next);
ubsec_callback2(sc, q2);
/*
* Don't send any more packet to chip if there has been
* a DMAERR.
*/
if (!(stat & BS_STAT_DMAERR))
ubsec_feed2(sc);
}
mtx_unlock(&sc->sc_mcr2lock);
}
/*
* Check to see if we got any DMA Error
*/
if (stat & BS_STAT_DMAERR) {
#ifdef UBSEC_DEBUG
if (ubsec_debug) {
volatile u_int32_t a = READ_REG(sc, BS_ERR);
printf("dmaerr %s@%08x\n",
(a & BS_ERR_READ) ? "read" : "write",
a & BS_ERR_ADDR);
}
#endif /* UBSEC_DEBUG */
ubsecstats.hst_dmaerr++;
mtx_lock(&sc->sc_mcr1lock);
ubsec_totalreset(sc);
ubsec_feed(sc);
mtx_unlock(&sc->sc_mcr1lock);
}
if (sc->sc_needwakeup) { /* XXX check high watermark */
int wakeup;
mtx_lock(&sc->sc_freeqlock);
wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
#ifdef UBSEC_DEBUG
if (ubsec_debug)
device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
sc->sc_needwakeup);
#endif /* UBSEC_DEBUG */
sc->sc_needwakeup &= ~wakeup;
mtx_unlock(&sc->sc_freeqlock);
crypto_unblock(sc->sc_cid, wakeup);
}
}
/*
* ubsec_feed() - aggregate and post requests to chip
*/
static void
ubsec_feed(struct ubsec_softc *sc)
{
struct ubsec_q *q, *q2;
int npkts, i;
void *v;
u_int32_t stat;
/*
* Decide how many ops to combine in a single MCR. We cannot
* aggregate more than UBS_MAX_AGGR because this is the number
* of slots defined in the data structure. Note that
* aggregation only happens if ops are marked batch'able.
* Aggregating ops reduces the number of interrupts to the host
* but also (potentially) increases the latency for processing
* completed ops as we only get an interrupt when all aggregated
* ops have completed.
*/
if (sc->sc_nqueue == 0)
return;
if (sc->sc_nqueue > 1) {
npkts = 0;
SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
npkts++;
if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
break;
}
} else
npkts = 1;
/*
* Check device status before going any further.
*/
if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
if (stat & BS_STAT_DMAERR) {
ubsec_totalreset(sc);
ubsecstats.hst_dmaerr++;
} else
ubsecstats.hst_mcr1full++;
return;
}
if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
ubsecstats.hst_maxqueue = sc->sc_nqueue;
if (npkts > UBS_MAX_AGGR)
npkts = UBS_MAX_AGGR;
if (npkts < 2) /* special case 1 op */
goto feed1;
ubsecstats.hst_totbatch += npkts-1;
#ifdef UBSEC_DEBUG
if (ubsec_debug)
printf("merging %d records\n", npkts);
#endif /* UBSEC_DEBUG */
q = SIMPLEQ_FIRST(&sc->sc_queue);
SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
--sc->sc_nqueue;
bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
if (q->q_dst_map != NULL)
bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */
for (i = 0; i < q->q_nstacked_mcrs; i++) {
q2 = SIMPLEQ_FIRST(&sc->sc_queue);
bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
BUS_DMASYNC_PREWRITE);
if (q2->q_dst_map != NULL)
bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
BUS_DMASYNC_PREREAD);
SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
--sc->sc_nqueue;
v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
sizeof(struct ubsec_mcr_add));
bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
q->q_stacked_mcr[i] = q2;
}
q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
sc->sc_nqchip += npkts;
if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
ubsecstats.hst_maxqchip = sc->sc_nqchip;
ubsec_dma_sync(&q->q_dma->d_alloc,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_mcr));
return;
feed1:
q = SIMPLEQ_FIRST(&sc->sc_queue);
bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
if (q->q_dst_map != NULL)
bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
ubsec_dma_sync(&q->q_dma->d_alloc,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_mcr));
#ifdef UBSEC_DEBUG
if (ubsec_debug)
printf("feed1: q->chip %p %08x stat %08x\n",
q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
stat);
#endif /* UBSEC_DEBUG */
SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
--sc->sc_nqueue;
SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
sc->sc_nqchip++;
if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
ubsecstats.hst_maxqchip = sc->sc_nqchip;
return;
}
static void
ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
{
/* Go ahead and compute key in ubsec's byte order */
if (algo == CRYPTO_DES_CBC) {
bcopy(key, &ses->ses_deskey[0], 8);
bcopy(key, &ses->ses_deskey[2], 8);
bcopy(key, &ses->ses_deskey[4], 8);
} else
bcopy(key, ses->ses_deskey, 24);
SWAP32(ses->ses_deskey[0]);
SWAP32(ses->ses_deskey[1]);
SWAP32(ses->ses_deskey[2]);
SWAP32(ses->ses_deskey[3]);
SWAP32(ses->ses_deskey[4]);
SWAP32(ses->ses_deskey[5]);
}
static void
ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
{
MD5_CTX md5ctx;
SHA1_CTX sha1ctx;
int i;
for (i = 0; i < klen; i++)
key[i] ^= HMAC_IPAD_VAL;
if (algo == CRYPTO_MD5_HMAC) {
MD5Init(&md5ctx);
MD5Update(&md5ctx, key, klen);
MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
} else {
SHA1Init(&sha1ctx);
SHA1Update(&sha1ctx, key, klen);
SHA1Update(&sha1ctx, hmac_ipad_buffer,
SHA1_HMAC_BLOCK_LEN - klen);
bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
}
for (i = 0; i < klen; i++)
key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
if (algo == CRYPTO_MD5_HMAC) {
MD5Init(&md5ctx);
MD5Update(&md5ctx, key, klen);
MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
} else {
SHA1Init(&sha1ctx);
SHA1Update(&sha1ctx, key, klen);
SHA1Update(&sha1ctx, hmac_opad_buffer,
SHA1_HMAC_BLOCK_LEN - klen);
bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
}
for (i = 0; i < klen; i++)
key[i] ^= HMAC_OPAD_VAL;
}
/*
* Allocate a new 'session' and return an encoded session id. 'sidp'
* contains our registration id, and should contain an encoded session
* id on successful allocation.
*/
static int
ubsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
{
struct ubsec_softc *sc = device_get_softc(dev);
struct cryptoini *c, *encini = NULL, *macini = NULL;
struct ubsec_session *ses = NULL;
int sesn;
if (sidp == NULL || cri == NULL || sc == NULL)
return (EINVAL);
for (c = cri; c != NULL; c = c->cri_next) {
if (c->cri_alg == CRYPTO_MD5_HMAC ||
c->cri_alg == CRYPTO_SHA1_HMAC) {
if (macini)
return (EINVAL);
macini = c;
} else if (c->cri_alg == CRYPTO_DES_CBC ||
c->cri_alg == CRYPTO_3DES_CBC) {
if (encini)
return (EINVAL);
encini = c;
} else
return (EINVAL);
}
if (encini == NULL && macini == NULL)
return (EINVAL);
if (sc->sc_sessions == NULL) {
ses = sc->sc_sessions = (struct ubsec_session *)malloc(
sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
if (ses == NULL)
return (ENOMEM);
sesn = 0;
sc->sc_nsessions = 1;
} else {
for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
if (sc->sc_sessions[sesn].ses_used == 0) {
ses = &sc->sc_sessions[sesn];
break;
}
}
if (ses == NULL) {
sesn = sc->sc_nsessions;
ses = (struct ubsec_session *)malloc((sesn + 1) *
sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
if (ses == NULL)
return (ENOMEM);
bcopy(sc->sc_sessions, ses, sesn *
sizeof(struct ubsec_session));
bzero(sc->sc_sessions, sesn *
sizeof(struct ubsec_session));
free(sc->sc_sessions, M_DEVBUF);
sc->sc_sessions = ses;
ses = &sc->sc_sessions[sesn];
sc->sc_nsessions++;
}
}
bzero(ses, sizeof(struct ubsec_session));
ses->ses_used = 1;
if (encini) {
/* get an IV, network byte order */
/* XXX may read fewer than requested */
read_random(ses->ses_iv, sizeof(ses->ses_iv));
if (encini->cri_key != NULL) {
ubsec_setup_enckey(ses, encini->cri_alg,
encini->cri_key);
}
}
if (macini) {
ses->ses_mlen = macini->cri_mlen;
if (ses->ses_mlen == 0) {
if (macini->cri_alg == CRYPTO_MD5_HMAC)
ses->ses_mlen = MD5_HASH_LEN;
else
ses->ses_mlen = SHA1_HASH_LEN;
}
if (macini->cri_key != NULL) {
ubsec_setup_mackey(ses, macini->cri_alg,
macini->cri_key, macini->cri_klen / 8);
}
}
*sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
return (0);
}
/*
* Deallocate a session.
*/
static int
ubsec_freesession(device_t dev, u_int64_t tid)
{
struct ubsec_softc *sc = device_get_softc(dev);
int session, ret;
u_int32_t sid = CRYPTO_SESID2LID(tid);
if (sc == NULL)
return (EINVAL);
session = UBSEC_SESSION(sid);
if (session < sc->sc_nsessions) {
bzero(&sc->sc_sessions[session],
sizeof(sc->sc_sessions[session]));
ret = 0;
} else
ret = EINVAL;
return (ret);
}
static void
ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
{
struct ubsec_operand *op = arg;
KASSERT(nsegs <= UBS_MAX_SCATTER,
("Too many DMA segments returned when mapping operand"));
#ifdef UBSEC_DEBUG
if (ubsec_debug)
printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n",
(u_int) mapsize, nsegs, error);
#endif
if (error != 0)
return;
op->mapsize = mapsize;
op->nsegs = nsegs;
bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
}
static int
ubsec_process(device_t dev, struct cryptop *crp, int hint)
{
struct ubsec_softc *sc = device_get_softc(dev);
struct ubsec_q *q = NULL;
int err = 0, i, j, nicealign;
struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
int encoffset = 0, macoffset = 0, cpskip, cpoffset;
int sskip, dskip, stheend, dtheend;
int16_t coffset;
struct ubsec_session *ses;
struct ubsec_pktctx ctx;
struct ubsec_dma *dmap = NULL;
if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
ubsecstats.hst_invalid++;
return (EINVAL);
}
if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
ubsecstats.hst_badsession++;
return (EINVAL);
}
mtx_lock(&sc->sc_freeqlock);
if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
ubsecstats.hst_queuefull++;
sc->sc_needwakeup |= CRYPTO_SYMQ;
mtx_unlock(&sc->sc_freeqlock);
return (ERESTART);
}
q = SIMPLEQ_FIRST(&sc->sc_freequeue);
SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
mtx_unlock(&sc->sc_freeqlock);
dmap = q->q_dma; /* Save dma pointer */
bzero(q, sizeof(struct ubsec_q));
bzero(&ctx, sizeof(ctx));
q->q_sesn = UBSEC_SESSION(crp->crp_sid);
q->q_dma = dmap;
ses = &sc->sc_sessions[q->q_sesn];
if (crp->crp_flags & CRYPTO_F_IMBUF) {
q->q_src_m = (struct mbuf *)crp->crp_buf;
q->q_dst_m = (struct mbuf *)crp->crp_buf;
} else if (crp->crp_flags & CRYPTO_F_IOV) {
q->q_src_io = (struct uio *)crp->crp_buf;
q->q_dst_io = (struct uio *)crp->crp_buf;
} else {
ubsecstats.hst_badflags++;
err = EINVAL;
goto errout; /* XXX we don't handle contiguous blocks! */
}
bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
dmap->d_dma->d_mcr.mcr_flags = 0;
q->q_crp = crp;
crd1 = crp->crp_desc;
if (crd1 == NULL) {
ubsecstats.hst_nodesc++;
err = EINVAL;
goto errout;
}
crd2 = crd1->crd_next;
if (crd2 == NULL) {
if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC) {
maccrd = crd1;
enccrd = NULL;
} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
crd1->crd_alg == CRYPTO_3DES_CBC) {
maccrd = NULL;
enccrd = crd1;
} else {
ubsecstats.hst_badalg++;
err = EINVAL;
goto errout;
}
} else {
if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
(crd2->crd_alg == CRYPTO_DES_CBC ||
crd2->crd_alg == CRYPTO_3DES_CBC) &&
((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
maccrd = crd1;
enccrd = crd2;
} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
crd1->crd_alg == CRYPTO_3DES_CBC) &&
(crd2->crd_alg == CRYPTO_MD5_HMAC ||
crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
(crd1->crd_flags & CRD_F_ENCRYPT)) {
enccrd = crd1;
maccrd = crd2;
} else {
/*
* We cannot order the ubsec as requested
*/
ubsecstats.hst_badalg++;
err = EINVAL;
goto errout;
}
}
if (enccrd) {
if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
ubsec_setup_enckey(ses, enccrd->crd_alg,
enccrd->crd_key);
}
encoffset = enccrd->crd_skip;
ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
if (enccrd->crd_flags & CRD_F_ENCRYPT) {
q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
else {
ctx.pc_iv[0] = ses->ses_iv[0];
ctx.pc_iv[1] = ses->ses_iv[1];
}
if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
crypto_copyback(crp->crp_flags, crp->crp_buf,
enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
}
} else {
ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
else {
crypto_copydata(crp->crp_flags, crp->crp_buf,
enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
}
}
ctx.pc_deskey[0] = ses->ses_deskey[0];
ctx.pc_deskey[1] = ses->ses_deskey[1];
ctx.pc_deskey[2] = ses->ses_deskey[2];
ctx.pc_deskey[3] = ses->ses_deskey[3];
ctx.pc_deskey[4] = ses->ses_deskey[4];
ctx.pc_deskey[5] = ses->ses_deskey[5];
SWAP32(ctx.pc_iv[0]);
SWAP32(ctx.pc_iv[1]);
}
if (maccrd) {
if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
ubsec_setup_mackey(ses, maccrd->crd_alg,
maccrd->crd_key, maccrd->crd_klen / 8);
}
macoffset = maccrd->crd_skip;
if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
else
ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
for (i = 0; i < 5; i++) {
ctx.pc_hminner[i] = ses->ses_hminner[i];
ctx.pc_hmouter[i] = ses->ses_hmouter[i];
HTOLE32(ctx.pc_hminner[i]);
HTOLE32(ctx.pc_hmouter[i]);
}
}
if (enccrd && maccrd) {
/*
* ubsec cannot handle packets where the end of encryption
* and authentication are not the same, or where the
* encrypted part begins before the authenticated part.
*/
if ((encoffset + enccrd->crd_len) !=
(macoffset + maccrd->crd_len)) {
ubsecstats.hst_lenmismatch++;
err = EINVAL;
goto errout;
}
if (enccrd->crd_skip < maccrd->crd_skip) {
ubsecstats.hst_skipmismatch++;
err = EINVAL;
goto errout;
}
sskip = maccrd->crd_skip;
cpskip = dskip = enccrd->crd_skip;
stheend = maccrd->crd_len;
dtheend = enccrd->crd_len;
coffset = enccrd->crd_skip - maccrd->crd_skip;
cpoffset = cpskip + dtheend;
#ifdef UBSEC_DEBUG
if (ubsec_debug) {
printf("mac: skip %d, len %d, inject %d\n",
maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
printf("enc: skip %d, len %d, inject %d\n",
enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
printf("src: skip %d, len %d\n", sskip, stheend);
printf("dst: skip %d, len %d\n", dskip, dtheend);
printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
coffset, stheend, cpskip, cpoffset);
}
#endif
} else {
cpskip = dskip = sskip = macoffset + encoffset;
dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
cpoffset = cpskip + dtheend;
coffset = 0;
}
ctx.pc_offset = htole16(coffset >> 2);
if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
ubsecstats.hst_nomap++;
err = ENOMEM;
goto errout;
}
if (crp->crp_flags & CRYPTO_F_IMBUF) {
if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
q->q_src_map = NULL;
ubsecstats.hst_noload++;
err = ENOMEM;
goto errout;
}
} else if (crp->crp_flags & CRYPTO_F_IOV) {
if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
q->q_src_map = NULL;
ubsecstats.hst_noload++;
err = ENOMEM;
goto errout;
}
}
nicealign = ubsec_dmamap_aligned(&q->q_src);
dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
#ifdef UBSEC_DEBUG
if (ubsec_debug)
printf("src skip: %d nicealign: %u\n", sskip, nicealign);
#endif
for (i = j = 0; i < q->q_src_nsegs; i++) {
struct ubsec_pktbuf *pb;
bus_size_t packl = q->q_src_segs[i].ds_len;
bus_addr_t packp = q->q_src_segs[i].ds_addr;
if (sskip >= packl) {
sskip -= packl;
continue;
}
packl -= sskip;
packp += sskip;
sskip = 0;
if (packl > 0xfffc) {
err = EIO;
goto errout;
}
if (j == 0)
pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
else
pb = &dmap->d_dma->d_sbuf[j - 1];
pb->pb_addr = htole32(packp);
if (stheend) {
if (packl > stheend) {
pb->pb_len = htole32(stheend);
stheend = 0;
} else {
pb->pb_len = htole32(packl);
stheend -= packl;
}
} else
pb->pb_len = htole32(packl);
if ((i + 1) == q->q_src_nsegs)
pb->pb_next = 0;
else
pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_sbuf[j]));
j++;
}
if (enccrd == NULL && maccrd != NULL) {
dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_macbuf[0]));
#ifdef UBSEC_DEBUG
if (ubsec_debug)
printf("opkt: %x %x %x\n",
dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
#endif
} else {
if (crp->crp_flags & CRYPTO_F_IOV) {
if (!nicealign) {
ubsecstats.hst_iovmisaligned++;
err = EINVAL;
goto errout;
}
if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
&q->q_dst_map)) {
ubsecstats.hst_nomap++;
err = ENOMEM;
goto errout;
}
if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
q->q_dst_map = NULL;
ubsecstats.hst_noload++;
err = ENOMEM;
goto errout;
}
} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
if (nicealign) {
q->q_dst = q->q_src;
} else {
int totlen, len;
struct mbuf *m, *top, **mp;
ubsecstats.hst_unaligned++;
totlen = q->q_src_mapsize;
if (totlen >= MINCLSIZE) {
m = m_getcl(M_NOWAIT, MT_DATA,
q->q_src_m->m_flags & M_PKTHDR);
len = MCLBYTES;
} else if (q->q_src_m->m_flags & M_PKTHDR) {
m = m_gethdr(M_NOWAIT, MT_DATA);
len = MHLEN;
} else {
m = m_get(M_NOWAIT, MT_DATA);
len = MLEN;
}
if (m && q->q_src_m->m_flags & M_PKTHDR &&
!m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) {
m_free(m);
m = NULL;
}
if (m == NULL) {
ubsecstats.hst_nombuf++;
err = sc->sc_nqueue ? ERESTART : ENOMEM;
goto errout;
}
m->m_len = len = min(totlen, len);
totlen -= len;
top = m;
mp = &top;
while (totlen > 0) {
if (totlen >= MINCLSIZE) {
m = m_getcl(M_NOWAIT,
MT_DATA, 0);
len = MCLBYTES;
} else {
m = m_get(M_NOWAIT, MT_DATA);
len = MLEN;
}
if (m == NULL) {
m_freem(top);
ubsecstats.hst_nombuf++;
err = sc->sc_nqueue ? ERESTART : ENOMEM;
goto errout;
}
m->m_len = len = min(totlen, len);
totlen -= len;
*mp = m;
mp = &m->m_next;
}
q->q_dst_m = top;
ubsec_mcopy(q->q_src_m, q->q_dst_m,
cpskip, cpoffset);
if (bus_dmamap_create(sc->sc_dmat,
BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
ubsecstats.hst_nomap++;
err = ENOMEM;
goto errout;
}
if (bus_dmamap_load_mbuf(sc->sc_dmat,
q->q_dst_map, q->q_dst_m,
ubsec_op_cb, &q->q_dst,
BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat,
q->q_dst_map);
q->q_dst_map = NULL;
ubsecstats.hst_noload++;
err = ENOMEM;
goto errout;
}
}
} else {
ubsecstats.hst_badflags++;
err = EINVAL;
goto errout;
}
#ifdef UBSEC_DEBUG
if (ubsec_debug)
printf("dst skip: %d\n", dskip);
#endif
for (i = j = 0; i < q->q_dst_nsegs; i++) {
struct ubsec_pktbuf *pb;
bus_size_t packl = q->q_dst_segs[i].ds_len;
bus_addr_t packp = q->q_dst_segs[i].ds_addr;
if (dskip >= packl) {
dskip -= packl;
continue;
}
packl -= dskip;
packp += dskip;
dskip = 0;
if (packl > 0xfffc) {
err = EIO;
goto errout;
}
if (j == 0)
pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
else
pb = &dmap->d_dma->d_dbuf[j - 1];
pb->pb_addr = htole32(packp);
if (dtheend) {
if (packl > dtheend) {
pb->pb_len = htole32(dtheend);
dtheend = 0;
} else {
pb->pb_len = htole32(packl);
dtheend -= packl;
}
} else
pb->pb_len = htole32(packl);
if ((i + 1) == q->q_dst_nsegs) {
if (maccrd)
pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_macbuf[0]));
else
pb->pb_next = 0;
} else
pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_dbuf[j]));
j++;
}
}
dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_ctx));
if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
struct ubsec_pktctx_long *ctxl;
ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
offsetof(struct ubsec_dmachunk, d_ctx));
/* transform small context into long context */
ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
ctxl->pc_flags = ctx.pc_flags;
ctxl->pc_offset = ctx.pc_offset;
for (i = 0; i < 6; i++)
ctxl->pc_deskey[i] = ctx.pc_deskey[i];
for (i = 0; i < 5; i++)
ctxl->pc_hminner[i] = ctx.pc_hminner[i];
for (i = 0; i < 5; i++)
ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
ctxl->pc_iv[0] = ctx.pc_iv[0];
ctxl->pc_iv[1] = ctx.pc_iv[1];
} else
bcopy(&ctx, dmap->d_alloc.dma_vaddr +
offsetof(struct ubsec_dmachunk, d_ctx),
sizeof(struct ubsec_pktctx));
mtx_lock(&sc->sc_mcr1lock);
SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
sc->sc_nqueue++;
ubsecstats.hst_ipackets++;
ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
ubsec_feed(sc);
mtx_unlock(&sc->sc_mcr1lock);
return (0);
errout:
if (q != NULL) {
if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
m_freem(q->q_dst_m);
if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
}
if (q->q_src_map != NULL) {
bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
}
}
if (q != NULL || err == ERESTART) {
mtx_lock(&sc->sc_freeqlock);
if (q != NULL)
SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
if (err == ERESTART)
sc->sc_needwakeup |= CRYPTO_SYMQ;
mtx_unlock(&sc->sc_freeqlock);
}
if (err != ERESTART) {
crp->crp_etype = err;
crypto_done(crp);
}
return (err);
}
static void
ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
{
struct cryptop *crp = (struct cryptop *)q->q_crp;
struct cryptodesc *crd;
struct ubsec_dma *dmap = q->q_dma;
ubsecstats.hst_opackets++;
ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
ubsec_dma_sync(&dmap->d_alloc,
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
}
bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
m_freem(q->q_src_m);
crp->crp_buf = (caddr_t)q->q_dst_m;
}
/* copy out IV for future use */
if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
if (crd->crd_alg != CRYPTO_DES_CBC &&
crd->crd_alg != CRYPTO_3DES_CBC)
continue;
crypto_copydata(crp->crp_flags, crp->crp_buf,
crd->crd_skip + crd->crd_len - 8, 8,
(caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
break;
}
}
for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
if (crd->crd_alg != CRYPTO_MD5_HMAC &&
crd->crd_alg != CRYPTO_SHA1_HMAC)
continue;
crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
sc->sc_sessions[q->q_sesn].ses_mlen,
(caddr_t)dmap->d_dma->d_macbuf);
break;
}
mtx_lock(&sc->sc_freeqlock);
SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
mtx_unlock(&sc->sc_freeqlock);
crypto_done(crp);
}
static void
ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
{
int i, j, dlen, slen;
caddr_t dptr, sptr;
j = 0;
sptr = srcm->m_data;
slen = srcm->m_len;
dptr = dstm->m_data;
dlen = dstm->m_len;
while (1) {
for (i = 0; i < min(slen, dlen); i++) {
if (j < hoffset || j >= toffset)
*dptr++ = *sptr++;
slen--;
dlen--;
j++;
}
if (slen == 0) {
srcm = srcm->m_next;
if (srcm == NULL)
return;
sptr = srcm->m_data;
slen = srcm->m_len;
}
if (dlen == 0) {
dstm = dstm->m_next;
if (dstm == NULL)
return;
dptr = dstm->m_data;
dlen = dstm->m_len;
}
}
}
/*
* feed the key generator, must be called at splimp() or higher.
*/
static int
ubsec_feed2(struct ubsec_softc *sc)
{
struct ubsec_q2 *q;
while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
break;
q = SIMPLEQ_FIRST(&sc->sc_queue2);
ubsec_dma_sync(&q->q_mcr,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next);
--sc->sc_nqueue2;
SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
}
return (0);
}
/*
* Callback for handling random numbers
*/
static void
ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
{
struct cryptkop *krp;
struct ubsec_ctx_keyop *ctx;
ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
switch (q->q_type) {
#ifndef UBSEC_NO_RNG
case UBS_CTXOP_RNGBYPASS: {
struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
(*sc->sc_harvest)(sc->sc_rndtest,
rng->rng_buf.dma_vaddr,
UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
rng->rng_used = 0;
callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
break;
}
#endif
case UBS_CTXOP_MODEXP: {
struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
u_int rlen, clen;
krp = me->me_krp;
rlen = (me->me_modbits + 7) / 8;
clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
if (clen < rlen)
krp->krp_status = E2BIG;
else {
if (sc->sc_flags & UBS_FLAGS_HWNORM) {
bzero(krp->krp_param[krp->krp_iparams].crp_p,
(krp->krp_param[krp->krp_iparams].crp_nbits
+ 7) / 8);
bcopy(me->me_C.dma_vaddr,
krp->krp_param[krp->krp_iparams].crp_p,
(me->me_modbits + 7) / 8);
} else
ubsec_kshift_l(me->me_shiftbits,
me->me_C.dma_vaddr, me->me_normbits,
krp->krp_param[krp->krp_iparams].crp_p,
krp->krp_param[krp->krp_iparams].crp_nbits);
}
crypto_kdone(krp);
/* bzero all potentially sensitive data */
bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
/* Can't free here, so put us on the free list. */
SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
break;
}
case UBS_CTXOP_RSAPRIV: {
struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
u_int len;
krp = rp->rpr_krp;
ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
bcopy(rp->rpr_msgout.dma_vaddr,
krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
crypto_kdone(krp);
bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
/* Can't free here, so put us on the free list. */
SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
break;
}
default:
device_printf(sc->sc_dev, "unknown ctx op: %x\n",
letoh16(ctx->ctx_op));
break;
}
}
#ifndef UBSEC_NO_RNG
static void
ubsec_rng(void *vsc)
{
struct ubsec_softc *sc = vsc;
struct ubsec_q2_rng *rng = &sc->sc_rng;
struct ubsec_mcr *mcr;
struct ubsec_ctx_rngbypass *ctx;
mtx_lock(&sc->sc_mcr2lock);
if (rng->rng_used) {
mtx_unlock(&sc->sc_mcr2lock);
return;
}
sc->sc_nqueue2++;
if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
goto out;
mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
mcr->mcr_pkts = htole16(1);
mcr->mcr_flags = 0;
mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
mcr->mcr_ipktbuf.pb_len = 0;
mcr->mcr_reserved = mcr->mcr_pktlen = 0;
mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
UBS_PKTBUF_LEN);
mcr->mcr_opktbuf.pb_next = 0;
ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
rng->rng_used = 1;
ubsec_feed2(sc);
ubsecstats.hst_rng++;
mtx_unlock(&sc->sc_mcr2lock);
return;
out:
/*
* Something weird happened, generate our own call back.
*/
sc->sc_nqueue2--;
mtx_unlock(&sc->sc_mcr2lock);
callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
}
#endif /* UBSEC_NO_RNG */
static void
ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
bus_addr_t *paddr = (bus_addr_t*) arg;
*paddr = segs->ds_addr;
}
static int
ubsec_dma_malloc(
struct ubsec_softc *sc,
bus_size_t size,
struct ubsec_dma_alloc *dma,
int mapflags
)
{
int r;
/* XXX could specify sc_dmat as parent but that just adds overhead */
r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
1, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
size, /* maxsize */
1, /* nsegments */
size, /* maxsegsize */
BUS_DMA_ALLOCNOW, /* flags */
NULL, NULL, /* lockfunc, lockarg */
&dma->dma_tag);
if (r != 0) {
device_printf(sc->sc_dev, "ubsec_dma_malloc: "
"bus_dma_tag_create failed; error %u\n", r);
goto fail_1;
}
r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
BUS_DMA_NOWAIT, &dma->dma_map);
if (r != 0) {
device_printf(sc->sc_dev, "ubsec_dma_malloc: "
"bus_dmammem_alloc failed; size %ju, error %u\n",
(intmax_t)size, r);
goto fail_2;
}
r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
size,
ubsec_dmamap_cb,
&dma->dma_paddr,
mapflags | BUS_DMA_NOWAIT);
if (r != 0) {
device_printf(sc->sc_dev, "ubsec_dma_malloc: "
"bus_dmamap_load failed; error %u\n", r);
goto fail_3;
}
dma->dma_size = size;
return (0);
fail_3:
bus_dmamap_unload(dma->dma_tag, dma->dma_map);
fail_2:
bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
fail_1:
bus_dma_tag_destroy(dma->dma_tag);
dma->dma_tag = NULL;
return (r);
}
static void
ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
{
bus_dmamap_unload(dma->dma_tag, dma->dma_map);
bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
bus_dma_tag_destroy(dma->dma_tag);
}
/*
* Resets the board. Values in the regesters are left as is
* from the reset (i.e. initial values are assigned elsewhere).
*/
static void
ubsec_reset_board(struct ubsec_softc *sc)
{
volatile u_int32_t ctrl;
ctrl = READ_REG(sc, BS_CTRL);
ctrl |= BS_CTRL_RESET;
WRITE_REG(sc, BS_CTRL, ctrl);
/*
* Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
*/
DELAY(10);
}
/*
* Init Broadcom registers
*/
static void
ubsec_init_board(struct ubsec_softc *sc)
{
u_int32_t ctrl;
ctrl = READ_REG(sc, BS_CTRL);
ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
ctrl |= BS_CTRL_MCR2INT;
else
ctrl &= ~BS_CTRL_MCR2INT;
if (sc->sc_flags & UBS_FLAGS_HWNORM)
ctrl &= ~BS_CTRL_SWNORM;
WRITE_REG(sc, BS_CTRL, ctrl);
}
/*
* Init Broadcom PCI registers
*/
static void
ubsec_init_pciregs(device_t dev)
{
#if 0
u_int32_t misc;
misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
| ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
| ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
#endif
/*
* This will set the cache line size to 1, this will
* force the BCM58xx chip just to do burst read/writes.
* Cache line read/writes are to slow
*/
pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
}
/*
* Clean up after a chip crash.
* It is assumed that the caller in splimp()
*/
static void
ubsec_cleanchip(struct ubsec_softc *sc)
{
struct ubsec_q *q;
while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
q = SIMPLEQ_FIRST(&sc->sc_qchip);
SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
ubsec_free_q(sc, q);
}
sc->sc_nqchip = 0;
}
/*
* free a ubsec_q
* It is assumed that the caller is within splimp().
*/
static int
ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
{
struct ubsec_q *q2;
struct cryptop *crp;
int npkts;
int i;
npkts = q->q_nstacked_mcrs;
for (i = 0; i < npkts; i++) {
if(q->q_stacked_mcr[i]) {
q2 = q->q_stacked_mcr[i];
if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
m_freem(q2->q_dst_m);
crp = (struct cryptop *)q2->q_crp;
SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
crp->crp_etype = EFAULT;
crypto_done(crp);
} else {
break;
}
}
/*
* Free header MCR
*/
if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
m_freem(q->q_dst_m);
crp = (struct cryptop *)q->q_crp;
SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
crp->crp_etype = EFAULT;
crypto_done(crp);
return(0);
}
/*
* Routine to reset the chip and clean up.
* It is assumed that the caller is in splimp()
*/
static void
ubsec_totalreset(struct ubsec_softc *sc)
{
ubsec_reset_board(sc);
ubsec_init_board(sc);
ubsec_cleanchip(sc);
}
static int
ubsec_dmamap_aligned(struct ubsec_operand *op)
{
int i;
for (i = 0; i < op->nsegs; i++) {
if (op->segs[i].ds_addr & 3)
return (0);
if ((i != (op->nsegs - 1)) &&
(op->segs[i].ds_len & 3))
return (0);
}
return (1);
}
static void
ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
{
switch (q->q_type) {
case UBS_CTXOP_MODEXP: {
struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
ubsec_dma_free(sc, &me->me_q.q_mcr);
ubsec_dma_free(sc, &me->me_q.q_ctx);
ubsec_dma_free(sc, &me->me_M);
ubsec_dma_free(sc, &me->me_E);
ubsec_dma_free(sc, &me->me_C);
ubsec_dma_free(sc, &me->me_epb);
free(me, M_DEVBUF);
break;
}
case UBS_CTXOP_RSAPRIV: {
struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
ubsec_dma_free(sc, &rp->rpr_msgin);
ubsec_dma_free(sc, &rp->rpr_msgout);
free(rp, M_DEVBUF);
break;
}
default:
device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
break;
}
}
static int
ubsec_kprocess(device_t dev, struct cryptkop *krp, int hint)
{
struct ubsec_softc *sc = device_get_softc(dev);
int r;
if (krp == NULL || krp->krp_callback == NULL)
return (EINVAL);
while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
struct ubsec_q2 *q;
q = SIMPLEQ_FIRST(&sc->sc_q2free);
SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next);
ubsec_kfree(sc, q);
}
switch (krp->krp_op) {
case CRK_MOD_EXP:
if (sc->sc_flags & UBS_FLAGS_HWNORM)
r = ubsec_kprocess_modexp_hw(sc, krp, hint);
else
r = ubsec_kprocess_modexp_sw(sc, krp, hint);
break;
case CRK_MOD_EXP_CRT:
return (ubsec_kprocess_rsapriv(sc, krp, hint));
default:
device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
krp->krp_op);
krp->krp_status = EOPNOTSUPP;
crypto_kdone(krp);
return (0);
}
return (0); /* silence compiler */
}
/*
* Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
*/
static int
ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
{
struct ubsec_q2_modexp *me;
struct ubsec_mcr *mcr;
struct ubsec_ctx_modexp *ctx;
struct ubsec_pktbuf *epb;
int err = 0;
u_int nbits, normbits, mbits, shiftbits, ebits;
me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
if (me == NULL) {
err = ENOMEM;
goto errout;
}
bzero(me, sizeof *me);
me->me_krp = krp;
me->me_q.q_type = UBS_CTXOP_MODEXP;
nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
if (nbits <= 512)
normbits = 512;
else if (nbits <= 768)
normbits = 768;
else if (nbits <= 1024)
normbits = 1024;
else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
normbits = 1536;
else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
normbits = 2048;
else {
err = E2BIG;
goto errout;
}
shiftbits = normbits - nbits;
me->me_modbits = nbits;
me->me_shiftbits = shiftbits;
me->me_normbits = normbits;
/* Sanity check: result bits must be >= true modulus bits. */
if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
err = ERANGE;
goto errout;
}
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
&me->me_q.q_mcr, 0)) {
err = ENOMEM;
goto errout;
}
mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
&me->me_q.q_ctx, 0)) {
err = ENOMEM;
goto errout;
}
mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
if (mbits > nbits) {
err = E2BIG;
goto errout;
}
if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
err = ENOMEM;
goto errout;
}
ubsec_kshift_r(shiftbits,
krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
me->me_M.dma_vaddr, normbits);
if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
err = ENOMEM;
goto errout;
}
bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
if (ebits > nbits) {
err = E2BIG;
goto errout;
}
if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
err = ENOMEM;
goto errout;
}
ubsec_kshift_r(shiftbits,
krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
me->me_E.dma_vaddr, normbits);
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
&me->me_epb, 0)) {
err = ENOMEM;
goto errout;
}
epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
epb->pb_addr = htole32(me->me_E.dma_paddr);
epb->pb_next = 0;
epb->pb_len = htole32(normbits / 8);
#ifdef UBSEC_DEBUG
if (ubsec_debug) {
printf("Epb ");
ubsec_dump_pb(epb);
}
#endif
mcr->mcr_pkts = htole16(1);
mcr->mcr_flags = 0;
mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
mcr->mcr_reserved = 0;
mcr->mcr_pktlen = 0;
mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
mcr->mcr_opktbuf.pb_next = 0;
mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
#ifdef DIAGNOSTIC
/* Misaligned output buffer will hang the chip. */
if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
panic("%s: modexp invalid addr 0x%x\n",
device_get_nameunit(sc->sc_dev),
letoh32(mcr->mcr_opktbuf.pb_addr));
if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
panic("%s: modexp invalid len 0x%x\n",
device_get_nameunit(sc->sc_dev),
letoh32(mcr->mcr_opktbuf.pb_len));
#endif
ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
bzero(ctx, sizeof(*ctx));
ubsec_kshift_r(shiftbits,
krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
ctx->me_N, normbits);
ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
ctx->me_op = htole16(UBS_CTXOP_MODEXP);
ctx->me_E_len = htole16(nbits);
ctx->me_N_len = htole16(nbits);
#ifdef UBSEC_DEBUG
if (ubsec_debug) {
ubsec_dump_mcr(mcr);
ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
}
#endif
/*
* ubsec_feed2 will sync mcr and ctx, we just need to sync
* everything else.
*/
ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
/* Enqueue and we're done... */
mtx_lock(&sc->sc_mcr2lock);
SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
ubsec_feed2(sc);
ubsecstats.hst_modexp++;
mtx_unlock(&sc->sc_mcr2lock);
return (0);
errout:
if (me != NULL) {
if (me->me_q.q_mcr.dma_tag != NULL)
ubsec_dma_free(sc, &me->me_q.q_mcr);
if (me->me_q.q_ctx.dma_tag != NULL) {
bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
ubsec_dma_free(sc, &me->me_q.q_ctx);
}
if (me->me_M.dma_tag != NULL) {
bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
ubsec_dma_free(sc, &me->me_M);
}
if (me->me_E.dma_tag != NULL) {
bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
ubsec_dma_free(sc, &me->me_E);
}
if (me->me_C.dma_tag != NULL) {
bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
ubsec_dma_free(sc, &me->me_C);
}
if (me->me_epb.dma_tag != NULL)
ubsec_dma_free(sc, &me->me_epb);
free(me, M_DEVBUF);
}
krp->krp_status = err;
crypto_kdone(krp);
return (0);
}
/*
* Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
*/
static int
ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
{
struct ubsec_q2_modexp *me;
struct ubsec_mcr *mcr;
struct ubsec_ctx_modexp *ctx;
struct ubsec_pktbuf *epb;
int err = 0;
u_int nbits, normbits, mbits, shiftbits, ebits;
me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
if (me == NULL) {
err = ENOMEM;
goto errout;
}
bzero(me, sizeof *me);
me->me_krp = krp;
me->me_q.q_type = UBS_CTXOP_MODEXP;
nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
if (nbits <= 512)
normbits = 512;
else if (nbits <= 768)
normbits = 768;
else if (nbits <= 1024)
normbits = 1024;
else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
normbits = 1536;
else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
normbits = 2048;
else {
err = E2BIG;
goto errout;
}
shiftbits = normbits - nbits;
/* XXX ??? */
me->me_modbits = nbits;
me->me_shiftbits = shiftbits;
me->me_normbits = normbits;
/* Sanity check: result bits must be >= true modulus bits. */
if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
err = ERANGE;
goto errout;
}
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
&me->me_q.q_mcr, 0)) {
err = ENOMEM;
goto errout;
}
mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
&me->me_q.q_ctx, 0)) {
err = ENOMEM;
goto errout;
}
mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
if (mbits > nbits) {
err = E2BIG;
goto errout;
}
if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
err = ENOMEM;
goto errout;
}
bzero(me->me_M.dma_vaddr, normbits / 8);
bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
me->me_M.dma_vaddr, (mbits + 7) / 8);
if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
err = ENOMEM;
goto errout;
}
bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
if (ebits > nbits) {
err = E2BIG;
goto errout;
}
if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
err = ENOMEM;
goto errout;
}
bzero(me->me_E.dma_vaddr, normbits / 8);
bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
me->me_E.dma_vaddr, (ebits + 7) / 8);
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
&me->me_epb, 0)) {
err = ENOMEM;
goto errout;
}
epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
epb->pb_addr = htole32(me->me_E.dma_paddr);
epb->pb_next = 0;
epb->pb_len = htole32((ebits + 7) / 8);
#ifdef UBSEC_DEBUG
if (ubsec_debug) {
printf("Epb ");
ubsec_dump_pb(epb);
}
#endif
mcr->mcr_pkts = htole16(1);
mcr->mcr_flags = 0;
mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
mcr->mcr_reserved = 0;
mcr->mcr_pktlen = 0;
mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
mcr->mcr_opktbuf.pb_next = 0;
mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
#ifdef DIAGNOSTIC
/* Misaligned output buffer will hang the chip. */
if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
panic("%s: modexp invalid addr 0x%x\n",
device_get_nameunit(sc->sc_dev),
letoh32(mcr->mcr_opktbuf.pb_addr));
if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
panic("%s: modexp invalid len 0x%x\n",
device_get_nameunit(sc->sc_dev),
letoh32(mcr->mcr_opktbuf.pb_len));
#endif
ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
bzero(ctx, sizeof(*ctx));
bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
(nbits + 7) / 8);
ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
ctx->me_op = htole16(UBS_CTXOP_MODEXP);
ctx->me_E_len = htole16(ebits);
ctx->me_N_len = htole16(nbits);
#ifdef UBSEC_DEBUG
if (ubsec_debug) {
ubsec_dump_mcr(mcr);
ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
}
#endif
/*
* ubsec_feed2 will sync mcr and ctx, we just need to sync
* everything else.
*/
ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
/* Enqueue and we're done... */
mtx_lock(&sc->sc_mcr2lock);
SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
ubsec_feed2(sc);
mtx_unlock(&sc->sc_mcr2lock);
return (0);
errout:
if (me != NULL) {
if (me->me_q.q_mcr.dma_tag != NULL)
ubsec_dma_free(sc, &me->me_q.q_mcr);
if (me->me_q.q_ctx.dma_tag != NULL) {
bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
ubsec_dma_free(sc, &me->me_q.q_ctx);
}
if (me->me_M.dma_tag != NULL) {
bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
ubsec_dma_free(sc, &me->me_M);
}
if (me->me_E.dma_tag != NULL) {
bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
ubsec_dma_free(sc, &me->me_E);
}
if (me->me_C.dma_tag != NULL) {
bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
ubsec_dma_free(sc, &me->me_C);
}
if (me->me_epb.dma_tag != NULL)
ubsec_dma_free(sc, &me->me_epb);
free(me, M_DEVBUF);
}
krp->krp_status = err;
crypto_kdone(krp);
return (0);
}
static int
ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
{
struct ubsec_q2_rsapriv *rp = NULL;
struct ubsec_mcr *mcr;
struct ubsec_ctx_rsapriv *ctx;
int err = 0;
u_int padlen, msglen;
msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
if (msglen > padlen)
padlen = msglen;
if (padlen <= 256)
padlen = 256;
else if (padlen <= 384)
padlen = 384;
else if (padlen <= 512)
padlen = 512;
else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
padlen = 768;
else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
padlen = 1024;
else {
err = E2BIG;
goto errout;
}
if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
err = E2BIG;
goto errout;
}
if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
err = E2BIG;
goto errout;
}
if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
err = E2BIG;
goto errout;
}
rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
if (rp == NULL)
return (ENOMEM);
bzero(rp, sizeof *rp);
rp->rpr_krp = krp;
rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
&rp->rpr_q.q_mcr, 0)) {
err = ENOMEM;
goto errout;
}
mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
&rp->rpr_q.q_ctx, 0)) {
err = ENOMEM;
goto errout;
}
ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
bzero(ctx, sizeof *ctx);
/* Copy in p */
bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
&ctx->rpr_buf[0 * (padlen / 8)],
(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
/* Copy in q */
bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
&ctx->rpr_buf[1 * (padlen / 8)],
(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
/* Copy in dp */
bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
&ctx->rpr_buf[2 * (padlen / 8)],
(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
/* Copy in dq */
bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
&ctx->rpr_buf[3 * (padlen / 8)],
(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
/* Copy in pinv */
bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
&ctx->rpr_buf[4 * (padlen / 8)],
(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
msglen = padlen * 2;
/* Copy in input message (aligned buffer/length). */
if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
/* Is this likely? */
err = E2BIG;
goto errout;
}
if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
err = ENOMEM;
goto errout;
}
bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
rp->rpr_msgin.dma_vaddr,
(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
/* Prepare space for output message (aligned buffer/length). */
if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
/* Is this likely? */
err = E2BIG;
goto errout;
}
if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
err = ENOMEM;
goto errout;
}
bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
mcr->mcr_pkts = htole16(1);
mcr->mcr_flags = 0;
mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
mcr->mcr_ipktbuf.pb_next = 0;
mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
mcr->mcr_reserved = 0;
mcr->mcr_pktlen = htole16(msglen);
mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
mcr->mcr_opktbuf.pb_next = 0;
mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
#ifdef DIAGNOSTIC
if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
panic("%s: rsapriv: invalid msgin %x(0x%jx)",
device_get_nameunit(sc->sc_dev),
rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size);
}
if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
panic("%s: rsapriv: invalid msgout %x(0x%jx)",
device_get_nameunit(sc->sc_dev),
rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size);
}
#endif
ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
ctx->rpr_q_len = htole16(padlen);
ctx->rpr_p_len = htole16(padlen);
/*
* ubsec_feed2 will sync mcr and ctx, we just need to sync
* everything else.
*/
ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
/* Enqueue and we're done... */
mtx_lock(&sc->sc_mcr2lock);
SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
ubsec_feed2(sc);
ubsecstats.hst_modexpcrt++;
mtx_unlock(&sc->sc_mcr2lock);
return (0);
errout:
if (rp != NULL) {
if (rp->rpr_q.q_mcr.dma_tag != NULL)
ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
if (rp->rpr_msgin.dma_tag != NULL) {
bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
ubsec_dma_free(sc, &rp->rpr_msgin);
}
if (rp->rpr_msgout.dma_tag != NULL) {
bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
ubsec_dma_free(sc, &rp->rpr_msgout);
}
free(rp, M_DEVBUF);
}
krp->krp_status = err;
crypto_kdone(krp);
return (0);
}
#ifdef UBSEC_DEBUG
static void
ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
{
printf("addr 0x%x (0x%x) next 0x%x\n",
pb->pb_addr, pb->pb_len, pb->pb_next);
}
static void
ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
{
printf("CTX (0x%x):\n", c->ctx_len);
switch (letoh16(c->ctx_op)) {
case UBS_CTXOP_RNGBYPASS:
case UBS_CTXOP_RNGSHA1:
break;
case UBS_CTXOP_MODEXP:
{
struct ubsec_ctx_modexp *cx = (void *)c;
int i, len;
printf(" Elen %u, Nlen %u\n",
letoh16(cx->me_E_len), letoh16(cx->me_N_len));
len = (cx->me_N_len + 7)/8;
for (i = 0; i < len; i++)
printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
printf("\n");
break;
}
default:
printf("unknown context: %x\n", c->ctx_op);
}
printf("END CTX\n");
}
static void
ubsec_dump_mcr(struct ubsec_mcr *mcr)
{
volatile struct ubsec_mcr_add *ma;
int i;
printf("MCR:\n");
printf(" pkts: %u, flags 0x%x\n",
letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
letoh16(ma->mcr_reserved));
printf(" %d: ipkt ", i);
ubsec_dump_pb(&ma->mcr_ipktbuf);
printf(" %d: opkt ", i);
ubsec_dump_pb(&ma->mcr_opktbuf);
ma++;
}
printf("END MCR\n");
}
#endif /* UBSEC_DEBUG */
/*
* Return the number of significant bits of a big number.
*/
static int
ubsec_ksigbits(struct crparam *cr)
{
u_int plen = (cr->crp_nbits + 7) / 8;
int i, sig = plen * 8;
u_int8_t c, *p = cr->crp_p;
for (i = plen - 1; i >= 0; i--) {
c = p[i];
if (c != 0) {
while ((c & 0x80) == 0) {
sig--;
c <<= 1;
}
break;
}
sig -= 8;
}
return (sig);
}
static void
ubsec_kshift_r(
u_int shiftbits,
u_int8_t *src, u_int srcbits,
u_int8_t *dst, u_int dstbits)
{
u_int slen, dlen;
int i, si, di, n;
slen = (srcbits + 7) / 8;
dlen = (dstbits + 7) / 8;
for (i = 0; i < slen; i++)
dst[i] = src[i];
for (i = 0; i < dlen - slen; i++)
dst[slen + i] = 0;
n = shiftbits / 8;
if (n != 0) {
si = dlen - n - 1;
di = dlen - 1;
while (si >= 0)
dst[di--] = dst[si--];
while (di >= 0)
dst[di--] = 0;
}
n = shiftbits % 8;
if (n != 0) {
for (i = dlen - 1; i > 0; i--)
dst[i] = (dst[i] << n) |
(dst[i - 1] >> (8 - n));
dst[0] = dst[0] << n;
}
}
static void
ubsec_kshift_l(
u_int shiftbits,
u_int8_t *src, u_int srcbits,
u_int8_t *dst, u_int dstbits)
{
int slen, dlen, i, n;
slen = (srcbits + 7) / 8;
dlen = (dstbits + 7) / 8;
n = shiftbits / 8;
for (i = 0; i < slen; i++)
dst[i] = src[i + n];
for (i = 0; i < dlen - slen; i++)
dst[slen + i] = 0;
n = shiftbits % 8;
if (n != 0) {
for (i = 0; i < (dlen - 1); i++)
dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
dst[dlen - 1] = dst[dlen - 1] >> n;
}
}
Index: head/sys/dev/virtio/random/virtio_random.c
===================================================================
--- head/sys/dev/virtio/random/virtio_random.c (revision 283290)
+++ head/sys/dev/virtio/random/virtio_random.c (revision 283291)
@@ -1,231 +1,231 @@
/*-
* Copyright (c) 2013, Bryan Venteicher <bryanv@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice unmodified, this list of conditions, and the following
* disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/* Driver for VirtIO entropy device. */
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/sglist.h>
#include <sys/callout.h>
#include <sys/random.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <dev/virtio/virtio.h>
#include <dev/virtio/virtqueue.h>
struct vtrnd_softc {
device_t vtrnd_dev;
uint64_t vtrnd_features;
struct callout vtrnd_callout;
struct virtqueue *vtrnd_vq;
};
static int vtrnd_modevent(module_t, int, void *);
static int vtrnd_probe(device_t);
static int vtrnd_attach(device_t);
static int vtrnd_detach(device_t);
static void vtrnd_negotiate_features(struct vtrnd_softc *);
static int vtrnd_alloc_virtqueue(struct vtrnd_softc *);
static void vtrnd_harvest(struct vtrnd_softc *);
static void vtrnd_timer(void *);
#define VTRND_FEATURES 0
static struct virtio_feature_desc vtrnd_feature_desc[] = {
{ 0, NULL }
};
static device_method_t vtrnd_methods[] = {
/* Device methods. */
DEVMETHOD(device_probe, vtrnd_probe),
DEVMETHOD(device_attach, vtrnd_attach),
DEVMETHOD(device_detach, vtrnd_detach),
DEVMETHOD_END
};
static driver_t vtrnd_driver = {
"vtrnd",
vtrnd_methods,
sizeof(struct vtrnd_softc)
};
static devclass_t vtrnd_devclass;
DRIVER_MODULE(virtio_random, virtio_pci, vtrnd_driver, vtrnd_devclass,
vtrnd_modevent, 0);
MODULE_VERSION(virtio_random, 1);
MODULE_DEPEND(virtio_random, virtio, 1, 1, 1);
static int
vtrnd_modevent(module_t mod, int type, void *unused)
{
int error;
switch (type) {
case MOD_LOAD:
case MOD_QUIESCE:
case MOD_UNLOAD:
case MOD_SHUTDOWN:
error = 0;
break;
default:
error = EOPNOTSUPP;
break;
}
return (error);
}
static int
vtrnd_probe(device_t dev)
{
if (virtio_get_device_type(dev) != VIRTIO_ID_ENTROPY)
return (ENXIO);
device_set_desc(dev, "VirtIO Entropy Adapter");
return (BUS_PROBE_DEFAULT);
}
static int
vtrnd_attach(device_t dev)
{
struct vtrnd_softc *sc;
int error;
sc = device_get_softc(dev);
sc->vtrnd_dev = dev;
- callout_init(&sc->vtrnd_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->vtrnd_callout, 1);
virtio_set_feature_desc(dev, vtrnd_feature_desc);
vtrnd_negotiate_features(sc);
error = vtrnd_alloc_virtqueue(sc);
if (error) {
device_printf(dev, "cannot allocate virtqueue\n");
goto fail;
}
callout_reset(&sc->vtrnd_callout, 5 * hz, vtrnd_timer, sc);
fail:
if (error)
vtrnd_detach(dev);
return (error);
}
static int
vtrnd_detach(device_t dev)
{
struct vtrnd_softc *sc;
sc = device_get_softc(dev);
callout_drain(&sc->vtrnd_callout);
return (0);
}
static void
vtrnd_negotiate_features(struct vtrnd_softc *sc)
{
device_t dev;
uint64_t features;
dev = sc->vtrnd_dev;
features = VTRND_FEATURES;
sc->vtrnd_features = virtio_negotiate_features(dev, features);
}
static int
vtrnd_alloc_virtqueue(struct vtrnd_softc *sc)
{
device_t dev;
struct vq_alloc_info vq_info;
dev = sc->vtrnd_dev;
VQ_ALLOC_INFO_INIT(&vq_info, 0, NULL, sc, &sc->vtrnd_vq,
"%s request", device_get_nameunit(dev));
return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info));
}
static void
vtrnd_harvest(struct vtrnd_softc *sc)
{
struct sglist_seg segs[1];
struct sglist sg;
struct virtqueue *vq;
uint32_t value;
int error;
vq = sc->vtrnd_vq;
sglist_init(&sg, 1, segs);
error = sglist_append(&sg, &value, sizeof(value));
KASSERT(error == 0 && sg.sg_nseg == 1,
("%s: error %d adding buffer to sglist", __func__, error));
if (!virtqueue_empty(vq))
return;
if (virtqueue_enqueue(vq, &value, &sg, 0, 1) != 0)
return;
/*
* Poll for the response, but the command is likely already
* done when we return from the notify.
*/
virtqueue_notify(vq);
virtqueue_poll(vq, NULL);
random_harvest(&value, sizeof(value), sizeof(value) * NBBY / 2,
RANDOM_PURE_VIRTIO);
}
static void
vtrnd_timer(void *xsc)
{
struct vtrnd_softc *sc;
sc = xsc;
vtrnd_harvest(sc);
callout_schedule(&sc->vtrnd_callout, 5 * hz);
}
Index: head/sys/dev/watchdog/watchdog.c
===================================================================
--- head/sys/dev/watchdog/watchdog.c (revision 283290)
+++ head/sys/dev/watchdog/watchdog.c (revision 283291)
@@ -1,411 +1,411 @@
/*-
* Copyright (c) 2004 Poul-Henning Kamp
* Copyright (c) 2013 iXsystems.com,
* author: Alfred Perlstein <alfred@freebsd.org>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "opt_ddb.h"
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/conf.h>
#include <sys/uio.h>
#include <sys/kernel.h>
#include <sys/kdb.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/watchdog.h>
#include <sys/bus.h>
#include <machine/bus.h>
#include <sys/syscallsubr.h> /* kern_clock_gettime() */
static int wd_set_pretimeout(int newtimeout, int disableiftoolong);
static void wd_timeout_cb(void *arg);
static struct callout wd_pretimeo_handle;
static int wd_pretimeout;
static int wd_pretimeout_act = WD_SOFT_LOG;
static struct callout wd_softtimeo_handle;
static int wd_softtimer; /* true = use softtimer instead of hardware
watchdog */
static int wd_softtimeout_act = WD_SOFT_LOG; /* action for the software timeout */
static struct cdev *wd_dev;
static volatile u_int wd_last_u; /* last timeout value set by kern_do_pat */
static u_int wd_last_u_sysctl; /* last timeout value set by kern_do_pat */
static u_int wd_last_u_sysctl_secs; /* wd_last_u in seconds */
SYSCTL_NODE(_hw, OID_AUTO, watchdog, CTLFLAG_RD, 0, "Main watchdog device");
SYSCTL_UINT(_hw_watchdog, OID_AUTO, wd_last_u, CTLFLAG_RD,
&wd_last_u_sysctl, 0, "Watchdog last update time");
SYSCTL_UINT(_hw_watchdog, OID_AUTO, wd_last_u_secs, CTLFLAG_RD,
&wd_last_u_sysctl_secs, 0, "Watchdog last update time");
static int wd_lastpat_valid = 0;
static time_t wd_lastpat = 0; /* when the watchdog was last patted */
static void
pow2ns_to_ts(int pow2ns, struct timespec *ts)
{
uint64_t ns;
ns = 1ULL << pow2ns;
ts->tv_sec = ns / 1000000000ULL;
ts->tv_nsec = ns % 1000000000ULL;
}
static int
pow2ns_to_ticks(int pow2ns)
{
struct timeval tv;
struct timespec ts;
pow2ns_to_ts(pow2ns, &ts);
TIMESPEC_TO_TIMEVAL(&tv, &ts);
return (tvtohz(&tv));
}
static int
seconds_to_pow2ns(int seconds)
{
uint64_t power;
uint64_t ns;
uint64_t shifted;
ns = ((uint64_t)seconds) * 1000000000ULL;
power = flsll(ns);
shifted = 1ULL << power;
if (shifted <= ns) {
power++;
}
return (power);
}
int
wdog_kern_pat(u_int utim)
{
int error;
if ((utim & WD_LASTVAL) != 0 && (utim & WD_INTERVAL) > 0)
return (EINVAL);
if ((utim & WD_LASTVAL) != 0) {
/*
* if WD_LASTVAL is set, fill in the bits for timeout
* from the saved value in wd_last_u.
*/
MPASS((wd_last_u & ~WD_INTERVAL) == 0);
utim &= ~WD_LASTVAL;
utim |= wd_last_u;
} else {
/*
* Otherwise save the new interval.
* This can be zero (to disable the watchdog)
*/
wd_last_u = (utim & WD_INTERVAL);
wd_last_u_sysctl = wd_last_u;
wd_last_u_sysctl_secs = pow2ns_to_ticks(wd_last_u) / hz;
}
if ((utim & WD_INTERVAL) == WD_TO_NEVER) {
utim = 0;
/* Assume all is well; watchdog signals failure. */
error = 0;
} else {
/* Assume no watchdog available; watchdog flags success */
error = EOPNOTSUPP;
}
if (wd_softtimer) {
if (utim == 0) {
callout_stop(&wd_softtimeo_handle);
} else {
(void) callout_reset(&wd_softtimeo_handle,
pow2ns_to_ticks(utim), wd_timeout_cb, "soft");
}
error = 0;
} else {
EVENTHANDLER_INVOKE(watchdog_list, utim, &error);
}
wd_set_pretimeout(wd_pretimeout, true);
/*
* If we were able to arm/strobe the watchdog, then
* update the last time it was strobed for WDIOC_GETTIMELEFT
*/
if (!error) {
struct timespec ts;
error = kern_clock_gettime(curthread /* XXX */,
CLOCK_MONOTONIC_FAST, &ts);
if (!error) {
wd_lastpat = ts.tv_sec;
wd_lastpat_valid = 1;
}
}
return (error);
}
static int
wd_valid_act(int act)
{
if ((act & ~(WD_SOFT_MASK)) != 0)
return false;
return true;
}
static int
wd_ioctl_patpat(caddr_t data)
{
u_int u;
u = *(u_int *)data;
if (u & ~(WD_ACTIVE | WD_PASSIVE | WD_LASTVAL | WD_INTERVAL))
return (EINVAL);
if ((u & (WD_ACTIVE | WD_PASSIVE)) == (WD_ACTIVE | WD_PASSIVE))
return (EINVAL);
if ((u & (WD_ACTIVE | WD_PASSIVE)) == 0 && ((u & WD_INTERVAL) > 0 ||
(u & WD_LASTVAL) != 0))
return (EINVAL);
if (u & WD_PASSIVE)
return (ENOSYS); /* XXX Not implemented yet */
u &= ~(WD_ACTIVE | WD_PASSIVE);
return (wdog_kern_pat(u));
}
static int
wd_get_time_left(struct thread *td, time_t *remainp)
{
struct timespec ts;
int error;
error = kern_clock_gettime(td, CLOCK_MONOTONIC_FAST, &ts);
if (error)
return (error);
if (!wd_lastpat_valid)
return (ENOENT);
*remainp = ts.tv_sec - wd_lastpat;
return (0);
}
static void
wd_timeout_cb(void *arg)
{
const char *type = arg;
#ifdef DDB
if ((wd_pretimeout_act & WD_SOFT_DDB)) {
char kdb_why[80];
snprintf(kdb_why, sizeof(kdb_why), "watchdog %s timeout", type);
kdb_backtrace();
kdb_enter(KDB_WHY_WATCHDOG, kdb_why);
}
#endif
if ((wd_pretimeout_act & WD_SOFT_LOG))
log(LOG_EMERG, "watchdog %s-timeout, WD_SOFT_LOG", type);
if ((wd_pretimeout_act & WD_SOFT_PRINTF))
printf("watchdog %s-timeout, WD_SOFT_PRINTF\n", type);
if ((wd_pretimeout_act & WD_SOFT_PANIC))
panic("watchdog %s-timeout, WD_SOFT_PANIC set", type);
}
/*
* Called to manage timeouts.
* newtimeout needs to be in the range of 0 to actual watchdog timeout.
* if 0, we disable the pre-timeout.
* otherwise we set the pre-timeout provided it's not greater than the
* current actual watchdog timeout.
*/
static int
wd_set_pretimeout(int newtimeout, int disableiftoolong)
{
u_int utime;
struct timespec utime_ts;
int timeout_ticks;
utime = wdog_kern_last_timeout();
pow2ns_to_ts(utime, &utime_ts);
/* do not permit a pre-timeout >= than the timeout. */
if (newtimeout >= utime_ts.tv_sec) {
/*
* If 'disableiftoolong' then just fall through
* so as to disable the pre-watchdog
*/
if (disableiftoolong)
newtimeout = 0;
else
return EINVAL;
}
/* disable the pre-timeout */
if (newtimeout == 0) {
wd_pretimeout = 0;
callout_stop(&wd_pretimeo_handle);
return 0;
}
timeout_ticks = pow2ns_to_ticks(utime) - (hz*newtimeout);
#if 0
printf("wd_set_pretimeout: "
"newtimeout: %d, "
"utime: %d -> utime_ticks: %d, "
"hz*newtimeout: %d, "
"timeout_ticks: %d -> sec: %d\n",
newtimeout,
utime, pow2ns_to_ticks(utime),
hz*newtimeout,
timeout_ticks, timeout_ticks / hz);
#endif
/* We determined the value is sane, so reset the callout */
(void) callout_reset(&wd_pretimeo_handle,
timeout_ticks,
wd_timeout_cb, "pre-timeout");
wd_pretimeout = newtimeout;
return 0;
}
static int
wd_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t data,
int flags __unused, struct thread *td)
{
u_int u;
time_t timeleft;
int error;
error = 0;
switch (cmd) {
case WDIOC_SETSOFT:
u = *(int *)data;
/* do nothing? */
if (u == wd_softtimer)
break;
/* If there is a pending timeout disallow this ioctl */
if (wd_last_u != 0) {
error = EINVAL;
break;
}
wd_softtimer = u;
break;
case WDIOC_SETSOFTTIMEOUTACT:
u = *(int *)data;
if (wd_valid_act(u)) {
wd_softtimeout_act = u;
} else {
error = EINVAL;
}
break;
case WDIOC_SETPRETIMEOUTACT:
u = *(int *)data;
if (wd_valid_act(u)) {
wd_pretimeout_act = u;
} else {
error = EINVAL;
}
break;
case WDIOC_GETPRETIMEOUT:
*(int *)data = (int)wd_pretimeout;
break;
case WDIOC_SETPRETIMEOUT:
error = wd_set_pretimeout(*(int *)data, false);
break;
case WDIOC_GETTIMELEFT:
error = wd_get_time_left(td, &timeleft);
if (error)
break;
*(int *)data = (int)timeleft;
break;
case WDIOC_SETTIMEOUT:
u = *(u_int *)data;
error = wdog_kern_pat(seconds_to_pow2ns(u));
break;
case WDIOC_GETTIMEOUT:
u = wdog_kern_last_timeout();
*(u_int *)data = u;
break;
case WDIOCPATPAT:
error = wd_ioctl_patpat(data);
break;
default:
error = ENOIOCTL;
break;
}
return (error);
}
/*
* Return the last timeout set, this is NOT the seconds from NOW until timeout,
* rather it is the amount of seconds passed to WDIOCPATPAT/WDIOC_SETTIMEOUT.
*/
u_int
wdog_kern_last_timeout(void)
{
return (wd_last_u);
}
static struct cdevsw wd_cdevsw = {
.d_version = D_VERSION,
.d_ioctl = wd_ioctl,
.d_name = "watchdog",
};
static int
watchdog_modevent(module_t mod __unused, int type, void *data __unused)
{
switch(type) {
case MOD_LOAD:
- callout_init(&wd_pretimeo_handle, true);
- callout_init(&wd_softtimeo_handle, true);
+ callout_init(&wd_pretimeo_handle, 1);
+ callout_init(&wd_softtimeo_handle, 1);
wd_dev = make_dev(&wd_cdevsw, 0,
UID_ROOT, GID_WHEEL, 0600, _PATH_WATCHDOG);
return 0;
case MOD_UNLOAD:
callout_stop(&wd_pretimeo_handle);
callout_stop(&wd_softtimeo_handle);
callout_drain(&wd_pretimeo_handle);
callout_drain(&wd_softtimeo_handle);
destroy_dev(wd_dev);
return 0;
case MOD_SHUTDOWN:
return 0;
default:
return EOPNOTSUPP;
}
}
DEV_MODULE(watchdog, watchdog_modevent, NULL);
Index: head/sys/dev/xen/netfront/netfront.c
===================================================================
--- head/sys/dev/xen/netfront/netfront.c (revision 283290)
+++ head/sys/dev/xen/netfront/netfront.c (revision 283291)
@@ -1,2220 +1,2220 @@
/*-
* Copyright (c) 2004-2006 Kip Macy
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sockio.h>
#include <sys/limits.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <sys/queue.h>
#include <sys/lock.h>
#include <sys/sx.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_arp.h>
#include <net/ethernet.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/bpf.h>
#include <net/if_types.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/if_ether.h>
#if __FreeBSD_version >= 700000
#include <netinet/tcp.h>
#include <netinet/tcp_lro.h>
#endif
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/clock.h> /* for DELAY */
#include <machine/bus.h>
#include <machine/resource.h>
#include <machine/frame.h>
#include <machine/vmparam.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <machine/intr_machdep.h>
#include <xen/xen-os.h>
#include <xen/hypervisor.h>
#include <xen/xen_intr.h>
#include <xen/gnttab.h>
#include <xen/interface/memory.h>
#include <xen/interface/io/netif.h>
#include <xen/xenbus/xenbusvar.h>
#include <machine/xen/xenvar.h>
#include "xenbus_if.h"
/* Features supported by all backends. TSO and LRO can be negotiated */
#define XN_CSUM_FEATURES (CSUM_TCP | CSUM_UDP)
#define NET_TX_RING_SIZE __RING_SIZE((netif_tx_sring_t *)0, PAGE_SIZE)
#define NET_RX_RING_SIZE __RING_SIZE((netif_rx_sring_t *)0, PAGE_SIZE)
#if __FreeBSD_version >= 700000
/*
* Should the driver do LRO on the RX end
* this can be toggled on the fly, but the
* interface must be reset (down/up) for it
* to take effect.
*/
static int xn_enable_lro = 1;
TUNABLE_INT("hw.xn.enable_lro", &xn_enable_lro);
#else
#define IFCAP_TSO4 0
#define CSUM_TSO 0
#endif
#ifdef CONFIG_XEN
static int MODPARM_rx_copy = 0;
module_param_named(rx_copy, MODPARM_rx_copy, bool, 0);
MODULE_PARM_DESC(rx_copy, "Copy packets from network card (rather than flip)");
static int MODPARM_rx_flip = 0;
module_param_named(rx_flip, MODPARM_rx_flip, bool, 0);
MODULE_PARM_DESC(rx_flip, "Flip packets from network card (rather than copy)");
#else
static const int MODPARM_rx_copy = 1;
static const int MODPARM_rx_flip = 0;
#endif
/**
* \brief The maximum allowed data fragments in a single transmit
* request.
*
* This limit is imposed by the backend driver. We assume here that
* we are dealing with a Linux driver domain and have set our limit
* to mirror the Linux MAX_SKB_FRAGS constant.
*/
#define MAX_TX_REQ_FRAGS (65536 / PAGE_SIZE + 2)
#define RX_COPY_THRESHOLD 256
#define net_ratelimit() 0
struct netfront_info;
struct netfront_rx_info;
static void xn_txeof(struct netfront_info *);
static void xn_rxeof(struct netfront_info *);
static void network_alloc_rx_buffers(struct netfront_info *);
static void xn_tick_locked(struct netfront_info *);
static void xn_tick(void *);
static void xn_intr(void *);
static inline int xn_count_frags(struct mbuf *m);
static int xn_assemble_tx_request(struct netfront_info *sc,
struct mbuf *m_head);
static void xn_start_locked(struct ifnet *);
static void xn_start(struct ifnet *);
static int xn_ioctl(struct ifnet *, u_long, caddr_t);
static void xn_ifinit_locked(struct netfront_info *);
static void xn_ifinit(void *);
static void xn_stop(struct netfront_info *);
static void xn_query_features(struct netfront_info *np);
static int xn_configure_features(struct netfront_info *np);
#ifdef notyet
static void xn_watchdog(struct ifnet *);
#endif
#ifdef notyet
static void netfront_closing(device_t dev);
#endif
static void netif_free(struct netfront_info *info);
static int netfront_detach(device_t dev);
static int talk_to_backend(device_t dev, struct netfront_info *info);
static int create_netdev(device_t dev);
static void netif_disconnect_backend(struct netfront_info *info);
static int setup_device(device_t dev, struct netfront_info *info);
static void free_ring(int *ref, void *ring_ptr_ref);
static int xn_ifmedia_upd(struct ifnet *ifp);
static void xn_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr);
/* Xenolinux helper functions */
int network_connect(struct netfront_info *);
static void xn_free_rx_ring(struct netfront_info *);
static void xn_free_tx_ring(struct netfront_info *);
static int xennet_get_responses(struct netfront_info *np,
struct netfront_rx_info *rinfo, RING_IDX rp, RING_IDX *cons,
struct mbuf **list, int *pages_flipped_p);
#define virt_to_mfn(x) (vtomach(x) >> PAGE_SHIFT)
#define INVALID_P2M_ENTRY (~0UL)
/*
* Mbuf pointers. We need these to keep track of the virtual addresses
* of our mbuf chains since we can only convert from virtual to physical,
* not the other way around. The size must track the free index arrays.
*/
struct xn_chain_data {
struct mbuf *xn_tx_chain[NET_TX_RING_SIZE+1];
int xn_tx_chain_cnt;
struct mbuf *xn_rx_chain[NET_RX_RING_SIZE+1];
};
struct net_device_stats
{
u_long rx_packets; /* total packets received */
u_long tx_packets; /* total packets transmitted */
u_long rx_bytes; /* total bytes received */
u_long tx_bytes; /* total bytes transmitted */
u_long rx_errors; /* bad packets received */
u_long tx_errors; /* packet transmit problems */
u_long rx_dropped; /* no space in linux buffers */
u_long tx_dropped; /* no space available in linux */
u_long multicast; /* multicast packets received */
u_long collisions;
/* detailed rx_errors: */
u_long rx_length_errors;
u_long rx_over_errors; /* receiver ring buff overflow */
u_long rx_crc_errors; /* recved pkt with crc error */
u_long rx_frame_errors; /* recv'd frame alignment error */
u_long rx_fifo_errors; /* recv'r fifo overrun */
u_long rx_missed_errors; /* receiver missed packet */
/* detailed tx_errors */
u_long tx_aborted_errors;
u_long tx_carrier_errors;
u_long tx_fifo_errors;
u_long tx_heartbeat_errors;
u_long tx_window_errors;
/* for cslip etc */
u_long rx_compressed;
u_long tx_compressed;
};
struct netfront_info {
struct ifnet *xn_ifp;
#if __FreeBSD_version >= 700000
struct lro_ctrl xn_lro;
#endif
struct net_device_stats stats;
u_int tx_full;
netif_tx_front_ring_t tx;
netif_rx_front_ring_t rx;
struct mtx tx_lock;
struct mtx rx_lock;
struct mtx sc_lock;
xen_intr_handle_t xen_intr_handle;
u_int copying_receiver;
u_int carrier;
u_int maxfrags;
/* Receive-ring batched refills. */
#define RX_MIN_TARGET 32
#define RX_MAX_TARGET NET_RX_RING_SIZE
int rx_min_target;
int rx_max_target;
int rx_target;
grant_ref_t gref_tx_head;
grant_ref_t grant_tx_ref[NET_TX_RING_SIZE + 1];
grant_ref_t gref_rx_head;
grant_ref_t grant_rx_ref[NET_TX_RING_SIZE + 1];
device_t xbdev;
int tx_ring_ref;
int rx_ring_ref;
uint8_t mac[ETHER_ADDR_LEN];
struct xn_chain_data xn_cdata; /* mbufs */
struct mbufq xn_rx_batch; /* batch queue */
int xn_if_flags;
struct callout xn_stat_ch;
u_long rx_pfn_array[NET_RX_RING_SIZE];
multicall_entry_t rx_mcl[NET_RX_RING_SIZE+1];
mmu_update_t rx_mmu[NET_RX_RING_SIZE];
struct ifmedia sc_media;
};
#define rx_mbufs xn_cdata.xn_rx_chain
#define tx_mbufs xn_cdata.xn_tx_chain
#define XN_LOCK_INIT(_sc, _name) \
mtx_init(&(_sc)->tx_lock, #_name"_tx", "network transmit lock", MTX_DEF); \
mtx_init(&(_sc)->rx_lock, #_name"_rx", "network receive lock", MTX_DEF); \
mtx_init(&(_sc)->sc_lock, #_name"_sc", "netfront softc lock", MTX_DEF)
#define XN_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_lock)
#define XN_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->rx_lock)
#define XN_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_lock)
#define XN_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_lock)
#define XN_LOCK(_sc) mtx_lock(&(_sc)->sc_lock);
#define XN_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_lock);
#define XN_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_lock, MA_OWNED);
#define XN_RX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->rx_lock, MA_OWNED);
#define XN_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_lock, MA_OWNED);
#define XN_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rx_lock); \
mtx_destroy(&(_sc)->tx_lock); \
mtx_destroy(&(_sc)->sc_lock);
struct netfront_rx_info {
struct netif_rx_response rx;
struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
};
#define netfront_carrier_on(netif) ((netif)->carrier = 1)
#define netfront_carrier_off(netif) ((netif)->carrier = 0)
#define netfront_carrier_ok(netif) ((netif)->carrier)
/* Access macros for acquiring freeing slots in xn_free_{tx,rx}_idxs[]. */
static inline void
add_id_to_freelist(struct mbuf **list, uintptr_t id)
{
KASSERT(id != 0,
("%s: the head item (0) must always be free.", __func__));
list[id] = list[0];
list[0] = (struct mbuf *)id;
}
static inline unsigned short
get_id_from_freelist(struct mbuf **list)
{
uintptr_t id;
id = (uintptr_t)list[0];
KASSERT(id != 0,
("%s: the head item (0) must always remain free.", __func__));
list[0] = list[id];
return (id);
}
static inline int
xennet_rxidx(RING_IDX idx)
{
return idx & (NET_RX_RING_SIZE - 1);
}
static inline struct mbuf *
xennet_get_rx_mbuf(struct netfront_info *np, RING_IDX ri)
{
int i = xennet_rxidx(ri);
struct mbuf *m;
m = np->rx_mbufs[i];
np->rx_mbufs[i] = NULL;
return (m);
}
static inline grant_ref_t
xennet_get_rx_ref(struct netfront_info *np, RING_IDX ri)
{
int i = xennet_rxidx(ri);
grant_ref_t ref = np->grant_rx_ref[i];
KASSERT(ref != GRANT_REF_INVALID, ("Invalid grant reference!\n"));
np->grant_rx_ref[i] = GRANT_REF_INVALID;
return ref;
}
#define IPRINTK(fmt, args...) \
printf("[XEN] " fmt, ##args)
#ifdef INVARIANTS
#define WPRINTK(fmt, args...) \
printf("[XEN] " fmt, ##args)
#else
#define WPRINTK(fmt, args...)
#endif
#ifdef DEBUG
#define DPRINTK(fmt, args...) \
printf("[XEN] %s: " fmt, __func__, ##args)
#else
#define DPRINTK(fmt, args...)
#endif
/**
* Read the 'mac' node at the given device's node in the store, and parse that
* as colon-separated octets, placing result the given mac array. mac must be
* a preallocated array of length ETH_ALEN (as declared in linux/if_ether.h).
* Return 0 on success, or errno on error.
*/
static int
xen_net_read_mac(device_t dev, uint8_t mac[])
{
int error, i;
char *s, *e, *macstr;
const char *path;
path = xenbus_get_node(dev);
error = xs_read(XST_NIL, path, "mac", NULL, (void **) &macstr);
if (error == ENOENT) {
/*
* Deal with missing mac XenStore nodes on devices with
* HVM emulation (the 'ioemu' configuration attribute)
* enabled.
*
* The HVM emulator may execute in a stub device model
* domain which lacks the permission, only given to Dom0,
* to update the guest's XenStore tree. For this reason,
* the HVM emulator doesn't even attempt to write the
* front-side mac node, even when operating in Dom0.
* However, there should always be a mac listed in the
* backend tree. Fallback to this version if our query
* of the front side XenStore location doesn't find
* anything.
*/
path = xenbus_get_otherend_path(dev);
error = xs_read(XST_NIL, path, "mac", NULL, (void **) &macstr);
}
if (error != 0) {
xenbus_dev_fatal(dev, error, "parsing %s/mac", path);
return (error);
}
s = macstr;
for (i = 0; i < ETHER_ADDR_LEN; i++) {
mac[i] = strtoul(s, &e, 16);
if (s == e || (e[0] != ':' && e[0] != 0)) {
free(macstr, M_XENBUS);
return (ENOENT);
}
s = &e[1];
}
free(macstr, M_XENBUS);
return (0);
}
/**
* Entry point to this code when a new device is created. Allocate the basic
* structures and the ring buffers for communication with the backend, and
* inform the backend of the appropriate details for those. Switch to
* Connected state.
*/
static int
netfront_probe(device_t dev)
{
if (!strcmp(xenbus_get_type(dev), "vif")) {
device_set_desc(dev, "Virtual Network Interface");
return (0);
}
return (ENXIO);
}
static int
netfront_attach(device_t dev)
{
int err;
err = create_netdev(dev);
if (err) {
xenbus_dev_fatal(dev, err, "creating netdev");
return (err);
}
#if __FreeBSD_version >= 700000
SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "enable_lro", CTLFLAG_RW,
&xn_enable_lro, 0, "Large Receive Offload");
#endif
return (0);
}
static int
netfront_suspend(device_t dev)
{
struct netfront_info *info = device_get_softc(dev);
XN_RX_LOCK(info);
XN_TX_LOCK(info);
netfront_carrier_off(info);
XN_TX_UNLOCK(info);
XN_RX_UNLOCK(info);
return (0);
}
/**
* We are reconnecting to the backend, due to a suspend/resume, or a backend
* driver restart. We tear down our netif structure and recreate it, but
* leave the device-layer structures intact so that this is transparent to the
* rest of the kernel.
*/
static int
netfront_resume(device_t dev)
{
struct netfront_info *info = device_get_softc(dev);
netif_disconnect_backend(info);
return (0);
}
/* Common code used when first setting up, and when resuming. */
static int
talk_to_backend(device_t dev, struct netfront_info *info)
{
const char *message;
struct xs_transaction xst;
const char *node = xenbus_get_node(dev);
int err;
err = xen_net_read_mac(dev, info->mac);
if (err) {
xenbus_dev_fatal(dev, err, "parsing %s/mac", node);
goto out;
}
/* Create shared ring, alloc event channel. */
err = setup_device(dev, info);
if (err)
goto out;
again:
err = xs_transaction_start(&xst);
if (err) {
xenbus_dev_fatal(dev, err, "starting transaction");
goto destroy_ring;
}
err = xs_printf(xst, node, "tx-ring-ref","%u",
info->tx_ring_ref);
if (err) {
message = "writing tx ring-ref";
goto abort_transaction;
}
err = xs_printf(xst, node, "rx-ring-ref","%u",
info->rx_ring_ref);
if (err) {
message = "writing rx ring-ref";
goto abort_transaction;
}
err = xs_printf(xst, node,
"event-channel", "%u",
xen_intr_port(info->xen_intr_handle));
if (err) {
message = "writing event-channel";
goto abort_transaction;
}
err = xs_printf(xst, node, "request-rx-copy", "%u",
info->copying_receiver);
if (err) {
message = "writing request-rx-copy";
goto abort_transaction;
}
err = xs_printf(xst, node, "feature-rx-notify", "%d", 1);
if (err) {
message = "writing feature-rx-notify";
goto abort_transaction;
}
err = xs_printf(xst, node, "feature-sg", "%d", 1);
if (err) {
message = "writing feature-sg";
goto abort_transaction;
}
#if __FreeBSD_version >= 700000
err = xs_printf(xst, node, "feature-gso-tcpv4", "%d", 1);
if (err) {
message = "writing feature-gso-tcpv4";
goto abort_transaction;
}
#endif
err = xs_transaction_end(xst, 0);
if (err) {
if (err == EAGAIN)
goto again;
xenbus_dev_fatal(dev, err, "completing transaction");
goto destroy_ring;
}
return 0;
abort_transaction:
xs_transaction_end(xst, 1);
xenbus_dev_fatal(dev, err, "%s", message);
destroy_ring:
netif_free(info);
out:
return err;
}
static int
setup_device(device_t dev, struct netfront_info *info)
{
netif_tx_sring_t *txs;
netif_rx_sring_t *rxs;
int error;
struct ifnet *ifp;
ifp = info->xn_ifp;
info->tx_ring_ref = GRANT_REF_INVALID;
info->rx_ring_ref = GRANT_REF_INVALID;
info->rx.sring = NULL;
info->tx.sring = NULL;
txs = (netif_tx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO);
if (!txs) {
error = ENOMEM;
xenbus_dev_fatal(dev, error, "allocating tx ring page");
goto fail;
}
SHARED_RING_INIT(txs);
FRONT_RING_INIT(&info->tx, txs, PAGE_SIZE);
error = xenbus_grant_ring(dev, virt_to_mfn(txs), &info->tx_ring_ref);
if (error)
goto fail;
rxs = (netif_rx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO);
if (!rxs) {
error = ENOMEM;
xenbus_dev_fatal(dev, error, "allocating rx ring page");
goto fail;
}
SHARED_RING_INIT(rxs);
FRONT_RING_INIT(&info->rx, rxs, PAGE_SIZE);
error = xenbus_grant_ring(dev, virt_to_mfn(rxs), &info->rx_ring_ref);
if (error)
goto fail;
error = xen_intr_alloc_and_bind_local_port(dev,
xenbus_get_otherend_id(dev), /*filter*/NULL, xn_intr, info,
INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY, &info->xen_intr_handle);
if (error) {
xenbus_dev_fatal(dev, error,
"xen_intr_alloc_and_bind_local_port failed");
goto fail;
}
return (0);
fail:
netif_free(info);
return (error);
}
#ifdef INET
/**
* If this interface has an ipv4 address, send an arp for it. This
* helps to get the network going again after migrating hosts.
*/
static void
netfront_send_fake_arp(device_t dev, struct netfront_info *info)
{
struct ifnet *ifp;
struct ifaddr *ifa;
ifp = info->xn_ifp;
TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family == AF_INET) {
arp_ifinit(ifp, ifa);
}
}
}
#endif
/**
* Callback received when the backend's state changes.
*/
static void
netfront_backend_changed(device_t dev, XenbusState newstate)
{
struct netfront_info *sc = device_get_softc(dev);
DPRINTK("newstate=%d\n", newstate);
switch (newstate) {
case XenbusStateInitialising:
case XenbusStateInitialised:
case XenbusStateUnknown:
case XenbusStateClosed:
case XenbusStateReconfigured:
case XenbusStateReconfiguring:
break;
case XenbusStateInitWait:
if (xenbus_get_state(dev) != XenbusStateInitialising)
break;
if (network_connect(sc) != 0)
break;
xenbus_set_state(dev, XenbusStateConnected);
break;
case XenbusStateClosing:
xenbus_set_state(dev, XenbusStateClosed);
break;
case XenbusStateConnected:
#ifdef INET
netfront_send_fake_arp(dev, sc);
#endif
break;
}
}
static void
xn_free_rx_ring(struct netfront_info *sc)
{
#if 0
int i;
for (i = 0; i < NET_RX_RING_SIZE; i++) {
if (sc->xn_cdata.rx_mbufs[i] != NULL) {
m_freem(sc->rx_mbufs[i]);
sc->rx_mbufs[i] = NULL;
}
}
sc->rx.rsp_cons = 0;
sc->xn_rx_if->req_prod = 0;
sc->xn_rx_if->event = sc->rx.rsp_cons ;
#endif
}
static void
xn_free_tx_ring(struct netfront_info *sc)
{
#if 0
int i;
for (i = 0; i < NET_TX_RING_SIZE; i++) {
if (sc->tx_mbufs[i] != NULL) {
m_freem(sc->tx_mbufs[i]);
sc->xn_cdata.xn_tx_chain[i] = NULL;
}
}
return;
#endif
}
/**
* \brief Verify that there is sufficient space in the Tx ring
* buffer for a maximally sized request to be enqueued.
*
* A transmit request requires a transmit descriptor for each packet
* fragment, plus up to 2 entries for "options" (e.g. TSO).
*/
static inline int
xn_tx_slot_available(struct netfront_info *np)
{
return (RING_FREE_REQUESTS(&np->tx) > (MAX_TX_REQ_FRAGS + 2));
}
static void
netif_release_tx_bufs(struct netfront_info *np)
{
int i;
for (i = 1; i <= NET_TX_RING_SIZE; i++) {
struct mbuf *m;
m = np->tx_mbufs[i];
/*
* We assume that no kernel addresses are
* less than NET_TX_RING_SIZE. Any entry
* in the table that is below this number
* must be an index from free-list tracking.
*/
if (((uintptr_t)m) <= NET_TX_RING_SIZE)
continue;
gnttab_end_foreign_access_ref(np->grant_tx_ref[i]);
gnttab_release_grant_reference(&np->gref_tx_head,
np->grant_tx_ref[i]);
np->grant_tx_ref[i] = GRANT_REF_INVALID;
add_id_to_freelist(np->tx_mbufs, i);
np->xn_cdata.xn_tx_chain_cnt--;
if (np->xn_cdata.xn_tx_chain_cnt < 0) {
panic("%s: tx_chain_cnt must be >= 0", __func__);
}
m_free(m);
}
}
static void
network_alloc_rx_buffers(struct netfront_info *sc)
{
int otherend_id = xenbus_get_otherend_id(sc->xbdev);
unsigned short id;
struct mbuf *m_new;
int i, batch_target, notify;
RING_IDX req_prod;
struct xen_memory_reservation reservation;
grant_ref_t ref;
int nr_flips;
netif_rx_request_t *req;
vm_offset_t vaddr;
u_long pfn;
req_prod = sc->rx.req_prod_pvt;
if (__predict_false(sc->carrier == 0))
return;
/*
* Allocate mbufs greedily, even though we batch updates to the
* receive ring. This creates a less bursty demand on the memory
* allocator, and so should reduce the chance of failed allocation
* requests both for ourself and for other kernel subsystems.
*
* Here we attempt to maintain rx_target buffers in flight, counting
* buffers that we have yet to process in the receive ring.
*/
batch_target = sc->rx_target - (req_prod - sc->rx.rsp_cons);
for (i = mbufq_len(&sc->xn_rx_batch); i < batch_target; i++) {
m_new = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUMPAGESIZE);
if (m_new == NULL) {
if (i != 0)
goto refill;
/*
* XXX set timer
*/
break;
}
m_new->m_len = m_new->m_pkthdr.len = MJUMPAGESIZE;
/* queue the mbufs allocated */
(void )mbufq_enqueue(&sc->xn_rx_batch, m_new);
}
/*
* If we've allocated at least half of our target number of entries,
* submit them to the backend - we have enough to make the overhead
* of submission worthwhile. Otherwise wait for more mbufs and
* request entries to become available.
*/
if (i < (sc->rx_target/2)) {
if (req_prod >sc->rx.sring->req_prod)
goto push;
return;
}
/*
* Double floating fill target if we risked having the backend
* run out of empty buffers for receive traffic. We define "running
* low" as having less than a fourth of our target buffers free
* at the time we refilled the queue.
*/
if ((req_prod - sc->rx.sring->rsp_prod) < (sc->rx_target / 4)) {
sc->rx_target *= 2;
if (sc->rx_target > sc->rx_max_target)
sc->rx_target = sc->rx_max_target;
}
refill:
for (nr_flips = i = 0; ; i++) {
if ((m_new = mbufq_dequeue(&sc->xn_rx_batch)) == NULL)
break;
m_new->m_ext.ext_arg1 = (vm_paddr_t *)(uintptr_t)(
vtophys(m_new->m_ext.ext_buf) >> PAGE_SHIFT);
id = xennet_rxidx(req_prod + i);
KASSERT(sc->rx_mbufs[id] == NULL, ("non-NULL xm_rx_chain"));
sc->rx_mbufs[id] = m_new;
ref = gnttab_claim_grant_reference(&sc->gref_rx_head);
KASSERT(ref != GNTTAB_LIST_END,
("reserved grant references exhuasted"));
sc->grant_rx_ref[id] = ref;
vaddr = mtod(m_new, vm_offset_t);
pfn = vtophys(vaddr) >> PAGE_SHIFT;
req = RING_GET_REQUEST(&sc->rx, req_prod + i);
if (sc->copying_receiver == 0) {
gnttab_grant_foreign_transfer_ref(ref,
otherend_id, pfn);
sc->rx_pfn_array[nr_flips] = pfn;
if (!xen_feature(XENFEAT_auto_translated_physmap)) {
/* Remove this page before passing
* back to Xen.
*/
MULTI_update_va_mapping(&sc->rx_mcl[i],
vaddr, 0, 0);
}
nr_flips++;
} else {
gnttab_grant_foreign_access_ref(ref,
otherend_id,
pfn, 0);
}
req->id = id;
req->gref = ref;
sc->rx_pfn_array[i] =
vtomach(mtod(m_new,vm_offset_t)) >> PAGE_SHIFT;
}
KASSERT(i, ("no mbufs processed")); /* should have returned earlier */
KASSERT(mbufq_len(&sc->xn_rx_batch) == 0, ("not all mbufs processed"));
/*
* We may have allocated buffers which have entries outstanding
* in the page * update queue -- make sure we flush those first!
*/
if (nr_flips != 0) {
#ifdef notyet
/* Tell the ballon driver what is going on. */
balloon_update_driver_allowance(i);
#endif
set_xen_guest_handle(reservation.extent_start, sc->rx_pfn_array);
reservation.nr_extents = i;
reservation.extent_order = 0;
reservation.address_bits = 0;
reservation.domid = DOMID_SELF;
if (!xen_feature(XENFEAT_auto_translated_physmap)) {
/* After all PTEs have been zapped, flush the TLB. */
sc->rx_mcl[i-1].args[MULTI_UVMFLAGS_INDEX] =
UVMF_TLB_FLUSH|UVMF_ALL;
/* Give away a batch of pages. */
sc->rx_mcl[i].op = __HYPERVISOR_memory_op;
sc->rx_mcl[i].args[0] = XENMEM_decrease_reservation;
sc->rx_mcl[i].args[1] = (u_long)&reservation;
/* Zap PTEs and give away pages in one big multicall. */
(void)HYPERVISOR_multicall(sc->rx_mcl, i+1);
if (__predict_false(sc->rx_mcl[i].result != i ||
HYPERVISOR_memory_op(XENMEM_decrease_reservation,
&reservation) != i))
panic("%s: unable to reduce memory "
"reservation\n", __func__);
}
} else {
wmb();
}
/* Above is a suitable barrier to ensure backend will see requests. */
sc->rx.req_prod_pvt = req_prod + i;
push:
RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->rx, notify);
if (notify)
xen_intr_signal(sc->xen_intr_handle);
}
static void
xn_rxeof(struct netfront_info *np)
{
struct ifnet *ifp;
#if __FreeBSD_version >= 700000 && (defined(INET) || defined(INET6))
struct lro_ctrl *lro = &np->xn_lro;
struct lro_entry *queued;
#endif
struct netfront_rx_info rinfo;
struct netif_rx_response *rx = &rinfo.rx;
struct netif_extra_info *extras = rinfo.extras;
RING_IDX i, rp;
multicall_entry_t *mcl;
struct mbuf *m;
struct mbufq rxq, errq;
int err, pages_flipped = 0, work_to_do;
do {
XN_RX_LOCK_ASSERT(np);
if (!netfront_carrier_ok(np))
return;
/* XXX: there should be some sane limit. */
mbufq_init(&errq, INT_MAX);
mbufq_init(&rxq, INT_MAX);
ifp = np->xn_ifp;
rp = np->rx.sring->rsp_prod;
rmb(); /* Ensure we see queued responses up to 'rp'. */
i = np->rx.rsp_cons;
while ((i != rp)) {
memcpy(rx, RING_GET_RESPONSE(&np->rx, i), sizeof(*rx));
memset(extras, 0, sizeof(rinfo.extras));
m = NULL;
err = xennet_get_responses(np, &rinfo, rp, &i, &m,
&pages_flipped);
if (__predict_false(err)) {
if (m)
(void )mbufq_enqueue(&errq, m);
np->stats.rx_errors++;
continue;
}
m->m_pkthdr.rcvif = ifp;
if ( rx->flags & NETRXF_data_validated ) {
/* Tell the stack the checksums are okay */
/*
* XXX this isn't necessarily the case - need to add
* check
*/
m->m_pkthdr.csum_flags |=
(CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID
| CSUM_PSEUDO_HDR);
m->m_pkthdr.csum_data = 0xffff;
}
np->stats.rx_packets++;
np->stats.rx_bytes += m->m_pkthdr.len;
(void )mbufq_enqueue(&rxq, m);
np->rx.rsp_cons = i;
}
if (pages_flipped) {
/* Some pages are no longer absent... */
#ifdef notyet
balloon_update_driver_allowance(-pages_flipped);
#endif
/* Do all the remapping work, and M->P updates, in one big
* hypercall.
*/
if (!!xen_feature(XENFEAT_auto_translated_physmap)) {
mcl = np->rx_mcl + pages_flipped;
mcl->op = __HYPERVISOR_mmu_update;
mcl->args[0] = (u_long)np->rx_mmu;
mcl->args[1] = pages_flipped;
mcl->args[2] = 0;
mcl->args[3] = DOMID_SELF;
(void)HYPERVISOR_multicall(np->rx_mcl,
pages_flipped + 1);
}
}
mbufq_drain(&errq);
/*
* Process all the mbufs after the remapping is complete.
* Break the mbuf chain first though.
*/
while ((m = mbufq_dequeue(&rxq)) != NULL) {
if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
/*
* Do we really need to drop the rx lock?
*/
XN_RX_UNLOCK(np);
#if __FreeBSD_version >= 700000 && (defined(INET) || defined(INET6))
/* Use LRO if possible */
if ((ifp->if_capenable & IFCAP_LRO) == 0 ||
lro->lro_cnt == 0 || tcp_lro_rx(lro, m, 0)) {
/*
* If LRO fails, pass up to the stack
* directly.
*/
(*ifp->if_input)(ifp, m);
}
#else
(*ifp->if_input)(ifp, m);
#endif
XN_RX_LOCK(np);
}
np->rx.rsp_cons = i;
#if __FreeBSD_version >= 700000 && (defined(INET) || defined(INET6))
/*
* Flush any outstanding LRO work
*/
while (!SLIST_EMPTY(&lro->lro_active)) {
queued = SLIST_FIRST(&lro->lro_active);
SLIST_REMOVE_HEAD(&lro->lro_active, next);
tcp_lro_flush(lro, queued);
}
#endif
#if 0
/* If we get a callback with very few responses, reduce fill target. */
/* NB. Note exponential increase, linear decrease. */
if (((np->rx.req_prod_pvt - np->rx.sring->rsp_prod) >
((3*np->rx_target) / 4)) && (--np->rx_target < np->rx_min_target))
np->rx_target = np->rx_min_target;
#endif
network_alloc_rx_buffers(np);
RING_FINAL_CHECK_FOR_RESPONSES(&np->rx, work_to_do);
} while (work_to_do);
}
static void
xn_txeof(struct netfront_info *np)
{
RING_IDX i, prod;
unsigned short id;
struct ifnet *ifp;
netif_tx_response_t *txr;
struct mbuf *m;
XN_TX_LOCK_ASSERT(np);
if (!netfront_carrier_ok(np))
return;
ifp = np->xn_ifp;
do {
prod = np->tx.sring->rsp_prod;
rmb(); /* Ensure we see responses up to 'rp'. */
for (i = np->tx.rsp_cons; i != prod; i++) {
txr = RING_GET_RESPONSE(&np->tx, i);
if (txr->status == NETIF_RSP_NULL)
continue;
if (txr->status != NETIF_RSP_OKAY) {
printf("%s: WARNING: response is %d!\n",
__func__, txr->status);
}
id = txr->id;
m = np->tx_mbufs[id];
KASSERT(m != NULL, ("mbuf not found in xn_tx_chain"));
KASSERT((uintptr_t)m > NET_TX_RING_SIZE,
("mbuf already on the free list, but we're "
"trying to free it again!"));
M_ASSERTVALID(m);
/*
* Increment packet count if this is the last
* mbuf of the chain.
*/
if (!m->m_next)
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
if (__predict_false(gnttab_query_foreign_access(
np->grant_tx_ref[id]) != 0)) {
panic("%s: grant id %u still in use by the "
"backend", __func__, id);
}
gnttab_end_foreign_access_ref(
np->grant_tx_ref[id]);
gnttab_release_grant_reference(
&np->gref_tx_head, np->grant_tx_ref[id]);
np->grant_tx_ref[id] = GRANT_REF_INVALID;
np->tx_mbufs[id] = NULL;
add_id_to_freelist(np->tx_mbufs, id);
np->xn_cdata.xn_tx_chain_cnt--;
m_free(m);
/* Only mark the queue active if we've freed up at least one slot to try */
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
}
np->tx.rsp_cons = prod;
/*
* Set a new event, then check for race with update of
* tx_cons. Note that it is essential to schedule a
* callback, no matter how few buffers are pending. Even if
* there is space in the transmit ring, higher layers may
* be blocked because too much data is outstanding: in such
* cases notification from Xen is likely to be the only kick
* that we'll get.
*/
np->tx.sring->rsp_event =
prod + ((np->tx.sring->req_prod - prod) >> 1) + 1;
mb();
} while (prod != np->tx.sring->rsp_prod);
if (np->tx_full &&
((np->tx.sring->req_prod - prod) < NET_TX_RING_SIZE)) {
np->tx_full = 0;
#if 0
if (np->user_state == UST_OPEN)
netif_wake_queue(dev);
#endif
}
}
static void
xn_intr(void *xsc)
{
struct netfront_info *np = xsc;
struct ifnet *ifp = np->xn_ifp;
#if 0
if (!(np->rx.rsp_cons != np->rx.sring->rsp_prod &&
likely(netfront_carrier_ok(np)) &&
ifp->if_drv_flags & IFF_DRV_RUNNING))
return;
#endif
if (RING_HAS_UNCONSUMED_RESPONSES(&np->tx)) {
XN_TX_LOCK(np);
xn_txeof(np);
XN_TX_UNLOCK(np);
}
XN_RX_LOCK(np);
xn_rxeof(np);
XN_RX_UNLOCK(np);
if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
xn_start(ifp);
}
static void
xennet_move_rx_slot(struct netfront_info *np, struct mbuf *m,
grant_ref_t ref)
{
int new = xennet_rxidx(np->rx.req_prod_pvt);
KASSERT(np->rx_mbufs[new] == NULL, ("rx_mbufs != NULL"));
np->rx_mbufs[new] = m;
np->grant_rx_ref[new] = ref;
RING_GET_REQUEST(&np->rx, np->rx.req_prod_pvt)->id = new;
RING_GET_REQUEST(&np->rx, np->rx.req_prod_pvt)->gref = ref;
np->rx.req_prod_pvt++;
}
static int
xennet_get_extras(struct netfront_info *np,
struct netif_extra_info *extras, RING_IDX rp, RING_IDX *cons)
{
struct netif_extra_info *extra;
int err = 0;
do {
struct mbuf *m;
grant_ref_t ref;
if (__predict_false(*cons + 1 == rp)) {
#if 0
if (net_ratelimit())
WPRINTK("Missing extra info\n");
#endif
err = EINVAL;
break;
}
extra = (struct netif_extra_info *)
RING_GET_RESPONSE(&np->rx, ++(*cons));
if (__predict_false(!extra->type ||
extra->type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
#if 0
if (net_ratelimit())
WPRINTK("Invalid extra type: %d\n",
extra->type);
#endif
err = EINVAL;
} else {
memcpy(&extras[extra->type - 1], extra, sizeof(*extra));
}
m = xennet_get_rx_mbuf(np, *cons);
ref = xennet_get_rx_ref(np, *cons);
xennet_move_rx_slot(np, m, ref);
} while (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE);
return err;
}
static int
xennet_get_responses(struct netfront_info *np,
struct netfront_rx_info *rinfo, RING_IDX rp, RING_IDX *cons,
struct mbuf **list,
int *pages_flipped_p)
{
int pages_flipped = *pages_flipped_p;
struct mmu_update *mmu;
struct multicall_entry *mcl;
struct netif_rx_response *rx = &rinfo->rx;
struct netif_extra_info *extras = rinfo->extras;
struct mbuf *m, *m0, *m_prev;
grant_ref_t ref = xennet_get_rx_ref(np, *cons);
RING_IDX ref_cons = *cons;
int frags = 1;
int err = 0;
u_long ret;
m0 = m = m_prev = xennet_get_rx_mbuf(np, *cons);
if (rx->flags & NETRXF_extra_info) {
err = xennet_get_extras(np, extras, rp, cons);
}
if (m0 != NULL) {
m0->m_pkthdr.len = 0;
m0->m_next = NULL;
}
for (;;) {
u_long mfn;
#if 0
DPRINTK("rx->status=%hd rx->offset=%hu frags=%u\n",
rx->status, rx->offset, frags);
#endif
if (__predict_false(rx->status < 0 ||
rx->offset + rx->status > PAGE_SIZE)) {
#if 0
if (net_ratelimit())
WPRINTK("rx->offset: %x, size: %u\n",
rx->offset, rx->status);
#endif
xennet_move_rx_slot(np, m, ref);
if (m0 == m)
m0 = NULL;
m = NULL;
err = EINVAL;
goto next_skip_queue;
}
/*
* This definitely indicates a bug, either in this driver or in
* the backend driver. In future this should flag the bad
* situation to the system controller to reboot the backed.
*/
if (ref == GRANT_REF_INVALID) {
#if 0
if (net_ratelimit())
WPRINTK("Bad rx response id %d.\n", rx->id);
#endif
printf("%s: Bad rx response id %d.\n", __func__,rx->id);
err = EINVAL;
goto next;
}
if (!np->copying_receiver) {
/* Memory pressure, insufficient buffer
* headroom, ...
*/
if (!(mfn = gnttab_end_foreign_transfer_ref(ref))) {
WPRINTK("Unfulfilled rx req (id=%d, st=%d).\n",
rx->id, rx->status);
xennet_move_rx_slot(np, m, ref);
err = ENOMEM;
goto next;
}
if (!xen_feature( XENFEAT_auto_translated_physmap)) {
/* Remap the page. */
void *vaddr = mtod(m, void *);
uint32_t pfn;
mcl = np->rx_mcl + pages_flipped;
mmu = np->rx_mmu + pages_flipped;
MULTI_update_va_mapping(mcl, (u_long)vaddr,
(((vm_paddr_t)mfn) << PAGE_SHIFT) | PG_RW |
PG_V | PG_M | PG_A, 0);
pfn = (uintptr_t)m->m_ext.ext_arg1;
mmu->ptr = ((vm_paddr_t)mfn << PAGE_SHIFT) |
MMU_MACHPHYS_UPDATE;
mmu->val = pfn;
}
pages_flipped++;
} else {
ret = gnttab_end_foreign_access_ref(ref);
KASSERT(ret, ("ret != 0"));
}
gnttab_release_grant_reference(&np->gref_rx_head, ref);
next:
if (m == NULL)
break;
m->m_len = rx->status;
m->m_data += rx->offset;
m0->m_pkthdr.len += rx->status;
next_skip_queue:
if (!(rx->flags & NETRXF_more_data))
break;
if (*cons + frags == rp) {
if (net_ratelimit())
WPRINTK("Need more frags\n");
err = ENOENT;
printf("%s: cons %u frags %u rp %u, not enough frags\n",
__func__, *cons, frags, rp);
break;
}
/*
* Note that m can be NULL, if rx->status < 0 or if
* rx->offset + rx->status > PAGE_SIZE above.
*/
m_prev = m;
rx = RING_GET_RESPONSE(&np->rx, *cons + frags);
m = xennet_get_rx_mbuf(np, *cons + frags);
/*
* m_prev == NULL can happen if rx->status < 0 or if
* rx->offset + * rx->status > PAGE_SIZE above.
*/
if (m_prev != NULL)
m_prev->m_next = m;
/*
* m0 can be NULL if rx->status < 0 or if * rx->offset +
* rx->status > PAGE_SIZE above.
*/
if (m0 == NULL)
m0 = m;
m->m_next = NULL;
ref = xennet_get_rx_ref(np, *cons + frags);
ref_cons = *cons + frags;
frags++;
}
*list = m0;
*cons += frags;
*pages_flipped_p = pages_flipped;
return (err);
}
static void
xn_tick_locked(struct netfront_info *sc)
{
XN_RX_LOCK_ASSERT(sc);
callout_reset(&sc->xn_stat_ch, hz, xn_tick, sc);
/* XXX placeholder for printing debug information */
}
static void
xn_tick(void *xsc)
{
struct netfront_info *sc;
sc = xsc;
XN_RX_LOCK(sc);
xn_tick_locked(sc);
XN_RX_UNLOCK(sc);
}
/**
* \brief Count the number of fragments in an mbuf chain.
*
* Surprisingly, there isn't an M* macro for this.
*/
static inline int
xn_count_frags(struct mbuf *m)
{
int nfrags;
for (nfrags = 0; m != NULL; m = m->m_next)
nfrags++;
return (nfrags);
}
/**
* Given an mbuf chain, make sure we have enough room and then push
* it onto the transmit ring.
*/
static int
xn_assemble_tx_request(struct netfront_info *sc, struct mbuf *m_head)
{
struct ifnet *ifp;
struct mbuf *m;
u_int nfrags;
int otherend_id;
ifp = sc->xn_ifp;
/**
* Defragment the mbuf if necessary.
*/
nfrags = xn_count_frags(m_head);
/*
* Check to see whether this request is longer than netback
* can handle, and try to defrag it.
*/
/**
* It is a bit lame, but the netback driver in Linux can't
* deal with nfrags > MAX_TX_REQ_FRAGS, which is a quirk of
* the Linux network stack.
*/
if (nfrags > sc->maxfrags) {
m = m_defrag(m_head, M_NOWAIT);
if (!m) {
/*
* Defrag failed, so free the mbuf and
* therefore drop the packet.
*/
m_freem(m_head);
return (EMSGSIZE);
}
m_head = m;
}
/* Determine how many fragments now exist */
nfrags = xn_count_frags(m_head);
/*
* Check to see whether the defragmented packet has too many
* segments for the Linux netback driver.
*/
/**
* The FreeBSD TCP stack, with TSO enabled, can produce a chain
* of mbufs longer than Linux can handle. Make sure we don't
* pass a too-long chain over to the other side by dropping the
* packet. It doesn't look like there is currently a way to
* tell the TCP stack to generate a shorter chain of packets.
*/
if (nfrags > MAX_TX_REQ_FRAGS) {
#ifdef DEBUG
printf("%s: nfrags %d > MAX_TX_REQ_FRAGS %d, netback "
"won't be able to handle it, dropping\n",
__func__, nfrags, MAX_TX_REQ_FRAGS);
#endif
m_freem(m_head);
return (EMSGSIZE);
}
/*
* This check should be redundant. We've already verified that we
* have enough slots in the ring to handle a packet of maximum
* size, and that our packet is less than the maximum size. Keep
* it in here as an assert for now just to make certain that
* xn_tx_chain_cnt is accurate.
*/
KASSERT((sc->xn_cdata.xn_tx_chain_cnt + nfrags) <= NET_TX_RING_SIZE,
("%s: xn_tx_chain_cnt (%d) + nfrags (%d) > NET_TX_RING_SIZE "
"(%d)!", __func__, (int) sc->xn_cdata.xn_tx_chain_cnt,
(int) nfrags, (int) NET_TX_RING_SIZE));
/*
* Start packing the mbufs in this chain into
* the fragment pointers. Stop when we run out
* of fragments or hit the end of the mbuf chain.
*/
m = m_head;
otherend_id = xenbus_get_otherend_id(sc->xbdev);
for (m = m_head; m; m = m->m_next) {
netif_tx_request_t *tx;
uintptr_t id;
grant_ref_t ref;
u_long mfn; /* XXX Wrong type? */
tx = RING_GET_REQUEST(&sc->tx, sc->tx.req_prod_pvt);
id = get_id_from_freelist(sc->tx_mbufs);
if (id == 0)
panic("%s: was allocated the freelist head!\n",
__func__);
sc->xn_cdata.xn_tx_chain_cnt++;
if (sc->xn_cdata.xn_tx_chain_cnt > NET_TX_RING_SIZE)
panic("%s: tx_chain_cnt must be <= NET_TX_RING_SIZE\n",
__func__);
sc->tx_mbufs[id] = m;
tx->id = id;
ref = gnttab_claim_grant_reference(&sc->gref_tx_head);
KASSERT((short)ref >= 0, ("Negative ref"));
mfn = virt_to_mfn(mtod(m, vm_offset_t));
gnttab_grant_foreign_access_ref(ref, otherend_id,
mfn, GNTMAP_readonly);
tx->gref = sc->grant_tx_ref[id] = ref;
tx->offset = mtod(m, vm_offset_t) & (PAGE_SIZE - 1);
tx->flags = 0;
if (m == m_head) {
/*
* The first fragment has the entire packet
* size, subsequent fragments have just the
* fragment size. The backend works out the
* true size of the first fragment by
* subtracting the sizes of the other
* fragments.
*/
tx->size = m->m_pkthdr.len;
/*
* The first fragment contains the checksum flags
* and is optionally followed by extra data for
* TSO etc.
*/
/**
* CSUM_TSO requires checksum offloading.
* Some versions of FreeBSD fail to
* set CSUM_TCP in the CSUM_TSO case,
* so we have to test for CSUM_TSO
* explicitly.
*/
if (m->m_pkthdr.csum_flags
& (CSUM_DELAY_DATA | CSUM_TSO)) {
tx->flags |= (NETTXF_csum_blank
| NETTXF_data_validated);
}
#if __FreeBSD_version >= 700000
if (m->m_pkthdr.csum_flags & CSUM_TSO) {
struct netif_extra_info *gso =
(struct netif_extra_info *)
RING_GET_REQUEST(&sc->tx,
++sc->tx.req_prod_pvt);
tx->flags |= NETTXF_extra_info;
gso->u.gso.size = m->m_pkthdr.tso_segsz;
gso->u.gso.type =
XEN_NETIF_GSO_TYPE_TCPV4;
gso->u.gso.pad = 0;
gso->u.gso.features = 0;
gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
gso->flags = 0;
}
#endif
} else {
tx->size = m->m_len;
}
if (m->m_next)
tx->flags |= NETTXF_more_data;
sc->tx.req_prod_pvt++;
}
BPF_MTAP(ifp, m_head);
sc->stats.tx_bytes += m_head->m_pkthdr.len;
sc->stats.tx_packets++;
return (0);
}
static void
xn_start_locked(struct ifnet *ifp)
{
struct netfront_info *sc;
struct mbuf *m_head;
int notify;
sc = ifp->if_softc;
if (!netfront_carrier_ok(sc))
return;
/*
* While we have enough transmit slots available for at least one
* maximum-sized packet, pull mbufs off the queue and put them on
* the transmit ring.
*/
while (xn_tx_slot_available(sc)) {
IF_DEQUEUE(&ifp->if_snd, m_head);
if (m_head == NULL)
break;
if (xn_assemble_tx_request(sc, m_head) != 0)
break;
}
RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->tx, notify);
if (notify)
xen_intr_signal(sc->xen_intr_handle);
if (RING_FULL(&sc->tx)) {
sc->tx_full = 1;
#if 0
netif_stop_queue(dev);
#endif
}
}
static void
xn_start(struct ifnet *ifp)
{
struct netfront_info *sc;
sc = ifp->if_softc;
XN_TX_LOCK(sc);
xn_start_locked(ifp);
XN_TX_UNLOCK(sc);
}
/* equivalent of network_open() in Linux */
static void
xn_ifinit_locked(struct netfront_info *sc)
{
struct ifnet *ifp;
XN_LOCK_ASSERT(sc);
ifp = sc->xn_ifp;
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
return;
xn_stop(sc);
network_alloc_rx_buffers(sc);
sc->rx.sring->rsp_event = sc->rx.rsp_cons + 1;
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
if_link_state_change(ifp, LINK_STATE_UP);
callout_reset(&sc->xn_stat_ch, hz, xn_tick, sc);
}
static void
xn_ifinit(void *xsc)
{
struct netfront_info *sc = xsc;
XN_LOCK(sc);
xn_ifinit_locked(sc);
XN_UNLOCK(sc);
}
static int
xn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
struct netfront_info *sc = ifp->if_softc;
struct ifreq *ifr = (struct ifreq *) data;
#ifdef INET
struct ifaddr *ifa = (struct ifaddr *)data;
#endif
int mask, error = 0;
switch(cmd) {
case SIOCSIFADDR:
#ifdef INET
XN_LOCK(sc);
if (ifa->ifa_addr->sa_family == AF_INET) {
ifp->if_flags |= IFF_UP;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
xn_ifinit_locked(sc);
arp_ifinit(ifp, ifa);
XN_UNLOCK(sc);
} else {
XN_UNLOCK(sc);
#endif
error = ether_ioctl(ifp, cmd, data);
#ifdef INET
}
#endif
break;
case SIOCSIFMTU:
/* XXX can we alter the MTU on a VN ?*/
#ifdef notyet
if (ifr->ifr_mtu > XN_JUMBO_MTU)
error = EINVAL;
else
#endif
{
ifp->if_mtu = ifr->ifr_mtu;
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
xn_ifinit(sc);
}
break;
case SIOCSIFFLAGS:
XN_LOCK(sc);
if (ifp->if_flags & IFF_UP) {
/*
* If only the state of the PROMISC flag changed,
* then just use the 'set promisc mode' command
* instead of reinitializing the entire NIC. Doing
* a full re-init means reloading the firmware and
* waiting for it to start up, which may take a
* second or two.
*/
#ifdef notyet
/* No promiscuous mode with Xen */
if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
ifp->if_flags & IFF_PROMISC &&
!(sc->xn_if_flags & IFF_PROMISC)) {
XN_SETBIT(sc, XN_RX_MODE,
XN_RXMODE_RX_PROMISC);
} else if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
!(ifp->if_flags & IFF_PROMISC) &&
sc->xn_if_flags & IFF_PROMISC) {
XN_CLRBIT(sc, XN_RX_MODE,
XN_RXMODE_RX_PROMISC);
} else
#endif
xn_ifinit_locked(sc);
} else {
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
xn_stop(sc);
}
}
sc->xn_if_flags = ifp->if_flags;
XN_UNLOCK(sc);
error = 0;
break;
case SIOCSIFCAP:
mask = ifr->ifr_reqcap ^ ifp->if_capenable;
if (mask & IFCAP_TXCSUM) {
if (IFCAP_TXCSUM & ifp->if_capenable) {
ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
| CSUM_IP | CSUM_TSO);
} else {
ifp->if_capenable |= IFCAP_TXCSUM;
ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP
| CSUM_IP);
}
}
if (mask & IFCAP_RXCSUM) {
ifp->if_capenable ^= IFCAP_RXCSUM;
}
#if __FreeBSD_version >= 700000
if (mask & IFCAP_TSO4) {
if (IFCAP_TSO4 & ifp->if_capenable) {
ifp->if_capenable &= ~IFCAP_TSO4;
ifp->if_hwassist &= ~CSUM_TSO;
} else if (IFCAP_TXCSUM & ifp->if_capenable) {
ifp->if_capenable |= IFCAP_TSO4;
ifp->if_hwassist |= CSUM_TSO;
} else {
IPRINTK("Xen requires tx checksum offload"
" be enabled to use TSO\n");
error = EINVAL;
}
}
if (mask & IFCAP_LRO) {
ifp->if_capenable ^= IFCAP_LRO;
}
#endif
error = 0;
break;
case SIOCADDMULTI:
case SIOCDELMULTI:
#ifdef notyet
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
XN_LOCK(sc);
xn_setmulti(sc);
XN_UNLOCK(sc);
error = 0;
}
#endif
/* FALLTHROUGH */
case SIOCSIFMEDIA:
case SIOCGIFMEDIA:
error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd);
break;
default:
error = ether_ioctl(ifp, cmd, data);
}
return (error);
}
static void
xn_stop(struct netfront_info *sc)
{
struct ifnet *ifp;
XN_LOCK_ASSERT(sc);
ifp = sc->xn_ifp;
callout_stop(&sc->xn_stat_ch);
xn_free_rx_ring(sc);
xn_free_tx_ring(sc);
ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
if_link_state_change(ifp, LINK_STATE_DOWN);
}
/* START of Xenolinux helper functions adapted to FreeBSD */
int
network_connect(struct netfront_info *np)
{
int i, requeue_idx, error;
grant_ref_t ref;
netif_rx_request_t *req;
u_int feature_rx_copy, feature_rx_flip;
error = xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
"feature-rx-copy", NULL, "%u", &feature_rx_copy);
if (error)
feature_rx_copy = 0;
error = xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
"feature-rx-flip", NULL, "%u", &feature_rx_flip);
if (error)
feature_rx_flip = 1;
/*
* Copy packets on receive path if:
* (a) This was requested by user, and the backend supports it; or
* (b) Flipping was requested, but this is unsupported by the backend.
*/
np->copying_receiver = ((MODPARM_rx_copy && feature_rx_copy) ||
(MODPARM_rx_flip && !feature_rx_flip));
/* Recovery procedure: */
error = talk_to_backend(np->xbdev, np);
if (error)
return (error);
/* Step 1: Reinitialise variables. */
xn_query_features(np);
xn_configure_features(np);
netif_release_tx_bufs(np);
/* Step 2: Rebuild the RX buffer freelist and the RX ring itself. */
for (requeue_idx = 0, i = 0; i < NET_RX_RING_SIZE; i++) {
struct mbuf *m;
u_long pfn;
if (np->rx_mbufs[i] == NULL)
continue;
m = np->rx_mbufs[requeue_idx] = xennet_get_rx_mbuf(np, i);
ref = np->grant_rx_ref[requeue_idx] = xennet_get_rx_ref(np, i);
req = RING_GET_REQUEST(&np->rx, requeue_idx);
pfn = vtophys(mtod(m, vm_offset_t)) >> PAGE_SHIFT;
if (!np->copying_receiver) {
gnttab_grant_foreign_transfer_ref(ref,
xenbus_get_otherend_id(np->xbdev),
pfn);
} else {
gnttab_grant_foreign_access_ref(ref,
xenbus_get_otherend_id(np->xbdev),
pfn, 0);
}
req->gref = ref;
req->id = requeue_idx;
requeue_idx++;
}
np->rx.req_prod_pvt = requeue_idx;
/* Step 3: All public and private state should now be sane. Get
* ready to start sending and receiving packets and give the driver
* domain a kick because we've probably just requeued some
* packets.
*/
netfront_carrier_on(np);
xen_intr_signal(np->xen_intr_handle);
XN_TX_LOCK(np);
xn_txeof(np);
XN_TX_UNLOCK(np);
network_alloc_rx_buffers(np);
return (0);
}
static void
xn_query_features(struct netfront_info *np)
{
int val;
device_printf(np->xbdev, "backend features:");
if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
"feature-sg", NULL, "%d", &val) < 0)
val = 0;
np->maxfrags = 1;
if (val) {
np->maxfrags = MAX_TX_REQ_FRAGS;
printf(" feature-sg");
}
if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
"feature-gso-tcpv4", NULL, "%d", &val) < 0)
val = 0;
np->xn_ifp->if_capabilities &= ~(IFCAP_TSO4|IFCAP_LRO);
if (val) {
np->xn_ifp->if_capabilities |= IFCAP_TSO4|IFCAP_LRO;
printf(" feature-gso-tcp4");
}
printf("\n");
}
static int
xn_configure_features(struct netfront_info *np)
{
int err;
err = 0;
#if __FreeBSD_version >= 700000 && (defined(INET) || defined(INET6))
if ((np->xn_ifp->if_capenable & IFCAP_LRO) != 0)
tcp_lro_free(&np->xn_lro);
#endif
np->xn_ifp->if_capenable =
np->xn_ifp->if_capabilities & ~(IFCAP_LRO|IFCAP_TSO4);
np->xn_ifp->if_hwassist &= ~CSUM_TSO;
#if __FreeBSD_version >= 700000 && (defined(INET) || defined(INET6))
if (xn_enable_lro && (np->xn_ifp->if_capabilities & IFCAP_LRO) != 0) {
err = tcp_lro_init(&np->xn_lro);
if (err) {
device_printf(np->xbdev, "LRO initialization failed\n");
} else {
np->xn_lro.ifp = np->xn_ifp;
np->xn_ifp->if_capenable |= IFCAP_LRO;
}
}
if ((np->xn_ifp->if_capabilities & IFCAP_TSO4) != 0) {
np->xn_ifp->if_capenable |= IFCAP_TSO4;
np->xn_ifp->if_hwassist |= CSUM_TSO;
}
#endif
return (err);
}
/**
* Create a network device.
* @param dev Newbus device representing this virtual NIC.
*/
int
create_netdev(device_t dev)
{
int i;
struct netfront_info *np;
int err;
struct ifnet *ifp;
np = device_get_softc(dev);
np->xbdev = dev;
XN_LOCK_INIT(np, xennetif);
ifmedia_init(&np->sc_media, 0, xn_ifmedia_upd, xn_ifmedia_sts);
ifmedia_add(&np->sc_media, IFM_ETHER|IFM_MANUAL, 0, NULL);
ifmedia_set(&np->sc_media, IFM_ETHER|IFM_MANUAL);
np->rx_target = RX_MIN_TARGET;
np->rx_min_target = RX_MIN_TARGET;
np->rx_max_target = RX_MAX_TARGET;
/* Initialise {tx,rx}_skbs to be a free chain containing every entry. */
for (i = 0; i <= NET_TX_RING_SIZE; i++) {
np->tx_mbufs[i] = (void *) ((u_long) i+1);
np->grant_tx_ref[i] = GRANT_REF_INVALID;
}
np->tx_mbufs[NET_TX_RING_SIZE] = (void *)0;
for (i = 0; i <= NET_RX_RING_SIZE; i++) {
np->rx_mbufs[i] = NULL;
np->grant_rx_ref[i] = GRANT_REF_INVALID;
}
mbufq_init(&np->xn_rx_batch, INT_MAX);
/* A grant for every tx ring slot */
if (gnttab_alloc_grant_references(NET_TX_RING_SIZE,
&np->gref_tx_head) != 0) {
IPRINTK("#### netfront can't alloc tx grant refs\n");
err = ENOMEM;
goto exit;
}
/* A grant for every rx ring slot */
if (gnttab_alloc_grant_references(RX_MAX_TARGET,
&np->gref_rx_head) != 0) {
WPRINTK("#### netfront can't alloc rx grant refs\n");
gnttab_free_grant_references(np->gref_tx_head);
err = ENOMEM;
goto exit;
}
err = xen_net_read_mac(dev, np->mac);
if (err)
goto out;
/* Set up ifnet structure */
ifp = np->xn_ifp = if_alloc(IFT_ETHER);
ifp->if_softc = np;
if_initname(ifp, "xn", device_get_unit(dev));
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = xn_ioctl;
ifp->if_output = ether_output;
ifp->if_start = xn_start;
#ifdef notyet
ifp->if_watchdog = xn_watchdog;
#endif
ifp->if_init = xn_ifinit;
ifp->if_snd.ifq_maxlen = NET_TX_RING_SIZE - 1;
ifp->if_hwassist = XN_CSUM_FEATURES;
ifp->if_capabilities = IFCAP_HWCSUM;
ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
ifp->if_hw_tsomaxsegcount = MAX_TX_REQ_FRAGS;
ifp->if_hw_tsomaxsegsize = PAGE_SIZE;
ether_ifattach(ifp, np->mac);
- callout_init(&np->xn_stat_ch, CALLOUT_MPSAFE);
+ callout_init(&np->xn_stat_ch, 1);
netfront_carrier_off(np);
return (0);
exit:
gnttab_free_grant_references(np->gref_tx_head);
out:
return (err);
}
/**
* Handle the change of state of the backend to Closing. We must delete our
* device-layer structures now, to ensure that writes are flushed through to
* the backend. Once is this done, we can switch to Closed in
* acknowledgement.
*/
#if 0
static void
netfront_closing(device_t dev)
{
#if 0
struct netfront_info *info = dev->dev_driver_data;
DPRINTK("netfront_closing: %s removed\n", dev->nodename);
close_netdev(info);
#endif
xenbus_switch_state(dev, XenbusStateClosed);
}
#endif
static int
netfront_detach(device_t dev)
{
struct netfront_info *info = device_get_softc(dev);
DPRINTK("%s\n", xenbus_get_node(dev));
netif_free(info);
return 0;
}
static void
netif_free(struct netfront_info *info)
{
XN_LOCK(info);
xn_stop(info);
XN_UNLOCK(info);
callout_drain(&info->xn_stat_ch);
netif_disconnect_backend(info);
if (info->xn_ifp != NULL) {
ether_ifdetach(info->xn_ifp);
if_free(info->xn_ifp);
info->xn_ifp = NULL;
}
ifmedia_removeall(&info->sc_media);
}
static void
netif_disconnect_backend(struct netfront_info *info)
{
XN_RX_LOCK(info);
XN_TX_LOCK(info);
netfront_carrier_off(info);
XN_TX_UNLOCK(info);
XN_RX_UNLOCK(info);
free_ring(&info->tx_ring_ref, &info->tx.sring);
free_ring(&info->rx_ring_ref, &info->rx.sring);
xen_intr_unbind(&info->xen_intr_handle);
}
static void
free_ring(int *ref, void *ring_ptr_ref)
{
void **ring_ptr_ptr = ring_ptr_ref;
if (*ref != GRANT_REF_INVALID) {
/* This API frees the associated storage. */
gnttab_end_foreign_access(*ref, *ring_ptr_ptr);
*ref = GRANT_REF_INVALID;
}
*ring_ptr_ptr = NULL;
}
static int
xn_ifmedia_upd(struct ifnet *ifp)
{
return (0);
}
static void
xn_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
{
ifmr->ifm_status = IFM_AVALID|IFM_ACTIVE;
ifmr->ifm_active = IFM_ETHER|IFM_MANUAL;
}
/* ** Driver registration ** */
static device_method_t netfront_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, netfront_probe),
DEVMETHOD(device_attach, netfront_attach),
DEVMETHOD(device_detach, netfront_detach),
DEVMETHOD(device_shutdown, bus_generic_shutdown),
DEVMETHOD(device_suspend, netfront_suspend),
DEVMETHOD(device_resume, netfront_resume),
/* Xenbus interface */
DEVMETHOD(xenbus_otherend_changed, netfront_backend_changed),
DEVMETHOD_END
};
static driver_t netfront_driver = {
"xn",
netfront_methods,
sizeof(struct netfront_info),
};
devclass_t netfront_devclass;
DRIVER_MODULE(xe, xenbusb_front, netfront_driver, netfront_devclass, NULL,
NULL);
Index: head/sys/fs/nfs/nfs_commonport.c
===================================================================
--- head/sys/fs/nfs/nfs_commonport.c (revision 283290)
+++ head/sys/fs/nfs/nfs_commonport.c (revision 283291)
@@ -1,637 +1,637 @@
/*-
* Copyright (c) 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* Rick Macklem at The University of Guelph.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* Functions that need to be different for different versions of BSD
* kernel should be kept here, along with any global storage specific
* to this BSD variant.
*/
#include <fs/nfs/nfsport.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_param.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
#include <vm/uma.h>
extern int nfscl_ticks;
extern int nfsrv_nfsuserd;
extern struct nfssockreq nfsrv_nfsuserdsock;
extern void (*nfsd_call_recall)(struct vnode *, int, struct ucred *,
struct thread *);
extern int nfsrv_useacl;
struct mount nfsv4root_mnt;
int newnfs_numnfsd = 0;
struct nfsstats newnfsstats;
int nfs_numnfscbd = 0;
int nfscl_debuglevel = 0;
char nfsv4_callbackaddr[INET6_ADDRSTRLEN];
struct callout newnfsd_callout;
void (*nfsd_call_servertimer)(void) = NULL;
void (*ncl_call_invalcaches)(struct vnode *) = NULL;
static int nfs_realign_test;
static int nfs_realign_count;
SYSCTL_NODE(_vfs, OID_AUTO, nfs, CTLFLAG_RW, 0, "NFS filesystem");
SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_test, CTLFLAG_RW, &nfs_realign_test,
0, "Number of realign tests done");
SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_count, CTLFLAG_RW, &nfs_realign_count,
0, "Number of mbuf realignments done");
SYSCTL_STRING(_vfs_nfs, OID_AUTO, callback_addr, CTLFLAG_RW,
nfsv4_callbackaddr, sizeof(nfsv4_callbackaddr),
"NFSv4 callback addr for server to use");
SYSCTL_INT(_vfs_nfs, OID_AUTO, debuglevel, CTLFLAG_RW, &nfscl_debuglevel,
0, "Debug level for NFS client");
/*
* Defines for malloc
* (Here for FreeBSD, since they allocate storage.)
*/
MALLOC_DEFINE(M_NEWNFSRVCACHE, "NFSD srvcache", "NFSD Server Request Cache");
MALLOC_DEFINE(M_NEWNFSDCLIENT, "NFSD V4client", "NFSD V4 Client Id");
MALLOC_DEFINE(M_NEWNFSDSTATE, "NFSD V4state",
"NFSD V4 State (Openowner, Open, Lockowner, Delegation");
MALLOC_DEFINE(M_NEWNFSDLOCK, "NFSD V4lock", "NFSD V4 byte range lock");
MALLOC_DEFINE(M_NEWNFSDLOCKFILE, "NFSD lckfile", "NFSD Open/Lock file");
MALLOC_DEFINE(M_NEWNFSSTRING, "NFSD string", "NFSD V4 long string");
MALLOC_DEFINE(M_NEWNFSUSERGROUP, "NFSD usrgroup", "NFSD V4 User/group map");
MALLOC_DEFINE(M_NEWNFSDREQ, "NFS req", "NFS request header");
MALLOC_DEFINE(M_NEWNFSFH, "NFS fh", "NFS file handle");
MALLOC_DEFINE(M_NEWNFSCLOWNER, "NFSCL owner", "NFSCL Open Owner");
MALLOC_DEFINE(M_NEWNFSCLOPEN, "NFSCL open", "NFSCL Open");
MALLOC_DEFINE(M_NEWNFSCLDELEG, "NFSCL deleg", "NFSCL Delegation");
MALLOC_DEFINE(M_NEWNFSCLCLIENT, "NFSCL client", "NFSCL Client");
MALLOC_DEFINE(M_NEWNFSCLLOCKOWNER, "NFSCL lckown", "NFSCL Lock Owner");
MALLOC_DEFINE(M_NEWNFSCLLOCK, "NFSCL lck", "NFSCL Lock");
MALLOC_DEFINE(M_NEWNFSV4NODE, "NEWNFSnode", "NFS vnode");
MALLOC_DEFINE(M_NEWNFSDIRECTIO, "NEWdirectio", "NFS Direct IO buffer");
MALLOC_DEFINE(M_NEWNFSDIROFF, "NFSCL diroffdiroff",
"NFS directory offset data");
MALLOC_DEFINE(M_NEWNFSDROLLBACK, "NFSD rollback",
"NFS local lock rollback");
MALLOC_DEFINE(M_NEWNFSLAYOUT, "NFSCL layout", "NFSv4.1 Layout");
MALLOC_DEFINE(M_NEWNFSFLAYOUT, "NFSCL flayout", "NFSv4.1 File Layout");
MALLOC_DEFINE(M_NEWNFSDEVINFO, "NFSCL devinfo", "NFSv4.1 Device Info");
MALLOC_DEFINE(M_NEWNFSSOCKREQ, "NFSCL sockreq", "NFS Sock Req");
MALLOC_DEFINE(M_NEWNFSCLDS, "NFSCL session", "NFSv4.1 Session");
MALLOC_DEFINE(M_NEWNFSLAYRECALL, "NFSCL layrecall", "NFSv4.1 Layout Recall");
MALLOC_DEFINE(M_NEWNFSDSESSION, "NFSD session", "NFSD Session for a client");
/*
* Definition of mutex locks.
* newnfsd_mtx is used in nfsrvd_nfsd() to protect the nfs socket list
* and assorted other nfsd structures.
*/
struct mtx newnfsd_mtx;
struct mtx nfs_sockl_mutex;
struct mtx nfs_state_mutex;
struct mtx nfs_nameid_mutex;
struct mtx nfs_req_mutex;
struct mtx nfs_slock_mutex;
/* local functions */
static int nfssvc_call(struct thread *, struct nfssvc_args *, struct ucred *);
#ifdef __NO_STRICT_ALIGNMENT
/*
* These architectures don't need re-alignment, so just return.
*/
int
newnfs_realign(struct mbuf **pm, int how)
{
return (0);
}
#else /* !__NO_STRICT_ALIGNMENT */
/*
* newnfs_realign:
*
* Check for badly aligned mbuf data and realign by copying the unaligned
* portion of the data into a new mbuf chain and freeing the portions
* of the old chain that were replaced.
*
* We cannot simply realign the data within the existing mbuf chain
* because the underlying buffers may contain other rpc commands and
* we cannot afford to overwrite them.
*
* We would prefer to avoid this situation entirely. The situation does
* not occur with NFS/UDP and is supposed to only occassionally occur
* with TCP. Use vfs.nfs.realign_count and realign_test to check this.
*
*/
int
newnfs_realign(struct mbuf **pm, int how)
{
struct mbuf *m, *n;
int off, space;
++nfs_realign_test;
while ((m = *pm) != NULL) {
if ((m->m_len & 0x3) || (mtod(m, intptr_t) & 0x3)) {
/*
* NB: we can't depend on m_pkthdr.len to help us
* decide what to do here. May not be worth doing
* the m_length calculation as m_copyback will
* expand the mbuf chain below as needed.
*/
space = m_length(m, NULL);
if (space >= MINCLSIZE) {
/* NB: m_copyback handles space > MCLBYTES */
n = m_getcl(how, MT_DATA, 0);
} else
n = m_get(how, MT_DATA);
if (n == NULL)
return (ENOMEM);
/*
* Align the remainder of the mbuf chain.
*/
n->m_len = 0;
off = 0;
while (m != NULL) {
m_copyback(n, off, m->m_len, mtod(m, caddr_t));
off += m->m_len;
m = m->m_next;
}
m_freem(*pm);
*pm = n;
++nfs_realign_count;
break;
}
pm = &m->m_next;
}
return (0);
}
#endif /* __NO_STRICT_ALIGNMENT */
#ifdef notdef
static void
nfsrv_object_create(struct vnode *vp, struct thread *td)
{
if (vp == NULL || vp->v_type != VREG)
return;
(void) vfs_object_create(vp, td, td->td_ucred);
}
#endif
/*
* Look up a file name. Basically just initialize stuff and call namei().
*/
int
nfsrv_lookupfilename(struct nameidata *ndp, char *fname, NFSPROC_T *p)
{
int error;
NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF, UIO_USERSPACE, fname,
p);
error = namei(ndp);
if (!error) {
NDFREE(ndp, NDF_ONLY_PNBUF);
}
return (error);
}
/*
* Copy NFS uid, gids to the cred structure.
*/
void
newnfs_copycred(struct nfscred *nfscr, struct ucred *cr)
{
KASSERT(nfscr->nfsc_ngroups >= 0,
("newnfs_copycred: negative nfsc_ngroups"));
cr->cr_uid = nfscr->nfsc_uid;
crsetgroups(cr, nfscr->nfsc_ngroups, nfscr->nfsc_groups);
}
/*
* Map args from nfsmsleep() to msleep().
*/
int
nfsmsleep(void *chan, void *mutex, int prio, const char *wmesg,
struct timespec *ts)
{
u_int64_t nsecval;
int error, timeo;
if (ts) {
timeo = hz * ts->tv_sec;
nsecval = (u_int64_t)ts->tv_nsec;
nsecval = ((nsecval * ((u_int64_t)hz)) + 500000000) /
1000000000;
timeo += (int)nsecval;
} else {
timeo = 0;
}
error = msleep(chan, (struct mtx *)mutex, prio, wmesg, timeo);
return (error);
}
/*
* Get the file system info for the server. For now, just assume FFS.
*/
void
nfsvno_getfs(struct nfsfsinfo *sip, int isdgram)
{
int pref;
/*
* XXX
* There should be file system VFS OP(s) to get this information.
* For now, assume ufs.
*/
if (isdgram)
pref = NFS_MAXDGRAMDATA;
else
pref = NFS_SRVMAXIO;
sip->fs_rtmax = NFS_SRVMAXIO;
sip->fs_rtpref = pref;
sip->fs_rtmult = NFS_FABLKSIZE;
sip->fs_wtmax = NFS_SRVMAXIO;
sip->fs_wtpref = pref;
sip->fs_wtmult = NFS_FABLKSIZE;
sip->fs_dtpref = pref;
sip->fs_maxfilesize = 0xffffffffffffffffull;
sip->fs_timedelta.tv_sec = 0;
sip->fs_timedelta.tv_nsec = 1;
sip->fs_properties = (NFSV3FSINFO_LINK |
NFSV3FSINFO_SYMLINK | NFSV3FSINFO_HOMOGENEOUS |
NFSV3FSINFO_CANSETTIME);
}
/*
* Do the pathconf vnode op.
*/
int
nfsvno_pathconf(struct vnode *vp, int flag, register_t *retf,
struct ucred *cred, struct thread *p)
{
int error;
error = VOP_PATHCONF(vp, flag, retf);
if (error == EOPNOTSUPP || error == EINVAL) {
/*
* Some file systems return EINVAL for name arguments not
* supported and some return EOPNOTSUPP for this case.
* So the NFSv3 Pathconf RPC doesn't fail for these cases,
* just fake them.
*/
switch (flag) {
case _PC_LINK_MAX:
*retf = LINK_MAX;
break;
case _PC_NAME_MAX:
*retf = NAME_MAX;
break;
case _PC_CHOWN_RESTRICTED:
*retf = 1;
break;
case _PC_NO_TRUNC:
*retf = 1;
break;
default:
/*
* Only happens if a _PC_xxx is added to the server,
* but this isn't updated.
*/
*retf = 0;
printf("nfsrvd pathconf flag=%d not supp\n", flag);
};
error = 0;
}
NFSEXITCODE(error);
return (error);
}
/* Fake nfsrv_atroot. Just return 0 */
int
nfsrv_atroot(struct vnode *vp, long *retp)
{
return (0);
}
/*
* Set the credentials to refer to root.
* If only the various BSDen could agree on whether cr_gid is a separate
* field or cr_groups[0]...
*/
void
newnfs_setroot(struct ucred *cred)
{
cred->cr_uid = 0;
cred->cr_groups[0] = 0;
cred->cr_ngroups = 1;
}
/*
* Get the client credential. Used for Renew and recovery.
*/
struct ucred *
newnfs_getcred(void)
{
struct ucred *cred;
struct thread *td = curthread;
cred = crdup(td->td_ucred);
newnfs_setroot(cred);
return (cred);
}
/*
* Nfs timer routine
* Call the nfsd's timer function once/sec.
*/
void
newnfs_timer(void *arg)
{
static time_t lasttime = 0;
/*
* Call the server timer, if set up.
* The argument indicates if it is the next second and therefore
* leases should be checked.
*/
if (lasttime != NFSD_MONOSEC) {
lasttime = NFSD_MONOSEC;
if (nfsd_call_servertimer != NULL)
(*nfsd_call_servertimer)();
}
callout_reset(&newnfsd_callout, nfscl_ticks, newnfs_timer, NULL);
}
/*
* Sleep for a short period of time unless errval == NFSERR_GRACE, where
* the sleep should be for 5 seconds.
* Since lbolt doesn't exist in FreeBSD-CURRENT, just use a timeout on
* an event that never gets a wakeup. Only return EINTR or 0.
*/
int
nfs_catnap(int prio, int errval, const char *wmesg)
{
static int non_event;
int ret;
if (errval == NFSERR_GRACE)
ret = tsleep(&non_event, prio, wmesg, 5 * hz);
else
ret = tsleep(&non_event, prio, wmesg, 1);
if (ret != EINTR)
ret = 0;
return (ret);
}
/*
* Get referral. For now, just fail.
*/
struct nfsreferral *
nfsv4root_getreferral(struct vnode *vp, struct vnode *dvp, u_int32_t fileno)
{
return (NULL);
}
static int
nfssvc_nfscommon(struct thread *td, struct nfssvc_args *uap)
{
int error;
error = nfssvc_call(td, uap, td->td_ucred);
NFSEXITCODE(error);
return (error);
}
static int
nfssvc_call(struct thread *p, struct nfssvc_args *uap, struct ucred *cred)
{
int error = EINVAL;
struct nfsd_idargs nid;
if (uap->flag & NFSSVC_IDNAME) {
error = copyin(uap->argp, (caddr_t)&nid, sizeof (nid));
if (error)
goto out;
error = nfssvc_idname(&nid);
goto out;
} else if (uap->flag & NFSSVC_GETSTATS) {
error = copyout(&newnfsstats,
CAST_USER_ADDR_T(uap->argp), sizeof (newnfsstats));
if (error == 0) {
if ((uap->flag & NFSSVC_ZEROCLTSTATS) != 0) {
newnfsstats.attrcache_hits = 0;
newnfsstats.attrcache_misses = 0;
newnfsstats.lookupcache_hits = 0;
newnfsstats.lookupcache_misses = 0;
newnfsstats.direofcache_hits = 0;
newnfsstats.direofcache_misses = 0;
newnfsstats.accesscache_hits = 0;
newnfsstats.accesscache_misses = 0;
newnfsstats.biocache_reads = 0;
newnfsstats.read_bios = 0;
newnfsstats.read_physios = 0;
newnfsstats.biocache_writes = 0;
newnfsstats.write_bios = 0;
newnfsstats.write_physios = 0;
newnfsstats.biocache_readlinks = 0;
newnfsstats.readlink_bios = 0;
newnfsstats.biocache_readdirs = 0;
newnfsstats.readdir_bios = 0;
newnfsstats.rpcretries = 0;
newnfsstats.rpcrequests = 0;
newnfsstats.rpctimeouts = 0;
newnfsstats.rpcunexpected = 0;
newnfsstats.rpcinvalid = 0;
bzero(newnfsstats.rpccnt,
sizeof(newnfsstats.rpccnt));
}
if ((uap->flag & NFSSVC_ZEROSRVSTATS) != 0) {
newnfsstats.srvrpc_errs = 0;
newnfsstats.srv_errs = 0;
newnfsstats.srvcache_inproghits = 0;
newnfsstats.srvcache_idemdonehits = 0;
newnfsstats.srvcache_nonidemdonehits = 0;
newnfsstats.srvcache_misses = 0;
newnfsstats.srvcache_tcppeak = 0;
newnfsstats.srvclients = 0;
newnfsstats.srvopenowners = 0;
newnfsstats.srvopens = 0;
newnfsstats.srvlockowners = 0;
newnfsstats.srvlocks = 0;
newnfsstats.srvdelegates = 0;
newnfsstats.clopenowners = 0;
newnfsstats.clopens = 0;
newnfsstats.cllockowners = 0;
newnfsstats.cllocks = 0;
newnfsstats.cldelegates = 0;
newnfsstats.cllocalopenowners = 0;
newnfsstats.cllocalopens = 0;
newnfsstats.cllocallockowners = 0;
newnfsstats.cllocallocks = 0;
bzero(newnfsstats.srvrpccnt,
sizeof(newnfsstats.srvrpccnt));
bzero(newnfsstats.cbrpccnt,
sizeof(newnfsstats.cbrpccnt));
}
}
goto out;
} else if (uap->flag & NFSSVC_NFSUSERDPORT) {
u_short sockport;
error = copyin(uap->argp, (caddr_t)&sockport,
sizeof (u_short));
if (!error)
error = nfsrv_nfsuserdport(sockport, p);
} else if (uap->flag & NFSSVC_NFSUSERDDELPORT) {
nfsrv_nfsuserddelport();
error = 0;
}
out:
NFSEXITCODE(error);
return (error);
}
/*
* called by all three modevent routines, so that it gets things
* initialized soon enough.
*/
void
newnfs_portinit(void)
{
static int inited = 0;
if (inited)
return;
inited = 1;
/* Initialize SMP locks used by both client and server. */
mtx_init(&newnfsd_mtx, "newnfsd_mtx", NULL, MTX_DEF);
mtx_init(&nfs_state_mutex, "nfs_state_mutex", NULL, MTX_DEF);
}
/*
* Determine if the file system supports NFSv4 ACLs.
* Return 1 if it does, 0 otherwise.
*/
int
nfs_supportsnfsv4acls(struct vnode *vp)
{
int error;
register_t retval;
ASSERT_VOP_LOCKED(vp, "nfs supports nfsv4acls");
if (nfsrv_useacl == 0)
return (0);
error = VOP_PATHCONF(vp, _PC_ACL_NFS4, &retval);
if (error == 0 && retval != 0)
return (1);
return (0);
}
extern int (*nfsd_call_nfscommon)(struct thread *, struct nfssvc_args *);
/*
* Called once to initialize data structures...
*/
static int
nfscommon_modevent(module_t mod, int type, void *data)
{
int error = 0;
static int loaded = 0;
switch (type) {
case MOD_LOAD:
if (loaded)
goto out;
newnfs_portinit();
mtx_init(&nfs_nameid_mutex, "nfs_nameid_mutex", NULL, MTX_DEF);
mtx_init(&nfs_sockl_mutex, "nfs_sockl_mutex", NULL, MTX_DEF);
mtx_init(&nfs_slock_mutex, "nfs_slock_mutex", NULL, MTX_DEF);
mtx_init(&nfs_req_mutex, "nfs_req_mutex", NULL, MTX_DEF);
mtx_init(&nfsrv_nfsuserdsock.nr_mtx, "nfsuserd", NULL,
MTX_DEF);
- callout_init(&newnfsd_callout, CALLOUT_MPSAFE);
+ callout_init(&newnfsd_callout, 1);
newnfs_init();
nfsd_call_nfscommon = nfssvc_nfscommon;
loaded = 1;
break;
case MOD_UNLOAD:
if (newnfs_numnfsd != 0 || nfsrv_nfsuserd != 0 ||
nfs_numnfscbd != 0) {
error = EBUSY;
break;
}
nfsd_call_nfscommon = NULL;
callout_drain(&newnfsd_callout);
/* and get rid of the mutexes */
mtx_destroy(&nfs_nameid_mutex);
mtx_destroy(&newnfsd_mtx);
mtx_destroy(&nfs_state_mutex);
mtx_destroy(&nfs_sockl_mutex);
mtx_destroy(&nfs_slock_mutex);
mtx_destroy(&nfs_req_mutex);
mtx_destroy(&nfsrv_nfsuserdsock.nr_mtx);
loaded = 0;
break;
default:
error = EOPNOTSUPP;
break;
}
out:
NFSEXITCODE(error);
return error;
}
static moduledata_t nfscommon_mod = {
"nfscommon",
nfscommon_modevent,
NULL,
};
DECLARE_MODULE(nfscommon, nfscommon_mod, SI_SUB_VFS, SI_ORDER_ANY);
/* So that loader and kldload(2) can find us, wherever we are.. */
MODULE_VERSION(nfscommon, 1);
MODULE_DEPEND(nfscommon, nfssvc, 1, 1, 1);
MODULE_DEPEND(nfscommon, krpc, 1, 1, 1);
Index: head/sys/gdb/gdb_cons.c
===================================================================
--- head/sys/gdb/gdb_cons.c (revision 283290)
+++ head/sys/gdb/gdb_cons.c (revision 283291)
@@ -1,184 +1,184 @@
/*-
* Copyright (c) 2006 Sam Leffler
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Support for redirecting console msgs to gdb. We register
* a pseudo console to hook cnputc and send stuff to the gdb
* port. The only trickiness here is buffering output so this
* isn't dog slow.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/cons.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/reboot.h>
#include <sys/sysctl.h>
#include <machine/gdb_machdep.h>
#include <machine/kdb.h>
#include <gdb/gdb.h>
#include <gdb/gdb_int.h>
struct gdbcons {
int npending;
/* /2 for hex conversion, -6 for protocol glue */
char buf[GDB_BUFSZ/2 - 6];
struct callout flush;
};
static struct gdbcons state = { -1 };
static int gdbcons_enable = 0;
SYSCTL_INT(_debug, OID_AUTO, gdbcons, CTLFLAG_RWTUN, &gdbcons_enable,
0, "copy console messages to GDB");
static void
gdb_cnprobe(struct consdev *cp)
{
sprintf(cp->cn_name, "gdb");
cp->cn_pri = CN_LOW; /* XXX no way to say "write only" */
}
static void
gdb_cninit(struct consdev *cp)
{
struct gdbcons *c = &state;
/* setup tx buffer and callout */
if (c->npending == -1) {
c->npending = 0;
- callout_init(&c->flush, CALLOUT_MPSAFE);
+ callout_init(&c->flush, 1);
cp->cn_arg = c;
}
}
static void
gdb_cnterm(struct consdev *cp)
{
}
static void
gdb_cngrab(struct consdev *cp)
{
}
static void
gdb_cnungrab(struct consdev *cp)
{
}
static int
gdb_cngetc(struct consdev *cp)
{
return -1;
}
static void
gdb_tx_puthex(int c)
{
const char *hex = "0123456789abcdef";
gdb_tx_char(hex[(c>>4)&0xf]);
gdb_tx_char(hex[(c>>0)&0xf]);
}
static void
gdb_cnflush(void *arg)
{
struct gdbcons *gc = arg;
int i;
gdb_tx_begin('O');
for (i = 0; i < gc->npending; i++)
gdb_tx_puthex(gc->buf[i]);
gdb_tx_end();
gc->npending = 0;
}
/*
* This glop is to figure out when it's safe to use callouts
* to defer buffer flushing. There's probably a better way
* and/or an earlier point in the boot process when it's ok.
*/
static int calloutok = 0;
static void
oktousecallout(void *data __unused)
{
calloutok = 1;
}
SYSINIT(gdbhack, SI_SUB_LAST, SI_ORDER_MIDDLE, oktousecallout, NULL);
static void
gdb_cnputc(struct consdev *cp, int c)
{
struct gdbcons *gc;
if (gdbcons_enable && gdb_cur != NULL && gdb_listening) {
gc = cp->cn_arg;
if (gc->npending != 0) {
/*
* Cancel any pending callout and flush the
* buffer if there's no space for this byte.
*/
if (calloutok)
callout_stop(&gc->flush);
if (gc->npending == sizeof(gc->buf))
gdb_cnflush(gc);
}
gc->buf[gc->npending++] = c;
/*
* Flush on end of line; this is especially helpful
* during boot when we don't have callouts to flush
* the buffer. Otherwise we defer flushing; a 1/4
* second is a guess.
*/
if (c == '\n')
gdb_cnflush(gc);
else if (calloutok)
callout_reset(&gc->flush, hz/4, gdb_cnflush, gc);
}
}
CONSOLE_DRIVER(gdb);
/*
* Our console device only gets attached if the system is booted
* with RB_MULTIPLE set so gdb_init also calls us to attach the
* console so we're setup regardless.
*/
void
gdb_consinit(void)
{
gdb_cnprobe(&gdb_consdev);
gdb_cninit(&gdb_consdev);
cnadd(&gdb_consdev);
}
Index: head/sys/geom/gate/g_gate.c
===================================================================
--- head/sys/geom/gate/g_gate.c (revision 283290)
+++ head/sys/geom/gate/g_gate.c (revision 283291)
@@ -1,965 +1,965 @@
/*-
* Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
* Copyright (c) 2009-2010 The FreeBSD Foundation
* All rights reserved.
*
* Portions of this software were developed by Pawel Jakub Dawidek
* under sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/conf.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/fcntl.h>
#include <sys/linker.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/limits.h>
#include <sys/queue.h>
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include <sys/signalvar.h>
#include <sys/time.h>
#include <machine/atomic.h>
#include <geom/geom.h>
#include <geom/gate/g_gate.h>
FEATURE(geom_gate, "GEOM Gate module");
static MALLOC_DEFINE(M_GATE, "gg_data", "GEOM Gate Data");
SYSCTL_DECL(_kern_geom);
static SYSCTL_NODE(_kern_geom, OID_AUTO, gate, CTLFLAG_RW, 0,
"GEOM_GATE configuration");
static int g_gate_debug = 0;
SYSCTL_INT(_kern_geom_gate, OID_AUTO, debug, CTLFLAG_RWTUN, &g_gate_debug, 0,
"Debug level");
static u_int g_gate_maxunits = 256;
SYSCTL_UINT(_kern_geom_gate, OID_AUTO, maxunits, CTLFLAG_RDTUN,
&g_gate_maxunits, 0, "Maximum number of ggate devices");
struct g_class g_gate_class = {
.name = G_GATE_CLASS_NAME,
.version = G_VERSION,
};
static struct cdev *status_dev;
static d_ioctl_t g_gate_ioctl;
static struct cdevsw g_gate_cdevsw = {
.d_version = D_VERSION,
.d_ioctl = g_gate_ioctl,
.d_name = G_GATE_CTL_NAME
};
static struct g_gate_softc **g_gate_units;
static u_int g_gate_nunits;
static struct mtx g_gate_units_lock;
static int
g_gate_destroy(struct g_gate_softc *sc, boolean_t force)
{
struct bio_queue_head queue;
struct g_provider *pp;
struct g_consumer *cp;
struct g_geom *gp;
struct bio *bp;
g_topology_assert();
mtx_assert(&g_gate_units_lock, MA_OWNED);
pp = sc->sc_provider;
if (!force && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
mtx_unlock(&g_gate_units_lock);
return (EBUSY);
}
mtx_unlock(&g_gate_units_lock);
mtx_lock(&sc->sc_queue_mtx);
if ((sc->sc_flags & G_GATE_FLAG_DESTROY) == 0)
sc->sc_flags |= G_GATE_FLAG_DESTROY;
wakeup(sc);
mtx_unlock(&sc->sc_queue_mtx);
gp = pp->geom;
pp->flags |= G_PF_WITHER;
g_orphan_provider(pp, ENXIO);
callout_drain(&sc->sc_callout);
bioq_init(&queue);
mtx_lock(&sc->sc_queue_mtx);
while ((bp = bioq_takefirst(&sc->sc_inqueue)) != NULL) {
sc->sc_queue_count--;
bioq_insert_tail(&queue, bp);
}
while ((bp = bioq_takefirst(&sc->sc_outqueue)) != NULL) {
sc->sc_queue_count--;
bioq_insert_tail(&queue, bp);
}
mtx_unlock(&sc->sc_queue_mtx);
g_topology_unlock();
while ((bp = bioq_takefirst(&queue)) != NULL) {
G_GATE_LOGREQ(1, bp, "Request canceled.");
g_io_deliver(bp, ENXIO);
}
mtx_lock(&g_gate_units_lock);
/* One reference is ours. */
sc->sc_ref--;
while (sc->sc_ref > 0)
msleep(&sc->sc_ref, &g_gate_units_lock, 0, "gg:destroy", 0);
g_gate_units[sc->sc_unit] = NULL;
KASSERT(g_gate_nunits > 0, ("negative g_gate_nunits?"));
g_gate_nunits--;
mtx_unlock(&g_gate_units_lock);
mtx_destroy(&sc->sc_queue_mtx);
g_topology_lock();
if ((cp = sc->sc_readcons) != NULL) {
sc->sc_readcons = NULL;
(void)g_access(cp, -1, 0, 0);
g_detach(cp);
g_destroy_consumer(cp);
}
G_GATE_DEBUG(1, "Device %s destroyed.", gp->name);
gp->softc = NULL;
g_wither_geom(gp, ENXIO);
sc->sc_provider = NULL;
free(sc, M_GATE);
return (0);
}
static int
g_gate_access(struct g_provider *pp, int dr, int dw, int de)
{
struct g_gate_softc *sc;
if (dr <= 0 && dw <= 0 && de <= 0)
return (0);
sc = pp->geom->softc;
if (sc == NULL || (sc->sc_flags & G_GATE_FLAG_DESTROY) != 0)
return (ENXIO);
/* XXX: Hack to allow read-only mounts. */
#if 0
if ((sc->sc_flags & G_GATE_FLAG_READONLY) != 0 && dw > 0)
return (EPERM);
#endif
if ((sc->sc_flags & G_GATE_FLAG_WRITEONLY) != 0 && dr > 0)
return (EPERM);
return (0);
}
static void
g_gate_queue_io(struct bio *bp)
{
struct g_gate_softc *sc;
sc = bp->bio_to->geom->softc;
if (sc == NULL || (sc->sc_flags & G_GATE_FLAG_DESTROY) != 0) {
g_io_deliver(bp, ENXIO);
return;
}
mtx_lock(&sc->sc_queue_mtx);
if (sc->sc_queue_size > 0 && sc->sc_queue_count > sc->sc_queue_size) {
mtx_unlock(&sc->sc_queue_mtx);
G_GATE_LOGREQ(1, bp, "Queue full, request canceled.");
g_io_deliver(bp, ENOMEM);
return;
}
bp->bio_driver1 = (void *)sc->sc_seq;
sc->sc_seq++;
sc->sc_queue_count++;
bioq_insert_tail(&sc->sc_inqueue, bp);
wakeup(sc);
mtx_unlock(&sc->sc_queue_mtx);
}
static void
g_gate_done(struct bio *cbp)
{
struct bio *pbp;
pbp = cbp->bio_parent;
if (cbp->bio_error == 0) {
pbp->bio_completed = cbp->bio_completed;
g_destroy_bio(cbp);
pbp->bio_inbed++;
g_io_deliver(pbp, 0);
} else {
/* If direct read failed, pass it through userland daemon. */
g_destroy_bio(cbp);
pbp->bio_children--;
g_gate_queue_io(pbp);
}
}
static void
g_gate_start(struct bio *pbp)
{
struct g_gate_softc *sc;
sc = pbp->bio_to->geom->softc;
if (sc == NULL || (sc->sc_flags & G_GATE_FLAG_DESTROY) != 0) {
g_io_deliver(pbp, ENXIO);
return;
}
G_GATE_LOGREQ(2, pbp, "Request received.");
switch (pbp->bio_cmd) {
case BIO_READ:
if (sc->sc_readcons != NULL) {
struct bio *cbp;
cbp = g_clone_bio(pbp);
if (cbp == NULL) {
g_io_deliver(pbp, ENOMEM);
return;
}
cbp->bio_done = g_gate_done;
cbp->bio_offset = pbp->bio_offset + sc->sc_readoffset;
cbp->bio_to = sc->sc_readcons->provider;
g_io_request(cbp, sc->sc_readcons);
return;
}
break;
case BIO_DELETE:
case BIO_WRITE:
case BIO_FLUSH:
/* XXX: Hack to allow read-only mounts. */
if ((sc->sc_flags & G_GATE_FLAG_READONLY) != 0) {
g_io_deliver(pbp, EPERM);
return;
}
break;
case BIO_GETATTR:
default:
G_GATE_LOGREQ(2, pbp, "Ignoring request.");
g_io_deliver(pbp, EOPNOTSUPP);
return;
}
g_gate_queue_io(pbp);
}
static struct g_gate_softc *
g_gate_hold(int unit, const char *name)
{
struct g_gate_softc *sc = NULL;
mtx_lock(&g_gate_units_lock);
if (unit >= 0 && unit < g_gate_maxunits)
sc = g_gate_units[unit];
else if (unit == G_GATE_NAME_GIVEN) {
KASSERT(name != NULL, ("name is NULL"));
for (unit = 0; unit < g_gate_maxunits; unit++) {
if (g_gate_units[unit] == NULL)
continue;
if (strcmp(name,
g_gate_units[unit]->sc_provider->name) != 0) {
continue;
}
sc = g_gate_units[unit];
break;
}
}
if (sc != NULL)
sc->sc_ref++;
mtx_unlock(&g_gate_units_lock);
return (sc);
}
static void
g_gate_release(struct g_gate_softc *sc)
{
g_topology_assert_not();
mtx_lock(&g_gate_units_lock);
sc->sc_ref--;
KASSERT(sc->sc_ref >= 0, ("Negative sc_ref for %s.", sc->sc_name));
if (sc->sc_ref == 0 && (sc->sc_flags & G_GATE_FLAG_DESTROY) != 0)
wakeup(&sc->sc_ref);
mtx_unlock(&g_gate_units_lock);
}
static int
g_gate_getunit(int unit, int *errorp)
{
mtx_assert(&g_gate_units_lock, MA_OWNED);
if (unit >= 0) {
if (unit >= g_gate_maxunits)
*errorp = EINVAL;
else if (g_gate_units[unit] == NULL)
return (unit);
else
*errorp = EEXIST;
} else {
for (unit = 0; unit < g_gate_maxunits; unit++) {
if (g_gate_units[unit] == NULL)
return (unit);
}
*errorp = ENFILE;
}
return (-1);
}
static void
g_gate_guard(void *arg)
{
struct bio_queue_head queue;
struct g_gate_softc *sc;
struct bintime curtime;
struct bio *bp, *bp2;
sc = arg;
binuptime(&curtime);
g_gate_hold(sc->sc_unit, NULL);
bioq_init(&queue);
mtx_lock(&sc->sc_queue_mtx);
TAILQ_FOREACH_SAFE(bp, &sc->sc_inqueue.queue, bio_queue, bp2) {
if (curtime.sec - bp->bio_t0.sec < 5)
continue;
bioq_remove(&sc->sc_inqueue, bp);
sc->sc_queue_count--;
bioq_insert_tail(&queue, bp);
}
TAILQ_FOREACH_SAFE(bp, &sc->sc_outqueue.queue, bio_queue, bp2) {
if (curtime.sec - bp->bio_t0.sec < 5)
continue;
bioq_remove(&sc->sc_outqueue, bp);
sc->sc_queue_count--;
bioq_insert_tail(&queue, bp);
}
mtx_unlock(&sc->sc_queue_mtx);
while ((bp = bioq_takefirst(&queue)) != NULL) {
G_GATE_LOGREQ(1, bp, "Request timeout.");
g_io_deliver(bp, EIO);
}
if ((sc->sc_flags & G_GATE_FLAG_DESTROY) == 0) {
callout_reset(&sc->sc_callout, sc->sc_timeout * hz,
g_gate_guard, sc);
}
g_gate_release(sc);
}
static void
g_gate_orphan(struct g_consumer *cp)
{
struct g_gate_softc *sc;
struct g_geom *gp;
g_topology_assert();
gp = cp->geom;
sc = gp->softc;
if (sc == NULL)
return;
KASSERT(cp == sc->sc_readcons, ("cp=%p sc_readcons=%p", cp,
sc->sc_readcons));
sc->sc_readcons = NULL;
G_GATE_DEBUG(1, "Destroying read consumer on provider %s orphan.",
cp->provider->name);
(void)g_access(cp, -1, 0, 0);
g_detach(cp);
g_destroy_consumer(cp);
}
static void
g_gate_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
struct g_consumer *cp, struct g_provider *pp)
{
struct g_gate_softc *sc;
sc = gp->softc;
if (sc == NULL || pp != NULL || cp != NULL)
return;
sc = g_gate_hold(sc->sc_unit, NULL);
if (sc == NULL)
return;
if ((sc->sc_flags & G_GATE_FLAG_READONLY) != 0) {
sbuf_printf(sb, "%s<access>%s</access>\n", indent, "read-only");
} else if ((sc->sc_flags & G_GATE_FLAG_WRITEONLY) != 0) {
sbuf_printf(sb, "%s<access>%s</access>\n", indent,
"write-only");
} else {
sbuf_printf(sb, "%s<access>%s</access>\n", indent,
"read-write");
}
if (sc->sc_readcons != NULL) {
sbuf_printf(sb, "%s<read_offset>%jd</read_offset>\n",
indent, (intmax_t)sc->sc_readoffset);
sbuf_printf(sb, "%s<read_provider>%s</read_provider>\n",
indent, sc->sc_readcons->provider->name);
}
sbuf_printf(sb, "%s<timeout>%u</timeout>\n", indent, sc->sc_timeout);
sbuf_printf(sb, "%s<info>%s</info>\n", indent, sc->sc_info);
sbuf_printf(sb, "%s<queue_count>%u</queue_count>\n", indent,
sc->sc_queue_count);
sbuf_printf(sb, "%s<queue_size>%u</queue_size>\n", indent,
sc->sc_queue_size);
sbuf_printf(sb, "%s<ref>%u</ref>\n", indent, sc->sc_ref);
sbuf_printf(sb, "%s<unit>%d</unit>\n", indent, sc->sc_unit);
g_topology_unlock();
g_gate_release(sc);
g_topology_lock();
}
static int
g_gate_create(struct g_gate_ctl_create *ggio)
{
struct g_gate_softc *sc;
struct g_geom *gp;
struct g_provider *pp, *ropp;
struct g_consumer *cp;
char name[NAME_MAX];
int error = 0, unit;
if (ggio->gctl_mediasize <= 0) {
G_GATE_DEBUG(1, "Invalid media size.");
return (EINVAL);
}
if (ggio->gctl_sectorsize <= 0) {
G_GATE_DEBUG(1, "Invalid sector size.");
return (EINVAL);
}
if (!powerof2(ggio->gctl_sectorsize)) {
G_GATE_DEBUG(1, "Invalid sector size.");
return (EINVAL);
}
if ((ggio->gctl_mediasize % ggio->gctl_sectorsize) != 0) {
G_GATE_DEBUG(1, "Invalid media size.");
return (EINVAL);
}
if ((ggio->gctl_flags & G_GATE_FLAG_READONLY) != 0 &&
(ggio->gctl_flags & G_GATE_FLAG_WRITEONLY) != 0) {
G_GATE_DEBUG(1, "Invalid flags.");
return (EINVAL);
}
if (ggio->gctl_unit != G_GATE_UNIT_AUTO &&
ggio->gctl_unit != G_GATE_NAME_GIVEN &&
ggio->gctl_unit < 0) {
G_GATE_DEBUG(1, "Invalid unit number.");
return (EINVAL);
}
if (ggio->gctl_unit == G_GATE_NAME_GIVEN &&
ggio->gctl_name[0] == '\0') {
G_GATE_DEBUG(1, "No device name.");
return (EINVAL);
}
sc = malloc(sizeof(*sc), M_GATE, M_WAITOK | M_ZERO);
sc->sc_flags = (ggio->gctl_flags & G_GATE_USERFLAGS);
strlcpy(sc->sc_info, ggio->gctl_info, sizeof(sc->sc_info));
sc->sc_seq = 1;
bioq_init(&sc->sc_inqueue);
bioq_init(&sc->sc_outqueue);
mtx_init(&sc->sc_queue_mtx, "gg:queue", NULL, MTX_DEF);
sc->sc_queue_count = 0;
sc->sc_queue_size = ggio->gctl_maxcount;
if (sc->sc_queue_size > G_GATE_MAX_QUEUE_SIZE)
sc->sc_queue_size = G_GATE_MAX_QUEUE_SIZE;
sc->sc_timeout = ggio->gctl_timeout;
- callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_callout, 1);
mtx_lock(&g_gate_units_lock);
sc->sc_unit = g_gate_getunit(ggio->gctl_unit, &error);
if (sc->sc_unit < 0)
goto fail1;
if (ggio->gctl_unit == G_GATE_NAME_GIVEN)
snprintf(name, sizeof(name), "%s", ggio->gctl_name);
else {
snprintf(name, sizeof(name), "%s%d", G_GATE_PROVIDER_NAME,
sc->sc_unit);
}
/* Check for name collision. */
for (unit = 0; unit < g_gate_maxunits; unit++) {
if (g_gate_units[unit] == NULL)
continue;
if (strcmp(name, g_gate_units[unit]->sc_name) != 0)
continue;
error = EEXIST;
goto fail1;
}
sc->sc_name = name;
g_gate_units[sc->sc_unit] = sc;
g_gate_nunits++;
mtx_unlock(&g_gate_units_lock);
g_topology_lock();
if (ggio->gctl_readprov[0] == '\0') {
ropp = NULL;
} else {
ropp = g_provider_by_name(ggio->gctl_readprov);
if (ropp == NULL) {
G_GATE_DEBUG(1, "Provider %s doesn't exist.",
ggio->gctl_readprov);
error = EINVAL;
goto fail2;
}
if ((ggio->gctl_readoffset % ggio->gctl_sectorsize) != 0) {
G_GATE_DEBUG(1, "Invalid read offset.");
error = EINVAL;
goto fail2;
}
if (ggio->gctl_mediasize + ggio->gctl_readoffset >
ropp->mediasize) {
G_GATE_DEBUG(1, "Invalid read offset or media size.");
error = EINVAL;
goto fail2;
}
}
gp = g_new_geomf(&g_gate_class, "%s", name);
gp->start = g_gate_start;
gp->access = g_gate_access;
gp->orphan = g_gate_orphan;
gp->dumpconf = g_gate_dumpconf;
gp->softc = sc;
if (ropp != NULL) {
cp = g_new_consumer(gp);
cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
error = g_attach(cp, ropp);
if (error != 0) {
G_GATE_DEBUG(1, "Unable to attach to %s.", ropp->name);
goto fail3;
}
error = g_access(cp, 1, 0, 0);
if (error != 0) {
G_GATE_DEBUG(1, "Unable to access %s.", ropp->name);
g_detach(cp);
goto fail3;
}
sc->sc_readcons = cp;
sc->sc_readoffset = ggio->gctl_readoffset;
}
ggio->gctl_unit = sc->sc_unit;
pp = g_new_providerf(gp, "%s", name);
pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE;
pp->mediasize = ggio->gctl_mediasize;
pp->sectorsize = ggio->gctl_sectorsize;
sc->sc_provider = pp;
g_error_provider(pp, 0);
g_topology_unlock();
mtx_lock(&g_gate_units_lock);
sc->sc_name = sc->sc_provider->name;
mtx_unlock(&g_gate_units_lock);
G_GATE_DEBUG(1, "Device %s created.", gp->name);
if (sc->sc_timeout > 0) {
callout_reset(&sc->sc_callout, sc->sc_timeout * hz,
g_gate_guard, sc);
}
return (0);
fail3:
g_destroy_consumer(cp);
g_destroy_geom(gp);
fail2:
g_topology_unlock();
mtx_lock(&g_gate_units_lock);
g_gate_units[sc->sc_unit] = NULL;
KASSERT(g_gate_nunits > 0, ("negative g_gate_nunits?"));
g_gate_nunits--;
fail1:
mtx_unlock(&g_gate_units_lock);
mtx_destroy(&sc->sc_queue_mtx);
free(sc, M_GATE);
return (error);
}
static int
g_gate_modify(struct g_gate_softc *sc, struct g_gate_ctl_modify *ggio)
{
struct g_provider *pp;
struct g_consumer *cp;
int error;
if ((ggio->gctl_modify & GG_MODIFY_MEDIASIZE) != 0) {
if (ggio->gctl_mediasize <= 0) {
G_GATE_DEBUG(1, "Invalid media size.");
return (EINVAL);
}
pp = sc->sc_provider;
if ((ggio->gctl_mediasize % pp->sectorsize) != 0) {
G_GATE_DEBUG(1, "Invalid media size.");
return (EINVAL);
}
/* TODO */
return (EOPNOTSUPP);
}
if ((ggio->gctl_modify & GG_MODIFY_INFO) != 0)
(void)strlcpy(sc->sc_info, ggio->gctl_info, sizeof(sc->sc_info));
cp = NULL;
if ((ggio->gctl_modify & GG_MODIFY_READPROV) != 0) {
g_topology_lock();
if (sc->sc_readcons != NULL) {
cp = sc->sc_readcons;
sc->sc_readcons = NULL;
(void)g_access(cp, -1, 0, 0);
g_detach(cp);
g_destroy_consumer(cp);
}
if (ggio->gctl_readprov[0] != '\0') {
pp = g_provider_by_name(ggio->gctl_readprov);
if (pp == NULL) {
g_topology_unlock();
G_GATE_DEBUG(1, "Provider %s doesn't exist.",
ggio->gctl_readprov);
return (EINVAL);
}
cp = g_new_consumer(sc->sc_provider->geom);
cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
error = g_attach(cp, pp);
if (error != 0) {
G_GATE_DEBUG(1, "Unable to attach to %s.",
pp->name);
} else {
error = g_access(cp, 1, 0, 0);
if (error != 0) {
G_GATE_DEBUG(1, "Unable to access %s.",
pp->name);
g_detach(cp);
}
}
if (error != 0) {
g_destroy_consumer(cp);
g_topology_unlock();
return (error);
}
}
} else {
cp = sc->sc_readcons;
}
if ((ggio->gctl_modify & GG_MODIFY_READOFFSET) != 0) {
if (cp == NULL) {
G_GATE_DEBUG(1, "No read provider.");
return (EINVAL);
}
pp = sc->sc_provider;
if ((ggio->gctl_readoffset % pp->sectorsize) != 0) {
G_GATE_DEBUG(1, "Invalid read offset.");
return (EINVAL);
}
if (pp->mediasize + ggio->gctl_readoffset >
cp->provider->mediasize) {
G_GATE_DEBUG(1, "Invalid read offset or media size.");
return (EINVAL);
}
sc->sc_readoffset = ggio->gctl_readoffset;
}
if ((ggio->gctl_modify & GG_MODIFY_READPROV) != 0) {
sc->sc_readcons = cp;
g_topology_unlock();
}
return (0);
}
#define G_GATE_CHECK_VERSION(ggio) do { \
if ((ggio)->gctl_version != G_GATE_VERSION) { \
printf("Version mismatch %d != %d.\n", \
ggio->gctl_version, G_GATE_VERSION); \
return (EINVAL); \
} \
} while (0)
static int
g_gate_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td)
{
struct g_gate_softc *sc;
struct bio *bp;
int error = 0;
G_GATE_DEBUG(4, "ioctl(%s, %lx, %p, %x, %p)", devtoname(dev), cmd, addr,
flags, td);
switch (cmd) {
case G_GATE_CMD_CREATE:
{
struct g_gate_ctl_create *ggio = (void *)addr;
G_GATE_CHECK_VERSION(ggio);
error = g_gate_create(ggio);
/*
* Reset TDP_GEOM flag.
* There are pending events for sure, because we just created
* new provider and other classes want to taste it, but we
* cannot answer on I/O requests until we're here.
*/
td->td_pflags &= ~TDP_GEOM;
return (error);
}
case G_GATE_CMD_MODIFY:
{
struct g_gate_ctl_modify *ggio = (void *)addr;
G_GATE_CHECK_VERSION(ggio);
sc = g_gate_hold(ggio->gctl_unit, NULL);
if (sc == NULL)
return (ENXIO);
error = g_gate_modify(sc, ggio);
g_gate_release(sc);
return (error);
}
case G_GATE_CMD_DESTROY:
{
struct g_gate_ctl_destroy *ggio = (void *)addr;
G_GATE_CHECK_VERSION(ggio);
sc = g_gate_hold(ggio->gctl_unit, ggio->gctl_name);
if (sc == NULL)
return (ENXIO);
g_topology_lock();
mtx_lock(&g_gate_units_lock);
error = g_gate_destroy(sc, ggio->gctl_force);
g_topology_unlock();
if (error != 0)
g_gate_release(sc);
return (error);
}
case G_GATE_CMD_CANCEL:
{
struct g_gate_ctl_cancel *ggio = (void *)addr;
struct bio *tbp, *lbp;
G_GATE_CHECK_VERSION(ggio);
sc = g_gate_hold(ggio->gctl_unit, ggio->gctl_name);
if (sc == NULL)
return (ENXIO);
lbp = NULL;
mtx_lock(&sc->sc_queue_mtx);
TAILQ_FOREACH_SAFE(bp, &sc->sc_outqueue.queue, bio_queue, tbp) {
if (ggio->gctl_seq == 0 ||
ggio->gctl_seq == (uintptr_t)bp->bio_driver1) {
G_GATE_LOGREQ(1, bp, "Request canceled.");
bioq_remove(&sc->sc_outqueue, bp);
/*
* Be sure to put requests back onto incoming
* queue in the proper order.
*/
if (lbp == NULL)
bioq_insert_head(&sc->sc_inqueue, bp);
else {
TAILQ_INSERT_AFTER(&sc->sc_inqueue.queue,
lbp, bp, bio_queue);
}
lbp = bp;
/*
* If only one request was canceled, leave now.
*/
if (ggio->gctl_seq != 0)
break;
}
}
if (ggio->gctl_unit == G_GATE_NAME_GIVEN)
ggio->gctl_unit = sc->sc_unit;
mtx_unlock(&sc->sc_queue_mtx);
g_gate_release(sc);
return (error);
}
case G_GATE_CMD_START:
{
struct g_gate_ctl_io *ggio = (void *)addr;
G_GATE_CHECK_VERSION(ggio);
sc = g_gate_hold(ggio->gctl_unit, NULL);
if (sc == NULL)
return (ENXIO);
error = 0;
for (;;) {
mtx_lock(&sc->sc_queue_mtx);
bp = bioq_first(&sc->sc_inqueue);
if (bp != NULL)
break;
if ((sc->sc_flags & G_GATE_FLAG_DESTROY) != 0) {
ggio->gctl_error = ECANCELED;
mtx_unlock(&sc->sc_queue_mtx);
goto start_end;
}
if (msleep(sc, &sc->sc_queue_mtx,
PPAUSE | PDROP | PCATCH, "ggwait", 0) != 0) {
ggio->gctl_error = ECANCELED;
goto start_end;
}
}
ggio->gctl_cmd = bp->bio_cmd;
if (bp->bio_cmd == BIO_WRITE &&
bp->bio_length > ggio->gctl_length) {
mtx_unlock(&sc->sc_queue_mtx);
ggio->gctl_length = bp->bio_length;
ggio->gctl_error = ENOMEM;
goto start_end;
}
bioq_remove(&sc->sc_inqueue, bp);
bioq_insert_tail(&sc->sc_outqueue, bp);
mtx_unlock(&sc->sc_queue_mtx);
ggio->gctl_seq = (uintptr_t)bp->bio_driver1;
ggio->gctl_offset = bp->bio_offset;
ggio->gctl_length = bp->bio_length;
switch (bp->bio_cmd) {
case BIO_READ:
case BIO_DELETE:
case BIO_FLUSH:
break;
case BIO_WRITE:
error = copyout(bp->bio_data, ggio->gctl_data,
bp->bio_length);
if (error != 0) {
mtx_lock(&sc->sc_queue_mtx);
bioq_remove(&sc->sc_outqueue, bp);
bioq_insert_head(&sc->sc_inqueue, bp);
mtx_unlock(&sc->sc_queue_mtx);
goto start_end;
}
break;
}
start_end:
g_gate_release(sc);
return (error);
}
case G_GATE_CMD_DONE:
{
struct g_gate_ctl_io *ggio = (void *)addr;
G_GATE_CHECK_VERSION(ggio);
sc = g_gate_hold(ggio->gctl_unit, NULL);
if (sc == NULL)
return (ENOENT);
error = 0;
mtx_lock(&sc->sc_queue_mtx);
TAILQ_FOREACH(bp, &sc->sc_outqueue.queue, bio_queue) {
if (ggio->gctl_seq == (uintptr_t)bp->bio_driver1)
break;
}
if (bp != NULL) {
bioq_remove(&sc->sc_outqueue, bp);
sc->sc_queue_count--;
}
mtx_unlock(&sc->sc_queue_mtx);
if (bp == NULL) {
/*
* Request was probably canceled.
*/
goto done_end;
}
if (ggio->gctl_error == EAGAIN) {
bp->bio_error = 0;
G_GATE_LOGREQ(1, bp, "Request desisted.");
mtx_lock(&sc->sc_queue_mtx);
sc->sc_queue_count++;
bioq_insert_head(&sc->sc_inqueue, bp);
wakeup(sc);
mtx_unlock(&sc->sc_queue_mtx);
} else {
bp->bio_error = ggio->gctl_error;
if (bp->bio_error == 0) {
bp->bio_completed = bp->bio_length;
switch (bp->bio_cmd) {
case BIO_READ:
error = copyin(ggio->gctl_data,
bp->bio_data, bp->bio_length);
if (error != 0)
bp->bio_error = error;
break;
case BIO_DELETE:
case BIO_WRITE:
case BIO_FLUSH:
break;
}
}
G_GATE_LOGREQ(2, bp, "Request done.");
g_io_deliver(bp, bp->bio_error);
}
done_end:
g_gate_release(sc);
return (error);
}
}
return (ENOIOCTL);
}
static void
g_gate_device(void)
{
status_dev = make_dev(&g_gate_cdevsw, 0x0, UID_ROOT, GID_WHEEL, 0600,
G_GATE_CTL_NAME);
}
static int
g_gate_modevent(module_t mod, int type, void *data)
{
int error = 0;
switch (type) {
case MOD_LOAD:
mtx_init(&g_gate_units_lock, "gg_units_lock", NULL, MTX_DEF);
g_gate_units = malloc(g_gate_maxunits * sizeof(g_gate_units[0]),
M_GATE, M_WAITOK | M_ZERO);
g_gate_nunits = 0;
g_gate_device();
break;
case MOD_UNLOAD:
mtx_lock(&g_gate_units_lock);
if (g_gate_nunits > 0) {
mtx_unlock(&g_gate_units_lock);
error = EBUSY;
break;
}
mtx_unlock(&g_gate_units_lock);
mtx_destroy(&g_gate_units_lock);
if (status_dev != 0)
destroy_dev(status_dev);
free(g_gate_units, M_GATE);
break;
default:
return (EOPNOTSUPP);
break;
}
return (error);
}
static moduledata_t g_gate_module = {
G_GATE_MOD_NAME,
g_gate_modevent,
NULL
};
DECLARE_MODULE(geom_gate, g_gate_module, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
DECLARE_GEOM_CLASS(g_gate_class, g_gate);
Index: head/sys/geom/journal/g_journal.c
===================================================================
--- head/sys/geom/journal/g_journal.c (revision 283290)
+++ head/sys/geom/journal/g_journal.c (revision 283291)
@@ -1,3048 +1,3048 @@
/*-
* Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/bio.h>
#include <sys/sysctl.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/eventhandler.h>
#include <sys/proc.h>
#include <sys/kthread.h>
#include <sys/sched.h>
#include <sys/taskqueue.h>
#include <sys/vnode.h>
#include <sys/sbuf.h>
#ifdef GJ_MEMDEBUG
#include <sys/stack.h>
#include <sys/kdb.h>
#endif
#include <vm/vm.h>
#include <vm/vm_kern.h>
#include <geom/geom.h>
#include <geom/journal/g_journal.h>
FEATURE(geom_journal, "GEOM journaling support");
/*
* On-disk journal format:
*
* JH - Journal header
* RH - Record header
*
* %%%%%% ****** +------+ +------+ ****** +------+ %%%%%%
* % JH % * RH * | Data | | Data | ... * RH * | Data | ... % JH % ...
* %%%%%% ****** +------+ +------+ ****** +------+ %%%%%%
*
*/
CTASSERT(sizeof(struct g_journal_header) <= 512);
CTASSERT(sizeof(struct g_journal_record_header) <= 512);
static MALLOC_DEFINE(M_JOURNAL, "journal_data", "GEOM_JOURNAL Data");
static struct mtx g_journal_cache_mtx;
MTX_SYSINIT(g_journal_cache, &g_journal_cache_mtx, "cache usage", MTX_DEF);
const struct g_journal_desc *g_journal_filesystems[] = {
&g_journal_ufs,
NULL
};
SYSCTL_DECL(_kern_geom);
int g_journal_debug = 0;
static u_int g_journal_switch_time = 10;
static u_int g_journal_force_switch = 70;
static u_int g_journal_parallel_flushes = 16;
static u_int g_journal_parallel_copies = 16;
static u_int g_journal_accept_immediately = 64;
static u_int g_journal_record_entries = GJ_RECORD_HEADER_NENTRIES;
static u_int g_journal_do_optimize = 1;
static SYSCTL_NODE(_kern_geom, OID_AUTO, journal, CTLFLAG_RW, 0,
"GEOM_JOURNAL stuff");
SYSCTL_INT(_kern_geom_journal, OID_AUTO, debug, CTLFLAG_RWTUN, &g_journal_debug, 0,
"Debug level");
SYSCTL_UINT(_kern_geom_journal, OID_AUTO, switch_time, CTLFLAG_RW,
&g_journal_switch_time, 0, "Switch journals every N seconds");
SYSCTL_UINT(_kern_geom_journal, OID_AUTO, force_switch, CTLFLAG_RW,
&g_journal_force_switch, 0, "Force switch when journal is N% full");
SYSCTL_UINT(_kern_geom_journal, OID_AUTO, parallel_flushes, CTLFLAG_RW,
&g_journal_parallel_flushes, 0,
"Number of flush I/O requests to send in parallel");
SYSCTL_UINT(_kern_geom_journal, OID_AUTO, accept_immediately, CTLFLAG_RW,
&g_journal_accept_immediately, 0,
"Number of I/O requests accepted immediately");
SYSCTL_UINT(_kern_geom_journal, OID_AUTO, parallel_copies, CTLFLAG_RW,
&g_journal_parallel_copies, 0,
"Number of copy I/O requests to send in parallel");
static int
g_journal_record_entries_sysctl(SYSCTL_HANDLER_ARGS)
{
u_int entries;
int error;
entries = g_journal_record_entries;
error = sysctl_handle_int(oidp, &entries, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
if (entries < 1 || entries > GJ_RECORD_HEADER_NENTRIES)
return (EINVAL);
g_journal_record_entries = entries;
return (0);
}
SYSCTL_PROC(_kern_geom_journal, OID_AUTO, record_entries,
CTLTYPE_UINT | CTLFLAG_RW, NULL, 0, g_journal_record_entries_sysctl, "I",
"Maximum number of entires in one journal record");
SYSCTL_UINT(_kern_geom_journal, OID_AUTO, optimize, CTLFLAG_RW,
&g_journal_do_optimize, 0, "Try to combine bios on flush and copy");
static u_int g_journal_cache_used = 0;
static u_int g_journal_cache_limit = 64 * 1024 * 1024;
static u_int g_journal_cache_divisor = 2;
static u_int g_journal_cache_switch = 90;
static u_int g_journal_cache_misses = 0;
static u_int g_journal_cache_alloc_failures = 0;
static u_int g_journal_cache_low = 0;
static SYSCTL_NODE(_kern_geom_journal, OID_AUTO, cache, CTLFLAG_RW, 0,
"GEOM_JOURNAL cache");
SYSCTL_UINT(_kern_geom_journal_cache, OID_AUTO, used, CTLFLAG_RD,
&g_journal_cache_used, 0, "Number of allocated bytes");
static int
g_journal_cache_limit_sysctl(SYSCTL_HANDLER_ARGS)
{
u_int limit;
int error;
limit = g_journal_cache_limit;
error = sysctl_handle_int(oidp, &limit, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
g_journal_cache_limit = limit;
g_journal_cache_low = (limit / 100) * g_journal_cache_switch;
return (0);
}
SYSCTL_PROC(_kern_geom_journal_cache, OID_AUTO, limit,
CTLTYPE_UINT | CTLFLAG_RWTUN, NULL, 0, g_journal_cache_limit_sysctl, "I",
"Maximum number of allocated bytes");
SYSCTL_UINT(_kern_geom_journal_cache, OID_AUTO, divisor, CTLFLAG_RDTUN,
&g_journal_cache_divisor, 0,
"(kmem_size / kern.geom.journal.cache.divisor) == cache size");
static int
g_journal_cache_switch_sysctl(SYSCTL_HANDLER_ARGS)
{
u_int cswitch;
int error;
cswitch = g_journal_cache_switch;
error = sysctl_handle_int(oidp, &cswitch, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
if (cswitch > 100)
return (EINVAL);
g_journal_cache_switch = cswitch;
g_journal_cache_low = (g_journal_cache_limit / 100) * cswitch;
return (0);
}
SYSCTL_PROC(_kern_geom_journal_cache, OID_AUTO, switch,
CTLTYPE_UINT | CTLFLAG_RW, NULL, 0, g_journal_cache_switch_sysctl, "I",
"Force switch when we hit this percent of cache use");
SYSCTL_UINT(_kern_geom_journal_cache, OID_AUTO, misses, CTLFLAG_RW,
&g_journal_cache_misses, 0, "Number of cache misses");
SYSCTL_UINT(_kern_geom_journal_cache, OID_AUTO, alloc_failures, CTLFLAG_RW,
&g_journal_cache_alloc_failures, 0, "Memory allocation failures");
static u_long g_journal_stats_bytes_skipped = 0;
static u_long g_journal_stats_combined_ios = 0;
static u_long g_journal_stats_switches = 0;
static u_long g_journal_stats_wait_for_copy = 0;
static u_long g_journal_stats_journal_full = 0;
static u_long g_journal_stats_low_mem = 0;
static SYSCTL_NODE(_kern_geom_journal, OID_AUTO, stats, CTLFLAG_RW, 0,
"GEOM_JOURNAL statistics");
SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, skipped_bytes, CTLFLAG_RW,
&g_journal_stats_bytes_skipped, 0, "Number of skipped bytes");
SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, combined_ios, CTLFLAG_RW,
&g_journal_stats_combined_ios, 0, "Number of combined I/O requests");
SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, switches, CTLFLAG_RW,
&g_journal_stats_switches, 0, "Number of journal switches");
SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, wait_for_copy, CTLFLAG_RW,
&g_journal_stats_wait_for_copy, 0, "Wait for journal copy on switch");
SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, journal_full, CTLFLAG_RW,
&g_journal_stats_journal_full, 0,
"Number of times journal was almost full.");
SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, low_mem, CTLFLAG_RW,
&g_journal_stats_low_mem, 0, "Number of times low_mem hook was called.");
static g_taste_t g_journal_taste;
static g_ctl_req_t g_journal_config;
static g_dumpconf_t g_journal_dumpconf;
static g_init_t g_journal_init;
static g_fini_t g_journal_fini;
struct g_class g_journal_class = {
.name = G_JOURNAL_CLASS_NAME,
.version = G_VERSION,
.taste = g_journal_taste,
.ctlreq = g_journal_config,
.dumpconf = g_journal_dumpconf,
.init = g_journal_init,
.fini = g_journal_fini
};
static int g_journal_destroy(struct g_journal_softc *sc);
static void g_journal_metadata_update(struct g_journal_softc *sc);
static void g_journal_switch_wait(struct g_journal_softc *sc);
#define GJ_SWITCHER_WORKING 0
#define GJ_SWITCHER_DIE 1
#define GJ_SWITCHER_DIED 2
static int g_journal_switcher_state = GJ_SWITCHER_WORKING;
static int g_journal_switcher_wokenup = 0;
static int g_journal_sync_requested = 0;
#ifdef GJ_MEMDEBUG
struct meminfo {
size_t mi_size;
struct stack mi_stack;
};
#endif
/*
* We use our own malloc/realloc/free funtions, so we can collect statistics
* and force journal switch when we're running out of cache.
*/
static void *
gj_malloc(size_t size, int flags)
{
void *p;
#ifdef GJ_MEMDEBUG
struct meminfo *mi;
#endif
mtx_lock(&g_journal_cache_mtx);
if (g_journal_cache_limit > 0 && !g_journal_switcher_wokenup &&
g_journal_cache_used + size > g_journal_cache_low) {
GJ_DEBUG(1, "No cache, waking up the switcher.");
g_journal_switcher_wokenup = 1;
wakeup(&g_journal_switcher_state);
}
if ((flags & M_NOWAIT) && g_journal_cache_limit > 0 &&
g_journal_cache_used + size > g_journal_cache_limit) {
mtx_unlock(&g_journal_cache_mtx);
g_journal_cache_alloc_failures++;
return (NULL);
}
g_journal_cache_used += size;
mtx_unlock(&g_journal_cache_mtx);
flags &= ~M_NOWAIT;
#ifndef GJ_MEMDEBUG
p = malloc(size, M_JOURNAL, flags | M_WAITOK);
#else
mi = malloc(sizeof(*mi) + size, M_JOURNAL, flags | M_WAITOK);
p = (u_char *)mi + sizeof(*mi);
mi->mi_size = size;
stack_save(&mi->mi_stack);
#endif
return (p);
}
static void
gj_free(void *p, size_t size)
{
#ifdef GJ_MEMDEBUG
struct meminfo *mi;
#endif
KASSERT(p != NULL, ("p=NULL"));
KASSERT(size > 0, ("size=0"));
mtx_lock(&g_journal_cache_mtx);
KASSERT(g_journal_cache_used >= size, ("Freeing too much?"));
g_journal_cache_used -= size;
mtx_unlock(&g_journal_cache_mtx);
#ifdef GJ_MEMDEBUG
mi = p = (void *)((u_char *)p - sizeof(*mi));
if (mi->mi_size != size) {
printf("GJOURNAL: Size mismatch! %zu != %zu\n", size,
mi->mi_size);
printf("GJOURNAL: Alloc backtrace:\n");
stack_print(&mi->mi_stack);
printf("GJOURNAL: Free backtrace:\n");
kdb_backtrace();
}
#endif
free(p, M_JOURNAL);
}
static void *
gj_realloc(void *p, size_t size, size_t oldsize)
{
void *np;
#ifndef GJ_MEMDEBUG
mtx_lock(&g_journal_cache_mtx);
g_journal_cache_used -= oldsize;
g_journal_cache_used += size;
mtx_unlock(&g_journal_cache_mtx);
np = realloc(p, size, M_JOURNAL, M_WAITOK);
#else
np = gj_malloc(size, M_WAITOK);
bcopy(p, np, MIN(oldsize, size));
gj_free(p, oldsize);
#endif
return (np);
}
static void
g_journal_check_overflow(struct g_journal_softc *sc)
{
off_t length, used;
if ((sc->sc_active.jj_offset < sc->sc_inactive.jj_offset &&
sc->sc_journal_offset >= sc->sc_inactive.jj_offset) ||
(sc->sc_active.jj_offset > sc->sc_inactive.jj_offset &&
sc->sc_journal_offset >= sc->sc_inactive.jj_offset &&
sc->sc_journal_offset < sc->sc_active.jj_offset)) {
panic("Journal overflow "
"(id = %u joffset=%jd active=%jd inactive=%jd)",
(unsigned)sc->sc_id,
(intmax_t)sc->sc_journal_offset,
(intmax_t)sc->sc_active.jj_offset,
(intmax_t)sc->sc_inactive.jj_offset);
}
if (sc->sc_active.jj_offset < sc->sc_inactive.jj_offset) {
length = sc->sc_inactive.jj_offset - sc->sc_active.jj_offset;
used = sc->sc_journal_offset - sc->sc_active.jj_offset;
} else {
length = sc->sc_jend - sc->sc_active.jj_offset;
length += sc->sc_inactive.jj_offset - sc->sc_jstart;
if (sc->sc_journal_offset >= sc->sc_active.jj_offset)
used = sc->sc_journal_offset - sc->sc_active.jj_offset;
else {
used = sc->sc_jend - sc->sc_active.jj_offset;
used += sc->sc_journal_offset - sc->sc_jstart;
}
}
/* Already woken up? */
if (g_journal_switcher_wokenup)
return;
/*
* If the active journal takes more than g_journal_force_switch precent
* of free journal space, we force journal switch.
*/
KASSERT(length > 0,
("length=%jd used=%jd active=%jd inactive=%jd joffset=%jd",
(intmax_t)length, (intmax_t)used,
(intmax_t)sc->sc_active.jj_offset,
(intmax_t)sc->sc_inactive.jj_offset,
(intmax_t)sc->sc_journal_offset));
if ((used * 100) / length > g_journal_force_switch) {
g_journal_stats_journal_full++;
GJ_DEBUG(1, "Journal %s %jd%% full, forcing journal switch.",
sc->sc_name, (used * 100) / length);
mtx_lock(&g_journal_cache_mtx);
g_journal_switcher_wokenup = 1;
wakeup(&g_journal_switcher_state);
mtx_unlock(&g_journal_cache_mtx);
}
}
static void
g_journal_orphan(struct g_consumer *cp)
{
struct g_journal_softc *sc;
char name[256];
int error;
g_topology_assert();
sc = cp->geom->softc;
strlcpy(name, cp->provider->name, sizeof(name));
GJ_DEBUG(0, "Lost provider %s.", name);
if (sc == NULL)
return;
error = g_journal_destroy(sc);
if (error == 0)
GJ_DEBUG(0, "Journal %s destroyed.", name);
else {
GJ_DEBUG(0, "Cannot destroy journal %s (error=%d). "
"Destroy it manually after last close.", sc->sc_name,
error);
}
}
static int
g_journal_access(struct g_provider *pp, int acr, int acw, int ace)
{
struct g_journal_softc *sc;
int dcr, dcw, dce;
g_topology_assert();
GJ_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name,
acr, acw, ace);
dcr = pp->acr + acr;
dcw = pp->acw + acw;
dce = pp->ace + ace;
sc = pp->geom->softc;
if (sc == NULL || (sc->sc_flags & GJF_DEVICE_DESTROY)) {
if (acr <= 0 && acw <= 0 && ace <= 0)
return (0);
else
return (ENXIO);
}
if (pp->acw == 0 && dcw > 0) {
GJ_DEBUG(1, "Marking %s as dirty.", sc->sc_name);
sc->sc_flags &= ~GJF_DEVICE_CLEAN;
g_topology_unlock();
g_journal_metadata_update(sc);
g_topology_lock();
} /* else if (pp->acw == 0 && dcw > 0 && JEMPTY(sc)) {
GJ_DEBUG(1, "Marking %s as clean.", sc->sc_name);
sc->sc_flags |= GJF_DEVICE_CLEAN;
g_topology_unlock();
g_journal_metadata_update(sc);
g_topology_lock();
} */
return (0);
}
static void
g_journal_header_encode(struct g_journal_header *hdr, u_char *data)
{
bcopy(GJ_HEADER_MAGIC, data, sizeof(GJ_HEADER_MAGIC));
data += sizeof(GJ_HEADER_MAGIC);
le32enc(data, hdr->jh_journal_id);
data += 4;
le32enc(data, hdr->jh_journal_next_id);
}
static int
g_journal_header_decode(const u_char *data, struct g_journal_header *hdr)
{
bcopy(data, hdr->jh_magic, sizeof(hdr->jh_magic));
data += sizeof(hdr->jh_magic);
if (bcmp(hdr->jh_magic, GJ_HEADER_MAGIC, sizeof(GJ_HEADER_MAGIC)) != 0)
return (EINVAL);
hdr->jh_journal_id = le32dec(data);
data += 4;
hdr->jh_journal_next_id = le32dec(data);
return (0);
}
static void
g_journal_flush_cache(struct g_journal_softc *sc)
{
struct bintime bt;
int error;
if (sc->sc_bio_flush == 0)
return;
GJ_TIMER_START(1, &bt);
if (sc->sc_bio_flush & GJ_FLUSH_JOURNAL) {
error = g_io_flush(sc->sc_jconsumer);
GJ_DEBUG(error == 0 ? 2 : 0, "Flush cache of %s: error=%d.",
sc->sc_jconsumer->provider->name, error);
}
if (sc->sc_bio_flush & GJ_FLUSH_DATA) {
/*
* TODO: This could be called in parallel with the
* previous call.
*/
error = g_io_flush(sc->sc_dconsumer);
GJ_DEBUG(error == 0 ? 2 : 0, "Flush cache of %s: error=%d.",
sc->sc_dconsumer->provider->name, error);
}
GJ_TIMER_STOP(1, &bt, "Cache flush time");
}
static int
g_journal_write_header(struct g_journal_softc *sc)
{
struct g_journal_header hdr;
struct g_consumer *cp;
u_char *buf;
int error;
cp = sc->sc_jconsumer;
buf = gj_malloc(cp->provider->sectorsize, M_WAITOK);
strlcpy(hdr.jh_magic, GJ_HEADER_MAGIC, sizeof(hdr.jh_magic));
hdr.jh_journal_id = sc->sc_journal_id;
hdr.jh_journal_next_id = sc->sc_journal_next_id;
g_journal_header_encode(&hdr, buf);
error = g_write_data(cp, sc->sc_journal_offset, buf,
cp->provider->sectorsize);
/* if (error == 0) */
sc->sc_journal_offset += cp->provider->sectorsize;
gj_free(buf, cp->provider->sectorsize);
return (error);
}
/*
* Every journal record has a header and data following it.
* Functions below are used to decode the header before storing it to
* little endian and to encode it after reading to system endianess.
*/
static void
g_journal_record_header_encode(struct g_journal_record_header *hdr,
u_char *data)
{
struct g_journal_entry *ent;
u_int i;
bcopy(GJ_RECORD_HEADER_MAGIC, data, sizeof(GJ_RECORD_HEADER_MAGIC));
data += sizeof(GJ_RECORD_HEADER_MAGIC);
le32enc(data, hdr->jrh_journal_id);
data += 8;
le16enc(data, hdr->jrh_nentries);
data += 2;
bcopy(hdr->jrh_sum, data, sizeof(hdr->jrh_sum));
data += 8;
for (i = 0; i < hdr->jrh_nentries; i++) {
ent = &hdr->jrh_entries[i];
le64enc(data, ent->je_joffset);
data += 8;
le64enc(data, ent->je_offset);
data += 8;
le64enc(data, ent->je_length);
data += 8;
}
}
static int
g_journal_record_header_decode(const u_char *data,
struct g_journal_record_header *hdr)
{
struct g_journal_entry *ent;
u_int i;
bcopy(data, hdr->jrh_magic, sizeof(hdr->jrh_magic));
data += sizeof(hdr->jrh_magic);
if (strcmp(hdr->jrh_magic, GJ_RECORD_HEADER_MAGIC) != 0)
return (EINVAL);
hdr->jrh_journal_id = le32dec(data);
data += 8;
hdr->jrh_nentries = le16dec(data);
data += 2;
if (hdr->jrh_nentries > GJ_RECORD_HEADER_NENTRIES)
return (EINVAL);
bcopy(data, hdr->jrh_sum, sizeof(hdr->jrh_sum));
data += 8;
for (i = 0; i < hdr->jrh_nentries; i++) {
ent = &hdr->jrh_entries[i];
ent->je_joffset = le64dec(data);
data += 8;
ent->je_offset = le64dec(data);
data += 8;
ent->je_length = le64dec(data);
data += 8;
}
return (0);
}
/*
* Function reads metadata from a provider (via the given consumer), decodes
* it to system endianess and verifies its correctness.
*/
static int
g_journal_metadata_read(struct g_consumer *cp, struct g_journal_metadata *md)
{
struct g_provider *pp;
u_char *buf;
int error;
g_topology_assert();
error = g_access(cp, 1, 0, 0);
if (error != 0)
return (error);
pp = cp->provider;
g_topology_unlock();
/* Metadata is stored in last sector. */
buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
&error);
g_topology_lock();
g_access(cp, -1, 0, 0);
if (buf == NULL) {
GJ_DEBUG(1, "Cannot read metadata from %s (error=%d).",
cp->provider->name, error);
return (error);
}
/* Decode metadata. */
error = journal_metadata_decode(buf, md);
g_free(buf);
/* Is this is gjournal provider at all? */
if (strcmp(md->md_magic, G_JOURNAL_MAGIC) != 0)
return (EINVAL);
/*
* Are we able to handle this version of metadata?
* We only maintain backward compatibility.
*/
if (md->md_version > G_JOURNAL_VERSION) {
GJ_DEBUG(0,
"Kernel module is too old to handle metadata from %s.",
cp->provider->name);
return (EINVAL);
}
/* Is checksum correct? */
if (error != 0) {
GJ_DEBUG(0, "MD5 metadata hash mismatch for provider %s.",
cp->provider->name);
return (error);
}
return (0);
}
/*
* Two functions below are responsible for updating metadata.
* Only metadata on the data provider is updated (we need to update
* information about active journal in there).
*/
static void
g_journal_metadata_done(struct bio *bp)
{
/*
* There is not much we can do on error except informing about it.
*/
if (bp->bio_error != 0) {
GJ_LOGREQ(0, bp, "Cannot update metadata (error=%d).",
bp->bio_error);
} else {
GJ_LOGREQ(2, bp, "Metadata updated.");
}
gj_free(bp->bio_data, bp->bio_length);
g_destroy_bio(bp);
}
static void
g_journal_metadata_update(struct g_journal_softc *sc)
{
struct g_journal_metadata md;
struct g_consumer *cp;
struct bio *bp;
u_char *sector;
cp = sc->sc_dconsumer;
sector = gj_malloc(cp->provider->sectorsize, M_WAITOK);
strlcpy(md.md_magic, G_JOURNAL_MAGIC, sizeof(md.md_magic));
md.md_version = G_JOURNAL_VERSION;
md.md_id = sc->sc_id;
md.md_type = sc->sc_orig_type;
md.md_jstart = sc->sc_jstart;
md.md_jend = sc->sc_jend;
md.md_joffset = sc->sc_inactive.jj_offset;
md.md_jid = sc->sc_journal_previous_id;
md.md_flags = 0;
if (sc->sc_flags & GJF_DEVICE_CLEAN)
md.md_flags |= GJ_FLAG_CLEAN;
if (sc->sc_flags & GJF_DEVICE_HARDCODED)
strlcpy(md.md_provider, sc->sc_name, sizeof(md.md_provider));
else
bzero(md.md_provider, sizeof(md.md_provider));
md.md_provsize = cp->provider->mediasize;
journal_metadata_encode(&md, sector);
/*
* Flush the cache, so we know all data are on disk.
* We write here informations like "journal is consistent", so we need
* to be sure it is. Without BIO_FLUSH here, we can end up in situation
* where metadata is stored on disk, but not all data.
*/
g_journal_flush_cache(sc);
bp = g_alloc_bio();
bp->bio_offset = cp->provider->mediasize - cp->provider->sectorsize;
bp->bio_length = cp->provider->sectorsize;
bp->bio_data = sector;
bp->bio_cmd = BIO_WRITE;
if (!(sc->sc_flags & GJF_DEVICE_DESTROY)) {
bp->bio_done = g_journal_metadata_done;
g_io_request(bp, cp);
} else {
bp->bio_done = NULL;
g_io_request(bp, cp);
biowait(bp, "gjmdu");
g_journal_metadata_done(bp);
}
/*
* Be sure metadata reached the disk.
*/
g_journal_flush_cache(sc);
}
/*
* This is where the I/O request comes from the GEOM.
*/
static void
g_journal_start(struct bio *bp)
{
struct g_journal_softc *sc;
sc = bp->bio_to->geom->softc;
GJ_LOGREQ(3, bp, "Request received.");
switch (bp->bio_cmd) {
case BIO_READ:
case BIO_WRITE:
mtx_lock(&sc->sc_mtx);
bioq_insert_tail(&sc->sc_regular_queue, bp);
wakeup(sc);
mtx_unlock(&sc->sc_mtx);
return;
case BIO_GETATTR:
if (strcmp(bp->bio_attribute, "GJOURNAL::provider") == 0) {
strlcpy(bp->bio_data, bp->bio_to->name, bp->bio_length);
bp->bio_completed = strlen(bp->bio_to->name) + 1;
g_io_deliver(bp, 0);
return;
}
/* FALLTHROUGH */
case BIO_DELETE:
default:
g_io_deliver(bp, EOPNOTSUPP);
return;
}
}
static void
g_journal_std_done(struct bio *bp)
{
struct g_journal_softc *sc;
sc = bp->bio_from->geom->softc;
mtx_lock(&sc->sc_mtx);
bioq_insert_tail(&sc->sc_back_queue, bp);
wakeup(sc);
mtx_unlock(&sc->sc_mtx);
}
static struct bio *
g_journal_new_bio(off_t start, off_t end, off_t joffset, u_char *data,
int flags)
{
struct bio *bp;
bp = g_alloc_bio();
bp->bio_offset = start;
bp->bio_joffset = joffset;
bp->bio_length = end - start;
bp->bio_cmd = BIO_WRITE;
bp->bio_done = g_journal_std_done;
if (data == NULL)
bp->bio_data = NULL;
else {
bp->bio_data = gj_malloc(bp->bio_length, flags);
if (bp->bio_data != NULL)
bcopy(data, bp->bio_data, bp->bio_length);
}
return (bp);
}
#define g_journal_insert_bio(head, bp, flags) \
g_journal_insert((head), (bp)->bio_offset, \
(bp)->bio_offset + (bp)->bio_length, (bp)->bio_joffset, \
(bp)->bio_data, flags)
/*
* The function below does a lot more than just inserting bio to the queue.
* It keeps the queue sorted by offset and ensures that there are no doubled
* data (it combines bios where ranges overlap).
*
* The function returns the number of bios inserted (as bio can be splitted).
*/
static int
g_journal_insert(struct bio **head, off_t nstart, off_t nend, off_t joffset,
u_char *data, int flags)
{
struct bio *nbp, *cbp, *pbp;
off_t cstart, cend;
u_char *tmpdata;
int n;
GJ_DEBUG(3, "INSERT(%p): (%jd, %jd, %jd)", *head, nstart, nend,
joffset);
n = 0;
pbp = NULL;
GJQ_FOREACH(*head, cbp) {
cstart = cbp->bio_offset;
cend = cbp->bio_offset + cbp->bio_length;
if (nstart >= cend) {
/*
* +-------------+
* | |
* | current | +-------------+
* | bio | | |
* | | | new |
* +-------------+ | bio |
* | |
* +-------------+
*/
GJ_DEBUG(3, "INSERT(%p): 1", *head);
} else if (nend <= cstart) {
/*
* +-------------+
* | |
* +-------------+ | current |
* | | | bio |
* | new | | |
* | bio | +-------------+
* | |
* +-------------+
*/
nbp = g_journal_new_bio(nstart, nend, joffset, data,
flags);
if (pbp == NULL)
*head = nbp;
else
pbp->bio_next = nbp;
nbp->bio_next = cbp;
n++;
GJ_DEBUG(3, "INSERT(%p): 2 (nbp=%p pbp=%p)", *head, nbp,
pbp);
goto end;
} else if (nstart <= cstart && nend >= cend) {
/*
* +-------------+ +-------------+
* | current bio | | current bio |
* +---+-------------+---+ +-------------+---+
* | | | | | | |
* | | | | | | |
* | +-------------+ | +-------------+ |
* | new bio | | new bio |
* +---------------------+ +-----------------+
*
* +-------------+ +-------------+
* | current bio | | current bio |
* +---+-------------+ +-------------+
* | | | | |
* | | | | |
* | +-------------+ +-------------+
* | new bio | | new bio |
* +-----------------+ +-------------+
*/
g_journal_stats_bytes_skipped += cbp->bio_length;
cbp->bio_offset = nstart;
cbp->bio_joffset = joffset;
cbp->bio_length = cend - nstart;
if (cbp->bio_data != NULL) {
gj_free(cbp->bio_data, cend - cstart);
cbp->bio_data = NULL;
}
if (data != NULL) {
cbp->bio_data = gj_malloc(cbp->bio_length,
flags);
if (cbp->bio_data != NULL) {
bcopy(data, cbp->bio_data,
cbp->bio_length);
}
data += cend - nstart;
}
joffset += cend - nstart;
nstart = cend;
GJ_DEBUG(3, "INSERT(%p): 3 (cbp=%p)", *head, cbp);
} else if (nstart > cstart && nend >= cend) {
/*
* +-----------------+ +-------------+
* | current bio | | current bio |
* | +-------------+ | +---------+---+
* | | | | | | |
* | | | | | | |
* +---+-------------+ +---+---------+ |
* | new bio | | new bio |
* +-------------+ +-------------+
*/
g_journal_stats_bytes_skipped += cend - nstart;
nbp = g_journal_new_bio(nstart, cend, joffset, data,
flags);
nbp->bio_next = cbp->bio_next;
cbp->bio_next = nbp;
cbp->bio_length = nstart - cstart;
if (cbp->bio_data != NULL) {
cbp->bio_data = gj_realloc(cbp->bio_data,
cbp->bio_length, cend - cstart);
}
if (data != NULL)
data += cend - nstart;
joffset += cend - nstart;
nstart = cend;
n++;
GJ_DEBUG(3, "INSERT(%p): 4 (cbp=%p)", *head, cbp);
} else if (nstart > cstart && nend < cend) {
/*
* +---------------------+
* | current bio |
* | +-------------+ |
* | | | |
* | | | |
* +---+-------------+---+
* | new bio |
* +-------------+
*/
g_journal_stats_bytes_skipped += nend - nstart;
nbp = g_journal_new_bio(nstart, nend, joffset, data,
flags);
nbp->bio_next = cbp->bio_next;
cbp->bio_next = nbp;
if (cbp->bio_data == NULL)
tmpdata = NULL;
else
tmpdata = cbp->bio_data + nend - cstart;
nbp = g_journal_new_bio(nend, cend,
cbp->bio_joffset + nend - cstart, tmpdata, flags);
nbp->bio_next = ((struct bio *)cbp->bio_next)->bio_next;
((struct bio *)cbp->bio_next)->bio_next = nbp;
cbp->bio_length = nstart - cstart;
if (cbp->bio_data != NULL) {
cbp->bio_data = gj_realloc(cbp->bio_data,
cbp->bio_length, cend - cstart);
}
n += 2;
GJ_DEBUG(3, "INSERT(%p): 5 (cbp=%p)", *head, cbp);
goto end;
} else if (nstart <= cstart && nend < cend) {
/*
* +-----------------+ +-------------+
* | current bio | | current bio |
* +-------------+ | +---+---------+ |
* | | | | | | |
* | | | | | | |
* +-------------+---+ | +---------+---+
* | new bio | | new bio |
* +-------------+ +-------------+
*/
g_journal_stats_bytes_skipped += nend - nstart;
nbp = g_journal_new_bio(nstart, nend, joffset, data,
flags);
if (pbp == NULL)
*head = nbp;
else
pbp->bio_next = nbp;
nbp->bio_next = cbp;
cbp->bio_offset = nend;
cbp->bio_length = cend - nend;
cbp->bio_joffset += nend - cstart;
tmpdata = cbp->bio_data;
if (tmpdata != NULL) {
cbp->bio_data = gj_malloc(cbp->bio_length,
flags);
if (cbp->bio_data != NULL) {
bcopy(tmpdata + nend - cstart,
cbp->bio_data, cbp->bio_length);
}
gj_free(tmpdata, cend - cstart);
}
n++;
GJ_DEBUG(3, "INSERT(%p): 6 (cbp=%p)", *head, cbp);
goto end;
}
if (nstart == nend)
goto end;
pbp = cbp;
}
nbp = g_journal_new_bio(nstart, nend, joffset, data, flags);
if (pbp == NULL)
*head = nbp;
else
pbp->bio_next = nbp;
nbp->bio_next = NULL;
n++;
GJ_DEBUG(3, "INSERT(%p): 8 (nbp=%p pbp=%p)", *head, nbp, pbp);
end:
if (g_journal_debug >= 3) {
GJQ_FOREACH(*head, cbp) {
GJ_DEBUG(3, "ELEMENT: %p (%jd, %jd, %jd, %p)", cbp,
(intmax_t)cbp->bio_offset,
(intmax_t)cbp->bio_length,
(intmax_t)cbp->bio_joffset, cbp->bio_data);
}
GJ_DEBUG(3, "INSERT(%p): DONE %d", *head, n);
}
return (n);
}
/*
* The function combines neighbour bios trying to squeeze as much data as
* possible into one bio.
*
* The function returns the number of bios combined (negative value).
*/
static int
g_journal_optimize(struct bio *head)
{
struct bio *cbp, *pbp;
int n;
n = 0;
pbp = NULL;
GJQ_FOREACH(head, cbp) {
/* Skip bios which has to be read first. */
if (cbp->bio_data == NULL) {
pbp = NULL;
continue;
}
/* There is no previous bio yet. */
if (pbp == NULL) {
pbp = cbp;
continue;
}
/* Is this a neighbour bio? */
if (pbp->bio_offset + pbp->bio_length != cbp->bio_offset) {
/* Be sure that bios queue is sorted. */
KASSERT(pbp->bio_offset + pbp->bio_length < cbp->bio_offset,
("poffset=%jd plength=%jd coffset=%jd",
(intmax_t)pbp->bio_offset,
(intmax_t)pbp->bio_length,
(intmax_t)cbp->bio_offset));
pbp = cbp;
continue;
}
/* Be sure we don't end up with too big bio. */
if (pbp->bio_length + cbp->bio_length > MAXPHYS) {
pbp = cbp;
continue;
}
/* Ok, we can join bios. */
GJ_LOGREQ(4, pbp, "Join: ");
GJ_LOGREQ(4, cbp, "and: ");
pbp->bio_data = gj_realloc(pbp->bio_data,
pbp->bio_length + cbp->bio_length, pbp->bio_length);
bcopy(cbp->bio_data, pbp->bio_data + pbp->bio_length,
cbp->bio_length);
gj_free(cbp->bio_data, cbp->bio_length);
pbp->bio_length += cbp->bio_length;
pbp->bio_next = cbp->bio_next;
g_destroy_bio(cbp);
cbp = pbp;
g_journal_stats_combined_ios++;
n--;
GJ_LOGREQ(4, pbp, "Got: ");
}
return (n);
}
/*
* TODO: Update comment.
* These are functions responsible for copying one portion of data from journal
* to the destination provider.
* The order goes like this:
* 1. Read the header, which contains informations about data blocks
* following it.
* 2. Read the data blocks from the journal.
* 3. Write the data blocks on the data provider.
*
* g_journal_copy_start()
* g_journal_copy_done() - got finished write request, logs potential errors.
*/
/*
* When there is no data in cache, this function is used to read it.
*/
static void
g_journal_read_first(struct g_journal_softc *sc, struct bio *bp)
{
struct bio *cbp;
/*
* We were short in memory, so data was freed.
* In that case we need to read it back from journal.
*/
cbp = g_alloc_bio();
cbp->bio_cflags = bp->bio_cflags;
cbp->bio_parent = bp;
cbp->bio_offset = bp->bio_joffset;
cbp->bio_length = bp->bio_length;
cbp->bio_data = gj_malloc(bp->bio_length, M_WAITOK);
cbp->bio_cmd = BIO_READ;
cbp->bio_done = g_journal_std_done;
GJ_LOGREQ(4, cbp, "READ FIRST");
g_io_request(cbp, sc->sc_jconsumer);
g_journal_cache_misses++;
}
static void
g_journal_copy_send(struct g_journal_softc *sc)
{
struct bio *bioq, *bp, *lbp;
bioq = lbp = NULL;
mtx_lock(&sc->sc_mtx);
for (; sc->sc_copy_in_progress < g_journal_parallel_copies;) {
bp = GJQ_FIRST(sc->sc_inactive.jj_queue);
if (bp == NULL)
break;
GJQ_REMOVE(sc->sc_inactive.jj_queue, bp);
sc->sc_copy_in_progress++;
GJQ_INSERT_AFTER(bioq, bp, lbp);
lbp = bp;
}
mtx_unlock(&sc->sc_mtx);
if (g_journal_do_optimize)
sc->sc_copy_in_progress += g_journal_optimize(bioq);
while ((bp = GJQ_FIRST(bioq)) != NULL) {
GJQ_REMOVE(bioq, bp);
GJQ_INSERT_HEAD(sc->sc_copy_queue, bp);
bp->bio_cflags = GJ_BIO_COPY;
if (bp->bio_data == NULL)
g_journal_read_first(sc, bp);
else {
bp->bio_joffset = 0;
GJ_LOGREQ(4, bp, "SEND");
g_io_request(bp, sc->sc_dconsumer);
}
}
}
static void
g_journal_copy_start(struct g_journal_softc *sc)
{
/*
* Remember in metadata that we're starting to copy journaled data
* to the data provider.
* In case of power failure, we will copy these data once again on boot.
*/
if (!sc->sc_journal_copying) {
sc->sc_journal_copying = 1;
GJ_DEBUG(1, "Starting copy of journal.");
g_journal_metadata_update(sc);
}
g_journal_copy_send(sc);
}
/*
* Data block has been read from the journal provider.
*/
static int
g_journal_copy_read_done(struct bio *bp)
{
struct g_journal_softc *sc;
struct g_consumer *cp;
struct bio *pbp;
KASSERT(bp->bio_cflags == GJ_BIO_COPY,
("Invalid bio (%d != %d).", bp->bio_cflags, GJ_BIO_COPY));
sc = bp->bio_from->geom->softc;
pbp = bp->bio_parent;
if (bp->bio_error != 0) {
GJ_DEBUG(0, "Error while reading data from %s (error=%d).",
bp->bio_to->name, bp->bio_error);
/*
* We will not be able to deliver WRITE request as well.
*/
gj_free(bp->bio_data, bp->bio_length);
g_destroy_bio(pbp);
g_destroy_bio(bp);
sc->sc_copy_in_progress--;
return (1);
}
pbp->bio_data = bp->bio_data;
cp = sc->sc_dconsumer;
g_io_request(pbp, cp);
GJ_LOGREQ(4, bp, "READ DONE");
g_destroy_bio(bp);
return (0);
}
/*
* Data block has been written to the data provider.
*/
static void
g_journal_copy_write_done(struct bio *bp)
{
struct g_journal_softc *sc;
KASSERT(bp->bio_cflags == GJ_BIO_COPY,
("Invalid bio (%d != %d).", bp->bio_cflags, GJ_BIO_COPY));
sc = bp->bio_from->geom->softc;
sc->sc_copy_in_progress--;
if (bp->bio_error != 0) {
GJ_LOGREQ(0, bp, "[copy] Error while writing data (error=%d)",
bp->bio_error);
}
GJQ_REMOVE(sc->sc_copy_queue, bp);
gj_free(bp->bio_data, bp->bio_length);
GJ_LOGREQ(4, bp, "DONE");
g_destroy_bio(bp);
if (sc->sc_copy_in_progress == 0) {
/*
* This was the last write request for this journal.
*/
GJ_DEBUG(1, "Data has been copied.");
sc->sc_journal_copying = 0;
}
}
static void g_journal_flush_done(struct bio *bp);
/*
* Flush one record onto active journal provider.
*/
static void
g_journal_flush(struct g_journal_softc *sc)
{
struct g_journal_record_header hdr;
struct g_journal_entry *ent;
struct g_provider *pp;
struct bio **bioq;
struct bio *bp, *fbp, *pbp;
off_t joffset, size;
u_char *data, hash[16];
MD5_CTX ctx;
u_int i;
if (sc->sc_current_count == 0)
return;
size = 0;
pp = sc->sc_jprovider;
GJ_VALIDATE_OFFSET(sc->sc_journal_offset, sc);
joffset = sc->sc_journal_offset;
GJ_DEBUG(2, "Storing %d journal entries on %s at %jd.",
sc->sc_current_count, pp->name, (intmax_t)joffset);
/*
* Store 'journal id', so we know to which journal this record belongs.
*/
hdr.jrh_journal_id = sc->sc_journal_id;
/* Could be less than g_journal_record_entries if called due timeout. */
hdr.jrh_nentries = MIN(sc->sc_current_count, g_journal_record_entries);
strlcpy(hdr.jrh_magic, GJ_RECORD_HEADER_MAGIC, sizeof(hdr.jrh_magic));
bioq = &sc->sc_active.jj_queue;
pbp = sc->sc_flush_queue;
fbp = g_alloc_bio();
fbp->bio_parent = NULL;
fbp->bio_cflags = GJ_BIO_JOURNAL;
fbp->bio_offset = -1;
fbp->bio_joffset = joffset;
fbp->bio_length = pp->sectorsize;
fbp->bio_cmd = BIO_WRITE;
fbp->bio_done = g_journal_std_done;
GJQ_INSERT_AFTER(sc->sc_flush_queue, fbp, pbp);
pbp = fbp;
fbp->bio_to = pp;
GJ_LOGREQ(4, fbp, "FLUSH_OUT");
joffset += pp->sectorsize;
sc->sc_flush_count++;
if (sc->sc_flags & GJF_DEVICE_CHECKSUM)
MD5Init(&ctx);
for (i = 0; i < hdr.jrh_nentries; i++) {
bp = sc->sc_current_queue;
KASSERT(bp != NULL, ("NULL bp"));
bp->bio_to = pp;
GJ_LOGREQ(4, bp, "FLUSHED");
sc->sc_current_queue = bp->bio_next;
bp->bio_next = NULL;
sc->sc_current_count--;
/* Add to the header. */
ent = &hdr.jrh_entries[i];
ent->je_offset = bp->bio_offset;
ent->je_joffset = joffset;
ent->je_length = bp->bio_length;
size += ent->je_length;
data = bp->bio_data;
if (sc->sc_flags & GJF_DEVICE_CHECKSUM)
MD5Update(&ctx, data, ent->je_length);
bzero(bp, sizeof(*bp));
bp->bio_cflags = GJ_BIO_JOURNAL;
bp->bio_offset = ent->je_offset;
bp->bio_joffset = ent->je_joffset;
bp->bio_length = ent->je_length;
bp->bio_data = data;
bp->bio_cmd = BIO_WRITE;
bp->bio_done = g_journal_std_done;
GJQ_INSERT_AFTER(sc->sc_flush_queue, bp, pbp);
pbp = bp;
bp->bio_to = pp;
GJ_LOGREQ(4, bp, "FLUSH_OUT");
joffset += bp->bio_length;
sc->sc_flush_count++;
/*
* Add request to the active sc_journal_queue queue.
* This is our cache. After journal switch we don't have to
* read the data from the inactive journal, because we keep
* it in memory.
*/
g_journal_insert(bioq, ent->je_offset,
ent->je_offset + ent->je_length, ent->je_joffset, data,
M_NOWAIT);
}
/*
* After all requests, store valid header.
*/
data = gj_malloc(pp->sectorsize, M_WAITOK);
if (sc->sc_flags & GJF_DEVICE_CHECKSUM) {
MD5Final(hash, &ctx);
bcopy(hash, hdr.jrh_sum, sizeof(hdr.jrh_sum));
}
g_journal_record_header_encode(&hdr, data);
fbp->bio_data = data;
sc->sc_journal_offset = joffset;
g_journal_check_overflow(sc);
}
/*
* Flush request finished.
*/
static void
g_journal_flush_done(struct bio *bp)
{
struct g_journal_softc *sc;
struct g_consumer *cp;
KASSERT((bp->bio_cflags & GJ_BIO_MASK) == GJ_BIO_JOURNAL,
("Invalid bio (%d != %d).", bp->bio_cflags, GJ_BIO_JOURNAL));
cp = bp->bio_from;
sc = cp->geom->softc;
sc->sc_flush_in_progress--;
if (bp->bio_error != 0) {
GJ_LOGREQ(0, bp, "[flush] Error while writing data (error=%d)",
bp->bio_error);
}
gj_free(bp->bio_data, bp->bio_length);
GJ_LOGREQ(4, bp, "DONE");
g_destroy_bio(bp);
}
static void g_journal_release_delayed(struct g_journal_softc *sc);
static void
g_journal_flush_send(struct g_journal_softc *sc)
{
struct g_consumer *cp;
struct bio *bioq, *bp, *lbp;
cp = sc->sc_jconsumer;
bioq = lbp = NULL;
while (sc->sc_flush_in_progress < g_journal_parallel_flushes) {
/* Send one flush requests to the active journal. */
bp = GJQ_FIRST(sc->sc_flush_queue);
if (bp != NULL) {
GJQ_REMOVE(sc->sc_flush_queue, bp);
sc->sc_flush_count--;
bp->bio_offset = bp->bio_joffset;
bp->bio_joffset = 0;
sc->sc_flush_in_progress++;
GJQ_INSERT_AFTER(bioq, bp, lbp);
lbp = bp;
}
/* Try to release delayed requests. */
g_journal_release_delayed(sc);
/* If there are no requests to flush, leave. */
if (GJQ_FIRST(sc->sc_flush_queue) == NULL)
break;
}
if (g_journal_do_optimize)
sc->sc_flush_in_progress += g_journal_optimize(bioq);
while ((bp = GJQ_FIRST(bioq)) != NULL) {
GJQ_REMOVE(bioq, bp);
GJ_LOGREQ(3, bp, "Flush request send");
g_io_request(bp, cp);
}
}
static void
g_journal_add_current(struct g_journal_softc *sc, struct bio *bp)
{
int n;
GJ_LOGREQ(4, bp, "CURRENT %d", sc->sc_current_count);
n = g_journal_insert_bio(&sc->sc_current_queue, bp, M_WAITOK);
sc->sc_current_count += n;
n = g_journal_optimize(sc->sc_current_queue);
sc->sc_current_count += n;
/*
* For requests which are added to the current queue we deliver
* response immediately.
*/
bp->bio_completed = bp->bio_length;
g_io_deliver(bp, 0);
if (sc->sc_current_count >= g_journal_record_entries) {
/*
* Let's flush one record onto active journal provider.
*/
g_journal_flush(sc);
}
}
static void
g_journal_release_delayed(struct g_journal_softc *sc)
{
struct bio *bp;
for (;;) {
/* The flush queue is full, exit. */
if (sc->sc_flush_count >= g_journal_accept_immediately)
return;
bp = bioq_takefirst(&sc->sc_delayed_queue);
if (bp == NULL)
return;
sc->sc_delayed_count--;
g_journal_add_current(sc, bp);
}
}
/*
* Add I/O request to the current queue. If we have enough requests for one
* journal record we flush them onto active journal provider.
*/
static void
g_journal_add_request(struct g_journal_softc *sc, struct bio *bp)
{
/*
* The flush queue is full, we need to delay the request.
*/
if (sc->sc_delayed_count > 0 ||
sc->sc_flush_count >= g_journal_accept_immediately) {
GJ_LOGREQ(4, bp, "DELAYED");
bioq_insert_tail(&sc->sc_delayed_queue, bp);
sc->sc_delayed_count++;
return;
}
KASSERT(TAILQ_EMPTY(&sc->sc_delayed_queue.queue),
("DELAYED queue not empty."));
g_journal_add_current(sc, bp);
}
static void g_journal_read_done(struct bio *bp);
/*
* Try to find requested data in cache.
*/
static struct bio *
g_journal_read_find(struct bio *head, int sorted, struct bio *pbp, off_t ostart,
off_t oend)
{
off_t cstart, cend;
struct bio *bp;
GJQ_FOREACH(head, bp) {
if (bp->bio_offset == -1)
continue;
cstart = MAX(ostart, bp->bio_offset);
cend = MIN(oend, bp->bio_offset + bp->bio_length);
if (cend <= ostart)
continue;
else if (cstart >= oend) {
if (!sorted)
continue;
else {
bp = NULL;
break;
}
}
if (bp->bio_data == NULL)
break;
GJ_DEBUG(3, "READ(%p): (%jd, %jd) (bp=%p)", head, cstart, cend,
bp);
bcopy(bp->bio_data + cstart - bp->bio_offset,
pbp->bio_data + cstart - pbp->bio_offset, cend - cstart);
pbp->bio_completed += cend - cstart;
if (pbp->bio_completed == pbp->bio_length) {
/*
* Cool, the whole request was in cache, deliver happy
* message.
*/
g_io_deliver(pbp, 0);
return (pbp);
}
break;
}
return (bp);
}
/*
* Try to find requested data in cache.
*/
static struct bio *
g_journal_read_queue_find(struct bio_queue *head, struct bio *pbp, off_t ostart,
off_t oend)
{
off_t cstart, cend;
struct bio *bp;
TAILQ_FOREACH(bp, head, bio_queue) {
cstart = MAX(ostart, bp->bio_offset);
cend = MIN(oend, bp->bio_offset + bp->bio_length);
if (cend <= ostart)
continue;
else if (cstart >= oend)
continue;
KASSERT(bp->bio_data != NULL,
("%s: bio_data == NULL", __func__));
GJ_DEBUG(3, "READ(%p): (%jd, %jd) (bp=%p)", head, cstart, cend,
bp);
bcopy(bp->bio_data + cstart - bp->bio_offset,
pbp->bio_data + cstart - pbp->bio_offset, cend - cstart);
pbp->bio_completed += cend - cstart;
if (pbp->bio_completed == pbp->bio_length) {
/*
* Cool, the whole request was in cache, deliver happy
* message.
*/
g_io_deliver(pbp, 0);
return (pbp);
}
break;
}
return (bp);
}
/*
* This function is used for colecting data on read.
* The complexity is because parts of the data can be stored in four different
* places:
* - in delayed requests
* - in memory - the data not yet send to the active journal provider
* - in requests which are going to be sent to the active journal
* - in the active journal
* - in the inactive journal
* - in the data provider
*/
static void
g_journal_read(struct g_journal_softc *sc, struct bio *pbp, off_t ostart,
off_t oend)
{
struct bio *bp, *nbp, *head;
off_t cstart, cend;
u_int i, sorted = 0;
GJ_DEBUG(3, "READ: (%jd, %jd)", ostart, oend);
cstart = cend = -1;
bp = NULL;
head = NULL;
for (i = 0; i <= 5; i++) {
switch (i) {
case 0: /* Delayed requests. */
head = NULL;
sorted = 0;
break;
case 1: /* Not-yet-send data. */
head = sc->sc_current_queue;
sorted = 1;
break;
case 2: /* In-flight to the active journal. */
head = sc->sc_flush_queue;
sorted = 0;
break;
case 3: /* Active journal. */
head = sc->sc_active.jj_queue;
sorted = 1;
break;
case 4: /* Inactive journal. */
/*
* XXX: Here could be a race with g_journal_lowmem().
*/
head = sc->sc_inactive.jj_queue;
sorted = 1;
break;
case 5: /* In-flight to the data provider. */
head = sc->sc_copy_queue;
sorted = 0;
break;
default:
panic("gjournal %s: i=%d", __func__, i);
}
if (i == 0)
bp = g_journal_read_queue_find(&sc->sc_delayed_queue.queue, pbp, ostart, oend);
else
bp = g_journal_read_find(head, sorted, pbp, ostart, oend);
if (bp == pbp) { /* Got the whole request. */
GJ_DEBUG(2, "Got the whole request from %u.", i);
return;
} else if (bp != NULL) {
cstart = MAX(ostart, bp->bio_offset);
cend = MIN(oend, bp->bio_offset + bp->bio_length);
GJ_DEBUG(2, "Got part of the request from %u (%jd-%jd).",
i, (intmax_t)cstart, (intmax_t)cend);
break;
}
}
if (bp != NULL) {
if (bp->bio_data == NULL) {
nbp = g_duplicate_bio(pbp);
nbp->bio_cflags = GJ_BIO_READ;
nbp->bio_data =
pbp->bio_data + cstart - pbp->bio_offset;
nbp->bio_offset =
bp->bio_joffset + cstart - bp->bio_offset;
nbp->bio_length = cend - cstart;
nbp->bio_done = g_journal_read_done;
g_io_request(nbp, sc->sc_jconsumer);
}
/*
* If we don't have the whole request yet, call g_journal_read()
* recursively.
*/
if (ostart < cstart)
g_journal_read(sc, pbp, ostart, cstart);
if (oend > cend)
g_journal_read(sc, pbp, cend, oend);
} else {
/*
* No data in memory, no data in journal.
* Its time for asking data provider.
*/
GJ_DEBUG(3, "READ(data): (%jd, %jd)", ostart, oend);
nbp = g_duplicate_bio(pbp);
nbp->bio_cflags = GJ_BIO_READ;
nbp->bio_data = pbp->bio_data + ostart - pbp->bio_offset;
nbp->bio_offset = ostart;
nbp->bio_length = oend - ostart;
nbp->bio_done = g_journal_read_done;
g_io_request(nbp, sc->sc_dconsumer);
/* We have the whole request, return here. */
return;
}
}
/*
* Function responsible for handling finished READ requests.
* Actually, g_std_done() could be used here, the only difference is that we
* log error.
*/
static void
g_journal_read_done(struct bio *bp)
{
struct bio *pbp;
KASSERT(bp->bio_cflags == GJ_BIO_READ,
("Invalid bio (%d != %d).", bp->bio_cflags, GJ_BIO_READ));
pbp = bp->bio_parent;
pbp->bio_inbed++;
pbp->bio_completed += bp->bio_length;
if (bp->bio_error != 0) {
if (pbp->bio_error == 0)
pbp->bio_error = bp->bio_error;
GJ_DEBUG(0, "Error while reading data from %s (error=%d).",
bp->bio_to->name, bp->bio_error);
}
g_destroy_bio(bp);
if (pbp->bio_children == pbp->bio_inbed &&
pbp->bio_completed == pbp->bio_length) {
/* We're done. */
g_io_deliver(pbp, 0);
}
}
/*
* Deactive current journal and active next one.
*/
static void
g_journal_switch(struct g_journal_softc *sc)
{
struct g_provider *pp;
if (JEMPTY(sc)) {
GJ_DEBUG(3, "No need for %s switch.", sc->sc_name);
pp = LIST_FIRST(&sc->sc_geom->provider);
if (!(sc->sc_flags & GJF_DEVICE_CLEAN) && pp->acw == 0) {
sc->sc_flags |= GJF_DEVICE_CLEAN;
GJ_DEBUG(1, "Marking %s as clean.", sc->sc_name);
g_journal_metadata_update(sc);
}
} else {
GJ_DEBUG(3, "Switching journal %s.", sc->sc_geom->name);
pp = sc->sc_jprovider;
sc->sc_journal_previous_id = sc->sc_journal_id;
sc->sc_journal_id = sc->sc_journal_next_id;
sc->sc_journal_next_id = arc4random();
GJ_VALIDATE_OFFSET(sc->sc_journal_offset, sc);
g_journal_write_header(sc);
sc->sc_inactive.jj_offset = sc->sc_active.jj_offset;
sc->sc_inactive.jj_queue = sc->sc_active.jj_queue;
sc->sc_active.jj_offset =
sc->sc_journal_offset - pp->sectorsize;
sc->sc_active.jj_queue = NULL;
/*
* Switch is done, start copying data from the (now) inactive
* journal to the data provider.
*/
g_journal_copy_start(sc);
}
mtx_lock(&sc->sc_mtx);
sc->sc_flags &= ~GJF_DEVICE_SWITCH;
mtx_unlock(&sc->sc_mtx);
}
static void
g_journal_initialize(struct g_journal_softc *sc)
{
sc->sc_journal_id = arc4random();
sc->sc_journal_next_id = arc4random();
sc->sc_journal_previous_id = sc->sc_journal_id;
sc->sc_journal_offset = sc->sc_jstart;
sc->sc_inactive.jj_offset = sc->sc_jstart;
g_journal_write_header(sc);
sc->sc_active.jj_offset = sc->sc_jstart;
}
static void
g_journal_mark_as_dirty(struct g_journal_softc *sc)
{
const struct g_journal_desc *desc;
int i;
GJ_DEBUG(1, "Marking file system %s as dirty.", sc->sc_name);
for (i = 0; (desc = g_journal_filesystems[i]) != NULL; i++)
desc->jd_dirty(sc->sc_dconsumer);
}
/*
* Function read record header from the given journal.
* It is very simlar to g_read_data(9), but it doesn't allocate memory for bio
* and data on every call.
*/
static int
g_journal_sync_read(struct g_consumer *cp, struct bio *bp, off_t offset,
void *data)
{
int error;
bzero(bp, sizeof(*bp));
bp->bio_cmd = BIO_READ;
bp->bio_done = NULL;
bp->bio_offset = offset;
bp->bio_length = cp->provider->sectorsize;
bp->bio_data = data;
g_io_request(bp, cp);
error = biowait(bp, "gjs_read");
return (error);
}
#if 0
/*
* Function is called when we start the journal device and we detect that
* one of the journals was not fully copied.
* The purpose of this function is to read all records headers from journal
* and placed them in the inactive queue, so we can start journal
* synchronization process and the journal provider itself.
* Design decision was taken to not synchronize the whole journal here as it
* can take too much time. Reading headers only and delaying synchronization
* process until after journal provider is started should be the best choice.
*/
#endif
static void
g_journal_sync(struct g_journal_softc *sc)
{
struct g_journal_record_header rhdr;
struct g_journal_entry *ent;
struct g_journal_header jhdr;
struct g_consumer *cp;
struct bio *bp, *fbp, *tbp;
off_t joffset, offset;
u_char *buf, sum[16];
uint64_t id;
MD5_CTX ctx;
int error, found, i;
found = 0;
fbp = NULL;
cp = sc->sc_jconsumer;
bp = g_alloc_bio();
buf = gj_malloc(cp->provider->sectorsize, M_WAITOK);
offset = joffset = sc->sc_inactive.jj_offset = sc->sc_journal_offset;
GJ_DEBUG(2, "Looking for termination at %jd.", (intmax_t)joffset);
/*
* Read and decode first journal header.
*/
error = g_journal_sync_read(cp, bp, offset, buf);
if (error != 0) {
GJ_DEBUG(0, "Error while reading journal header from %s.",
cp->provider->name);
goto end;
}
error = g_journal_header_decode(buf, &jhdr);
if (error != 0) {
GJ_DEBUG(0, "Cannot decode journal header from %s.",
cp->provider->name);
goto end;
}
id = sc->sc_journal_id;
if (jhdr.jh_journal_id != sc->sc_journal_id) {
GJ_DEBUG(1, "Journal ID mismatch at %jd (0x%08x != 0x%08x).",
(intmax_t)offset, (u_int)jhdr.jh_journal_id, (u_int)id);
goto end;
}
offset += cp->provider->sectorsize;
id = sc->sc_journal_next_id = jhdr.jh_journal_next_id;
for (;;) {
/*
* If the biggest record won't fit, look for a record header or
* journal header from the begining.
*/
GJ_VALIDATE_OFFSET(offset, sc);
error = g_journal_sync_read(cp, bp, offset, buf);
if (error != 0) {
/*
* Not good. Having an error while reading header
* means, that we cannot read next headers and in
* consequence we cannot find termination.
*/
GJ_DEBUG(0,
"Error while reading record header from %s.",
cp->provider->name);
break;
}
error = g_journal_record_header_decode(buf, &rhdr);
if (error != 0) {
GJ_DEBUG(2, "Not a record header at %jd (error=%d).",
(intmax_t)offset, error);
/*
* This is not a record header.
* If we are lucky, this is next journal header.
*/
error = g_journal_header_decode(buf, &jhdr);
if (error != 0) {
GJ_DEBUG(1, "Not a journal header at %jd (error=%d).",
(intmax_t)offset, error);
/*
* Nope, this is not journal header, which
* bascially means that journal is not
* terminated properly.
*/
error = ENOENT;
break;
}
/*
* Ok. This is header of _some_ journal. Now we need to
* verify if this is header of the _next_ journal.
*/
if (jhdr.jh_journal_id != id) {
GJ_DEBUG(1, "Journal ID mismatch at %jd "
"(0x%08x != 0x%08x).", (intmax_t)offset,
(u_int)jhdr.jh_journal_id, (u_int)id);
error = ENOENT;
break;
}
/* Found termination. */
found++;
GJ_DEBUG(1, "Found termination at %jd (id=0x%08x).",
(intmax_t)offset, (u_int)id);
sc->sc_active.jj_offset = offset;
sc->sc_journal_offset =
offset + cp->provider->sectorsize;
sc->sc_journal_id = id;
id = sc->sc_journal_next_id = jhdr.jh_journal_next_id;
while ((tbp = fbp) != NULL) {
fbp = tbp->bio_next;
GJ_LOGREQ(3, tbp, "Adding request.");
g_journal_insert_bio(&sc->sc_inactive.jj_queue,
tbp, M_WAITOK);
}
/* Skip journal's header. */
offset += cp->provider->sectorsize;
continue;
}
/* Skip record's header. */
offset += cp->provider->sectorsize;
/*
* Add information about every record entry to the inactive
* queue.
*/
if (sc->sc_flags & GJF_DEVICE_CHECKSUM)
MD5Init(&ctx);
for (i = 0; i < rhdr.jrh_nentries; i++) {
ent = &rhdr.jrh_entries[i];
GJ_DEBUG(3, "Insert entry: %jd %jd.",
(intmax_t)ent->je_offset, (intmax_t)ent->je_length);
g_journal_insert(&fbp, ent->je_offset,
ent->je_offset + ent->je_length, ent->je_joffset,
NULL, M_WAITOK);
if (sc->sc_flags & GJF_DEVICE_CHECKSUM) {
u_char *buf2;
/*
* TODO: Should use faster function (like
* g_journal_sync_read()).
*/
buf2 = g_read_data(cp, offset, ent->je_length,
NULL);
if (buf2 == NULL)
GJ_DEBUG(0, "Cannot read data at %jd.",
(intmax_t)offset);
else {
MD5Update(&ctx, buf2, ent->je_length);
g_free(buf2);
}
}
/* Skip entry's data. */
offset += ent->je_length;
}
if (sc->sc_flags & GJF_DEVICE_CHECKSUM) {
MD5Final(sum, &ctx);
if (bcmp(sum, rhdr.jrh_sum, sizeof(rhdr.jrh_sum)) != 0) {
GJ_DEBUG(0, "MD5 hash mismatch at %jd!",
(intmax_t)offset);
}
}
}
end:
gj_free(bp->bio_data, cp->provider->sectorsize);
g_destroy_bio(bp);
/* Remove bios from unterminated journal. */
while ((tbp = fbp) != NULL) {
fbp = tbp->bio_next;
g_destroy_bio(tbp);
}
if (found < 1 && joffset > 0) {
GJ_DEBUG(0, "Journal on %s is broken/corrupted. Initializing.",
sc->sc_name);
while ((tbp = sc->sc_inactive.jj_queue) != NULL) {
sc->sc_inactive.jj_queue = tbp->bio_next;
g_destroy_bio(tbp);
}
g_journal_initialize(sc);
g_journal_mark_as_dirty(sc);
} else {
GJ_DEBUG(0, "Journal %s consistent.", sc->sc_name);
g_journal_copy_start(sc);
}
}
/*
* Wait for requests.
* If we have requests in the current queue, flush them after 3 seconds from the
* last flush. In this way we don't wait forever (or for journal switch) with
* storing not full records on journal.
*/
static void
g_journal_wait(struct g_journal_softc *sc, time_t last_write)
{
int error, timeout;
GJ_DEBUG(3, "%s: enter", __func__);
if (sc->sc_current_count == 0) {
if (g_journal_debug < 2)
msleep(sc, &sc->sc_mtx, PRIBIO | PDROP, "gj:work", 0);
else {
/*
* If we have debug turned on, show number of elements
* in various queues.
*/
for (;;) {
error = msleep(sc, &sc->sc_mtx, PRIBIO,
"gj:work", hz * 3);
if (error == 0) {
mtx_unlock(&sc->sc_mtx);
break;
}
GJ_DEBUG(3, "Report: current count=%d",
sc->sc_current_count);
GJ_DEBUG(3, "Report: flush count=%d",
sc->sc_flush_count);
GJ_DEBUG(3, "Report: flush in progress=%d",
sc->sc_flush_in_progress);
GJ_DEBUG(3, "Report: copy in progress=%d",
sc->sc_copy_in_progress);
GJ_DEBUG(3, "Report: delayed=%d",
sc->sc_delayed_count);
}
}
GJ_DEBUG(3, "%s: exit 1", __func__);
return;
}
/*
* Flush even not full records every 3 seconds.
*/
timeout = (last_write + 3 - time_second) * hz;
if (timeout <= 0) {
mtx_unlock(&sc->sc_mtx);
g_journal_flush(sc);
g_journal_flush_send(sc);
GJ_DEBUG(3, "%s: exit 2", __func__);
return;
}
error = msleep(sc, &sc->sc_mtx, PRIBIO | PDROP, "gj:work", timeout);
if (error == EWOULDBLOCK)
g_journal_flush_send(sc);
GJ_DEBUG(3, "%s: exit 3", __func__);
}
/*
* Worker thread.
*/
static void
g_journal_worker(void *arg)
{
struct g_journal_softc *sc;
struct g_geom *gp;
struct g_provider *pp;
struct bio *bp;
time_t last_write;
int type;
thread_lock(curthread);
sched_prio(curthread, PRIBIO);
thread_unlock(curthread);
sc = arg;
type = 0; /* gcc */
if (sc->sc_flags & GJF_DEVICE_CLEAN) {
GJ_DEBUG(0, "Journal %s clean.", sc->sc_name);
g_journal_initialize(sc);
} else {
g_journal_sync(sc);
}
/*
* Check if we can use BIO_FLUSH.
*/
sc->sc_bio_flush = 0;
if (g_io_flush(sc->sc_jconsumer) == 0) {
sc->sc_bio_flush |= GJ_FLUSH_JOURNAL;
GJ_DEBUG(1, "BIO_FLUSH supported by %s.",
sc->sc_jconsumer->provider->name);
} else {
GJ_DEBUG(0, "BIO_FLUSH not supported by %s.",
sc->sc_jconsumer->provider->name);
}
if (sc->sc_jconsumer != sc->sc_dconsumer) {
if (g_io_flush(sc->sc_dconsumer) == 0) {
sc->sc_bio_flush |= GJ_FLUSH_DATA;
GJ_DEBUG(1, "BIO_FLUSH supported by %s.",
sc->sc_dconsumer->provider->name);
} else {
GJ_DEBUG(0, "BIO_FLUSH not supported by %s.",
sc->sc_dconsumer->provider->name);
}
}
gp = sc->sc_geom;
g_topology_lock();
pp = g_new_providerf(gp, "%s.journal", sc->sc_name);
pp->mediasize = sc->sc_mediasize;
/*
* There could be a problem when data provider and journal providers
* have different sectorsize, but such scenario is prevented on journal
* creation.
*/
pp->sectorsize = sc->sc_sectorsize;
g_error_provider(pp, 0);
g_topology_unlock();
last_write = time_second;
if (sc->sc_rootmount != NULL) {
GJ_DEBUG(1, "root_mount_rel %p", sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
}
for (;;) {
/* Get first request from the queue. */
mtx_lock(&sc->sc_mtx);
bp = bioq_first(&sc->sc_back_queue);
if (bp != NULL)
type = (bp->bio_cflags & GJ_BIO_MASK);
if (bp == NULL) {
bp = bioq_first(&sc->sc_regular_queue);
if (bp != NULL)
type = GJ_BIO_REGULAR;
}
if (bp == NULL) {
try_switch:
if ((sc->sc_flags & GJF_DEVICE_SWITCH) ||
(sc->sc_flags & GJF_DEVICE_DESTROY)) {
if (sc->sc_current_count > 0) {
mtx_unlock(&sc->sc_mtx);
g_journal_flush(sc);
g_journal_flush_send(sc);
continue;
}
if (sc->sc_flush_in_progress > 0)
goto sleep;
if (sc->sc_copy_in_progress > 0)
goto sleep;
}
if (sc->sc_flags & GJF_DEVICE_SWITCH) {
mtx_unlock(&sc->sc_mtx);
g_journal_switch(sc);
wakeup(&sc->sc_journal_copying);
continue;
}
if (sc->sc_flags & GJF_DEVICE_DESTROY) {
GJ_DEBUG(1, "Shutting down worker "
"thread for %s.", gp->name);
sc->sc_worker = NULL;
wakeup(&sc->sc_worker);
mtx_unlock(&sc->sc_mtx);
kproc_exit(0);
}
sleep:
g_journal_wait(sc, last_write);
continue;
}
/*
* If we're in switch process, we need to delay all new
* write requests until its done.
*/
if ((sc->sc_flags & GJF_DEVICE_SWITCH) &&
type == GJ_BIO_REGULAR && bp->bio_cmd == BIO_WRITE) {
GJ_LOGREQ(2, bp, "WRITE on SWITCH");
goto try_switch;
}
if (type == GJ_BIO_REGULAR)
bioq_remove(&sc->sc_regular_queue, bp);
else
bioq_remove(&sc->sc_back_queue, bp);
mtx_unlock(&sc->sc_mtx);
switch (type) {
case GJ_BIO_REGULAR:
/* Regular request. */
switch (bp->bio_cmd) {
case BIO_READ:
g_journal_read(sc, bp, bp->bio_offset,
bp->bio_offset + bp->bio_length);
break;
case BIO_WRITE:
last_write = time_second;
g_journal_add_request(sc, bp);
g_journal_flush_send(sc);
break;
default:
panic("Invalid bio_cmd (%d).", bp->bio_cmd);
}
break;
case GJ_BIO_COPY:
switch (bp->bio_cmd) {
case BIO_READ:
if (g_journal_copy_read_done(bp))
g_journal_copy_send(sc);
break;
case BIO_WRITE:
g_journal_copy_write_done(bp);
g_journal_copy_send(sc);
break;
default:
panic("Invalid bio_cmd (%d).", bp->bio_cmd);
}
break;
case GJ_BIO_JOURNAL:
g_journal_flush_done(bp);
g_journal_flush_send(sc);
break;
case GJ_BIO_READ:
default:
panic("Invalid bio (%d).", type);
}
}
}
static void
g_journal_destroy_event(void *arg, int flags __unused)
{
struct g_journal_softc *sc;
g_topology_assert();
sc = arg;
g_journal_destroy(sc);
}
static void
g_journal_timeout(void *arg)
{
struct g_journal_softc *sc;
sc = arg;
GJ_DEBUG(0, "Timeout. Journal %s cannot be completed.",
sc->sc_geom->name);
g_post_event(g_journal_destroy_event, sc, M_NOWAIT, NULL);
}
static struct g_geom *
g_journal_create(struct g_class *mp, struct g_provider *pp,
const struct g_journal_metadata *md)
{
struct g_journal_softc *sc;
struct g_geom *gp;
struct g_consumer *cp;
int error;
sc = NULL; /* gcc */
g_topology_assert();
/*
* There are two possibilities:
* 1. Data and both journals are on the same provider.
* 2. Data and journals are all on separated providers.
*/
/* Look for journal device with the same ID. */
LIST_FOREACH(gp, &mp->geom, geom) {
sc = gp->softc;
if (sc == NULL)
continue;
if (sc->sc_id == md->md_id)
break;
}
if (gp == NULL)
sc = NULL;
else if (sc != NULL && (sc->sc_type & md->md_type) != 0) {
GJ_DEBUG(1, "Journal device %u already configured.", sc->sc_id);
return (NULL);
}
if (md->md_type == 0 || (md->md_type & ~GJ_TYPE_COMPLETE) != 0) {
GJ_DEBUG(0, "Invalid type on %s.", pp->name);
return (NULL);
}
if (md->md_type & GJ_TYPE_DATA) {
GJ_DEBUG(0, "Journal %u: %s contains data.", md->md_id,
pp->name);
}
if (md->md_type & GJ_TYPE_JOURNAL) {
GJ_DEBUG(0, "Journal %u: %s contains journal.", md->md_id,
pp->name);
}
if (sc == NULL) {
/* Action geom. */
sc = malloc(sizeof(*sc), M_JOURNAL, M_WAITOK | M_ZERO);
sc->sc_id = md->md_id;
sc->sc_type = 0;
sc->sc_flags = 0;
sc->sc_worker = NULL;
gp = g_new_geomf(mp, "gjournal %u", sc->sc_id);
gp->start = g_journal_start;
gp->orphan = g_journal_orphan;
gp->access = g_journal_access;
gp->softc = sc;
gp->flags |= G_GEOM_VOLATILE_BIO;
sc->sc_geom = gp;
mtx_init(&sc->sc_mtx, "gjournal", NULL, MTX_DEF);
bioq_init(&sc->sc_back_queue);
bioq_init(&sc->sc_regular_queue);
bioq_init(&sc->sc_delayed_queue);
sc->sc_delayed_count = 0;
sc->sc_current_queue = NULL;
sc->sc_current_count = 0;
sc->sc_flush_queue = NULL;
sc->sc_flush_count = 0;
sc->sc_flush_in_progress = 0;
sc->sc_copy_queue = NULL;
sc->sc_copy_in_progress = 0;
sc->sc_inactive.jj_queue = NULL;
sc->sc_active.jj_queue = NULL;
sc->sc_rootmount = root_mount_hold("GJOURNAL");
GJ_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
- callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_callout, 1);
if (md->md_type != GJ_TYPE_COMPLETE) {
/*
* Journal and data are on separate providers.
* At this point we have only one of them.
* We setup a timeout in case the other part will not
* appear, so we won't wait forever.
*/
callout_reset(&sc->sc_callout, 5 * hz,
g_journal_timeout, sc);
}
}
/* Remember type of the data provider. */
if (md->md_type & GJ_TYPE_DATA)
sc->sc_orig_type = md->md_type;
sc->sc_type |= md->md_type;
cp = NULL;
if (md->md_type & GJ_TYPE_DATA) {
if (md->md_flags & GJ_FLAG_CLEAN)
sc->sc_flags |= GJF_DEVICE_CLEAN;
if (md->md_flags & GJ_FLAG_CHECKSUM)
sc->sc_flags |= GJF_DEVICE_CHECKSUM;
cp = g_new_consumer(gp);
error = g_attach(cp, pp);
KASSERT(error == 0, ("Cannot attach to %s (error=%d).",
pp->name, error));
error = g_access(cp, 1, 1, 1);
if (error != 0) {
GJ_DEBUG(0, "Cannot access %s (error=%d).", pp->name,
error);
g_journal_destroy(sc);
return (NULL);
}
sc->sc_dconsumer = cp;
sc->sc_mediasize = pp->mediasize - pp->sectorsize;
sc->sc_sectorsize = pp->sectorsize;
sc->sc_jstart = md->md_jstart;
sc->sc_jend = md->md_jend;
if (md->md_provider[0] != '\0')
sc->sc_flags |= GJF_DEVICE_HARDCODED;
sc->sc_journal_offset = md->md_joffset;
sc->sc_journal_id = md->md_jid;
sc->sc_journal_previous_id = md->md_jid;
}
if (md->md_type & GJ_TYPE_JOURNAL) {
if (cp == NULL) {
cp = g_new_consumer(gp);
error = g_attach(cp, pp);
KASSERT(error == 0, ("Cannot attach to %s (error=%d).",
pp->name, error));
error = g_access(cp, 1, 1, 1);
if (error != 0) {
GJ_DEBUG(0, "Cannot access %s (error=%d).",
pp->name, error);
g_journal_destroy(sc);
return (NULL);
}
} else {
/*
* Journal is on the same provider as data, which means
* that data provider ends where journal starts.
*/
sc->sc_mediasize = md->md_jstart;
}
sc->sc_jconsumer = cp;
}
if ((sc->sc_type & GJ_TYPE_COMPLETE) != GJ_TYPE_COMPLETE) {
/* Journal is not complete yet. */
return (gp);
} else {
/* Journal complete, cancel timeout. */
callout_drain(&sc->sc_callout);
}
error = kproc_create(g_journal_worker, sc, &sc->sc_worker, 0, 0,
"g_journal %s", sc->sc_name);
if (error != 0) {
GJ_DEBUG(0, "Cannot create worker thread for %s.journal.",
sc->sc_name);
g_journal_destroy(sc);
return (NULL);
}
return (gp);
}
static void
g_journal_destroy_consumer(void *arg, int flags __unused)
{
struct g_consumer *cp;
g_topology_assert();
cp = arg;
g_detach(cp);
g_destroy_consumer(cp);
}
static int
g_journal_destroy(struct g_journal_softc *sc)
{
struct g_geom *gp;
struct g_provider *pp;
struct g_consumer *cp;
g_topology_assert();
if (sc == NULL)
return (ENXIO);
gp = sc->sc_geom;
pp = LIST_FIRST(&gp->provider);
if (pp != NULL) {
if (pp->acr != 0 || pp->acw != 0 || pp->ace != 0) {
GJ_DEBUG(1, "Device %s is still open (r%dw%de%d).",
pp->name, pp->acr, pp->acw, pp->ace);
return (EBUSY);
}
g_error_provider(pp, ENXIO);
g_journal_flush(sc);
g_journal_flush_send(sc);
g_journal_switch(sc);
}
sc->sc_flags |= (GJF_DEVICE_DESTROY | GJF_DEVICE_CLEAN);
g_topology_unlock();
if (sc->sc_rootmount != NULL) {
GJ_DEBUG(1, "root_mount_rel %p", sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
}
callout_drain(&sc->sc_callout);
mtx_lock(&sc->sc_mtx);
wakeup(sc);
while (sc->sc_worker != NULL)
msleep(&sc->sc_worker, &sc->sc_mtx, PRIBIO, "gj:destroy", 0);
mtx_unlock(&sc->sc_mtx);
if (pp != NULL) {
GJ_DEBUG(1, "Marking %s as clean.", sc->sc_name);
g_journal_metadata_update(sc);
g_topology_lock();
pp->flags |= G_PF_WITHER;
g_orphan_provider(pp, ENXIO);
} else {
g_topology_lock();
}
mtx_destroy(&sc->sc_mtx);
if (sc->sc_current_count != 0) {
GJ_DEBUG(0, "Warning! Number of current requests %d.",
sc->sc_current_count);
}
LIST_FOREACH(cp, &gp->consumer, consumer) {
if (cp->acr + cp->acw + cp->ace > 0)
g_access(cp, -1, -1, -1);
/*
* We keep all consumers open for writting, so if I'll detach
* and destroy consumer here, I'll get providers for taste, so
* journal will be started again.
* Sending an event here, prevents this from happening.
*/
g_post_event(g_journal_destroy_consumer, cp, M_WAITOK, NULL);
}
gp->softc = NULL;
g_wither_geom(gp, ENXIO);
free(sc, M_JOURNAL);
return (0);
}
static void
g_journal_taste_orphan(struct g_consumer *cp)
{
KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
cp->provider->name));
}
static struct g_geom *
g_journal_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
{
struct g_journal_metadata md;
struct g_consumer *cp;
struct g_geom *gp;
int error;
g_topology_assert();
g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
GJ_DEBUG(2, "Tasting %s.", pp->name);
if (pp->geom->class == mp)
return (NULL);
gp = g_new_geomf(mp, "journal:taste");
/* This orphan function should be never called. */
gp->orphan = g_journal_taste_orphan;
cp = g_new_consumer(gp);
g_attach(cp, pp);
error = g_journal_metadata_read(cp, &md);
g_detach(cp);
g_destroy_consumer(cp);
g_destroy_geom(gp);
if (error != 0)
return (NULL);
gp = NULL;
if (md.md_provider[0] != '\0' &&
!g_compare_names(md.md_provider, pp->name))
return (NULL);
if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
return (NULL);
if (g_journal_debug >= 2)
journal_metadata_dump(&md);
gp = g_journal_create(mp, pp, &md);
return (gp);
}
static struct g_journal_softc *
g_journal_find_device(struct g_class *mp, const char *name)
{
struct g_journal_softc *sc;
struct g_geom *gp;
struct g_provider *pp;
if (strncmp(name, "/dev/", 5) == 0)
name += 5;
LIST_FOREACH(gp, &mp->geom, geom) {
sc = gp->softc;
if (sc == NULL)
continue;
if (sc->sc_flags & GJF_DEVICE_DESTROY)
continue;
if ((sc->sc_type & GJ_TYPE_COMPLETE) != GJ_TYPE_COMPLETE)
continue;
pp = LIST_FIRST(&gp->provider);
if (strcmp(sc->sc_name, name) == 0)
return (sc);
if (pp != NULL && strcmp(pp->name, name) == 0)
return (sc);
}
return (NULL);
}
static void
g_journal_ctl_destroy(struct gctl_req *req, struct g_class *mp)
{
struct g_journal_softc *sc;
const char *name;
char param[16];
int *nargs;
int error, i;
g_topology_assert();
nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
if (nargs == NULL) {
gctl_error(req, "No '%s' argument.", "nargs");
return;
}
if (*nargs <= 0) {
gctl_error(req, "Missing device(s).");
return;
}
for (i = 0; i < *nargs; i++) {
snprintf(param, sizeof(param), "arg%d", i);
name = gctl_get_asciiparam(req, param);
if (name == NULL) {
gctl_error(req, "No 'arg%d' argument.", i);
return;
}
sc = g_journal_find_device(mp, name);
if (sc == NULL) {
gctl_error(req, "No such device: %s.", name);
return;
}
error = g_journal_destroy(sc);
if (error != 0) {
gctl_error(req, "Cannot destroy device %s (error=%d).",
LIST_FIRST(&sc->sc_geom->provider)->name, error);
return;
}
}
}
static void
g_journal_ctl_sync(struct gctl_req *req __unused, struct g_class *mp __unused)
{
g_topology_assert();
g_topology_unlock();
g_journal_sync_requested++;
wakeup(&g_journal_switcher_state);
while (g_journal_sync_requested > 0)
tsleep(&g_journal_sync_requested, PRIBIO, "j:sreq", hz / 2);
g_topology_lock();
}
static void
g_journal_config(struct gctl_req *req, struct g_class *mp, const char *verb)
{
uint32_t *version;
g_topology_assert();
version = gctl_get_paraml(req, "version", sizeof(*version));
if (version == NULL) {
gctl_error(req, "No '%s' argument.", "version");
return;
}
if (*version != G_JOURNAL_VERSION) {
gctl_error(req, "Userland and kernel parts are out of sync.");
return;
}
if (strcmp(verb, "destroy") == 0 || strcmp(verb, "stop") == 0) {
g_journal_ctl_destroy(req, mp);
return;
} else if (strcmp(verb, "sync") == 0) {
g_journal_ctl_sync(req, mp);
return;
}
gctl_error(req, "Unknown verb.");
}
static void
g_journal_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
struct g_consumer *cp, struct g_provider *pp)
{
struct g_journal_softc *sc;
g_topology_assert();
sc = gp->softc;
if (sc == NULL)
return;
if (pp != NULL) {
/* Nothing here. */
} else if (cp != NULL) {
int first = 1;
sbuf_printf(sb, "%s<Role>", indent);
if (cp == sc->sc_dconsumer) {
sbuf_printf(sb, "Data");
first = 0;
}
if (cp == sc->sc_jconsumer) {
if (!first)
sbuf_printf(sb, ",");
sbuf_printf(sb, "Journal");
}
sbuf_printf(sb, "</Role>\n");
if (cp == sc->sc_jconsumer) {
sbuf_printf(sb, "<Jstart>%jd</Jstart>\n",
(intmax_t)sc->sc_jstart);
sbuf_printf(sb, "<Jend>%jd</Jend>\n",
(intmax_t)sc->sc_jend);
}
} else {
sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
}
}
static eventhandler_tag g_journal_event_shutdown = NULL;
static eventhandler_tag g_journal_event_lowmem = NULL;
static void
g_journal_shutdown(void *arg, int howto __unused)
{
struct g_class *mp;
struct g_geom *gp, *gp2;
if (panicstr != NULL)
return;
mp = arg;
DROP_GIANT();
g_topology_lock();
LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
if (gp->softc == NULL)
continue;
GJ_DEBUG(0, "Shutting down geom %s.", gp->name);
g_journal_destroy(gp->softc);
}
g_topology_unlock();
PICKUP_GIANT();
}
/*
* Free cached requests from inactive queue in case of low memory.
* We free GJ_FREE_AT_ONCE elements at once.
*/
#define GJ_FREE_AT_ONCE 4
static void
g_journal_lowmem(void *arg, int howto __unused)
{
struct g_journal_softc *sc;
struct g_class *mp;
struct g_geom *gp;
struct bio *bp;
u_int nfree = GJ_FREE_AT_ONCE;
g_journal_stats_low_mem++;
mp = arg;
DROP_GIANT();
g_topology_lock();
LIST_FOREACH(gp, &mp->geom, geom) {
sc = gp->softc;
if (sc == NULL || (sc->sc_flags & GJF_DEVICE_DESTROY))
continue;
mtx_lock(&sc->sc_mtx);
for (bp = sc->sc_inactive.jj_queue; nfree > 0 && bp != NULL;
nfree--, bp = bp->bio_next) {
/*
* This is safe to free the bio_data, because:
* 1. If bio_data is NULL it will be read from the
* inactive journal.
* 2. If bp is sent down, it is first removed from the
* inactive queue, so it's impossible to free the
* data from under in-flight bio.
* On the other hand, freeing elements from the active
* queue, is not safe.
*/
if (bp->bio_data != NULL) {
GJ_DEBUG(2, "Freeing data from %s.",
sc->sc_name);
gj_free(bp->bio_data, bp->bio_length);
bp->bio_data = NULL;
}
}
mtx_unlock(&sc->sc_mtx);
if (nfree == 0)
break;
}
g_topology_unlock();
PICKUP_GIANT();
}
static void g_journal_switcher(void *arg);
static void
g_journal_init(struct g_class *mp)
{
int error;
/* Pick a conservative value if provided value sucks. */
if (g_journal_cache_divisor <= 0 ||
(vm_kmem_size / g_journal_cache_divisor == 0)) {
g_journal_cache_divisor = 5;
}
if (g_journal_cache_limit > 0) {
g_journal_cache_limit = vm_kmem_size / g_journal_cache_divisor;
g_journal_cache_low =
(g_journal_cache_limit / 100) * g_journal_cache_switch;
}
g_journal_event_shutdown = EVENTHANDLER_REGISTER(shutdown_post_sync,
g_journal_shutdown, mp, EVENTHANDLER_PRI_FIRST);
if (g_journal_event_shutdown == NULL)
GJ_DEBUG(0, "Warning! Cannot register shutdown event.");
g_journal_event_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
g_journal_lowmem, mp, EVENTHANDLER_PRI_FIRST);
if (g_journal_event_lowmem == NULL)
GJ_DEBUG(0, "Warning! Cannot register lowmem event.");
error = kproc_create(g_journal_switcher, mp, NULL, 0, 0,
"g_journal switcher");
KASSERT(error == 0, ("Cannot create switcher thread."));
}
static void
g_journal_fini(struct g_class *mp)
{
if (g_journal_event_shutdown != NULL) {
EVENTHANDLER_DEREGISTER(shutdown_post_sync,
g_journal_event_shutdown);
}
if (g_journal_event_lowmem != NULL)
EVENTHANDLER_DEREGISTER(vm_lowmem, g_journal_event_lowmem);
g_journal_switcher_state = GJ_SWITCHER_DIE;
wakeup(&g_journal_switcher_state);
while (g_journal_switcher_state != GJ_SWITCHER_DIED)
tsleep(&g_journal_switcher_state, PRIBIO, "jfini:wait", hz / 5);
GJ_DEBUG(1, "Switcher died.");
}
DECLARE_GEOM_CLASS(g_journal_class, g_journal);
static const struct g_journal_desc *
g_journal_find_desc(const char *fstype)
{
const struct g_journal_desc *desc;
int i;
for (desc = g_journal_filesystems[i = 0]; desc != NULL;
desc = g_journal_filesystems[++i]) {
if (strcmp(desc->jd_fstype, fstype) == 0)
break;
}
return (desc);
}
static void
g_journal_switch_wait(struct g_journal_softc *sc)
{
struct bintime bt;
mtx_assert(&sc->sc_mtx, MA_OWNED);
if (g_journal_debug >= 2) {
if (sc->sc_flush_in_progress > 0) {
GJ_DEBUG(2, "%d requests flushing.",
sc->sc_flush_in_progress);
}
if (sc->sc_copy_in_progress > 0) {
GJ_DEBUG(2, "%d requests copying.",
sc->sc_copy_in_progress);
}
if (sc->sc_flush_count > 0) {
GJ_DEBUG(2, "%d requests to flush.",
sc->sc_flush_count);
}
if (sc->sc_delayed_count > 0) {
GJ_DEBUG(2, "%d requests delayed.",
sc->sc_delayed_count);
}
}
g_journal_stats_switches++;
if (sc->sc_copy_in_progress > 0)
g_journal_stats_wait_for_copy++;
GJ_TIMER_START(1, &bt);
sc->sc_flags &= ~GJF_DEVICE_BEFORE_SWITCH;
sc->sc_flags |= GJF_DEVICE_SWITCH;
wakeup(sc);
while (sc->sc_flags & GJF_DEVICE_SWITCH) {
msleep(&sc->sc_journal_copying, &sc->sc_mtx, PRIBIO,
"gj:switch", 0);
}
GJ_TIMER_STOP(1, &bt, "Switch time of %s", sc->sc_name);
}
static void
g_journal_do_switch(struct g_class *classp)
{
struct g_journal_softc *sc;
const struct g_journal_desc *desc;
struct g_geom *gp;
struct mount *mp;
struct bintime bt;
char *mountpoint;
int error, save;
DROP_GIANT();
g_topology_lock();
LIST_FOREACH(gp, &classp->geom, geom) {
sc = gp->softc;
if (sc == NULL)
continue;
if (sc->sc_flags & GJF_DEVICE_DESTROY)
continue;
if ((sc->sc_type & GJ_TYPE_COMPLETE) != GJ_TYPE_COMPLETE)
continue;
mtx_lock(&sc->sc_mtx);
sc->sc_flags |= GJF_DEVICE_BEFORE_SWITCH;
mtx_unlock(&sc->sc_mtx);
}
g_topology_unlock();
PICKUP_GIANT();
mtx_lock(&mountlist_mtx);
TAILQ_FOREACH(mp, &mountlist, mnt_list) {
if (mp->mnt_gjprovider == NULL)
continue;
if (mp->mnt_flag & MNT_RDONLY)
continue;
desc = g_journal_find_desc(mp->mnt_stat.f_fstypename);
if (desc == NULL)
continue;
if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK))
continue;
/* mtx_unlock(&mountlist_mtx) was done inside vfs_busy() */
DROP_GIANT();
g_topology_lock();
sc = g_journal_find_device(classp, mp->mnt_gjprovider);
g_topology_unlock();
PICKUP_GIANT();
if (sc == NULL) {
GJ_DEBUG(0, "Cannot find journal geom for %s.",
mp->mnt_gjprovider);
goto next;
} else if (JEMPTY(sc)) {
mtx_lock(&sc->sc_mtx);
sc->sc_flags &= ~GJF_DEVICE_BEFORE_SWITCH;
mtx_unlock(&sc->sc_mtx);
GJ_DEBUG(3, "No need for %s switch.", sc->sc_name);
goto next;
}
mountpoint = mp->mnt_stat.f_mntonname;
error = vn_start_write(NULL, &mp, V_WAIT);
if (error != 0) {
GJ_DEBUG(0, "vn_start_write(%s) failed (error=%d).",
mountpoint, error);
goto next;
}
save = curthread_pflags_set(TDP_SYNCIO);
GJ_TIMER_START(1, &bt);
vfs_msync(mp, MNT_NOWAIT);
GJ_TIMER_STOP(1, &bt, "Msync time of %s", mountpoint);
GJ_TIMER_START(1, &bt);
error = VFS_SYNC(mp, MNT_NOWAIT);
if (error == 0)
GJ_TIMER_STOP(1, &bt, "Sync time of %s", mountpoint);
else {
GJ_DEBUG(0, "Cannot sync file system %s (error=%d).",
mountpoint, error);
}
curthread_pflags_restore(save);
vn_finished_write(mp);
if (error != 0)
goto next;
/*
* Send BIO_FLUSH before freezing the file system, so it can be
* faster after the freeze.
*/
GJ_TIMER_START(1, &bt);
g_journal_flush_cache(sc);
GJ_TIMER_STOP(1, &bt, "BIO_FLUSH time of %s", sc->sc_name);
GJ_TIMER_START(1, &bt);
error = vfs_write_suspend(mp, VS_SKIP_UNMOUNT);
GJ_TIMER_STOP(1, &bt, "Suspend time of %s", mountpoint);
if (error != 0) {
GJ_DEBUG(0, "Cannot suspend file system %s (error=%d).",
mountpoint, error);
goto next;
}
error = desc->jd_clean(mp);
if (error != 0)
goto next;
mtx_lock(&sc->sc_mtx);
g_journal_switch_wait(sc);
mtx_unlock(&sc->sc_mtx);
vfs_write_resume(mp, 0);
next:
mtx_lock(&mountlist_mtx);
vfs_unbusy(mp);
}
mtx_unlock(&mountlist_mtx);
sc = NULL;
for (;;) {
DROP_GIANT();
g_topology_lock();
LIST_FOREACH(gp, &g_journal_class.geom, geom) {
sc = gp->softc;
if (sc == NULL)
continue;
mtx_lock(&sc->sc_mtx);
if ((sc->sc_type & GJ_TYPE_COMPLETE) == GJ_TYPE_COMPLETE &&
!(sc->sc_flags & GJF_DEVICE_DESTROY) &&
(sc->sc_flags & GJF_DEVICE_BEFORE_SWITCH)) {
break;
}
mtx_unlock(&sc->sc_mtx);
sc = NULL;
}
g_topology_unlock();
PICKUP_GIANT();
if (sc == NULL)
break;
mtx_assert(&sc->sc_mtx, MA_OWNED);
g_journal_switch_wait(sc);
mtx_unlock(&sc->sc_mtx);
}
}
/*
* TODO: Switcher thread should be started on first geom creation and killed on
* last geom destruction.
*/
static void
g_journal_switcher(void *arg)
{
struct g_class *mp;
struct bintime bt;
int error;
mp = arg;
curthread->td_pflags |= TDP_NORUNNINGBUF;
for (;;) {
g_journal_switcher_wokenup = 0;
error = tsleep(&g_journal_switcher_state, PRIBIO, "jsw:wait",
g_journal_switch_time * hz);
if (g_journal_switcher_state == GJ_SWITCHER_DIE) {
g_journal_switcher_state = GJ_SWITCHER_DIED;
GJ_DEBUG(1, "Switcher exiting.");
wakeup(&g_journal_switcher_state);
kproc_exit(0);
}
if (error == 0 && g_journal_sync_requested == 0) {
GJ_DEBUG(1, "Out of cache, force switch (used=%u "
"limit=%u).", g_journal_cache_used,
g_journal_cache_limit);
}
GJ_TIMER_START(1, &bt);
g_journal_do_switch(mp);
GJ_TIMER_STOP(1, &bt, "Entire switch time");
if (g_journal_sync_requested > 0) {
g_journal_sync_requested = 0;
wakeup(&g_journal_sync_requested);
}
}
}
Index: head/sys/geom/mirror/g_mirror.c
===================================================================
--- head/sys/geom/mirror/g_mirror.c (revision 283290)
+++ head/sys/geom/mirror/g_mirror.c (revision 283291)
@@ -1,3353 +1,3353 @@
/*-
* Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/bio.h>
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include <sys/malloc.h>
#include <sys/eventhandler.h>
#include <vm/uma.h>
#include <geom/geom.h>
#include <sys/proc.h>
#include <sys/kthread.h>
#include <sys/sched.h>
#include <geom/mirror/g_mirror.h>
FEATURE(geom_mirror, "GEOM mirroring support");
static MALLOC_DEFINE(M_MIRROR, "mirror_data", "GEOM_MIRROR Data");
SYSCTL_DECL(_kern_geom);
static SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0,
"GEOM_MIRROR stuff");
u_int g_mirror_debug = 0;
SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RWTUN, &g_mirror_debug, 0,
"Debug level");
static u_int g_mirror_timeout = 4;
SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, timeout, CTLFLAG_RWTUN, &g_mirror_timeout,
0, "Time to wait on all mirror components");
static u_int g_mirror_idletime = 5;
SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, idletime, CTLFLAG_RWTUN,
&g_mirror_idletime, 0, "Mark components as clean when idling");
static u_int g_mirror_disconnect_on_failure = 1;
SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, disconnect_on_failure, CTLFLAG_RWTUN,
&g_mirror_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
static u_int g_mirror_syncreqs = 2;
SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
&g_mirror_syncreqs, 0, "Parallel synchronization I/O requests.");
#define MSLEEP(ident, mtx, priority, wmesg, timeout) do { \
G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, (ident)); \
msleep((ident), (mtx), (priority), (wmesg), (timeout)); \
G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident)); \
} while (0)
static eventhandler_tag g_mirror_post_sync = NULL;
static int g_mirror_shutdown = 0;
static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp,
struct g_geom *gp);
static g_taste_t g_mirror_taste;
static g_resize_t g_mirror_resize;
static void g_mirror_init(struct g_class *mp);
static void g_mirror_fini(struct g_class *mp);
struct g_class g_mirror_class = {
.name = G_MIRROR_CLASS_NAME,
.version = G_VERSION,
.ctlreq = g_mirror_config,
.taste = g_mirror_taste,
.destroy_geom = g_mirror_destroy_geom,
.init = g_mirror_init,
.fini = g_mirror_fini,
.resize = g_mirror_resize
};
static void g_mirror_destroy_provider(struct g_mirror_softc *sc);
static int g_mirror_update_disk(struct g_mirror_disk *disk, u_int state);
static void g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force);
static void g_mirror_dumpconf(struct sbuf *sb, const char *indent,
struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
static void g_mirror_sync_stop(struct g_mirror_disk *disk, int type);
static void g_mirror_register_request(struct bio *bp);
static void g_mirror_sync_release(struct g_mirror_softc *sc);
static const char *
g_mirror_disk_state2str(int state)
{
switch (state) {
case G_MIRROR_DISK_STATE_NONE:
return ("NONE");
case G_MIRROR_DISK_STATE_NEW:
return ("NEW");
case G_MIRROR_DISK_STATE_ACTIVE:
return ("ACTIVE");
case G_MIRROR_DISK_STATE_STALE:
return ("STALE");
case G_MIRROR_DISK_STATE_SYNCHRONIZING:
return ("SYNCHRONIZING");
case G_MIRROR_DISK_STATE_DISCONNECTED:
return ("DISCONNECTED");
case G_MIRROR_DISK_STATE_DESTROY:
return ("DESTROY");
default:
return ("INVALID");
}
}
static const char *
g_mirror_device_state2str(int state)
{
switch (state) {
case G_MIRROR_DEVICE_STATE_STARTING:
return ("STARTING");
case G_MIRROR_DEVICE_STATE_RUNNING:
return ("RUNNING");
default:
return ("INVALID");
}
}
static const char *
g_mirror_get_diskname(struct g_mirror_disk *disk)
{
if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
return ("[unknown]");
return (disk->d_name);
}
/*
* --- Events handling functions ---
* Events in geom_mirror are used to maintain disks and device status
* from one thread to simplify locking.
*/
static void
g_mirror_event_free(struct g_mirror_event *ep)
{
free(ep, M_MIRROR);
}
int
g_mirror_event_send(void *arg, int state, int flags)
{
struct g_mirror_softc *sc;
struct g_mirror_disk *disk;
struct g_mirror_event *ep;
int error;
ep = malloc(sizeof(*ep), M_MIRROR, M_WAITOK);
G_MIRROR_DEBUG(4, "%s: Sending event %p.", __func__, ep);
if ((flags & G_MIRROR_EVENT_DEVICE) != 0) {
disk = NULL;
sc = arg;
} else {
disk = arg;
sc = disk->d_softc;
}
ep->e_disk = disk;
ep->e_state = state;
ep->e_flags = flags;
ep->e_error = 0;
mtx_lock(&sc->sc_events_mtx);
TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
mtx_unlock(&sc->sc_events_mtx);
G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
mtx_lock(&sc->sc_queue_mtx);
wakeup(sc);
mtx_unlock(&sc->sc_queue_mtx);
if ((flags & G_MIRROR_EVENT_DONTWAIT) != 0)
return (0);
sx_assert(&sc->sc_lock, SX_XLOCKED);
G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
sx_xunlock(&sc->sc_lock);
while ((ep->e_flags & G_MIRROR_EVENT_DONE) == 0) {
mtx_lock(&sc->sc_events_mtx);
MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "m:event",
hz * 5);
}
error = ep->e_error;
g_mirror_event_free(ep);
sx_xlock(&sc->sc_lock);
return (error);
}
static struct g_mirror_event *
g_mirror_event_get(struct g_mirror_softc *sc)
{
struct g_mirror_event *ep;
mtx_lock(&sc->sc_events_mtx);
ep = TAILQ_FIRST(&sc->sc_events);
mtx_unlock(&sc->sc_events_mtx);
return (ep);
}
static void
g_mirror_event_remove(struct g_mirror_softc *sc, struct g_mirror_event *ep)
{
mtx_lock(&sc->sc_events_mtx);
TAILQ_REMOVE(&sc->sc_events, ep, e_next);
mtx_unlock(&sc->sc_events_mtx);
}
static void
g_mirror_event_cancel(struct g_mirror_disk *disk)
{
struct g_mirror_softc *sc;
struct g_mirror_event *ep, *tmpep;
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_XLOCKED);
mtx_lock(&sc->sc_events_mtx);
TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0)
continue;
if (ep->e_disk != disk)
continue;
TAILQ_REMOVE(&sc->sc_events, ep, e_next);
if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
g_mirror_event_free(ep);
else {
ep->e_error = ECANCELED;
wakeup(ep);
}
}
mtx_unlock(&sc->sc_events_mtx);
}
/*
* Return the number of disks in given state.
* If state is equal to -1, count all connected disks.
*/
u_int
g_mirror_ndisks(struct g_mirror_softc *sc, int state)
{
struct g_mirror_disk *disk;
u_int n = 0;
sx_assert(&sc->sc_lock, SX_LOCKED);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (state == -1 || disk->d_state == state)
n++;
}
return (n);
}
/*
* Find a disk in mirror by its disk ID.
*/
static struct g_mirror_disk *
g_mirror_id2disk(struct g_mirror_softc *sc, uint32_t id)
{
struct g_mirror_disk *disk;
sx_assert(&sc->sc_lock, SX_XLOCKED);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_id == id)
return (disk);
}
return (NULL);
}
static u_int
g_mirror_nrequests(struct g_mirror_softc *sc, struct g_consumer *cp)
{
struct bio *bp;
u_int nreqs = 0;
mtx_lock(&sc->sc_queue_mtx);
TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
if (bp->bio_from == cp)
nreqs++;
}
mtx_unlock(&sc->sc_queue_mtx);
return (nreqs);
}
static int
g_mirror_is_busy(struct g_mirror_softc *sc, struct g_consumer *cp)
{
if (cp->index > 0) {
G_MIRROR_DEBUG(2,
"I/O requests for %s exist, can't destroy it now.",
cp->provider->name);
return (1);
}
if (g_mirror_nrequests(sc, cp) > 0) {
G_MIRROR_DEBUG(2,
"I/O requests for %s in queue, can't destroy it now.",
cp->provider->name);
return (1);
}
return (0);
}
static void
g_mirror_destroy_consumer(void *arg, int flags __unused)
{
struct g_consumer *cp;
g_topology_assert();
cp = arg;
G_MIRROR_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
g_detach(cp);
g_destroy_consumer(cp);
}
static void
g_mirror_kill_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
{
struct g_provider *pp;
int retaste_wait;
g_topology_assert();
cp->private = NULL;
if (g_mirror_is_busy(sc, cp))
return;
pp = cp->provider;
retaste_wait = 0;
if (cp->acw == 1) {
if ((pp->geom->flags & G_GEOM_WITHER) == 0)
retaste_wait = 1;
}
G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
-cp->acw, -cp->ace, 0);
if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
g_access(cp, -cp->acr, -cp->acw, -cp->ace);
if (retaste_wait) {
/*
* After retaste event was send (inside g_access()), we can send
* event to detach and destroy consumer.
* A class, which has consumer to the given provider connected
* will not receive retaste event for the provider.
* This is the way how I ignore retaste events when I close
* consumers opened for write: I detach and destroy consumer
* after retaste event is sent.
*/
g_post_event(g_mirror_destroy_consumer, cp, M_WAITOK, NULL);
return;
}
G_MIRROR_DEBUG(1, "Consumer %s destroyed.", pp->name);
g_detach(cp);
g_destroy_consumer(cp);
}
static int
g_mirror_connect_disk(struct g_mirror_disk *disk, struct g_provider *pp)
{
struct g_consumer *cp;
int error;
g_topology_assert_not();
KASSERT(disk->d_consumer == NULL,
("Disk already connected (device %s).", disk->d_softc->sc_name));
g_topology_lock();
cp = g_new_consumer(disk->d_softc->sc_geom);
cp->flags |= G_CF_DIRECT_RECEIVE;
error = g_attach(cp, pp);
if (error != 0) {
g_destroy_consumer(cp);
g_topology_unlock();
return (error);
}
error = g_access(cp, 1, 1, 1);
if (error != 0) {
g_detach(cp);
g_destroy_consumer(cp);
g_topology_unlock();
G_MIRROR_DEBUG(0, "Cannot open consumer %s (error=%d).",
pp->name, error);
return (error);
}
g_topology_unlock();
disk->d_consumer = cp;
disk->d_consumer->private = disk;
disk->d_consumer->index = 0;
G_MIRROR_DEBUG(2, "Disk %s connected.", g_mirror_get_diskname(disk));
return (0);
}
static void
g_mirror_disconnect_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
{
g_topology_assert();
if (cp == NULL)
return;
if (cp->provider != NULL)
g_mirror_kill_consumer(sc, cp);
else
g_destroy_consumer(cp);
}
/*
* Initialize disk. This means allocate memory, create consumer, attach it
* to the provider and open access (r1w1e1) to it.
*/
static struct g_mirror_disk *
g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
struct g_mirror_metadata *md, int *errorp)
{
struct g_mirror_disk *disk;
int i, error;
disk = malloc(sizeof(*disk), M_MIRROR, M_NOWAIT | M_ZERO);
if (disk == NULL) {
error = ENOMEM;
goto fail;
}
disk->d_softc = sc;
error = g_mirror_connect_disk(disk, pp);
if (error != 0)
goto fail;
disk->d_id = md->md_did;
disk->d_state = G_MIRROR_DISK_STATE_NONE;
disk->d_priority = md->md_priority;
disk->d_flags = md->md_dflags;
error = g_getattr("GEOM::candelete", disk->d_consumer, &i);
if (error == 0 && i != 0)
disk->d_flags |= G_MIRROR_DISK_FLAG_CANDELETE;
if (md->md_provider[0] != '\0')
disk->d_flags |= G_MIRROR_DISK_FLAG_HARDCODED;
disk->d_sync.ds_consumer = NULL;
disk->d_sync.ds_offset = md->md_sync_offset;
disk->d_sync.ds_offset_done = md->md_sync_offset;
disk->d_genid = md->md_genid;
disk->d_sync.ds_syncid = md->md_syncid;
if (errorp != NULL)
*errorp = 0;
return (disk);
fail:
if (errorp != NULL)
*errorp = error;
if (disk != NULL)
free(disk, M_MIRROR);
return (NULL);
}
static void
g_mirror_destroy_disk(struct g_mirror_disk *disk)
{
struct g_mirror_softc *sc;
g_topology_assert_not();
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_XLOCKED);
LIST_REMOVE(disk, d_next);
g_mirror_event_cancel(disk);
if (sc->sc_hint == disk)
sc->sc_hint = NULL;
switch (disk->d_state) {
case G_MIRROR_DISK_STATE_SYNCHRONIZING:
g_mirror_sync_stop(disk, 1);
/* FALLTHROUGH */
case G_MIRROR_DISK_STATE_NEW:
case G_MIRROR_DISK_STATE_STALE:
case G_MIRROR_DISK_STATE_ACTIVE:
g_topology_lock();
g_mirror_disconnect_consumer(sc, disk->d_consumer);
g_topology_unlock();
free(disk, M_MIRROR);
break;
default:
KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
}
}
static void
g_mirror_destroy_device(struct g_mirror_softc *sc)
{
struct g_mirror_disk *disk;
struct g_mirror_event *ep;
struct g_geom *gp;
struct g_consumer *cp, *tmpcp;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
gp = sc->sc_geom;
if (sc->sc_provider != NULL)
g_mirror_destroy_provider(sc);
for (disk = LIST_FIRST(&sc->sc_disks); disk != NULL;
disk = LIST_FIRST(&sc->sc_disks)) {
disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
g_mirror_update_metadata(disk);
g_mirror_destroy_disk(disk);
}
while ((ep = g_mirror_event_get(sc)) != NULL) {
g_mirror_event_remove(sc, ep);
if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
g_mirror_event_free(ep);
else {
ep->e_error = ECANCELED;
ep->e_flags |= G_MIRROR_EVENT_DONE;
G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, ep);
mtx_lock(&sc->sc_events_mtx);
wakeup(ep);
mtx_unlock(&sc->sc_events_mtx);
}
}
callout_drain(&sc->sc_callout);
g_topology_lock();
LIST_FOREACH_SAFE(cp, &sc->sc_sync.ds_geom->consumer, consumer, tmpcp) {
g_mirror_disconnect_consumer(sc, cp);
}
g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
G_MIRROR_DEBUG(0, "Device %s destroyed.", gp->name);
g_wither_geom(gp, ENXIO);
g_topology_unlock();
mtx_destroy(&sc->sc_queue_mtx);
mtx_destroy(&sc->sc_events_mtx);
mtx_destroy(&sc->sc_done_mtx);
sx_xunlock(&sc->sc_lock);
sx_destroy(&sc->sc_lock);
}
static void
g_mirror_orphan(struct g_consumer *cp)
{
struct g_mirror_disk *disk;
g_topology_assert();
disk = cp->private;
if (disk == NULL)
return;
disk->d_softc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
G_MIRROR_EVENT_DONTWAIT);
}
/*
* Function should return the next active disk on the list.
* It is possible that it will be the same disk as given.
* If there are no active disks on list, NULL is returned.
*/
static __inline struct g_mirror_disk *
g_mirror_find_next(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
{
struct g_mirror_disk *dp;
for (dp = LIST_NEXT(disk, d_next); dp != disk;
dp = LIST_NEXT(dp, d_next)) {
if (dp == NULL)
dp = LIST_FIRST(&sc->sc_disks);
if (dp->d_state == G_MIRROR_DISK_STATE_ACTIVE)
break;
}
if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
return (NULL);
return (dp);
}
static struct g_mirror_disk *
g_mirror_get_disk(struct g_mirror_softc *sc)
{
struct g_mirror_disk *disk;
if (sc->sc_hint == NULL) {
sc->sc_hint = LIST_FIRST(&sc->sc_disks);
if (sc->sc_hint == NULL)
return (NULL);
}
disk = sc->sc_hint;
if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) {
disk = g_mirror_find_next(sc, disk);
if (disk == NULL)
return (NULL);
}
sc->sc_hint = g_mirror_find_next(sc, disk);
return (disk);
}
static int
g_mirror_write_metadata(struct g_mirror_disk *disk,
struct g_mirror_metadata *md)
{
struct g_mirror_softc *sc;
struct g_consumer *cp;
off_t offset, length;
u_char *sector;
int error = 0;
g_topology_assert_not();
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_LOCKED);
cp = disk->d_consumer;
KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
cp->acw, cp->ace));
length = cp->provider->sectorsize;
offset = cp->provider->mediasize - length;
sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO);
if (md != NULL &&
(sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0) {
/*
* Handle the case, when the size of parent provider reduced.
*/
if (offset < md->md_mediasize)
error = ENOSPC;
else
mirror_metadata_encode(md, sector);
}
if (error == 0)
error = g_write_data(cp, offset, sector, length);
free(sector, M_MIRROR);
if (error != 0) {
if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
G_MIRROR_DEBUG(0, "Cannot write metadata on %s "
"(device=%s, error=%d).",
g_mirror_get_diskname(disk), sc->sc_name, error);
} else {
G_MIRROR_DEBUG(1, "Cannot write metadata on %s "
"(device=%s, error=%d).",
g_mirror_get_diskname(disk), sc->sc_name, error);
}
if (g_mirror_disconnect_on_failure &&
g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1) {
sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
g_mirror_event_send(disk,
G_MIRROR_DISK_STATE_DISCONNECTED,
G_MIRROR_EVENT_DONTWAIT);
}
}
return (error);
}
static int
g_mirror_clear_metadata(struct g_mirror_disk *disk)
{
int error;
g_topology_assert_not();
sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
error = g_mirror_write_metadata(disk, NULL);
if (error == 0) {
G_MIRROR_DEBUG(2, "Metadata on %s cleared.",
g_mirror_get_diskname(disk));
} else {
G_MIRROR_DEBUG(0,
"Cannot clear metadata on disk %s (error=%d).",
g_mirror_get_diskname(disk), error);
}
return (error);
}
void
g_mirror_fill_metadata(struct g_mirror_softc *sc, struct g_mirror_disk *disk,
struct g_mirror_metadata *md)
{
strlcpy(md->md_magic, G_MIRROR_MAGIC, sizeof(md->md_magic));
md->md_version = G_MIRROR_VERSION;
strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
md->md_mid = sc->sc_id;
md->md_all = sc->sc_ndisks;
md->md_slice = sc->sc_slice;
md->md_balance = sc->sc_balance;
md->md_genid = sc->sc_genid;
md->md_mediasize = sc->sc_mediasize;
md->md_sectorsize = sc->sc_sectorsize;
md->md_mflags = (sc->sc_flags & G_MIRROR_DEVICE_FLAG_MASK);
bzero(md->md_provider, sizeof(md->md_provider));
if (disk == NULL) {
md->md_did = arc4random();
md->md_priority = 0;
md->md_syncid = 0;
md->md_dflags = 0;
md->md_sync_offset = 0;
md->md_provsize = 0;
} else {
md->md_did = disk->d_id;
md->md_priority = disk->d_priority;
md->md_syncid = disk->d_sync.ds_syncid;
md->md_dflags = (disk->d_flags & G_MIRROR_DISK_FLAG_MASK);
if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
md->md_sync_offset = disk->d_sync.ds_offset_done;
else
md->md_sync_offset = 0;
if ((disk->d_flags & G_MIRROR_DISK_FLAG_HARDCODED) != 0) {
strlcpy(md->md_provider,
disk->d_consumer->provider->name,
sizeof(md->md_provider));
}
md->md_provsize = disk->d_consumer->provider->mediasize;
}
}
void
g_mirror_update_metadata(struct g_mirror_disk *disk)
{
struct g_mirror_softc *sc;
struct g_mirror_metadata md;
int error;
g_topology_assert_not();
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_LOCKED);
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0)
g_mirror_fill_metadata(sc, disk, &md);
error = g_mirror_write_metadata(disk, &md);
if (error == 0) {
G_MIRROR_DEBUG(2, "Metadata on %s updated.",
g_mirror_get_diskname(disk));
} else {
G_MIRROR_DEBUG(0,
"Cannot update metadata on disk %s (error=%d).",
g_mirror_get_diskname(disk), error);
}
}
static void
g_mirror_bump_syncid(struct g_mirror_softc *sc)
{
struct g_mirror_disk *disk;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
("%s called with no active disks (device=%s).", __func__,
sc->sc_name));
sc->sc_syncid++;
G_MIRROR_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
sc->sc_syncid);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
disk->d_sync.ds_syncid = sc->sc_syncid;
g_mirror_update_metadata(disk);
}
}
}
static void
g_mirror_bump_genid(struct g_mirror_softc *sc)
{
struct g_mirror_disk *disk;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
("%s called with no active disks (device=%s).", __func__,
sc->sc_name));
sc->sc_genid++;
G_MIRROR_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
sc->sc_genid);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
disk->d_genid = sc->sc_genid;
g_mirror_update_metadata(disk);
}
}
}
static int
g_mirror_idle(struct g_mirror_softc *sc, int acw)
{
struct g_mirror_disk *disk;
int timeout;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
if (sc->sc_provider == NULL)
return (0);
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
return (0);
if (sc->sc_idle)
return (0);
if (sc->sc_writes > 0)
return (0);
if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
timeout = g_mirror_idletime - (time_uptime - sc->sc_last_write);
if (!g_mirror_shutdown && timeout > 0)
return (timeout);
}
sc->sc_idle = 1;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
continue;
G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
g_mirror_get_diskname(disk), sc->sc_name);
disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
g_mirror_update_metadata(disk);
}
return (0);
}
static void
g_mirror_unidle(struct g_mirror_softc *sc)
{
struct g_mirror_disk *disk;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
return;
sc->sc_idle = 0;
sc->sc_last_write = time_uptime;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
continue;
G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
g_mirror_get_diskname(disk), sc->sc_name);
disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
g_mirror_update_metadata(disk);
}
}
static void
g_mirror_flush_done(struct bio *bp)
{
struct g_mirror_softc *sc;
struct bio *pbp;
pbp = bp->bio_parent;
sc = pbp->bio_to->geom->softc;
mtx_lock(&sc->sc_done_mtx);
if (pbp->bio_error == 0)
pbp->bio_error = bp->bio_error;
pbp->bio_completed += bp->bio_completed;
pbp->bio_inbed++;
if (pbp->bio_children == pbp->bio_inbed) {
mtx_unlock(&sc->sc_done_mtx);
g_io_deliver(pbp, pbp->bio_error);
} else
mtx_unlock(&sc->sc_done_mtx);
g_destroy_bio(bp);
}
static void
g_mirror_done(struct bio *bp)
{
struct g_mirror_softc *sc;
sc = bp->bio_from->geom->softc;
bp->bio_cflags = G_MIRROR_BIO_FLAG_REGULAR;
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_tail(&sc->sc_queue, bp);
mtx_unlock(&sc->sc_queue_mtx);
wakeup(sc);
}
static void
g_mirror_regular_request(struct bio *bp)
{
struct g_mirror_softc *sc;
struct g_mirror_disk *disk;
struct bio *pbp;
g_topology_assert_not();
pbp = bp->bio_parent;
sc = pbp->bio_to->geom->softc;
bp->bio_from->index--;
if (bp->bio_cmd == BIO_WRITE)
sc->sc_writes--;
disk = bp->bio_from->private;
if (disk == NULL) {
g_topology_lock();
g_mirror_kill_consumer(sc, bp->bio_from);
g_topology_unlock();
}
pbp->bio_inbed++;
KASSERT(pbp->bio_inbed <= pbp->bio_children,
("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
pbp->bio_children));
if (bp->bio_error == 0 && pbp->bio_error == 0) {
G_MIRROR_LOGREQ(3, bp, "Request delivered.");
g_destroy_bio(bp);
if (pbp->bio_children == pbp->bio_inbed) {
G_MIRROR_LOGREQ(3, pbp, "Request delivered.");
pbp->bio_completed = pbp->bio_length;
if (pbp->bio_cmd == BIO_WRITE ||
pbp->bio_cmd == BIO_DELETE) {
bioq_remove(&sc->sc_inflight, pbp);
/* Release delayed sync requests if possible. */
g_mirror_sync_release(sc);
}
g_io_deliver(pbp, pbp->bio_error);
}
return;
} else if (bp->bio_error != 0) {
if (pbp->bio_error == 0)
pbp->bio_error = bp->bio_error;
if (disk != NULL) {
if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
G_MIRROR_LOGREQ(0, bp,
"Request failed (error=%d).",
bp->bio_error);
} else {
G_MIRROR_LOGREQ(1, bp,
"Request failed (error=%d).",
bp->bio_error);
}
if (g_mirror_disconnect_on_failure &&
g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1)
{
sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
g_mirror_event_send(disk,
G_MIRROR_DISK_STATE_DISCONNECTED,
G_MIRROR_EVENT_DONTWAIT);
}
}
switch (pbp->bio_cmd) {
case BIO_DELETE:
case BIO_WRITE:
pbp->bio_inbed--;
pbp->bio_children--;
break;
}
}
g_destroy_bio(bp);
switch (pbp->bio_cmd) {
case BIO_READ:
if (pbp->bio_inbed < pbp->bio_children)
break;
if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 1)
g_io_deliver(pbp, pbp->bio_error);
else {
pbp->bio_error = 0;
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_tail(&sc->sc_queue, pbp);
mtx_unlock(&sc->sc_queue_mtx);
G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
wakeup(sc);
}
break;
case BIO_DELETE:
case BIO_WRITE:
if (pbp->bio_children == 0) {
/*
* All requests failed.
*/
} else if (pbp->bio_inbed < pbp->bio_children) {
/* Do nothing. */
break;
} else if (pbp->bio_children == pbp->bio_inbed) {
/* Some requests succeeded. */
pbp->bio_error = 0;
pbp->bio_completed = pbp->bio_length;
}
bioq_remove(&sc->sc_inflight, pbp);
/* Release delayed sync requests if possible. */
g_mirror_sync_release(sc);
g_io_deliver(pbp, pbp->bio_error);
break;
default:
KASSERT(1 == 0, ("Invalid request: %u.", pbp->bio_cmd));
break;
}
}
static void
g_mirror_sync_done(struct bio *bp)
{
struct g_mirror_softc *sc;
G_MIRROR_LOGREQ(3, bp, "Synchronization request delivered.");
sc = bp->bio_from->geom->softc;
bp->bio_cflags = G_MIRROR_BIO_FLAG_SYNC;
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_tail(&sc->sc_queue, bp);
mtx_unlock(&sc->sc_queue_mtx);
wakeup(sc);
}
static void
g_mirror_candelete(struct bio *bp)
{
struct g_mirror_softc *sc;
struct g_mirror_disk *disk;
int *val;
sc = bp->bio_to->geom->softc;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE)
break;
}
val = (int *)bp->bio_data;
*val = (disk != NULL);
g_io_deliver(bp, 0);
}
static void
g_mirror_kernel_dump(struct bio *bp)
{
struct g_mirror_softc *sc;
struct g_mirror_disk *disk;
struct bio *cbp;
struct g_kerneldump *gkd;
/*
* We configure dumping to the first component, because this component
* will be used for reading with 'prefer' balance algorithm.
* If the component with the higest priority is currently disconnected
* we will not be able to read the dump after the reboot if it will be
* connected and synchronized later. Can we do something better?
*/
sc = bp->bio_to->geom->softc;
disk = LIST_FIRST(&sc->sc_disks);
gkd = (struct g_kerneldump *)bp->bio_data;
if (gkd->length > bp->bio_to->mediasize)
gkd->length = bp->bio_to->mediasize;
cbp = g_clone_bio(bp);
if (cbp == NULL) {
g_io_deliver(bp, ENOMEM);
return;
}
cbp->bio_done = g_std_done;
g_io_request(cbp, disk->d_consumer);
G_MIRROR_DEBUG(1, "Kernel dump will go to %s.",
g_mirror_get_diskname(disk));
}
static void
g_mirror_flush(struct g_mirror_softc *sc, struct bio *bp)
{
struct bio_queue_head queue;
struct g_mirror_disk *disk;
struct g_consumer *cp;
struct bio *cbp;
bioq_init(&queue);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
continue;
cbp = g_clone_bio(bp);
if (cbp == NULL) {
while ((cbp = bioq_takefirst(&queue)) != NULL)
g_destroy_bio(cbp);
if (bp->bio_error == 0)
bp->bio_error = ENOMEM;
g_io_deliver(bp, bp->bio_error);
return;
}
bioq_insert_tail(&queue, cbp);
cbp->bio_done = g_mirror_flush_done;
cbp->bio_caller1 = disk;
cbp->bio_to = disk->d_consumer->provider;
}
while ((cbp = bioq_takefirst(&queue)) != NULL) {
G_MIRROR_LOGREQ(3, cbp, "Sending request.");
disk = cbp->bio_caller1;
cbp->bio_caller1 = NULL;
cp = disk->d_consumer;
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
cp->acr, cp->acw, cp->ace));
g_io_request(cbp, disk->d_consumer);
}
}
static void
g_mirror_start(struct bio *bp)
{
struct g_mirror_softc *sc;
sc = bp->bio_to->geom->softc;
/*
* If sc == NULL or there are no valid disks, provider's error
* should be set and g_mirror_start() should not be called at all.
*/
KASSERT(sc != NULL && sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
("Provider's error should be set (error=%d)(mirror=%s).",
bp->bio_to->error, bp->bio_to->name));
G_MIRROR_LOGREQ(3, bp, "Request received.");
switch (bp->bio_cmd) {
case BIO_READ:
case BIO_WRITE:
case BIO_DELETE:
break;
case BIO_FLUSH:
g_mirror_flush(sc, bp);
return;
case BIO_GETATTR:
if (!strcmp(bp->bio_attribute, "GEOM::candelete")) {
g_mirror_candelete(bp);
return;
} else if (strcmp("GEOM::kerneldump", bp->bio_attribute) == 0) {
g_mirror_kernel_dump(bp);
return;
}
/* FALLTHROUGH */
default:
g_io_deliver(bp, EOPNOTSUPP);
return;
}
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_tail(&sc->sc_queue, bp);
mtx_unlock(&sc->sc_queue_mtx);
G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
wakeup(sc);
}
/*
* Return TRUE if the given request is colliding with a in-progress
* synchronization request.
*/
static int
g_mirror_sync_collision(struct g_mirror_softc *sc, struct bio *bp)
{
struct g_mirror_disk *disk;
struct bio *sbp;
off_t rstart, rend, sstart, send;
int i;
if (sc->sc_sync.ds_ndisks == 0)
return (0);
rstart = bp->bio_offset;
rend = bp->bio_offset + bp->bio_length;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state != G_MIRROR_DISK_STATE_SYNCHRONIZING)
continue;
for (i = 0; i < g_mirror_syncreqs; i++) {
sbp = disk->d_sync.ds_bios[i];
if (sbp == NULL)
continue;
sstart = sbp->bio_offset;
send = sbp->bio_offset + sbp->bio_length;
if (rend > sstart && rstart < send)
return (1);
}
}
return (0);
}
/*
* Return TRUE if the given sync request is colliding with a in-progress regular
* request.
*/
static int
g_mirror_regular_collision(struct g_mirror_softc *sc, struct bio *sbp)
{
off_t rstart, rend, sstart, send;
struct bio *bp;
if (sc->sc_sync.ds_ndisks == 0)
return (0);
sstart = sbp->bio_offset;
send = sbp->bio_offset + sbp->bio_length;
TAILQ_FOREACH(bp, &sc->sc_inflight.queue, bio_queue) {
rstart = bp->bio_offset;
rend = bp->bio_offset + bp->bio_length;
if (rend > sstart && rstart < send)
return (1);
}
return (0);
}
/*
* Puts request onto delayed queue.
*/
static void
g_mirror_regular_delay(struct g_mirror_softc *sc, struct bio *bp)
{
G_MIRROR_LOGREQ(2, bp, "Delaying request.");
bioq_insert_head(&sc->sc_regular_delayed, bp);
}
/*
* Puts synchronization request onto delayed queue.
*/
static void
g_mirror_sync_delay(struct g_mirror_softc *sc, struct bio *bp)
{
G_MIRROR_LOGREQ(2, bp, "Delaying synchronization request.");
bioq_insert_tail(&sc->sc_sync_delayed, bp);
}
/*
* Releases delayed regular requests which don't collide anymore with sync
* requests.
*/
static void
g_mirror_regular_release(struct g_mirror_softc *sc)
{
struct bio *bp, *bp2;
TAILQ_FOREACH_SAFE(bp, &sc->sc_regular_delayed.queue, bio_queue, bp2) {
if (g_mirror_sync_collision(sc, bp))
continue;
bioq_remove(&sc->sc_regular_delayed, bp);
G_MIRROR_LOGREQ(2, bp, "Releasing delayed request (%p).", bp);
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_head(&sc->sc_queue, bp);
#if 0
/*
* wakeup() is not needed, because this function is called from
* the worker thread.
*/
wakeup(&sc->sc_queue);
#endif
mtx_unlock(&sc->sc_queue_mtx);
}
}
/*
* Releases delayed sync requests which don't collide anymore with regular
* requests.
*/
static void
g_mirror_sync_release(struct g_mirror_softc *sc)
{
struct bio *bp, *bp2;
TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed.queue, bio_queue, bp2) {
if (g_mirror_regular_collision(sc, bp))
continue;
bioq_remove(&sc->sc_sync_delayed, bp);
G_MIRROR_LOGREQ(2, bp,
"Releasing delayed synchronization request.");
g_io_request(bp, bp->bio_from);
}
}
/*
* Handle synchronization requests.
* Every synchronization request is two-steps process: first, READ request is
* send to active provider and then WRITE request (with read data) to the provider
* beeing synchronized. When WRITE is finished, new synchronization request is
* send.
*/
static void
g_mirror_sync_request(struct bio *bp)
{
struct g_mirror_softc *sc;
struct g_mirror_disk *disk;
bp->bio_from->index--;
sc = bp->bio_from->geom->softc;
disk = bp->bio_from->private;
if (disk == NULL) {
sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
g_topology_lock();
g_mirror_kill_consumer(sc, bp->bio_from);
g_topology_unlock();
free(bp->bio_data, M_MIRROR);
g_destroy_bio(bp);
sx_xlock(&sc->sc_lock);
return;
}
/*
* Synchronization request.
*/
switch (bp->bio_cmd) {
case BIO_READ:
{
struct g_consumer *cp;
if (bp->bio_error != 0) {
G_MIRROR_LOGREQ(0, bp,
"Synchronization request failed (error=%d).",
bp->bio_error);
g_destroy_bio(bp);
return;
}
G_MIRROR_LOGREQ(3, bp,
"Synchronization request half-finished.");
bp->bio_cmd = BIO_WRITE;
bp->bio_cflags = 0;
cp = disk->d_consumer;
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
cp->acr, cp->acw, cp->ace));
cp->index++;
g_io_request(bp, cp);
return;
}
case BIO_WRITE:
{
struct g_mirror_disk_sync *sync;
off_t offset;
void *data;
int i;
if (bp->bio_error != 0) {
G_MIRROR_LOGREQ(0, bp,
"Synchronization request failed (error=%d).",
bp->bio_error);
g_destroy_bio(bp);
sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
g_mirror_event_send(disk,
G_MIRROR_DISK_STATE_DISCONNECTED,
G_MIRROR_EVENT_DONTWAIT);
return;
}
G_MIRROR_LOGREQ(3, bp, "Synchronization request finished.");
sync = &disk->d_sync;
if (sync->ds_offset >= sc->sc_mediasize ||
sync->ds_consumer == NULL ||
(sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
/* Don't send more synchronization requests. */
sync->ds_inflight--;
if (sync->ds_bios != NULL) {
i = (int)(uintptr_t)bp->bio_caller1;
sync->ds_bios[i] = NULL;
}
free(bp->bio_data, M_MIRROR);
g_destroy_bio(bp);
if (sync->ds_inflight > 0)
return;
if (sync->ds_consumer == NULL ||
(sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
return;
}
/* Disk up-to-date, activate it. */
g_mirror_event_send(disk, G_MIRROR_DISK_STATE_ACTIVE,
G_MIRROR_EVENT_DONTWAIT);
return;
}
/* Send next synchronization request. */
data = bp->bio_data;
bzero(bp, sizeof(*bp));
bp->bio_cmd = BIO_READ;
bp->bio_offset = sync->ds_offset;
bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
sync->ds_offset += bp->bio_length;
bp->bio_done = g_mirror_sync_done;
bp->bio_data = data;
bp->bio_from = sync->ds_consumer;
bp->bio_to = sc->sc_provider;
G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
sync->ds_consumer->index++;
/*
* Delay the request if it is colliding with a regular request.
*/
if (g_mirror_regular_collision(sc, bp))
g_mirror_sync_delay(sc, bp);
else
g_io_request(bp, sync->ds_consumer);
/* Release delayed requests if possible. */
g_mirror_regular_release(sc);
/* Find the smallest offset */
offset = sc->sc_mediasize;
for (i = 0; i < g_mirror_syncreqs; i++) {
bp = sync->ds_bios[i];
if (bp->bio_offset < offset)
offset = bp->bio_offset;
}
if (sync->ds_offset_done + (MAXPHYS * 100) < offset) {
/* Update offset_done on every 100 blocks. */
sync->ds_offset_done = offset;
g_mirror_update_metadata(disk);
}
return;
}
default:
KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
bp->bio_cmd, sc->sc_name));
break;
}
}
static void
g_mirror_request_prefer(struct g_mirror_softc *sc, struct bio *bp)
{
struct g_mirror_disk *disk;
struct g_consumer *cp;
struct bio *cbp;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE)
break;
}
if (disk == NULL) {
if (bp->bio_error == 0)
bp->bio_error = ENXIO;
g_io_deliver(bp, bp->bio_error);
return;
}
cbp = g_clone_bio(bp);
if (cbp == NULL) {
if (bp->bio_error == 0)
bp->bio_error = ENOMEM;
g_io_deliver(bp, bp->bio_error);
return;
}
/*
* Fill in the component buf structure.
*/
cp = disk->d_consumer;
cbp->bio_done = g_mirror_done;
cbp->bio_to = cp->provider;
G_MIRROR_LOGREQ(3, cbp, "Sending request.");
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
cp->acw, cp->ace));
cp->index++;
g_io_request(cbp, cp);
}
static void
g_mirror_request_round_robin(struct g_mirror_softc *sc, struct bio *bp)
{
struct g_mirror_disk *disk;
struct g_consumer *cp;
struct bio *cbp;
disk = g_mirror_get_disk(sc);
if (disk == NULL) {
if (bp->bio_error == 0)
bp->bio_error = ENXIO;
g_io_deliver(bp, bp->bio_error);
return;
}
cbp = g_clone_bio(bp);
if (cbp == NULL) {
if (bp->bio_error == 0)
bp->bio_error = ENOMEM;
g_io_deliver(bp, bp->bio_error);
return;
}
/*
* Fill in the component buf structure.
*/
cp = disk->d_consumer;
cbp->bio_done = g_mirror_done;
cbp->bio_to = cp->provider;
G_MIRROR_LOGREQ(3, cbp, "Sending request.");
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
cp->acw, cp->ace));
cp->index++;
g_io_request(cbp, cp);
}
#define TRACK_SIZE (1 * 1024 * 1024)
#define LOAD_SCALE 256
#define ABS(x) (((x) >= 0) ? (x) : (-(x)))
static void
g_mirror_request_load(struct g_mirror_softc *sc, struct bio *bp)
{
struct g_mirror_disk *disk, *dp;
struct g_consumer *cp;
struct bio *cbp;
int prio, best;
/* Find a disk with the smallest load. */
disk = NULL;
best = INT_MAX;
LIST_FOREACH(dp, &sc->sc_disks, d_next) {
if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
continue;
prio = dp->load;
/* If disk head is precisely in position - highly prefer it. */
if (dp->d_last_offset == bp->bio_offset)
prio -= 2 * LOAD_SCALE;
else
/* If disk head is close to position - prefer it. */
if (ABS(dp->d_last_offset - bp->bio_offset) < TRACK_SIZE)
prio -= 1 * LOAD_SCALE;
if (prio <= best) {
disk = dp;
best = prio;
}
}
KASSERT(disk != NULL, ("NULL disk for %s.", sc->sc_name));
cbp = g_clone_bio(bp);
if (cbp == NULL) {
if (bp->bio_error == 0)
bp->bio_error = ENOMEM;
g_io_deliver(bp, bp->bio_error);
return;
}
/*
* Fill in the component buf structure.
*/
cp = disk->d_consumer;
cbp->bio_done = g_mirror_done;
cbp->bio_to = cp->provider;
G_MIRROR_LOGREQ(3, cbp, "Sending request.");
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
cp->acw, cp->ace));
cp->index++;
/* Remember last head position */
disk->d_last_offset = bp->bio_offset + bp->bio_length;
/* Update loads. */
LIST_FOREACH(dp, &sc->sc_disks, d_next) {
dp->load = (dp->d_consumer->index * LOAD_SCALE +
dp->load * 7) / 8;
}
g_io_request(cbp, cp);
}
static void
g_mirror_request_split(struct g_mirror_softc *sc, struct bio *bp)
{
struct bio_queue_head queue;
struct g_mirror_disk *disk;
struct g_consumer *cp;
struct bio *cbp;
off_t left, mod, offset, slice;
u_char *data;
u_int ndisks;
if (bp->bio_length <= sc->sc_slice) {
g_mirror_request_round_robin(sc, bp);
return;
}
ndisks = g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE);
slice = bp->bio_length / ndisks;
mod = slice % sc->sc_provider->sectorsize;
if (mod != 0)
slice += sc->sc_provider->sectorsize - mod;
/*
* Allocate all bios before sending any request, so we can
* return ENOMEM in nice and clean way.
*/
left = bp->bio_length;
offset = bp->bio_offset;
data = bp->bio_data;
bioq_init(&queue);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
continue;
cbp = g_clone_bio(bp);
if (cbp == NULL) {
while ((cbp = bioq_takefirst(&queue)) != NULL)
g_destroy_bio(cbp);
if (bp->bio_error == 0)
bp->bio_error = ENOMEM;
g_io_deliver(bp, bp->bio_error);
return;
}
bioq_insert_tail(&queue, cbp);
cbp->bio_done = g_mirror_done;
cbp->bio_caller1 = disk;
cbp->bio_to = disk->d_consumer->provider;
cbp->bio_offset = offset;
cbp->bio_data = data;
cbp->bio_length = MIN(left, slice);
left -= cbp->bio_length;
if (left == 0)
break;
offset += cbp->bio_length;
data += cbp->bio_length;
}
while ((cbp = bioq_takefirst(&queue)) != NULL) {
G_MIRROR_LOGREQ(3, cbp, "Sending request.");
disk = cbp->bio_caller1;
cbp->bio_caller1 = NULL;
cp = disk->d_consumer;
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
cp->acr, cp->acw, cp->ace));
disk->d_consumer->index++;
g_io_request(cbp, disk->d_consumer);
}
}
static void
g_mirror_register_request(struct bio *bp)
{
struct g_mirror_softc *sc;
sc = bp->bio_to->geom->softc;
switch (bp->bio_cmd) {
case BIO_READ:
switch (sc->sc_balance) {
case G_MIRROR_BALANCE_LOAD:
g_mirror_request_load(sc, bp);
break;
case G_MIRROR_BALANCE_PREFER:
g_mirror_request_prefer(sc, bp);
break;
case G_MIRROR_BALANCE_ROUND_ROBIN:
g_mirror_request_round_robin(sc, bp);
break;
case G_MIRROR_BALANCE_SPLIT:
g_mirror_request_split(sc, bp);
break;
}
return;
case BIO_WRITE:
case BIO_DELETE:
{
struct g_mirror_disk *disk;
struct g_mirror_disk_sync *sync;
struct bio_queue_head queue;
struct g_consumer *cp;
struct bio *cbp;
/*
* Delay the request if it is colliding with a synchronization
* request.
*/
if (g_mirror_sync_collision(sc, bp)) {
g_mirror_regular_delay(sc, bp);
return;
}
if (sc->sc_idle)
g_mirror_unidle(sc);
else
sc->sc_last_write = time_uptime;
/*
* Allocate all bios before sending any request, so we can
* return ENOMEM in nice and clean way.
*/
bioq_init(&queue);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
sync = &disk->d_sync;
switch (disk->d_state) {
case G_MIRROR_DISK_STATE_ACTIVE:
break;
case G_MIRROR_DISK_STATE_SYNCHRONIZING:
if (bp->bio_offset >= sync->ds_offset)
continue;
break;
default:
continue;
}
if (bp->bio_cmd == BIO_DELETE &&
(disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE) == 0)
continue;
cbp = g_clone_bio(bp);
if (cbp == NULL) {
while ((cbp = bioq_takefirst(&queue)) != NULL)
g_destroy_bio(cbp);
if (bp->bio_error == 0)
bp->bio_error = ENOMEM;
g_io_deliver(bp, bp->bio_error);
return;
}
bioq_insert_tail(&queue, cbp);
cbp->bio_done = g_mirror_done;
cp = disk->d_consumer;
cbp->bio_caller1 = cp;
cbp->bio_to = cp->provider;
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).",
cp->provider->name, cp->acr, cp->acw, cp->ace));
}
if (bioq_first(&queue) == NULL) {
g_io_deliver(bp, EOPNOTSUPP);
return;
}
while ((cbp = bioq_takefirst(&queue)) != NULL) {
G_MIRROR_LOGREQ(3, cbp, "Sending request.");
cp = cbp->bio_caller1;
cbp->bio_caller1 = NULL;
cp->index++;
sc->sc_writes++;
g_io_request(cbp, cp);
}
/*
* Put request onto inflight queue, so we can check if new
* synchronization requests don't collide with it.
*/
bioq_insert_tail(&sc->sc_inflight, bp);
/*
* Bump syncid on first write.
*/
if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0) {
sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
g_mirror_bump_syncid(sc);
}
return;
}
default:
KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
bp->bio_cmd, sc->sc_name));
break;
}
}
static int
g_mirror_can_destroy(struct g_mirror_softc *sc)
{
struct g_geom *gp;
struct g_consumer *cp;
g_topology_assert();
gp = sc->sc_geom;
if (gp->softc == NULL)
return (1);
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_TASTING) != 0)
return (0);
LIST_FOREACH(cp, &gp->consumer, consumer) {
if (g_mirror_is_busy(sc, cp))
return (0);
}
gp = sc->sc_sync.ds_geom;
LIST_FOREACH(cp, &gp->consumer, consumer) {
if (g_mirror_is_busy(sc, cp))
return (0);
}
G_MIRROR_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
sc->sc_name);
return (1);
}
static int
g_mirror_try_destroy(struct g_mirror_softc *sc)
{
if (sc->sc_rootmount != NULL) {
G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
}
g_topology_lock();
if (!g_mirror_can_destroy(sc)) {
g_topology_unlock();
return (0);
}
sc->sc_geom->softc = NULL;
sc->sc_sync.ds_geom->softc = NULL;
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WAIT) != 0) {
g_topology_unlock();
G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
&sc->sc_worker);
/* Unlock sc_lock here, as it can be destroyed after wakeup. */
sx_xunlock(&sc->sc_lock);
wakeup(&sc->sc_worker);
sc->sc_worker = NULL;
} else {
g_topology_unlock();
g_mirror_destroy_device(sc);
free(sc, M_MIRROR);
}
return (1);
}
/*
* Worker thread.
*/
static void
g_mirror_worker(void *arg)
{
struct g_mirror_softc *sc;
struct g_mirror_event *ep;
struct bio *bp;
int timeout;
sc = arg;
thread_lock(curthread);
sched_prio(curthread, PRIBIO);
thread_unlock(curthread);
sx_xlock(&sc->sc_lock);
for (;;) {
G_MIRROR_DEBUG(5, "%s: Let's see...", __func__);
/*
* First take a look at events.
* This is important to handle events before any I/O requests.
*/
ep = g_mirror_event_get(sc);
if (ep != NULL) {
g_mirror_event_remove(sc, ep);
if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) {
/* Update only device status. */
G_MIRROR_DEBUG(3,
"Running event for device %s.",
sc->sc_name);
ep->e_error = 0;
g_mirror_update_device(sc, 1);
} else {
/* Update disk status. */
G_MIRROR_DEBUG(3, "Running event for disk %s.",
g_mirror_get_diskname(ep->e_disk));
ep->e_error = g_mirror_update_disk(ep->e_disk,
ep->e_state);
if (ep->e_error == 0)
g_mirror_update_device(sc, 0);
}
if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) {
KASSERT(ep->e_error == 0,
("Error cannot be handled."));
g_mirror_event_free(ep);
} else {
ep->e_flags |= G_MIRROR_EVENT_DONE;
G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
ep);
mtx_lock(&sc->sc_events_mtx);
wakeup(ep);
mtx_unlock(&sc->sc_events_mtx);
}
if ((sc->sc_flags &
G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
if (g_mirror_try_destroy(sc)) {
curthread->td_pflags &= ~TDP_GEOM;
G_MIRROR_DEBUG(1, "Thread exiting.");
kproc_exit(0);
}
}
G_MIRROR_DEBUG(5, "%s: I'm here 1.", __func__);
continue;
}
/*
* Check if we can mark array as CLEAN and if we can't take
* how much seconds should we wait.
*/
timeout = g_mirror_idle(sc, -1);
/*
* Now I/O requests.
*/
/* Get first request from the queue. */
mtx_lock(&sc->sc_queue_mtx);
bp = bioq_takefirst(&sc->sc_queue);
if (bp == NULL) {
if ((sc->sc_flags &
G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
mtx_unlock(&sc->sc_queue_mtx);
if (g_mirror_try_destroy(sc)) {
curthread->td_pflags &= ~TDP_GEOM;
G_MIRROR_DEBUG(1, "Thread exiting.");
kproc_exit(0);
}
mtx_lock(&sc->sc_queue_mtx);
}
sx_xunlock(&sc->sc_lock);
/*
* XXX: We can miss an event here, because an event
* can be added without sx-device-lock and without
* mtx-queue-lock. Maybe I should just stop using
* dedicated mutex for events synchronization and
* stick with the queue lock?
* The event will hang here until next I/O request
* or next event is received.
*/
MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w1",
timeout * hz);
sx_xlock(&sc->sc_lock);
G_MIRROR_DEBUG(5, "%s: I'm here 4.", __func__);
continue;
}
mtx_unlock(&sc->sc_queue_mtx);
if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
(bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0) {
g_mirror_sync_request(bp); /* READ */
} else if (bp->bio_to != sc->sc_provider) {
if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_REGULAR) != 0)
g_mirror_regular_request(bp);
else if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
g_mirror_sync_request(bp); /* WRITE */
else {
KASSERT(0,
("Invalid request cflags=0x%hhx to=%s.",
bp->bio_cflags, bp->bio_to->name));
}
} else {
g_mirror_register_request(bp);
}
G_MIRROR_DEBUG(5, "%s: I'm here 9.", __func__);
}
}
static void
g_mirror_update_idle(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
{
sx_assert(&sc->sc_lock, SX_LOCKED);
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
return;
if (!sc->sc_idle && (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) {
G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
g_mirror_get_diskname(disk), sc->sc_name);
disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
} else if (sc->sc_idle &&
(disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
g_mirror_get_diskname(disk), sc->sc_name);
disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
}
}
static void
g_mirror_sync_start(struct g_mirror_disk *disk)
{
struct g_mirror_softc *sc;
struct g_consumer *cp;
struct bio *bp;
int error, i;
g_topology_assert_not();
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_LOCKED);
KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
("Disk %s is not marked for synchronization.",
g_mirror_get_diskname(disk)));
KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
("Device not in RUNNING state (%s, %u).", sc->sc_name,
sc->sc_state));
sx_xunlock(&sc->sc_lock);
g_topology_lock();
cp = g_new_consumer(sc->sc_sync.ds_geom);
cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
error = g_attach(cp, sc->sc_provider);
KASSERT(error == 0,
("Cannot attach to %s (error=%d).", sc->sc_name, error));
error = g_access(cp, 1, 0, 0);
KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
g_topology_unlock();
sx_xlock(&sc->sc_lock);
G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
g_mirror_get_diskname(disk));
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) == 0)
disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
KASSERT(disk->d_sync.ds_consumer == NULL,
("Sync consumer already exists (device=%s, disk=%s).",
sc->sc_name, g_mirror_get_diskname(disk)));
disk->d_sync.ds_consumer = cp;
disk->d_sync.ds_consumer->private = disk;
disk->d_sync.ds_consumer->index = 0;
/*
* Allocate memory for synchronization bios and initialize them.
*/
disk->d_sync.ds_bios = malloc(sizeof(struct bio *) * g_mirror_syncreqs,
M_MIRROR, M_WAITOK);
for (i = 0; i < g_mirror_syncreqs; i++) {
bp = g_alloc_bio();
disk->d_sync.ds_bios[i] = bp;
bp->bio_parent = NULL;
bp->bio_cmd = BIO_READ;
bp->bio_data = malloc(MAXPHYS, M_MIRROR, M_WAITOK);
bp->bio_cflags = 0;
bp->bio_offset = disk->d_sync.ds_offset;
bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
disk->d_sync.ds_offset += bp->bio_length;
bp->bio_done = g_mirror_sync_done;
bp->bio_from = disk->d_sync.ds_consumer;
bp->bio_to = sc->sc_provider;
bp->bio_caller1 = (void *)(uintptr_t)i;
}
/* Increase the number of disks in SYNCHRONIZING state. */
sc->sc_sync.ds_ndisks++;
/* Set the number of in-flight synchronization requests. */
disk->d_sync.ds_inflight = g_mirror_syncreqs;
/*
* Fire off first synchronization requests.
*/
for (i = 0; i < g_mirror_syncreqs; i++) {
bp = disk->d_sync.ds_bios[i];
G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
disk->d_sync.ds_consumer->index++;
/*
* Delay the request if it is colliding with a regular request.
*/
if (g_mirror_regular_collision(sc, bp))
g_mirror_sync_delay(sc, bp);
else
g_io_request(bp, disk->d_sync.ds_consumer);
}
}
/*
* Stop synchronization process.
* type: 0 - synchronization finished
* 1 - synchronization stopped
*/
static void
g_mirror_sync_stop(struct g_mirror_disk *disk, int type)
{
struct g_mirror_softc *sc;
struct g_consumer *cp;
g_topology_assert_not();
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_LOCKED);
KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
if (disk->d_sync.ds_consumer == NULL)
return;
if (type == 0) {
G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s finished.",
sc->sc_name, g_mirror_get_diskname(disk));
} else /* if (type == 1) */ {
G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
sc->sc_name, g_mirror_get_diskname(disk));
}
free(disk->d_sync.ds_bios, M_MIRROR);
disk->d_sync.ds_bios = NULL;
cp = disk->d_sync.ds_consumer;
disk->d_sync.ds_consumer = NULL;
disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
sc->sc_sync.ds_ndisks--;
sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
g_topology_lock();
g_mirror_kill_consumer(sc, cp);
g_topology_unlock();
sx_xlock(&sc->sc_lock);
}
static void
g_mirror_launch_provider(struct g_mirror_softc *sc)
{
struct g_mirror_disk *disk;
struct g_provider *pp, *dp;
sx_assert(&sc->sc_lock, SX_LOCKED);
g_topology_lock();
pp = g_new_providerf(sc->sc_geom, "mirror/%s", sc->sc_name);
pp->flags |= G_PF_DIRECT_RECEIVE;
pp->mediasize = sc->sc_mediasize;
pp->sectorsize = sc->sc_sectorsize;
pp->stripesize = 0;
pp->stripeoffset = 0;
/* Splitting of unmapped BIO's could work but isn't implemented now */
if (sc->sc_balance != G_MIRROR_BALANCE_SPLIT)
pp->flags |= G_PF_ACCEPT_UNMAPPED;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_consumer && disk->d_consumer->provider) {
dp = disk->d_consumer->provider;
if (dp->stripesize > pp->stripesize) {
pp->stripesize = dp->stripesize;
pp->stripeoffset = dp->stripeoffset;
}
/* A provider underneath us doesn't support unmapped */
if ((dp->flags & G_PF_ACCEPT_UNMAPPED) == 0) {
G_MIRROR_DEBUG(0, "Cancelling unmapped "
"because of %s.", dp->name);
pp->flags &= ~G_PF_ACCEPT_UNMAPPED;
}
}
}
sc->sc_provider = pp;
g_error_provider(pp, 0);
g_topology_unlock();
G_MIRROR_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE), sc->sc_ndisks);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
g_mirror_sync_start(disk);
}
}
static void
g_mirror_destroy_provider(struct g_mirror_softc *sc)
{
struct g_mirror_disk *disk;
struct bio *bp;
g_topology_assert_not();
KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
sc->sc_name));
g_topology_lock();
g_error_provider(sc->sc_provider, ENXIO);
mtx_lock(&sc->sc_queue_mtx);
while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL)
g_io_deliver(bp, ENXIO);
mtx_unlock(&sc->sc_queue_mtx);
G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name,
sc->sc_provider->name);
sc->sc_provider->flags |= G_PF_WITHER;
g_orphan_provider(sc->sc_provider, ENXIO);
g_topology_unlock();
sc->sc_provider = NULL;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
g_mirror_sync_stop(disk, 1);
}
}
static void
g_mirror_go(void *arg)
{
struct g_mirror_softc *sc;
sc = arg;
G_MIRROR_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
g_mirror_event_send(sc, 0,
G_MIRROR_EVENT_DONTWAIT | G_MIRROR_EVENT_DEVICE);
}
static u_int
g_mirror_determine_state(struct g_mirror_disk *disk)
{
struct g_mirror_softc *sc;
u_int state;
sc = disk->d_softc;
if (sc->sc_syncid == disk->d_sync.ds_syncid) {
if ((disk->d_flags &
G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
/* Disk does not need synchronization. */
state = G_MIRROR_DISK_STATE_ACTIVE;
} else {
if ((sc->sc_flags &
G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
(disk->d_flags &
G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
/*
* We can start synchronization from
* the stored offset.
*/
state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
} else {
state = G_MIRROR_DISK_STATE_STALE;
}
}
} else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
/*
* Reset all synchronization data for this disk,
* because if it even was synchronized, it was
* synchronized to disks with different syncid.
*/
disk->d_flags |= G_MIRROR_DISK_FLAG_SYNCHRONIZING;
disk->d_sync.ds_offset = 0;
disk->d_sync.ds_offset_done = 0;
disk->d_sync.ds_syncid = sc->sc_syncid;
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
(disk->d_flags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
} else {
state = G_MIRROR_DISK_STATE_STALE;
}
} else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
/*
* Not good, NOT GOOD!
* It means that mirror was started on stale disks
* and more fresh disk just arrive.
* If there were writes, mirror is broken, sorry.
* I think the best choice here is don't touch
* this disk and inform the user loudly.
*/
G_MIRROR_DEBUG(0, "Device %s was started before the freshest "
"disk (%s) arrives!! It will not be connected to the "
"running device.", sc->sc_name,
g_mirror_get_diskname(disk));
g_mirror_destroy_disk(disk);
state = G_MIRROR_DISK_STATE_NONE;
/* Return immediately, because disk was destroyed. */
return (state);
}
G_MIRROR_DEBUG(3, "State for %s disk: %s.",
g_mirror_get_diskname(disk), g_mirror_disk_state2str(state));
return (state);
}
/*
* Update device state.
*/
static void
g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force)
{
struct g_mirror_disk *disk;
u_int state;
sx_assert(&sc->sc_lock, SX_XLOCKED);
switch (sc->sc_state) {
case G_MIRROR_DEVICE_STATE_STARTING:
{
struct g_mirror_disk *pdisk, *tdisk;
u_int dirty, ndisks, genid, syncid;
KASSERT(sc->sc_provider == NULL,
("Non-NULL provider in STARTING state (%s).", sc->sc_name));
/*
* Are we ready? We are, if all disks are connected or
* if we have any disks and 'force' is true.
*/
ndisks = g_mirror_ndisks(sc, -1);
if (sc->sc_ndisks == ndisks || (force && ndisks > 0)) {
;
} else if (ndisks == 0) {
/*
* Disks went down in starting phase, so destroy
* device.
*/
callout_drain(&sc->sc_callout);
sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
return;
} else {
return;
}
/*
* Activate all disks with the biggest syncid.
*/
if (force) {
/*
* If 'force' is true, we have been called due to
* timeout, so don't bother canceling timeout.
*/
ndisks = 0;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if ((disk->d_flags &
G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
ndisks++;
}
}
if (ndisks == 0) {
/* No valid disks found, destroy device. */
sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
__LINE__, sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
return;
}
} else {
/* Cancel timeout. */
callout_drain(&sc->sc_callout);
}
/*
* Find the biggest genid.
*/
genid = 0;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_genid > genid)
genid = disk->d_genid;
}
sc->sc_genid = genid;
/*
* Remove all disks without the biggest genid.
*/
LIST_FOREACH_SAFE(disk, &sc->sc_disks, d_next, tdisk) {
if (disk->d_genid < genid) {
G_MIRROR_DEBUG(0,
"Component %s (device %s) broken, skipping.",
g_mirror_get_diskname(disk), sc->sc_name);
g_mirror_destroy_disk(disk);
}
}
/*
* Find the biggest syncid.
*/
syncid = 0;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_sync.ds_syncid > syncid)
syncid = disk->d_sync.ds_syncid;
}
/*
* Here we need to look for dirty disks and if all disks
* with the biggest syncid are dirty, we have to choose
* one with the biggest priority and rebuild the rest.
*/
/*
* Find the number of dirty disks with the biggest syncid.
* Find the number of disks with the biggest syncid.
* While here, find a disk with the biggest priority.
*/
dirty = ndisks = 0;
pdisk = NULL;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_sync.ds_syncid != syncid)
continue;
if ((disk->d_flags &
G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
continue;
}
ndisks++;
if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
dirty++;
if (pdisk == NULL ||
pdisk->d_priority < disk->d_priority) {
pdisk = disk;
}
}
}
if (dirty == 0) {
/* No dirty disks at all, great. */
} else if (dirty == ndisks) {
/*
* Force synchronization for all dirty disks except one
* with the biggest priority.
*/
KASSERT(pdisk != NULL, ("pdisk == NULL"));
G_MIRROR_DEBUG(1, "Using disk %s (device %s) as a "
"master disk for synchronization.",
g_mirror_get_diskname(pdisk), sc->sc_name);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_sync.ds_syncid != syncid)
continue;
if ((disk->d_flags &
G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
continue;
}
KASSERT((disk->d_flags &
G_MIRROR_DISK_FLAG_DIRTY) != 0,
("Disk %s isn't marked as dirty.",
g_mirror_get_diskname(disk)));
/* Skip the disk with the biggest priority. */
if (disk == pdisk)
continue;
disk->d_sync.ds_syncid = 0;
}
} else if (dirty < ndisks) {
/*
* Force synchronization for all dirty disks.
* We have some non-dirty disks.
*/
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_sync.ds_syncid != syncid)
continue;
if ((disk->d_flags &
G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
continue;
}
if ((disk->d_flags &
G_MIRROR_DISK_FLAG_DIRTY) == 0) {
continue;
}
disk->d_sync.ds_syncid = 0;
}
}
/* Reset hint. */
sc->sc_hint = NULL;
sc->sc_syncid = syncid;
if (force) {
/* Remember to bump syncid on first write. */
sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
}
state = G_MIRROR_DEVICE_STATE_RUNNING;
G_MIRROR_DEBUG(1, "Device %s state changed from %s to %s.",
sc->sc_name, g_mirror_device_state2str(sc->sc_state),
g_mirror_device_state2str(state));
sc->sc_state = state;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
state = g_mirror_determine_state(disk);
g_mirror_event_send(disk, state,
G_MIRROR_EVENT_DONTWAIT);
if (state == G_MIRROR_DISK_STATE_STALE)
sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
}
break;
}
case G_MIRROR_DEVICE_STATE_RUNNING:
if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 0 &&
g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
/*
* No active disks or no disks at all,
* so destroy device.
*/
if (sc->sc_provider != NULL)
g_mirror_destroy_provider(sc);
sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
break;
} else if (g_mirror_ndisks(sc,
G_MIRROR_DISK_STATE_ACTIVE) > 0 &&
g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
/*
* We have active disks, launch provider if it doesn't
* exist.
*/
if (sc->sc_provider == NULL)
g_mirror_launch_provider(sc);
if (sc->sc_rootmount != NULL) {
G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
__LINE__, sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
}
}
/*
* Genid should be bumped immediately, so do it here.
*/
if ((sc->sc_bump_id & G_MIRROR_BUMP_GENID) != 0) {
sc->sc_bump_id &= ~G_MIRROR_BUMP_GENID;
g_mirror_bump_genid(sc);
}
break;
default:
KASSERT(1 == 0, ("Wrong device state (%s, %s).",
sc->sc_name, g_mirror_device_state2str(sc->sc_state)));
break;
}
}
/*
* Update disk state and device state if needed.
*/
#define DISK_STATE_CHANGED() G_MIRROR_DEBUG(1, \
"Disk %s state changed from %s to %s (device %s).", \
g_mirror_get_diskname(disk), \
g_mirror_disk_state2str(disk->d_state), \
g_mirror_disk_state2str(state), sc->sc_name)
static int
g_mirror_update_disk(struct g_mirror_disk *disk, u_int state)
{
struct g_mirror_softc *sc;
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_XLOCKED);
again:
G_MIRROR_DEBUG(3, "Changing disk %s state from %s to %s.",
g_mirror_get_diskname(disk), g_mirror_disk_state2str(disk->d_state),
g_mirror_disk_state2str(state));
switch (state) {
case G_MIRROR_DISK_STATE_NEW:
/*
* Possible scenarios:
* 1. New disk arrive.
*/
/* Previous state should be NONE. */
KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NONE,
("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
DISK_STATE_CHANGED();
disk->d_state = state;
if (LIST_EMPTY(&sc->sc_disks))
LIST_INSERT_HEAD(&sc->sc_disks, disk, d_next);
else {
struct g_mirror_disk *dp;
LIST_FOREACH(dp, &sc->sc_disks, d_next) {
if (disk->d_priority >= dp->d_priority) {
LIST_INSERT_BEFORE(dp, disk, d_next);
dp = NULL;
break;
}
if (LIST_NEXT(dp, d_next) == NULL)
break;
}
if (dp != NULL)
LIST_INSERT_AFTER(dp, disk, d_next);
}
G_MIRROR_DEBUG(1, "Device %s: provider %s detected.",
sc->sc_name, g_mirror_get_diskname(disk));
if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
break;
KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_mirror_device_state2str(sc->sc_state),
g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
state = g_mirror_determine_state(disk);
if (state != G_MIRROR_DISK_STATE_NONE)
goto again;
break;
case G_MIRROR_DISK_STATE_ACTIVE:
/*
* Possible scenarios:
* 1. New disk does not need synchronization.
* 2. Synchronization process finished successfully.
*/
KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_mirror_device_state2str(sc->sc_state),
g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
/* Previous state should be NEW or SYNCHRONIZING. */
KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW ||
disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
DISK_STATE_CHANGED();
if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
disk->d_flags &= ~G_MIRROR_DISK_FLAG_SYNCHRONIZING;
disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC;
g_mirror_sync_stop(disk, 0);
}
disk->d_state = state;
disk->d_sync.ds_offset = 0;
disk->d_sync.ds_offset_done = 0;
g_mirror_update_idle(sc, disk);
g_mirror_update_metadata(disk);
G_MIRROR_DEBUG(1, "Device %s: provider %s activated.",
sc->sc_name, g_mirror_get_diskname(disk));
break;
case G_MIRROR_DISK_STATE_STALE:
/*
* Possible scenarios:
* 1. Stale disk was connected.
*/
/* Previous state should be NEW. */
KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_mirror_device_state2str(sc->sc_state),
g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
/*
* STALE state is only possible if device is marked
* NOAUTOSYNC.
*/
KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_mirror_device_state2str(sc->sc_state),
g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
DISK_STATE_CHANGED();
disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
disk->d_state = state;
g_mirror_update_metadata(disk);
G_MIRROR_DEBUG(0, "Device %s: provider %s is stale.",
sc->sc_name, g_mirror_get_diskname(disk));
break;
case G_MIRROR_DISK_STATE_SYNCHRONIZING:
/*
* Possible scenarios:
* 1. Disk which needs synchronization was connected.
*/
/* Previous state should be NEW. */
KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_mirror_device_state2str(sc->sc_state),
g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
DISK_STATE_CHANGED();
if (disk->d_state == G_MIRROR_DISK_STATE_NEW)
disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
disk->d_state = state;
if (sc->sc_provider != NULL) {
g_mirror_sync_start(disk);
g_mirror_update_metadata(disk);
}
break;
case G_MIRROR_DISK_STATE_DISCONNECTED:
/*
* Possible scenarios:
* 1. Device wasn't running yet, but disk disappear.
* 2. Disk was active and disapppear.
* 3. Disk disappear during synchronization process.
*/
if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING) {
/*
* Previous state should be ACTIVE, STALE or
* SYNCHRONIZING.
*/
KASSERT(disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
disk->d_state == G_MIRROR_DISK_STATE_STALE ||
disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
("Wrong disk state (%s, %s).",
g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
} else if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) {
/* Previous state should be NEW. */
KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
("Wrong disk state (%s, %s).",
g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
/*
* Reset bumping syncid if disk disappeared in STARTING
* state.
*/
if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0)
sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
#ifdef INVARIANTS
} else {
KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
sc->sc_name,
g_mirror_device_state2str(sc->sc_state),
g_mirror_get_diskname(disk),
g_mirror_disk_state2str(disk->d_state)));
#endif
}
DISK_STATE_CHANGED();
G_MIRROR_DEBUG(0, "Device %s: provider %s disconnected.",
sc->sc_name, g_mirror_get_diskname(disk));
g_mirror_destroy_disk(disk);
break;
case G_MIRROR_DISK_STATE_DESTROY:
{
int error;
error = g_mirror_clear_metadata(disk);
if (error != 0)
return (error);
DISK_STATE_CHANGED();
G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.",
sc->sc_name, g_mirror_get_diskname(disk));
g_mirror_destroy_disk(disk);
sc->sc_ndisks--;
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
g_mirror_update_metadata(disk);
}
break;
}
default:
KASSERT(1 == 0, ("Unknown state (%u).", state));
break;
}
return (0);
}
#undef DISK_STATE_CHANGED
int
g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md)
{
struct g_provider *pp;
u_char *buf;
int error;
g_topology_assert();
error = g_access(cp, 1, 0, 0);
if (error != 0)
return (error);
pp = cp->provider;
g_topology_unlock();
/* Metadata are stored on last sector. */
buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
&error);
g_topology_lock();
g_access(cp, -1, 0, 0);
if (buf == NULL) {
G_MIRROR_DEBUG(1, "Cannot read metadata from %s (error=%d).",
cp->provider->name, error);
return (error);
}
/* Decode metadata. */
error = mirror_metadata_decode(buf, md);
g_free(buf);
if (strcmp(md->md_magic, G_MIRROR_MAGIC) != 0)
return (EINVAL);
if (md->md_version > G_MIRROR_VERSION) {
G_MIRROR_DEBUG(0,
"Kernel module is too old to handle metadata from %s.",
cp->provider->name);
return (EINVAL);
}
if (error != 0) {
G_MIRROR_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
cp->provider->name);
return (error);
}
return (0);
}
static int
g_mirror_check_metadata(struct g_mirror_softc *sc, struct g_provider *pp,
struct g_mirror_metadata *md)
{
if (g_mirror_id2disk(sc, md->md_did) != NULL) {
G_MIRROR_DEBUG(1, "Disk %s (id=%u) already exists, skipping.",
pp->name, md->md_did);
return (EEXIST);
}
if (md->md_all != sc->sc_ndisks) {
G_MIRROR_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_all", pp->name, sc->sc_name);
return (EINVAL);
}
if (md->md_slice != sc->sc_slice) {
G_MIRROR_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_slice", pp->name, sc->sc_name);
return (EINVAL);
}
if (md->md_balance != sc->sc_balance) {
G_MIRROR_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_balance", pp->name, sc->sc_name);
return (EINVAL);
}
#if 0
if (md->md_mediasize != sc->sc_mediasize) {
G_MIRROR_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_mediasize", pp->name, sc->sc_name);
return (EINVAL);
}
#endif
if (sc->sc_mediasize > pp->mediasize) {
G_MIRROR_DEBUG(1,
"Invalid size of disk %s (device %s), skipping.", pp->name,
sc->sc_name);
return (EINVAL);
}
if (md->md_sectorsize != sc->sc_sectorsize) {
G_MIRROR_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_sectorsize", pp->name, sc->sc_name);
return (EINVAL);
}
if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
G_MIRROR_DEBUG(1,
"Invalid sector size of disk %s (device %s), skipping.",
pp->name, sc->sc_name);
return (EINVAL);
}
if ((md->md_mflags & ~G_MIRROR_DEVICE_FLAG_MASK) != 0) {
G_MIRROR_DEBUG(1,
"Invalid device flags on disk %s (device %s), skipping.",
pp->name, sc->sc_name);
return (EINVAL);
}
if ((md->md_dflags & ~G_MIRROR_DISK_FLAG_MASK) != 0) {
G_MIRROR_DEBUG(1,
"Invalid disk flags on disk %s (device %s), skipping.",
pp->name, sc->sc_name);
return (EINVAL);
}
return (0);
}
int
g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
struct g_mirror_metadata *md)
{
struct g_mirror_disk *disk;
int error;
g_topology_assert_not();
G_MIRROR_DEBUG(2, "Adding disk %s.", pp->name);
error = g_mirror_check_metadata(sc, pp, md);
if (error != 0)
return (error);
if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING &&
md->md_genid < sc->sc_genid) {
G_MIRROR_DEBUG(0, "Component %s (device %s) broken, skipping.",
pp->name, sc->sc_name);
return (EINVAL);
}
disk = g_mirror_init_disk(sc, pp, md, &error);
if (disk == NULL)
return (error);
error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW,
G_MIRROR_EVENT_WAIT);
if (error != 0)
return (error);
if (md->md_version < G_MIRROR_VERSION) {
G_MIRROR_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
pp->name, md->md_version, G_MIRROR_VERSION);
g_mirror_update_metadata(disk);
}
return (0);
}
static void
g_mirror_destroy_delayed(void *arg, int flag)
{
struct g_mirror_softc *sc;
int error;
if (flag == EV_CANCEL) {
G_MIRROR_DEBUG(1, "Destroying canceled.");
return;
}
sc = arg;
g_topology_unlock();
sx_xlock(&sc->sc_lock);
KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) == 0,
("DESTROY flag set on %s.", sc->sc_name));
KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0,
("DESTROYING flag not set on %s.", sc->sc_name));
G_MIRROR_DEBUG(1, "Destroying %s (delayed).", sc->sc_name);
error = g_mirror_destroy(sc, G_MIRROR_DESTROY_SOFT);
if (error != 0) {
G_MIRROR_DEBUG(0, "Cannot destroy %s (error=%d).",
sc->sc_name, error);
sx_xunlock(&sc->sc_lock);
}
g_topology_lock();
}
static int
g_mirror_access(struct g_provider *pp, int acr, int acw, int ace)
{
struct g_mirror_softc *sc;
int dcr, dcw, dce, error = 0;
g_topology_assert();
G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
acw, ace);
sc = pp->geom->softc;
if (sc == NULL && acr <= 0 && acw <= 0 && ace <= 0)
return (0);
KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
dcr = pp->acr + acr;
dcw = pp->acw + acw;
dce = pp->ace + ace;
g_topology_unlock();
sx_xlock(&sc->sc_lock);
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0 ||
LIST_EMPTY(&sc->sc_disks)) {
if (acr > 0 || acw > 0 || ace > 0)
error = ENXIO;
goto end;
}
if (dcw == 0)
g_mirror_idle(sc, dcw);
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0) {
if (acr > 0 || acw > 0 || ace > 0) {
error = ENXIO;
goto end;
}
if (dcr == 0 && dcw == 0 && dce == 0) {
g_post_event(g_mirror_destroy_delayed, sc, M_WAITOK,
sc, NULL);
}
}
end:
sx_xunlock(&sc->sc_lock);
g_topology_lock();
return (error);
}
static struct g_geom *
g_mirror_create(struct g_class *mp, const struct g_mirror_metadata *md)
{
struct g_mirror_softc *sc;
struct g_geom *gp;
int error, timeout;
g_topology_assert();
G_MIRROR_DEBUG(1, "Creating device %s (id=%u).", md->md_name,
md->md_mid);
/* One disk is minimum. */
if (md->md_all < 1)
return (NULL);
/*
* Action geom.
*/
gp = g_new_geomf(mp, "%s", md->md_name);
sc = malloc(sizeof(*sc), M_MIRROR, M_WAITOK | M_ZERO);
gp->start = g_mirror_start;
gp->orphan = g_mirror_orphan;
gp->access = g_mirror_access;
gp->dumpconf = g_mirror_dumpconf;
sc->sc_id = md->md_mid;
sc->sc_slice = md->md_slice;
sc->sc_balance = md->md_balance;
sc->sc_mediasize = md->md_mediasize;
sc->sc_sectorsize = md->md_sectorsize;
sc->sc_ndisks = md->md_all;
sc->sc_flags = md->md_mflags;
sc->sc_bump_id = 0;
sc->sc_idle = 1;
sc->sc_last_write = time_uptime;
sc->sc_writes = 0;
sx_init(&sc->sc_lock, "gmirror:lock");
bioq_init(&sc->sc_queue);
mtx_init(&sc->sc_queue_mtx, "gmirror:queue", NULL, MTX_DEF);
bioq_init(&sc->sc_regular_delayed);
bioq_init(&sc->sc_inflight);
bioq_init(&sc->sc_sync_delayed);
LIST_INIT(&sc->sc_disks);
TAILQ_INIT(&sc->sc_events);
mtx_init(&sc->sc_events_mtx, "gmirror:events", NULL, MTX_DEF);
- callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_callout, 1);
mtx_init(&sc->sc_done_mtx, "gmirror:done", NULL, MTX_DEF);
sc->sc_state = G_MIRROR_DEVICE_STATE_STARTING;
gp->softc = sc;
sc->sc_geom = gp;
sc->sc_provider = NULL;
/*
* Synchronization geom.
*/
gp = g_new_geomf(mp, "%s.sync", md->md_name);
gp->softc = sc;
gp->orphan = g_mirror_orphan;
sc->sc_sync.ds_geom = gp;
sc->sc_sync.ds_ndisks = 0;
error = kproc_create(g_mirror_worker, sc, &sc->sc_worker, 0, 0,
"g_mirror %s", md->md_name);
if (error != 0) {
G_MIRROR_DEBUG(1, "Cannot create kernel thread for %s.",
sc->sc_name);
g_destroy_geom(sc->sc_sync.ds_geom);
mtx_destroy(&sc->sc_done_mtx);
mtx_destroy(&sc->sc_events_mtx);
mtx_destroy(&sc->sc_queue_mtx);
sx_destroy(&sc->sc_lock);
g_destroy_geom(sc->sc_geom);
free(sc, M_MIRROR);
return (NULL);
}
G_MIRROR_DEBUG(1, "Device %s created (%u components, id=%u).",
sc->sc_name, sc->sc_ndisks, sc->sc_id);
sc->sc_rootmount = root_mount_hold("GMIRROR");
G_MIRROR_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
/*
* Run timeout.
*/
timeout = g_mirror_timeout * hz;
callout_reset(&sc->sc_callout, timeout, g_mirror_go, sc);
return (sc->sc_geom);
}
int
g_mirror_destroy(struct g_mirror_softc *sc, int how)
{
struct g_mirror_disk *disk;
struct g_provider *pp;
g_topology_assert_not();
if (sc == NULL)
return (ENXIO);
sx_assert(&sc->sc_lock, SX_XLOCKED);
pp = sc->sc_provider;
if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
switch (how) {
case G_MIRROR_DESTROY_SOFT:
G_MIRROR_DEBUG(1,
"Device %s is still open (r%dw%de%d).", pp->name,
pp->acr, pp->acw, pp->ace);
return (EBUSY);
case G_MIRROR_DESTROY_DELAYED:
G_MIRROR_DEBUG(1,
"Device %s will be destroyed on last close.",
pp->name);
LIST_FOREACH(disk, &sc->sc_disks, d_next) {
if (disk->d_state ==
G_MIRROR_DISK_STATE_SYNCHRONIZING) {
g_mirror_sync_stop(disk, 1);
}
}
sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROYING;
return (EBUSY);
case G_MIRROR_DESTROY_HARD:
G_MIRROR_DEBUG(1, "Device %s is still open, so it "
"can't be definitely removed.", pp->name);
}
}
g_topology_lock();
if (sc->sc_geom->softc == NULL) {
g_topology_unlock();
return (0);
}
sc->sc_geom->softc = NULL;
sc->sc_sync.ds_geom->softc = NULL;
g_topology_unlock();
sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
sc->sc_flags |= G_MIRROR_DEVICE_FLAG_WAIT;
G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
sx_xunlock(&sc->sc_lock);
mtx_lock(&sc->sc_queue_mtx);
wakeup(sc);
mtx_unlock(&sc->sc_queue_mtx);
G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
while (sc->sc_worker != NULL)
tsleep(&sc->sc_worker, PRIBIO, "m:destroy", hz / 5);
G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
sx_xlock(&sc->sc_lock);
g_mirror_destroy_device(sc);
free(sc, M_MIRROR);
return (0);
}
static void
g_mirror_taste_orphan(struct g_consumer *cp)
{
KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
cp->provider->name));
}
static struct g_geom *
g_mirror_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
{
struct g_mirror_metadata md;
struct g_mirror_softc *sc;
struct g_consumer *cp;
struct g_geom *gp;
int error;
g_topology_assert();
g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
G_MIRROR_DEBUG(2, "Tasting %s.", pp->name);
gp = g_new_geomf(mp, "mirror:taste");
/*
* This orphan function should be never called.
*/
gp->orphan = g_mirror_taste_orphan;
cp = g_new_consumer(gp);
g_attach(cp, pp);
error = g_mirror_read_metadata(cp, &md);
g_detach(cp);
g_destroy_consumer(cp);
g_destroy_geom(gp);
if (error != 0)
return (NULL);
gp = NULL;
if (md.md_provider[0] != '\0' &&
!g_compare_names(md.md_provider, pp->name))
return (NULL);
if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
return (NULL);
if ((md.md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) {
G_MIRROR_DEBUG(0,
"Device %s: provider %s marked as inactive, skipping.",
md.md_name, pp->name);
return (NULL);
}
if (g_mirror_debug >= 2)
mirror_metadata_dump(&md);
/*
* Let's check if device already exists.
*/
sc = NULL;
LIST_FOREACH(gp, &mp->geom, geom) {
sc = gp->softc;
if (sc == NULL)
continue;
if (sc->sc_sync.ds_geom == gp)
continue;
if (strcmp(md.md_name, sc->sc_name) != 0)
continue;
if (md.md_mid != sc->sc_id) {
G_MIRROR_DEBUG(0, "Device %s already configured.",
sc->sc_name);
return (NULL);
}
break;
}
if (gp == NULL) {
gp = g_mirror_create(mp, &md);
if (gp == NULL) {
G_MIRROR_DEBUG(0, "Cannot create device %s.",
md.md_name);
return (NULL);
}
sc = gp->softc;
}
G_MIRROR_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
g_topology_unlock();
sx_xlock(&sc->sc_lock);
sc->sc_flags |= G_MIRROR_DEVICE_FLAG_TASTING;
error = g_mirror_add_disk(sc, pp, &md);
if (error != 0) {
G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
pp->name, gp->name, error);
if (LIST_EMPTY(&sc->sc_disks)) {
g_cancel_event(sc);
g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
g_topology_lock();
return (NULL);
}
gp = NULL;
}
sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_TASTING;
if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
g_topology_lock();
return (NULL);
}
sx_xunlock(&sc->sc_lock);
g_topology_lock();
return (gp);
}
static void
g_mirror_resize(struct g_consumer *cp)
{
struct g_mirror_disk *disk;
g_topology_assert();
g_trace(G_T_TOPOLOGY, "%s(%s)", __func__, cp->provider->name);
disk = cp->private;
if (disk == NULL)
return;
g_topology_unlock();
g_mirror_update_metadata(disk);
g_topology_lock();
}
static int
g_mirror_destroy_geom(struct gctl_req *req __unused,
struct g_class *mp __unused, struct g_geom *gp)
{
struct g_mirror_softc *sc;
int error;
g_topology_unlock();
sc = gp->softc;
sx_xlock(&sc->sc_lock);
g_cancel_event(sc);
error = g_mirror_destroy(gp->softc, G_MIRROR_DESTROY_SOFT);
if (error != 0)
sx_xunlock(&sc->sc_lock);
g_topology_lock();
return (error);
}
static void
g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
struct g_consumer *cp, struct g_provider *pp)
{
struct g_mirror_softc *sc;
g_topology_assert();
sc = gp->softc;
if (sc == NULL)
return;
/* Skip synchronization geom. */
if (gp == sc->sc_sync.ds_geom)
return;
if (pp != NULL) {
/* Nothing here. */
} else if (cp != NULL) {
struct g_mirror_disk *disk;
disk = cp->private;
if (disk == NULL)
return;
g_topology_unlock();
sx_xlock(&sc->sc_lock);
sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)disk->d_id);
if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
sbuf_printf(sb, "%s<Synchronized>", indent);
if (disk->d_sync.ds_offset == 0)
sbuf_printf(sb, "0%%");
else {
sbuf_printf(sb, "%u%%",
(u_int)((disk->d_sync.ds_offset * 100) /
sc->sc_provider->mediasize));
}
sbuf_printf(sb, "</Synchronized>\n");
if (disk->d_sync.ds_offset > 0) {
sbuf_printf(sb, "%s<BytesSynced>%jd"
"</BytesSynced>\n", indent,
(intmax_t)disk->d_sync.ds_offset);
}
}
sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
disk->d_sync.ds_syncid);
sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent,
disk->d_genid);
sbuf_printf(sb, "%s<Flags>", indent);
if (disk->d_flags == 0)
sbuf_printf(sb, "NONE");
else {
int first = 1;
#define ADD_FLAG(flag, name) do { \
if ((disk->d_flags & (flag)) != 0) { \
if (!first) \
sbuf_printf(sb, ", "); \
else \
first = 0; \
sbuf_printf(sb, name); \
} \
} while (0)
ADD_FLAG(G_MIRROR_DISK_FLAG_DIRTY, "DIRTY");
ADD_FLAG(G_MIRROR_DISK_FLAG_HARDCODED, "HARDCODED");
ADD_FLAG(G_MIRROR_DISK_FLAG_INACTIVE, "INACTIVE");
ADD_FLAG(G_MIRROR_DISK_FLAG_SYNCHRONIZING,
"SYNCHRONIZING");
ADD_FLAG(G_MIRROR_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
ADD_FLAG(G_MIRROR_DISK_FLAG_BROKEN, "BROKEN");
#undef ADD_FLAG
}
sbuf_printf(sb, "</Flags>\n");
sbuf_printf(sb, "%s<Priority>%u</Priority>\n", indent,
disk->d_priority);
sbuf_printf(sb, "%s<State>%s</State>\n", indent,
g_mirror_disk_state2str(disk->d_state));
sx_xunlock(&sc->sc_lock);
g_topology_lock();
} else {
g_topology_unlock();
sx_xlock(&sc->sc_lock);
sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
sbuf_printf(sb, "%s<Flags>", indent);
if (sc->sc_flags == 0)
sbuf_printf(sb, "NONE");
else {
int first = 1;
#define ADD_FLAG(flag, name) do { \
if ((sc->sc_flags & (flag)) != 0) { \
if (!first) \
sbuf_printf(sb, ", "); \
else \
first = 0; \
sbuf_printf(sb, name); \
} \
} while (0)
ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
#undef ADD_FLAG
}
sbuf_printf(sb, "</Flags>\n");
sbuf_printf(sb, "%s<Slice>%u</Slice>\n", indent,
(u_int)sc->sc_slice);
sbuf_printf(sb, "%s<Balance>%s</Balance>\n", indent,
balance_name(sc->sc_balance));
sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
sc->sc_ndisks);
sbuf_printf(sb, "%s<State>", indent);
if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
sbuf_printf(sb, "%s", "STARTING");
else if (sc->sc_ndisks ==
g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE))
sbuf_printf(sb, "%s", "COMPLETE");
else
sbuf_printf(sb, "%s", "DEGRADED");
sbuf_printf(sb, "</State>\n");
sx_xunlock(&sc->sc_lock);
g_topology_lock();
}
}
static void
g_mirror_shutdown_post_sync(void *arg, int howto)
{
struct g_class *mp;
struct g_geom *gp, *gp2;
struct g_mirror_softc *sc;
int error;
mp = arg;
DROP_GIANT();
g_topology_lock();
g_mirror_shutdown = 1;
LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
if ((sc = gp->softc) == NULL)
continue;
/* Skip synchronization geom. */
if (gp == sc->sc_sync.ds_geom)
continue;
g_topology_unlock();
sx_xlock(&sc->sc_lock);
g_mirror_idle(sc, -1);
g_cancel_event(sc);
error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED);
if (error != 0)
sx_xunlock(&sc->sc_lock);
g_topology_lock();
}
g_topology_unlock();
PICKUP_GIANT();
}
static void
g_mirror_init(struct g_class *mp)
{
g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
if (g_mirror_post_sync == NULL)
G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event.");
}
static void
g_mirror_fini(struct g_class *mp)
{
if (g_mirror_post_sync != NULL)
EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync);
}
DECLARE_GEOM_CLASS(g_mirror_class, g_mirror);
Index: head/sys/geom/raid3/g_raid3.c
===================================================================
--- head/sys/geom/raid3/g_raid3.c (revision 283290)
+++ head/sys/geom/raid3/g_raid3.c (revision 283291)
@@ -1,3586 +1,3586 @@
/*-
* Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/bio.h>
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include <sys/malloc.h>
#include <sys/eventhandler.h>
#include <vm/uma.h>
#include <geom/geom.h>
#include <sys/proc.h>
#include <sys/kthread.h>
#include <sys/sched.h>
#include <geom/raid3/g_raid3.h>
FEATURE(geom_raid3, "GEOM RAID-3 functionality");
static MALLOC_DEFINE(M_RAID3, "raid3_data", "GEOM_RAID3 Data");
SYSCTL_DECL(_kern_geom);
static SYSCTL_NODE(_kern_geom, OID_AUTO, raid3, CTLFLAG_RW, 0,
"GEOM_RAID3 stuff");
u_int g_raid3_debug = 0;
SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, debug, CTLFLAG_RWTUN, &g_raid3_debug, 0,
"Debug level");
static u_int g_raid3_timeout = 4;
SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, timeout, CTLFLAG_RWTUN, &g_raid3_timeout,
0, "Time to wait on all raid3 components");
static u_int g_raid3_idletime = 5;
SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, idletime, CTLFLAG_RWTUN,
&g_raid3_idletime, 0, "Mark components as clean when idling");
static u_int g_raid3_disconnect_on_failure = 1;
SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, disconnect_on_failure, CTLFLAG_RWTUN,
&g_raid3_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
static u_int g_raid3_syncreqs = 2;
SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
&g_raid3_syncreqs, 0, "Parallel synchronization I/O requests.");
static u_int g_raid3_use_malloc = 0;
SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, use_malloc, CTLFLAG_RDTUN,
&g_raid3_use_malloc, 0, "Use malloc(9) instead of uma(9).");
static u_int g_raid3_n64k = 50;
SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, n64k, CTLFLAG_RDTUN, &g_raid3_n64k, 0,
"Maximum number of 64kB allocations");
static u_int g_raid3_n16k = 200;
SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, n16k, CTLFLAG_RDTUN, &g_raid3_n16k, 0,
"Maximum number of 16kB allocations");
static u_int g_raid3_n4k = 1200;
SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, n4k, CTLFLAG_RDTUN, &g_raid3_n4k, 0,
"Maximum number of 4kB allocations");
static SYSCTL_NODE(_kern_geom_raid3, OID_AUTO, stat, CTLFLAG_RW, 0,
"GEOM_RAID3 statistics");
static u_int g_raid3_parity_mismatch = 0;
SYSCTL_UINT(_kern_geom_raid3_stat, OID_AUTO, parity_mismatch, CTLFLAG_RD,
&g_raid3_parity_mismatch, 0, "Number of failures in VERIFY mode");
#define MSLEEP(ident, mtx, priority, wmesg, timeout) do { \
G_RAID3_DEBUG(4, "%s: Sleeping %p.", __func__, (ident)); \
msleep((ident), (mtx), (priority), (wmesg), (timeout)); \
G_RAID3_DEBUG(4, "%s: Woken up %p.", __func__, (ident)); \
} while (0)
static eventhandler_tag g_raid3_post_sync = NULL;
static int g_raid3_shutdown = 0;
static int g_raid3_destroy_geom(struct gctl_req *req, struct g_class *mp,
struct g_geom *gp);
static g_taste_t g_raid3_taste;
static void g_raid3_init(struct g_class *mp);
static void g_raid3_fini(struct g_class *mp);
struct g_class g_raid3_class = {
.name = G_RAID3_CLASS_NAME,
.version = G_VERSION,
.ctlreq = g_raid3_config,
.taste = g_raid3_taste,
.destroy_geom = g_raid3_destroy_geom,
.init = g_raid3_init,
.fini = g_raid3_fini
};
static void g_raid3_destroy_provider(struct g_raid3_softc *sc);
static int g_raid3_update_disk(struct g_raid3_disk *disk, u_int state);
static void g_raid3_update_device(struct g_raid3_softc *sc, boolean_t force);
static void g_raid3_dumpconf(struct sbuf *sb, const char *indent,
struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
static void g_raid3_sync_stop(struct g_raid3_softc *sc, int type);
static int g_raid3_register_request(struct bio *pbp);
static void g_raid3_sync_release(struct g_raid3_softc *sc);
static const char *
g_raid3_disk_state2str(int state)
{
switch (state) {
case G_RAID3_DISK_STATE_NODISK:
return ("NODISK");
case G_RAID3_DISK_STATE_NONE:
return ("NONE");
case G_RAID3_DISK_STATE_NEW:
return ("NEW");
case G_RAID3_DISK_STATE_ACTIVE:
return ("ACTIVE");
case G_RAID3_DISK_STATE_STALE:
return ("STALE");
case G_RAID3_DISK_STATE_SYNCHRONIZING:
return ("SYNCHRONIZING");
case G_RAID3_DISK_STATE_DISCONNECTED:
return ("DISCONNECTED");
default:
return ("INVALID");
}
}
static const char *
g_raid3_device_state2str(int state)
{
switch (state) {
case G_RAID3_DEVICE_STATE_STARTING:
return ("STARTING");
case G_RAID3_DEVICE_STATE_DEGRADED:
return ("DEGRADED");
case G_RAID3_DEVICE_STATE_COMPLETE:
return ("COMPLETE");
default:
return ("INVALID");
}
}
const char *
g_raid3_get_diskname(struct g_raid3_disk *disk)
{
if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
return ("[unknown]");
return (disk->d_name);
}
static void *
g_raid3_alloc(struct g_raid3_softc *sc, size_t size, int flags)
{
void *ptr;
enum g_raid3_zones zone;
if (g_raid3_use_malloc ||
(zone = g_raid3_zone(size)) == G_RAID3_NUM_ZONES)
ptr = malloc(size, M_RAID3, flags);
else {
ptr = uma_zalloc_arg(sc->sc_zones[zone].sz_zone,
&sc->sc_zones[zone], flags);
sc->sc_zones[zone].sz_requested++;
if (ptr == NULL)
sc->sc_zones[zone].sz_failed++;
}
return (ptr);
}
static void
g_raid3_free(struct g_raid3_softc *sc, void *ptr, size_t size)
{
enum g_raid3_zones zone;
if (g_raid3_use_malloc ||
(zone = g_raid3_zone(size)) == G_RAID3_NUM_ZONES)
free(ptr, M_RAID3);
else {
uma_zfree_arg(sc->sc_zones[zone].sz_zone,
ptr, &sc->sc_zones[zone]);
}
}
static int
g_raid3_uma_ctor(void *mem, int size, void *arg, int flags)
{
struct g_raid3_zone *sz = arg;
if (sz->sz_max > 0 && sz->sz_inuse == sz->sz_max)
return (ENOMEM);
sz->sz_inuse++;
return (0);
}
static void
g_raid3_uma_dtor(void *mem, int size, void *arg)
{
struct g_raid3_zone *sz = arg;
sz->sz_inuse--;
}
#define g_raid3_xor(src, dst, size) \
_g_raid3_xor((uint64_t *)(src), \
(uint64_t *)(dst), (size_t)size)
static void
_g_raid3_xor(uint64_t *src, uint64_t *dst, size_t size)
{
KASSERT((size % 128) == 0, ("Invalid size: %zu.", size));
for (; size > 0; size -= 128) {
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
*dst++ ^= (*src++);
}
}
static int
g_raid3_is_zero(struct bio *bp)
{
static const uint64_t zeros[] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
u_char *addr;
ssize_t size;
size = bp->bio_length;
addr = (u_char *)bp->bio_data;
for (; size > 0; size -= sizeof(zeros), addr += sizeof(zeros)) {
if (bcmp(addr, zeros, sizeof(zeros)) != 0)
return (0);
}
return (1);
}
/*
* --- Events handling functions ---
* Events in geom_raid3 are used to maintain disks and device status
* from one thread to simplify locking.
*/
static void
g_raid3_event_free(struct g_raid3_event *ep)
{
free(ep, M_RAID3);
}
int
g_raid3_event_send(void *arg, int state, int flags)
{
struct g_raid3_softc *sc;
struct g_raid3_disk *disk;
struct g_raid3_event *ep;
int error;
ep = malloc(sizeof(*ep), M_RAID3, M_WAITOK);
G_RAID3_DEBUG(4, "%s: Sending event %p.", __func__, ep);
if ((flags & G_RAID3_EVENT_DEVICE) != 0) {
disk = NULL;
sc = arg;
} else {
disk = arg;
sc = disk->d_softc;
}
ep->e_disk = disk;
ep->e_state = state;
ep->e_flags = flags;
ep->e_error = 0;
mtx_lock(&sc->sc_events_mtx);
TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
mtx_unlock(&sc->sc_events_mtx);
G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__, sc);
mtx_lock(&sc->sc_queue_mtx);
wakeup(sc);
wakeup(&sc->sc_queue);
mtx_unlock(&sc->sc_queue_mtx);
if ((flags & G_RAID3_EVENT_DONTWAIT) != 0)
return (0);
sx_assert(&sc->sc_lock, SX_XLOCKED);
G_RAID3_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
sx_xunlock(&sc->sc_lock);
while ((ep->e_flags & G_RAID3_EVENT_DONE) == 0) {
mtx_lock(&sc->sc_events_mtx);
MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "r3:event",
hz * 5);
}
error = ep->e_error;
g_raid3_event_free(ep);
sx_xlock(&sc->sc_lock);
return (error);
}
static struct g_raid3_event *
g_raid3_event_get(struct g_raid3_softc *sc)
{
struct g_raid3_event *ep;
mtx_lock(&sc->sc_events_mtx);
ep = TAILQ_FIRST(&sc->sc_events);
mtx_unlock(&sc->sc_events_mtx);
return (ep);
}
static void
g_raid3_event_remove(struct g_raid3_softc *sc, struct g_raid3_event *ep)
{
mtx_lock(&sc->sc_events_mtx);
TAILQ_REMOVE(&sc->sc_events, ep, e_next);
mtx_unlock(&sc->sc_events_mtx);
}
static void
g_raid3_event_cancel(struct g_raid3_disk *disk)
{
struct g_raid3_softc *sc;
struct g_raid3_event *ep, *tmpep;
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_XLOCKED);
mtx_lock(&sc->sc_events_mtx);
TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
if ((ep->e_flags & G_RAID3_EVENT_DEVICE) != 0)
continue;
if (ep->e_disk != disk)
continue;
TAILQ_REMOVE(&sc->sc_events, ep, e_next);
if ((ep->e_flags & G_RAID3_EVENT_DONTWAIT) != 0)
g_raid3_event_free(ep);
else {
ep->e_error = ECANCELED;
wakeup(ep);
}
}
mtx_unlock(&sc->sc_events_mtx);
}
/*
* Return the number of disks in the given state.
* If state is equal to -1, count all connected disks.
*/
u_int
g_raid3_ndisks(struct g_raid3_softc *sc, int state)
{
struct g_raid3_disk *disk;
u_int n, ndisks;
sx_assert(&sc->sc_lock, SX_LOCKED);
for (n = ndisks = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
continue;
if (state == -1 || disk->d_state == state)
ndisks++;
}
return (ndisks);
}
static u_int
g_raid3_nrequests(struct g_raid3_softc *sc, struct g_consumer *cp)
{
struct bio *bp;
u_int nreqs = 0;
mtx_lock(&sc->sc_queue_mtx);
TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
if (bp->bio_from == cp)
nreqs++;
}
mtx_unlock(&sc->sc_queue_mtx);
return (nreqs);
}
static int
g_raid3_is_busy(struct g_raid3_softc *sc, struct g_consumer *cp)
{
if (cp->index > 0) {
G_RAID3_DEBUG(2,
"I/O requests for %s exist, can't destroy it now.",
cp->provider->name);
return (1);
}
if (g_raid3_nrequests(sc, cp) > 0) {
G_RAID3_DEBUG(2,
"I/O requests for %s in queue, can't destroy it now.",
cp->provider->name);
return (1);
}
return (0);
}
static void
g_raid3_destroy_consumer(void *arg, int flags __unused)
{
struct g_consumer *cp;
g_topology_assert();
cp = arg;
G_RAID3_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
g_detach(cp);
g_destroy_consumer(cp);
}
static void
g_raid3_kill_consumer(struct g_raid3_softc *sc, struct g_consumer *cp)
{
struct g_provider *pp;
int retaste_wait;
g_topology_assert();
cp->private = NULL;
if (g_raid3_is_busy(sc, cp))
return;
G_RAID3_DEBUG(2, "Consumer %s destroyed.", cp->provider->name);
pp = cp->provider;
retaste_wait = 0;
if (cp->acw == 1) {
if ((pp->geom->flags & G_GEOM_WITHER) == 0)
retaste_wait = 1;
}
G_RAID3_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
-cp->acw, -cp->ace, 0);
if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
g_access(cp, -cp->acr, -cp->acw, -cp->ace);
if (retaste_wait) {
/*
* After retaste event was send (inside g_access()), we can send
* event to detach and destroy consumer.
* A class, which has consumer to the given provider connected
* will not receive retaste event for the provider.
* This is the way how I ignore retaste events when I close
* consumers opened for write: I detach and destroy consumer
* after retaste event is sent.
*/
g_post_event(g_raid3_destroy_consumer, cp, M_WAITOK, NULL);
return;
}
G_RAID3_DEBUG(1, "Consumer %s destroyed.", pp->name);
g_detach(cp);
g_destroy_consumer(cp);
}
static int
g_raid3_connect_disk(struct g_raid3_disk *disk, struct g_provider *pp)
{
struct g_consumer *cp;
int error;
g_topology_assert_not();
KASSERT(disk->d_consumer == NULL,
("Disk already connected (device %s).", disk->d_softc->sc_name));
g_topology_lock();
cp = g_new_consumer(disk->d_softc->sc_geom);
error = g_attach(cp, pp);
if (error != 0) {
g_destroy_consumer(cp);
g_topology_unlock();
return (error);
}
error = g_access(cp, 1, 1, 1);
g_topology_unlock();
if (error != 0) {
g_detach(cp);
g_destroy_consumer(cp);
G_RAID3_DEBUG(0, "Cannot open consumer %s (error=%d).",
pp->name, error);
return (error);
}
disk->d_consumer = cp;
disk->d_consumer->private = disk;
disk->d_consumer->index = 0;
G_RAID3_DEBUG(2, "Disk %s connected.", g_raid3_get_diskname(disk));
return (0);
}
static void
g_raid3_disconnect_consumer(struct g_raid3_softc *sc, struct g_consumer *cp)
{
g_topology_assert();
if (cp == NULL)
return;
if (cp->provider != NULL)
g_raid3_kill_consumer(sc, cp);
else
g_destroy_consumer(cp);
}
/*
* Initialize disk. This means allocate memory, create consumer, attach it
* to the provider and open access (r1w1e1) to it.
*/
static struct g_raid3_disk *
g_raid3_init_disk(struct g_raid3_softc *sc, struct g_provider *pp,
struct g_raid3_metadata *md, int *errorp)
{
struct g_raid3_disk *disk;
int error;
disk = &sc->sc_disks[md->md_no];
error = g_raid3_connect_disk(disk, pp);
if (error != 0) {
if (errorp != NULL)
*errorp = error;
return (NULL);
}
disk->d_state = G_RAID3_DISK_STATE_NONE;
disk->d_flags = md->md_dflags;
if (md->md_provider[0] != '\0')
disk->d_flags |= G_RAID3_DISK_FLAG_HARDCODED;
disk->d_sync.ds_consumer = NULL;
disk->d_sync.ds_offset = md->md_sync_offset;
disk->d_sync.ds_offset_done = md->md_sync_offset;
disk->d_genid = md->md_genid;
disk->d_sync.ds_syncid = md->md_syncid;
if (errorp != NULL)
*errorp = 0;
return (disk);
}
static void
g_raid3_destroy_disk(struct g_raid3_disk *disk)
{
struct g_raid3_softc *sc;
g_topology_assert_not();
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_XLOCKED);
if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
return;
g_raid3_event_cancel(disk);
switch (disk->d_state) {
case G_RAID3_DISK_STATE_SYNCHRONIZING:
if (sc->sc_syncdisk != NULL)
g_raid3_sync_stop(sc, 1);
/* FALLTHROUGH */
case G_RAID3_DISK_STATE_NEW:
case G_RAID3_DISK_STATE_STALE:
case G_RAID3_DISK_STATE_ACTIVE:
g_topology_lock();
g_raid3_disconnect_consumer(sc, disk->d_consumer);
g_topology_unlock();
disk->d_consumer = NULL;
break;
default:
KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
}
disk->d_state = G_RAID3_DISK_STATE_NODISK;
}
static void
g_raid3_destroy_device(struct g_raid3_softc *sc)
{
struct g_raid3_event *ep;
struct g_raid3_disk *disk;
struct g_geom *gp;
struct g_consumer *cp;
u_int n;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
gp = sc->sc_geom;
if (sc->sc_provider != NULL)
g_raid3_destroy_provider(sc);
for (n = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if (disk->d_state != G_RAID3_DISK_STATE_NODISK) {
disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
g_raid3_update_metadata(disk);
g_raid3_destroy_disk(disk);
}
}
while ((ep = g_raid3_event_get(sc)) != NULL) {
g_raid3_event_remove(sc, ep);
if ((ep->e_flags & G_RAID3_EVENT_DONTWAIT) != 0)
g_raid3_event_free(ep);
else {
ep->e_error = ECANCELED;
ep->e_flags |= G_RAID3_EVENT_DONE;
G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__, ep);
mtx_lock(&sc->sc_events_mtx);
wakeup(ep);
mtx_unlock(&sc->sc_events_mtx);
}
}
callout_drain(&sc->sc_callout);
cp = LIST_FIRST(&sc->sc_sync.ds_geom->consumer);
g_topology_lock();
if (cp != NULL)
g_raid3_disconnect_consumer(sc, cp);
g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
G_RAID3_DEBUG(0, "Device %s destroyed.", gp->name);
g_wither_geom(gp, ENXIO);
g_topology_unlock();
if (!g_raid3_use_malloc) {
uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_64K].sz_zone);
uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_16K].sz_zone);
uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_4K].sz_zone);
}
mtx_destroy(&sc->sc_queue_mtx);
mtx_destroy(&sc->sc_events_mtx);
sx_xunlock(&sc->sc_lock);
sx_destroy(&sc->sc_lock);
}
static void
g_raid3_orphan(struct g_consumer *cp)
{
struct g_raid3_disk *disk;
g_topology_assert();
disk = cp->private;
if (disk == NULL)
return;
disk->d_softc->sc_bump_id = G_RAID3_BUMP_SYNCID;
g_raid3_event_send(disk, G_RAID3_DISK_STATE_DISCONNECTED,
G_RAID3_EVENT_DONTWAIT);
}
static int
g_raid3_write_metadata(struct g_raid3_disk *disk, struct g_raid3_metadata *md)
{
struct g_raid3_softc *sc;
struct g_consumer *cp;
off_t offset, length;
u_char *sector;
int error = 0;
g_topology_assert_not();
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_LOCKED);
cp = disk->d_consumer;
KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
cp->acw, cp->ace));
length = cp->provider->sectorsize;
offset = cp->provider->mediasize - length;
sector = malloc((size_t)length, M_RAID3, M_WAITOK | M_ZERO);
if (md != NULL)
raid3_metadata_encode(md, sector);
error = g_write_data(cp, offset, sector, length);
free(sector, M_RAID3);
if (error != 0) {
if ((disk->d_flags & G_RAID3_DISK_FLAG_BROKEN) == 0) {
G_RAID3_DEBUG(0, "Cannot write metadata on %s "
"(device=%s, error=%d).",
g_raid3_get_diskname(disk), sc->sc_name, error);
disk->d_flags |= G_RAID3_DISK_FLAG_BROKEN;
} else {
G_RAID3_DEBUG(1, "Cannot write metadata on %s "
"(device=%s, error=%d).",
g_raid3_get_diskname(disk), sc->sc_name, error);
}
if (g_raid3_disconnect_on_failure &&
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
sc->sc_bump_id |= G_RAID3_BUMP_GENID;
g_raid3_event_send(disk,
G_RAID3_DISK_STATE_DISCONNECTED,
G_RAID3_EVENT_DONTWAIT);
}
}
return (error);
}
int
g_raid3_clear_metadata(struct g_raid3_disk *disk)
{
int error;
g_topology_assert_not();
sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
error = g_raid3_write_metadata(disk, NULL);
if (error == 0) {
G_RAID3_DEBUG(2, "Metadata on %s cleared.",
g_raid3_get_diskname(disk));
} else {
G_RAID3_DEBUG(0,
"Cannot clear metadata on disk %s (error=%d).",
g_raid3_get_diskname(disk), error);
}
return (error);
}
void
g_raid3_fill_metadata(struct g_raid3_disk *disk, struct g_raid3_metadata *md)
{
struct g_raid3_softc *sc;
struct g_provider *pp;
sc = disk->d_softc;
strlcpy(md->md_magic, G_RAID3_MAGIC, sizeof(md->md_magic));
md->md_version = G_RAID3_VERSION;
strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
md->md_id = sc->sc_id;
md->md_all = sc->sc_ndisks;
md->md_genid = sc->sc_genid;
md->md_mediasize = sc->sc_mediasize;
md->md_sectorsize = sc->sc_sectorsize;
md->md_mflags = (sc->sc_flags & G_RAID3_DEVICE_FLAG_MASK);
md->md_no = disk->d_no;
md->md_syncid = disk->d_sync.ds_syncid;
md->md_dflags = (disk->d_flags & G_RAID3_DISK_FLAG_MASK);
if (disk->d_state != G_RAID3_DISK_STATE_SYNCHRONIZING)
md->md_sync_offset = 0;
else {
md->md_sync_offset =
disk->d_sync.ds_offset_done / (sc->sc_ndisks - 1);
}
if (disk->d_consumer != NULL && disk->d_consumer->provider != NULL)
pp = disk->d_consumer->provider;
else
pp = NULL;
if ((disk->d_flags & G_RAID3_DISK_FLAG_HARDCODED) != 0 && pp != NULL)
strlcpy(md->md_provider, pp->name, sizeof(md->md_provider));
else
bzero(md->md_provider, sizeof(md->md_provider));
if (pp != NULL)
md->md_provsize = pp->mediasize;
else
md->md_provsize = 0;
}
void
g_raid3_update_metadata(struct g_raid3_disk *disk)
{
struct g_raid3_softc *sc;
struct g_raid3_metadata md;
int error;
g_topology_assert_not();
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_LOCKED);
g_raid3_fill_metadata(disk, &md);
error = g_raid3_write_metadata(disk, &md);
if (error == 0) {
G_RAID3_DEBUG(2, "Metadata on %s updated.",
g_raid3_get_diskname(disk));
} else {
G_RAID3_DEBUG(0,
"Cannot update metadata on disk %s (error=%d).",
g_raid3_get_diskname(disk), error);
}
}
static void
g_raid3_bump_syncid(struct g_raid3_softc *sc)
{
struct g_raid3_disk *disk;
u_int n;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
KASSERT(g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) > 0,
("%s called with no active disks (device=%s).", __func__,
sc->sc_name));
sc->sc_syncid++;
G_RAID3_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
sc->sc_syncid);
for (n = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if (disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
disk->d_sync.ds_syncid = sc->sc_syncid;
g_raid3_update_metadata(disk);
}
}
}
static void
g_raid3_bump_genid(struct g_raid3_softc *sc)
{
struct g_raid3_disk *disk;
u_int n;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
KASSERT(g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) > 0,
("%s called with no active disks (device=%s).", __func__,
sc->sc_name));
sc->sc_genid++;
G_RAID3_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
sc->sc_genid);
for (n = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if (disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
disk->d_genid = sc->sc_genid;
g_raid3_update_metadata(disk);
}
}
}
static int
g_raid3_idle(struct g_raid3_softc *sc, int acw)
{
struct g_raid3_disk *disk;
u_int i;
int timeout;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
if (sc->sc_provider == NULL)
return (0);
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOFAILSYNC) != 0)
return (0);
if (sc->sc_idle)
return (0);
if (sc->sc_writes > 0)
return (0);
if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
timeout = g_raid3_idletime - (time_uptime - sc->sc_last_write);
if (!g_raid3_shutdown && timeout > 0)
return (timeout);
}
sc->sc_idle = 1;
for (i = 0; i < sc->sc_ndisks; i++) {
disk = &sc->sc_disks[i];
if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE)
continue;
G_RAID3_DEBUG(1, "Disk %s (device %s) marked as clean.",
g_raid3_get_diskname(disk), sc->sc_name);
disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
g_raid3_update_metadata(disk);
}
return (0);
}
static void
g_raid3_unidle(struct g_raid3_softc *sc)
{
struct g_raid3_disk *disk;
u_int i;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOFAILSYNC) != 0)
return;
sc->sc_idle = 0;
sc->sc_last_write = time_uptime;
for (i = 0; i < sc->sc_ndisks; i++) {
disk = &sc->sc_disks[i];
if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE)
continue;
G_RAID3_DEBUG(1, "Disk %s (device %s) marked as dirty.",
g_raid3_get_diskname(disk), sc->sc_name);
disk->d_flags |= G_RAID3_DISK_FLAG_DIRTY;
g_raid3_update_metadata(disk);
}
}
/*
* Treat bio_driver1 field in parent bio as list head and field bio_caller1
* in child bio as pointer to the next element on the list.
*/
#define G_RAID3_HEAD_BIO(pbp) (pbp)->bio_driver1
#define G_RAID3_NEXT_BIO(cbp) (cbp)->bio_caller1
#define G_RAID3_FOREACH_BIO(pbp, bp) \
for ((bp) = G_RAID3_HEAD_BIO(pbp); (bp) != NULL; \
(bp) = G_RAID3_NEXT_BIO(bp))
#define G_RAID3_FOREACH_SAFE_BIO(pbp, bp, tmpbp) \
for ((bp) = G_RAID3_HEAD_BIO(pbp); \
(bp) != NULL && ((tmpbp) = G_RAID3_NEXT_BIO(bp), 1); \
(bp) = (tmpbp))
static void
g_raid3_init_bio(struct bio *pbp)
{
G_RAID3_HEAD_BIO(pbp) = NULL;
}
static void
g_raid3_remove_bio(struct bio *cbp)
{
struct bio *pbp, *bp;
pbp = cbp->bio_parent;
if (G_RAID3_HEAD_BIO(pbp) == cbp)
G_RAID3_HEAD_BIO(pbp) = G_RAID3_NEXT_BIO(cbp);
else {
G_RAID3_FOREACH_BIO(pbp, bp) {
if (G_RAID3_NEXT_BIO(bp) == cbp) {
G_RAID3_NEXT_BIO(bp) = G_RAID3_NEXT_BIO(cbp);
break;
}
}
}
G_RAID3_NEXT_BIO(cbp) = NULL;
}
static void
g_raid3_replace_bio(struct bio *sbp, struct bio *dbp)
{
struct bio *pbp, *bp;
g_raid3_remove_bio(sbp);
pbp = dbp->bio_parent;
G_RAID3_NEXT_BIO(sbp) = G_RAID3_NEXT_BIO(dbp);
if (G_RAID3_HEAD_BIO(pbp) == dbp)
G_RAID3_HEAD_BIO(pbp) = sbp;
else {
G_RAID3_FOREACH_BIO(pbp, bp) {
if (G_RAID3_NEXT_BIO(bp) == dbp) {
G_RAID3_NEXT_BIO(bp) = sbp;
break;
}
}
}
G_RAID3_NEXT_BIO(dbp) = NULL;
}
static void
g_raid3_destroy_bio(struct g_raid3_softc *sc, struct bio *cbp)
{
struct bio *bp, *pbp;
size_t size;
pbp = cbp->bio_parent;
pbp->bio_children--;
KASSERT(cbp->bio_data != NULL, ("NULL bio_data"));
size = pbp->bio_length / (sc->sc_ndisks - 1);
g_raid3_free(sc, cbp->bio_data, size);
if (G_RAID3_HEAD_BIO(pbp) == cbp) {
G_RAID3_HEAD_BIO(pbp) = G_RAID3_NEXT_BIO(cbp);
G_RAID3_NEXT_BIO(cbp) = NULL;
g_destroy_bio(cbp);
} else {
G_RAID3_FOREACH_BIO(pbp, bp) {
if (G_RAID3_NEXT_BIO(bp) == cbp)
break;
}
if (bp != NULL) {
KASSERT(G_RAID3_NEXT_BIO(bp) != NULL,
("NULL bp->bio_driver1"));
G_RAID3_NEXT_BIO(bp) = G_RAID3_NEXT_BIO(cbp);
G_RAID3_NEXT_BIO(cbp) = NULL;
}
g_destroy_bio(cbp);
}
}
static struct bio *
g_raid3_clone_bio(struct g_raid3_softc *sc, struct bio *pbp)
{
struct bio *bp, *cbp;
size_t size;
int memflag;
cbp = g_clone_bio(pbp);
if (cbp == NULL)
return (NULL);
size = pbp->bio_length / (sc->sc_ndisks - 1);
if ((pbp->bio_cflags & G_RAID3_BIO_CFLAG_REGULAR) != 0)
memflag = M_WAITOK;
else
memflag = M_NOWAIT;
cbp->bio_data = g_raid3_alloc(sc, size, memflag);
if (cbp->bio_data == NULL) {
pbp->bio_children--;
g_destroy_bio(cbp);
return (NULL);
}
G_RAID3_NEXT_BIO(cbp) = NULL;
if (G_RAID3_HEAD_BIO(pbp) == NULL)
G_RAID3_HEAD_BIO(pbp) = cbp;
else {
G_RAID3_FOREACH_BIO(pbp, bp) {
if (G_RAID3_NEXT_BIO(bp) == NULL) {
G_RAID3_NEXT_BIO(bp) = cbp;
break;
}
}
}
return (cbp);
}
static void
g_raid3_scatter(struct bio *pbp)
{
struct g_raid3_softc *sc;
struct g_raid3_disk *disk;
struct bio *bp, *cbp, *tmpbp;
off_t atom, cadd, padd, left;
int first;
sc = pbp->bio_to->geom->softc;
bp = NULL;
if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_NOPARITY) == 0) {
/*
* Find bio for which we should calculate data.
*/
G_RAID3_FOREACH_BIO(pbp, cbp) {
if ((cbp->bio_cflags & G_RAID3_BIO_CFLAG_PARITY) != 0) {
bp = cbp;
break;
}
}
KASSERT(bp != NULL, ("NULL parity bio."));
}
atom = sc->sc_sectorsize / (sc->sc_ndisks - 1);
cadd = padd = 0;
for (left = pbp->bio_length; left > 0; left -= sc->sc_sectorsize) {
G_RAID3_FOREACH_BIO(pbp, cbp) {
if (cbp == bp)
continue;
bcopy(pbp->bio_data + padd, cbp->bio_data + cadd, atom);
padd += atom;
}
cadd += atom;
}
if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_NOPARITY) == 0) {
/*
* Calculate parity.
*/
first = 1;
G_RAID3_FOREACH_SAFE_BIO(pbp, cbp, tmpbp) {
if (cbp == bp)
continue;
if (first) {
bcopy(cbp->bio_data, bp->bio_data,
bp->bio_length);
first = 0;
} else {
g_raid3_xor(cbp->bio_data, bp->bio_data,
bp->bio_length);
}
if ((cbp->bio_cflags & G_RAID3_BIO_CFLAG_NODISK) != 0)
g_raid3_destroy_bio(sc, cbp);
}
}
G_RAID3_FOREACH_SAFE_BIO(pbp, cbp, tmpbp) {
struct g_consumer *cp;
disk = cbp->bio_caller2;
cp = disk->d_consumer;
cbp->bio_to = cp->provider;
G_RAID3_LOGREQ(3, cbp, "Sending request.");
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
cp->acr, cp->acw, cp->ace));
cp->index++;
sc->sc_writes++;
g_io_request(cbp, cp);
}
}
static void
g_raid3_gather(struct bio *pbp)
{
struct g_raid3_softc *sc;
struct g_raid3_disk *disk;
struct bio *xbp, *fbp, *cbp;
off_t atom, cadd, padd, left;
sc = pbp->bio_to->geom->softc;
/*
* Find bio for which we have to calculate data.
* While going through this path, check if all requests
* succeeded, if not, deny whole request.
* If we're in COMPLETE mode, we allow one request to fail,
* so if we find one, we're sending it to the parity consumer.
* If there are more failed requests, we deny whole request.
*/
xbp = fbp = NULL;
G_RAID3_FOREACH_BIO(pbp, cbp) {
if ((cbp->bio_cflags & G_RAID3_BIO_CFLAG_PARITY) != 0) {
KASSERT(xbp == NULL, ("More than one parity bio."));
xbp = cbp;
}
if (cbp->bio_error == 0)
continue;
/*
* Found failed request.
*/
if (fbp == NULL) {
if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_DEGRADED) != 0) {
/*
* We are already in degraded mode, so we can't
* accept any failures.
*/
if (pbp->bio_error == 0)
pbp->bio_error = cbp->bio_error;
} else {
fbp = cbp;
}
} else {
/*
* Next failed request, that's too many.
*/
if (pbp->bio_error == 0)
pbp->bio_error = fbp->bio_error;
}
disk = cbp->bio_caller2;
if (disk == NULL)
continue;
if ((disk->d_flags & G_RAID3_DISK_FLAG_BROKEN) == 0) {
disk->d_flags |= G_RAID3_DISK_FLAG_BROKEN;
G_RAID3_LOGREQ(0, cbp, "Request failed (error=%d).",
cbp->bio_error);
} else {
G_RAID3_LOGREQ(1, cbp, "Request failed (error=%d).",
cbp->bio_error);
}
if (g_raid3_disconnect_on_failure &&
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
sc->sc_bump_id |= G_RAID3_BUMP_GENID;
g_raid3_event_send(disk,
G_RAID3_DISK_STATE_DISCONNECTED,
G_RAID3_EVENT_DONTWAIT);
}
}
if (pbp->bio_error != 0)
goto finish;
if (fbp != NULL && (pbp->bio_pflags & G_RAID3_BIO_PFLAG_VERIFY) != 0) {
pbp->bio_pflags &= ~G_RAID3_BIO_PFLAG_VERIFY;
if (xbp != fbp)
g_raid3_replace_bio(xbp, fbp);
g_raid3_destroy_bio(sc, fbp);
} else if (fbp != NULL) {
struct g_consumer *cp;
/*
* One request failed, so send the same request to
* the parity consumer.
*/
disk = pbp->bio_driver2;
if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE) {
pbp->bio_error = fbp->bio_error;
goto finish;
}
pbp->bio_pflags |= G_RAID3_BIO_PFLAG_DEGRADED;
pbp->bio_inbed--;
fbp->bio_flags &= ~(BIO_DONE | BIO_ERROR);
if (disk->d_no == sc->sc_ndisks - 1)
fbp->bio_cflags |= G_RAID3_BIO_CFLAG_PARITY;
fbp->bio_error = 0;
fbp->bio_completed = 0;
fbp->bio_children = 0;
fbp->bio_inbed = 0;
cp = disk->d_consumer;
fbp->bio_caller2 = disk;
fbp->bio_to = cp->provider;
G_RAID3_LOGREQ(3, fbp, "Sending request (recover).");
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
cp->acr, cp->acw, cp->ace));
cp->index++;
g_io_request(fbp, cp);
return;
}
if (xbp != NULL) {
/*
* Calculate parity.
*/
G_RAID3_FOREACH_BIO(pbp, cbp) {
if ((cbp->bio_cflags & G_RAID3_BIO_CFLAG_PARITY) != 0)
continue;
g_raid3_xor(cbp->bio_data, xbp->bio_data,
xbp->bio_length);
}
xbp->bio_cflags &= ~G_RAID3_BIO_CFLAG_PARITY;
if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_VERIFY) != 0) {
if (!g_raid3_is_zero(xbp)) {
g_raid3_parity_mismatch++;
pbp->bio_error = EIO;
goto finish;
}
g_raid3_destroy_bio(sc, xbp);
}
}
atom = sc->sc_sectorsize / (sc->sc_ndisks - 1);
cadd = padd = 0;
for (left = pbp->bio_length; left > 0; left -= sc->sc_sectorsize) {
G_RAID3_FOREACH_BIO(pbp, cbp) {
bcopy(cbp->bio_data + cadd, pbp->bio_data + padd, atom);
pbp->bio_completed += atom;
padd += atom;
}
cadd += atom;
}
finish:
if (pbp->bio_error == 0)
G_RAID3_LOGREQ(3, pbp, "Request finished.");
else {
if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_VERIFY) != 0)
G_RAID3_LOGREQ(1, pbp, "Verification error.");
else
G_RAID3_LOGREQ(0, pbp, "Request failed.");
}
pbp->bio_pflags &= ~G_RAID3_BIO_PFLAG_MASK;
while ((cbp = G_RAID3_HEAD_BIO(pbp)) != NULL)
g_raid3_destroy_bio(sc, cbp);
g_io_deliver(pbp, pbp->bio_error);
}
static void
g_raid3_done(struct bio *bp)
{
struct g_raid3_softc *sc;
sc = bp->bio_from->geom->softc;
bp->bio_cflags |= G_RAID3_BIO_CFLAG_REGULAR;
G_RAID3_LOGREQ(3, bp, "Regular request done (error=%d).", bp->bio_error);
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_head(&sc->sc_queue, bp);
mtx_unlock(&sc->sc_queue_mtx);
wakeup(sc);
wakeup(&sc->sc_queue);
}
static void
g_raid3_regular_request(struct bio *cbp)
{
struct g_raid3_softc *sc;
struct g_raid3_disk *disk;
struct bio *pbp;
g_topology_assert_not();
pbp = cbp->bio_parent;
sc = pbp->bio_to->geom->softc;
cbp->bio_from->index--;
if (cbp->bio_cmd == BIO_WRITE)
sc->sc_writes--;
disk = cbp->bio_from->private;
if (disk == NULL) {
g_topology_lock();
g_raid3_kill_consumer(sc, cbp->bio_from);
g_topology_unlock();
}
G_RAID3_LOGREQ(3, cbp, "Request finished.");
pbp->bio_inbed++;
KASSERT(pbp->bio_inbed <= pbp->bio_children,
("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
pbp->bio_children));
if (pbp->bio_inbed != pbp->bio_children)
return;
switch (pbp->bio_cmd) {
case BIO_READ:
g_raid3_gather(pbp);
break;
case BIO_WRITE:
case BIO_DELETE:
{
int error = 0;
pbp->bio_completed = pbp->bio_length;
while ((cbp = G_RAID3_HEAD_BIO(pbp)) != NULL) {
if (cbp->bio_error == 0) {
g_raid3_destroy_bio(sc, cbp);
continue;
}
if (error == 0)
error = cbp->bio_error;
else if (pbp->bio_error == 0) {
/*
* Next failed request, that's too many.
*/
pbp->bio_error = error;
}
disk = cbp->bio_caller2;
if (disk == NULL) {
g_raid3_destroy_bio(sc, cbp);
continue;
}
if ((disk->d_flags & G_RAID3_DISK_FLAG_BROKEN) == 0) {
disk->d_flags |= G_RAID3_DISK_FLAG_BROKEN;
G_RAID3_LOGREQ(0, cbp,
"Request failed (error=%d).",
cbp->bio_error);
} else {
G_RAID3_LOGREQ(1, cbp,
"Request failed (error=%d).",
cbp->bio_error);
}
if (g_raid3_disconnect_on_failure &&
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
sc->sc_bump_id |= G_RAID3_BUMP_GENID;
g_raid3_event_send(disk,
G_RAID3_DISK_STATE_DISCONNECTED,
G_RAID3_EVENT_DONTWAIT);
}
g_raid3_destroy_bio(sc, cbp);
}
if (pbp->bio_error == 0)
G_RAID3_LOGREQ(3, pbp, "Request finished.");
else
G_RAID3_LOGREQ(0, pbp, "Request failed.");
pbp->bio_pflags &= ~G_RAID3_BIO_PFLAG_DEGRADED;
pbp->bio_pflags &= ~G_RAID3_BIO_PFLAG_NOPARITY;
bioq_remove(&sc->sc_inflight, pbp);
/* Release delayed sync requests if possible. */
g_raid3_sync_release(sc);
g_io_deliver(pbp, pbp->bio_error);
break;
}
}
}
static void
g_raid3_sync_done(struct bio *bp)
{
struct g_raid3_softc *sc;
G_RAID3_LOGREQ(3, bp, "Synchronization request delivered.");
sc = bp->bio_from->geom->softc;
bp->bio_cflags |= G_RAID3_BIO_CFLAG_SYNC;
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_head(&sc->sc_queue, bp);
mtx_unlock(&sc->sc_queue_mtx);
wakeup(sc);
wakeup(&sc->sc_queue);
}
static void
g_raid3_flush(struct g_raid3_softc *sc, struct bio *bp)
{
struct bio_queue_head queue;
struct g_raid3_disk *disk;
struct g_consumer *cp;
struct bio *cbp;
u_int i;
bioq_init(&queue);
for (i = 0; i < sc->sc_ndisks; i++) {
disk = &sc->sc_disks[i];
if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE)
continue;
cbp = g_clone_bio(bp);
if (cbp == NULL) {
for (cbp = bioq_first(&queue); cbp != NULL;
cbp = bioq_first(&queue)) {
bioq_remove(&queue, cbp);
g_destroy_bio(cbp);
}
if (bp->bio_error == 0)
bp->bio_error = ENOMEM;
g_io_deliver(bp, bp->bio_error);
return;
}
bioq_insert_tail(&queue, cbp);
cbp->bio_done = g_std_done;
cbp->bio_caller1 = disk;
cbp->bio_to = disk->d_consumer->provider;
}
for (cbp = bioq_first(&queue); cbp != NULL; cbp = bioq_first(&queue)) {
bioq_remove(&queue, cbp);
G_RAID3_LOGREQ(3, cbp, "Sending request.");
disk = cbp->bio_caller1;
cbp->bio_caller1 = NULL;
cp = disk->d_consumer;
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
cp->acr, cp->acw, cp->ace));
g_io_request(cbp, disk->d_consumer);
}
}
static void
g_raid3_start(struct bio *bp)
{
struct g_raid3_softc *sc;
sc = bp->bio_to->geom->softc;
/*
* If sc == NULL or there are no valid disks, provider's error
* should be set and g_raid3_start() should not be called at all.
*/
KASSERT(sc != NULL && (sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE),
("Provider's error should be set (error=%d)(device=%s).",
bp->bio_to->error, bp->bio_to->name));
G_RAID3_LOGREQ(3, bp, "Request received.");
switch (bp->bio_cmd) {
case BIO_READ:
case BIO_WRITE:
case BIO_DELETE:
break;
case BIO_FLUSH:
g_raid3_flush(sc, bp);
return;
case BIO_GETATTR:
default:
g_io_deliver(bp, EOPNOTSUPP);
return;
}
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_tail(&sc->sc_queue, bp);
mtx_unlock(&sc->sc_queue_mtx);
G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__, sc);
wakeup(sc);
}
/*
* Return TRUE if the given request is colliding with a in-progress
* synchronization request.
*/
static int
g_raid3_sync_collision(struct g_raid3_softc *sc, struct bio *bp)
{
struct g_raid3_disk *disk;
struct bio *sbp;
off_t rstart, rend, sstart, send;
int i;
disk = sc->sc_syncdisk;
if (disk == NULL)
return (0);
rstart = bp->bio_offset;
rend = bp->bio_offset + bp->bio_length;
for (i = 0; i < g_raid3_syncreqs; i++) {
sbp = disk->d_sync.ds_bios[i];
if (sbp == NULL)
continue;
sstart = sbp->bio_offset;
send = sbp->bio_length;
if (sbp->bio_cmd == BIO_WRITE) {
sstart *= sc->sc_ndisks - 1;
send *= sc->sc_ndisks - 1;
}
send += sstart;
if (rend > sstart && rstart < send)
return (1);
}
return (0);
}
/*
* Return TRUE if the given sync request is colliding with a in-progress regular
* request.
*/
static int
g_raid3_regular_collision(struct g_raid3_softc *sc, struct bio *sbp)
{
off_t rstart, rend, sstart, send;
struct bio *bp;
if (sc->sc_syncdisk == NULL)
return (0);
sstart = sbp->bio_offset;
send = sstart + sbp->bio_length;
TAILQ_FOREACH(bp, &sc->sc_inflight.queue, bio_queue) {
rstart = bp->bio_offset;
rend = bp->bio_offset + bp->bio_length;
if (rend > sstart && rstart < send)
return (1);
}
return (0);
}
/*
* Puts request onto delayed queue.
*/
static void
g_raid3_regular_delay(struct g_raid3_softc *sc, struct bio *bp)
{
G_RAID3_LOGREQ(2, bp, "Delaying request.");
bioq_insert_head(&sc->sc_regular_delayed, bp);
}
/*
* Puts synchronization request onto delayed queue.
*/
static void
g_raid3_sync_delay(struct g_raid3_softc *sc, struct bio *bp)
{
G_RAID3_LOGREQ(2, bp, "Delaying synchronization request.");
bioq_insert_tail(&sc->sc_sync_delayed, bp);
}
/*
* Releases delayed regular requests which don't collide anymore with sync
* requests.
*/
static void
g_raid3_regular_release(struct g_raid3_softc *sc)
{
struct bio *bp, *bp2;
TAILQ_FOREACH_SAFE(bp, &sc->sc_regular_delayed.queue, bio_queue, bp2) {
if (g_raid3_sync_collision(sc, bp))
continue;
bioq_remove(&sc->sc_regular_delayed, bp);
G_RAID3_LOGREQ(2, bp, "Releasing delayed request (%p).", bp);
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_head(&sc->sc_queue, bp);
#if 0
/*
* wakeup() is not needed, because this function is called from
* the worker thread.
*/
wakeup(&sc->sc_queue);
#endif
mtx_unlock(&sc->sc_queue_mtx);
}
}
/*
* Releases delayed sync requests which don't collide anymore with regular
* requests.
*/
static void
g_raid3_sync_release(struct g_raid3_softc *sc)
{
struct bio *bp, *bp2;
TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed.queue, bio_queue, bp2) {
if (g_raid3_regular_collision(sc, bp))
continue;
bioq_remove(&sc->sc_sync_delayed, bp);
G_RAID3_LOGREQ(2, bp,
"Releasing delayed synchronization request.");
g_io_request(bp, bp->bio_from);
}
}
/*
* Handle synchronization requests.
* Every synchronization request is two-steps process: first, READ request is
* send to active provider and then WRITE request (with read data) to the provider
* beeing synchronized. When WRITE is finished, new synchronization request is
* send.
*/
static void
g_raid3_sync_request(struct bio *bp)
{
struct g_raid3_softc *sc;
struct g_raid3_disk *disk;
bp->bio_from->index--;
sc = bp->bio_from->geom->softc;
disk = bp->bio_from->private;
if (disk == NULL) {
sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
g_topology_lock();
g_raid3_kill_consumer(sc, bp->bio_from);
g_topology_unlock();
free(bp->bio_data, M_RAID3);
g_destroy_bio(bp);
sx_xlock(&sc->sc_lock);
return;
}
/*
* Synchronization request.
*/
switch (bp->bio_cmd) {
case BIO_READ:
{
struct g_consumer *cp;
u_char *dst, *src;
off_t left;
u_int atom;
if (bp->bio_error != 0) {
G_RAID3_LOGREQ(0, bp,
"Synchronization request failed (error=%d).",
bp->bio_error);
g_destroy_bio(bp);
return;
}
G_RAID3_LOGREQ(3, bp, "Synchronization request finished.");
atom = sc->sc_sectorsize / (sc->sc_ndisks - 1);
dst = src = bp->bio_data;
if (disk->d_no == sc->sc_ndisks - 1) {
u_int n;
/* Parity component. */
for (left = bp->bio_length; left > 0;
left -= sc->sc_sectorsize) {
bcopy(src, dst, atom);
src += atom;
for (n = 1; n < sc->sc_ndisks - 1; n++) {
g_raid3_xor(src, dst, atom);
src += atom;
}
dst += atom;
}
} else {
/* Regular component. */
src += atom * disk->d_no;
for (left = bp->bio_length; left > 0;
left -= sc->sc_sectorsize) {
bcopy(src, dst, atom);
src += sc->sc_sectorsize;
dst += atom;
}
}
bp->bio_driver1 = bp->bio_driver2 = NULL;
bp->bio_pflags = 0;
bp->bio_offset /= sc->sc_ndisks - 1;
bp->bio_length /= sc->sc_ndisks - 1;
bp->bio_cmd = BIO_WRITE;
bp->bio_cflags = 0;
bp->bio_children = bp->bio_inbed = 0;
cp = disk->d_consumer;
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
cp->acr, cp->acw, cp->ace));
cp->index++;
g_io_request(bp, cp);
return;
}
case BIO_WRITE:
{
struct g_raid3_disk_sync *sync;
off_t boffset, moffset;
void *data;
int i;
if (bp->bio_error != 0) {
G_RAID3_LOGREQ(0, bp,
"Synchronization request failed (error=%d).",
bp->bio_error);
g_destroy_bio(bp);
sc->sc_bump_id |= G_RAID3_BUMP_GENID;
g_raid3_event_send(disk,
G_RAID3_DISK_STATE_DISCONNECTED,
G_RAID3_EVENT_DONTWAIT);
return;
}
G_RAID3_LOGREQ(3, bp, "Synchronization request finished.");
sync = &disk->d_sync;
if (sync->ds_offset == sc->sc_mediasize / (sc->sc_ndisks - 1) ||
sync->ds_consumer == NULL ||
(sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0) {
/* Don't send more synchronization requests. */
sync->ds_inflight--;
if (sync->ds_bios != NULL) {
i = (int)(uintptr_t)bp->bio_caller1;
sync->ds_bios[i] = NULL;
}
free(bp->bio_data, M_RAID3);
g_destroy_bio(bp);
if (sync->ds_inflight > 0)
return;
if (sync->ds_consumer == NULL ||
(sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0) {
return;
}
/*
* Disk up-to-date, activate it.
*/
g_raid3_event_send(disk, G_RAID3_DISK_STATE_ACTIVE,
G_RAID3_EVENT_DONTWAIT);
return;
}
/* Send next synchronization request. */
data = bp->bio_data;
bzero(bp, sizeof(*bp));
bp->bio_cmd = BIO_READ;
bp->bio_offset = sync->ds_offset * (sc->sc_ndisks - 1);
bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
sync->ds_offset += bp->bio_length / (sc->sc_ndisks - 1);
bp->bio_done = g_raid3_sync_done;
bp->bio_data = data;
bp->bio_from = sync->ds_consumer;
bp->bio_to = sc->sc_provider;
G_RAID3_LOGREQ(3, bp, "Sending synchronization request.");
sync->ds_consumer->index++;
/*
* Delay the request if it is colliding with a regular request.
*/
if (g_raid3_regular_collision(sc, bp))
g_raid3_sync_delay(sc, bp);
else
g_io_request(bp, sync->ds_consumer);
/* Release delayed requests if possible. */
g_raid3_regular_release(sc);
/* Find the smallest offset. */
moffset = sc->sc_mediasize;
for (i = 0; i < g_raid3_syncreqs; i++) {
bp = sync->ds_bios[i];
boffset = bp->bio_offset;
if (bp->bio_cmd == BIO_WRITE)
boffset *= sc->sc_ndisks - 1;
if (boffset < moffset)
moffset = boffset;
}
if (sync->ds_offset_done + (MAXPHYS * 100) < moffset) {
/* Update offset_done on every 100 blocks. */
sync->ds_offset_done = moffset;
g_raid3_update_metadata(disk);
}
return;
}
default:
KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
bp->bio_cmd, sc->sc_name));
break;
}
}
static int
g_raid3_register_request(struct bio *pbp)
{
struct g_raid3_softc *sc;
struct g_raid3_disk *disk;
struct g_consumer *cp;
struct bio *cbp, *tmpbp;
off_t offset, length;
u_int n, ndisks;
int round_robin, verify;
ndisks = 0;
sc = pbp->bio_to->geom->softc;
if ((pbp->bio_cflags & G_RAID3_BIO_CFLAG_REGSYNC) != 0 &&
sc->sc_syncdisk == NULL) {
g_io_deliver(pbp, EIO);
return (0);
}
g_raid3_init_bio(pbp);
length = pbp->bio_length / (sc->sc_ndisks - 1);
offset = pbp->bio_offset / (sc->sc_ndisks - 1);
round_robin = verify = 0;
switch (pbp->bio_cmd) {
case BIO_READ:
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_VERIFY) != 0 &&
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
pbp->bio_pflags |= G_RAID3_BIO_PFLAG_VERIFY;
verify = 1;
ndisks = sc->sc_ndisks;
} else {
verify = 0;
ndisks = sc->sc_ndisks - 1;
}
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_ROUND_ROBIN) != 0 &&
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
round_robin = 1;
} else {
round_robin = 0;
}
KASSERT(!round_robin || !verify,
("ROUND-ROBIN and VERIFY are mutually exclusive."));
pbp->bio_driver2 = &sc->sc_disks[sc->sc_ndisks - 1];
break;
case BIO_WRITE:
case BIO_DELETE:
/*
* Delay the request if it is colliding with a synchronization
* request.
*/
if (g_raid3_sync_collision(sc, pbp)) {
g_raid3_regular_delay(sc, pbp);
return (0);
}
if (sc->sc_idle)
g_raid3_unidle(sc);
else
sc->sc_last_write = time_uptime;
ndisks = sc->sc_ndisks;
break;
}
for (n = 0; n < ndisks; n++) {
disk = &sc->sc_disks[n];
cbp = g_raid3_clone_bio(sc, pbp);
if (cbp == NULL) {
while ((cbp = G_RAID3_HEAD_BIO(pbp)) != NULL)
g_raid3_destroy_bio(sc, cbp);
/*
* To prevent deadlock, we must run back up
* with the ENOMEM for failed requests of any
* of our consumers. Our own sync requests
* can stick around, as they are finite.
*/
if ((pbp->bio_cflags &
G_RAID3_BIO_CFLAG_REGULAR) != 0) {
g_io_deliver(pbp, ENOMEM);
return (0);
}
return (ENOMEM);
}
cbp->bio_offset = offset;
cbp->bio_length = length;
cbp->bio_done = g_raid3_done;
switch (pbp->bio_cmd) {
case BIO_READ:
if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE) {
/*
* Replace invalid component with the parity
* component.
*/
disk = &sc->sc_disks[sc->sc_ndisks - 1];
cbp->bio_cflags |= G_RAID3_BIO_CFLAG_PARITY;
pbp->bio_pflags |= G_RAID3_BIO_PFLAG_DEGRADED;
} else if (round_robin &&
disk->d_no == sc->sc_round_robin) {
/*
* In round-robin mode skip one data component
* and use parity component when reading.
*/
pbp->bio_driver2 = disk;
disk = &sc->sc_disks[sc->sc_ndisks - 1];
cbp->bio_cflags |= G_RAID3_BIO_CFLAG_PARITY;
sc->sc_round_robin++;
round_robin = 0;
} else if (verify && disk->d_no == sc->sc_ndisks - 1) {
cbp->bio_cflags |= G_RAID3_BIO_CFLAG_PARITY;
}
break;
case BIO_WRITE:
case BIO_DELETE:
if (disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
if (n == ndisks - 1) {
/*
* Active parity component, mark it as such.
*/
cbp->bio_cflags |=
G_RAID3_BIO_CFLAG_PARITY;
}
} else {
pbp->bio_pflags |= G_RAID3_BIO_PFLAG_DEGRADED;
if (n == ndisks - 1) {
/*
* Parity component is not connected,
* so destroy its request.
*/
pbp->bio_pflags |=
G_RAID3_BIO_PFLAG_NOPARITY;
g_raid3_destroy_bio(sc, cbp);
cbp = NULL;
} else {
cbp->bio_cflags |=
G_RAID3_BIO_CFLAG_NODISK;
disk = NULL;
}
}
break;
}
if (cbp != NULL)
cbp->bio_caller2 = disk;
}
switch (pbp->bio_cmd) {
case BIO_READ:
if (round_robin) {
/*
* If we are in round-robin mode and 'round_robin' is
* still 1, it means, that we skipped parity component
* for this read and must reset sc_round_robin field.
*/
sc->sc_round_robin = 0;
}
G_RAID3_FOREACH_SAFE_BIO(pbp, cbp, tmpbp) {
disk = cbp->bio_caller2;
cp = disk->d_consumer;
cbp->bio_to = cp->provider;
G_RAID3_LOGREQ(3, cbp, "Sending request.");
KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
("Consumer %s not opened (r%dw%de%d).",
cp->provider->name, cp->acr, cp->acw, cp->ace));
cp->index++;
g_io_request(cbp, cp);
}
break;
case BIO_WRITE:
case BIO_DELETE:
/*
* Put request onto inflight queue, so we can check if new
* synchronization requests don't collide with it.
*/
bioq_insert_tail(&sc->sc_inflight, pbp);
/*
* Bump syncid on first write.
*/
if ((sc->sc_bump_id & G_RAID3_BUMP_SYNCID) != 0) {
sc->sc_bump_id &= ~G_RAID3_BUMP_SYNCID;
g_raid3_bump_syncid(sc);
}
g_raid3_scatter(pbp);
break;
}
return (0);
}
static int
g_raid3_can_destroy(struct g_raid3_softc *sc)
{
struct g_geom *gp;
struct g_consumer *cp;
g_topology_assert();
gp = sc->sc_geom;
if (gp->softc == NULL)
return (1);
LIST_FOREACH(cp, &gp->consumer, consumer) {
if (g_raid3_is_busy(sc, cp))
return (0);
}
gp = sc->sc_sync.ds_geom;
LIST_FOREACH(cp, &gp->consumer, consumer) {
if (g_raid3_is_busy(sc, cp))
return (0);
}
G_RAID3_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
sc->sc_name);
return (1);
}
static int
g_raid3_try_destroy(struct g_raid3_softc *sc)
{
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
if (sc->sc_rootmount != NULL) {
G_RAID3_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
}
g_topology_lock();
if (!g_raid3_can_destroy(sc)) {
g_topology_unlock();
return (0);
}
sc->sc_geom->softc = NULL;
sc->sc_sync.ds_geom->softc = NULL;
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_WAIT) != 0) {
g_topology_unlock();
G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__,
&sc->sc_worker);
/* Unlock sc_lock here, as it can be destroyed after wakeup. */
sx_xunlock(&sc->sc_lock);
wakeup(&sc->sc_worker);
sc->sc_worker = NULL;
} else {
g_topology_unlock();
g_raid3_destroy_device(sc);
free(sc->sc_disks, M_RAID3);
free(sc, M_RAID3);
}
return (1);
}
/*
* Worker thread.
*/
static void
g_raid3_worker(void *arg)
{
struct g_raid3_softc *sc;
struct g_raid3_event *ep;
struct bio *bp;
int timeout;
sc = arg;
thread_lock(curthread);
sched_prio(curthread, PRIBIO);
thread_unlock(curthread);
sx_xlock(&sc->sc_lock);
for (;;) {
G_RAID3_DEBUG(5, "%s: Let's see...", __func__);
/*
* First take a look at events.
* This is important to handle events before any I/O requests.
*/
ep = g_raid3_event_get(sc);
if (ep != NULL) {
g_raid3_event_remove(sc, ep);
if ((ep->e_flags & G_RAID3_EVENT_DEVICE) != 0) {
/* Update only device status. */
G_RAID3_DEBUG(3,
"Running event for device %s.",
sc->sc_name);
ep->e_error = 0;
g_raid3_update_device(sc, 1);
} else {
/* Update disk status. */
G_RAID3_DEBUG(3, "Running event for disk %s.",
g_raid3_get_diskname(ep->e_disk));
ep->e_error = g_raid3_update_disk(ep->e_disk,
ep->e_state);
if (ep->e_error == 0)
g_raid3_update_device(sc, 0);
}
if ((ep->e_flags & G_RAID3_EVENT_DONTWAIT) != 0) {
KASSERT(ep->e_error == 0,
("Error cannot be handled."));
g_raid3_event_free(ep);
} else {
ep->e_flags |= G_RAID3_EVENT_DONE;
G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__,
ep);
mtx_lock(&sc->sc_events_mtx);
wakeup(ep);
mtx_unlock(&sc->sc_events_mtx);
}
if ((sc->sc_flags &
G_RAID3_DEVICE_FLAG_DESTROY) != 0) {
if (g_raid3_try_destroy(sc)) {
curthread->td_pflags &= ~TDP_GEOM;
G_RAID3_DEBUG(1, "Thread exiting.");
kproc_exit(0);
}
}
G_RAID3_DEBUG(5, "%s: I'm here 1.", __func__);
continue;
}
/*
* Check if we can mark array as CLEAN and if we can't take
* how much seconds should we wait.
*/
timeout = g_raid3_idle(sc, -1);
/*
* Now I/O requests.
*/
/* Get first request from the queue. */
mtx_lock(&sc->sc_queue_mtx);
bp = bioq_first(&sc->sc_queue);
if (bp == NULL) {
if ((sc->sc_flags &
G_RAID3_DEVICE_FLAG_DESTROY) != 0) {
mtx_unlock(&sc->sc_queue_mtx);
if (g_raid3_try_destroy(sc)) {
curthread->td_pflags &= ~TDP_GEOM;
G_RAID3_DEBUG(1, "Thread exiting.");
kproc_exit(0);
}
mtx_lock(&sc->sc_queue_mtx);
}
sx_xunlock(&sc->sc_lock);
/*
* XXX: We can miss an event here, because an event
* can be added without sx-device-lock and without
* mtx-queue-lock. Maybe I should just stop using
* dedicated mutex for events synchronization and
* stick with the queue lock?
* The event will hang here until next I/O request
* or next event is received.
*/
MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "r3:w1",
timeout * hz);
sx_xlock(&sc->sc_lock);
G_RAID3_DEBUG(5, "%s: I'm here 4.", __func__);
continue;
}
process:
bioq_remove(&sc->sc_queue, bp);
mtx_unlock(&sc->sc_queue_mtx);
if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
(bp->bio_cflags & G_RAID3_BIO_CFLAG_SYNC) != 0) {
g_raid3_sync_request(bp); /* READ */
} else if (bp->bio_to != sc->sc_provider) {
if ((bp->bio_cflags & G_RAID3_BIO_CFLAG_REGULAR) != 0)
g_raid3_regular_request(bp);
else if ((bp->bio_cflags & G_RAID3_BIO_CFLAG_SYNC) != 0)
g_raid3_sync_request(bp); /* WRITE */
else {
KASSERT(0,
("Invalid request cflags=0x%hhx to=%s.",
bp->bio_cflags, bp->bio_to->name));
}
} else if (g_raid3_register_request(bp) != 0) {
mtx_lock(&sc->sc_queue_mtx);
bioq_insert_head(&sc->sc_queue, bp);
/*
* We are short in memory, let see if there are finished
* request we can free.
*/
TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
if (bp->bio_cflags & G_RAID3_BIO_CFLAG_REGULAR)
goto process;
}
/*
* No finished regular request, so at least keep
* synchronization running.
*/
TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
if (bp->bio_cflags & G_RAID3_BIO_CFLAG_SYNC)
goto process;
}
sx_xunlock(&sc->sc_lock);
MSLEEP(&sc->sc_queue, &sc->sc_queue_mtx, PRIBIO | PDROP,
"r3:lowmem", hz / 10);
sx_xlock(&sc->sc_lock);
}
G_RAID3_DEBUG(5, "%s: I'm here 9.", __func__);
}
}
static void
g_raid3_update_idle(struct g_raid3_softc *sc, struct g_raid3_disk *disk)
{
sx_assert(&sc->sc_lock, SX_LOCKED);
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOFAILSYNC) != 0)
return;
if (!sc->sc_idle && (disk->d_flags & G_RAID3_DISK_FLAG_DIRTY) == 0) {
G_RAID3_DEBUG(1, "Disk %s (device %s) marked as dirty.",
g_raid3_get_diskname(disk), sc->sc_name);
disk->d_flags |= G_RAID3_DISK_FLAG_DIRTY;
} else if (sc->sc_idle &&
(disk->d_flags & G_RAID3_DISK_FLAG_DIRTY) != 0) {
G_RAID3_DEBUG(1, "Disk %s (device %s) marked as clean.",
g_raid3_get_diskname(disk), sc->sc_name);
disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
}
}
static void
g_raid3_sync_start(struct g_raid3_softc *sc)
{
struct g_raid3_disk *disk;
struct g_consumer *cp;
struct bio *bp;
int error;
u_int n;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_XLOCKED);
KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED,
("Device not in DEGRADED state (%s, %u).", sc->sc_name,
sc->sc_state));
KASSERT(sc->sc_syncdisk == NULL, ("Syncdisk is not NULL (%s, %u).",
sc->sc_name, sc->sc_state));
disk = NULL;
for (n = 0; n < sc->sc_ndisks; n++) {
if (sc->sc_disks[n].d_state != G_RAID3_DISK_STATE_SYNCHRONIZING)
continue;
disk = &sc->sc_disks[n];
break;
}
if (disk == NULL)
return;
sx_xunlock(&sc->sc_lock);
g_topology_lock();
cp = g_new_consumer(sc->sc_sync.ds_geom);
error = g_attach(cp, sc->sc_provider);
KASSERT(error == 0,
("Cannot attach to %s (error=%d).", sc->sc_name, error));
error = g_access(cp, 1, 0, 0);
KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
g_topology_unlock();
sx_xlock(&sc->sc_lock);
G_RAID3_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
g_raid3_get_diskname(disk));
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOFAILSYNC) == 0)
disk->d_flags |= G_RAID3_DISK_FLAG_DIRTY;
KASSERT(disk->d_sync.ds_consumer == NULL,
("Sync consumer already exists (device=%s, disk=%s).",
sc->sc_name, g_raid3_get_diskname(disk)));
disk->d_sync.ds_consumer = cp;
disk->d_sync.ds_consumer->private = disk;
disk->d_sync.ds_consumer->index = 0;
sc->sc_syncdisk = disk;
/*
* Allocate memory for synchronization bios and initialize them.
*/
disk->d_sync.ds_bios = malloc(sizeof(struct bio *) * g_raid3_syncreqs,
M_RAID3, M_WAITOK);
for (n = 0; n < g_raid3_syncreqs; n++) {
bp = g_alloc_bio();
disk->d_sync.ds_bios[n] = bp;
bp->bio_parent = NULL;
bp->bio_cmd = BIO_READ;
bp->bio_data = malloc(MAXPHYS, M_RAID3, M_WAITOK);
bp->bio_cflags = 0;
bp->bio_offset = disk->d_sync.ds_offset * (sc->sc_ndisks - 1);
bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
disk->d_sync.ds_offset += bp->bio_length / (sc->sc_ndisks - 1);
bp->bio_done = g_raid3_sync_done;
bp->bio_from = disk->d_sync.ds_consumer;
bp->bio_to = sc->sc_provider;
bp->bio_caller1 = (void *)(uintptr_t)n;
}
/* Set the number of in-flight synchronization requests. */
disk->d_sync.ds_inflight = g_raid3_syncreqs;
/*
* Fire off first synchronization requests.
*/
for (n = 0; n < g_raid3_syncreqs; n++) {
bp = disk->d_sync.ds_bios[n];
G_RAID3_LOGREQ(3, bp, "Sending synchronization request.");
disk->d_sync.ds_consumer->index++;
/*
* Delay the request if it is colliding with a regular request.
*/
if (g_raid3_regular_collision(sc, bp))
g_raid3_sync_delay(sc, bp);
else
g_io_request(bp, disk->d_sync.ds_consumer);
}
}
/*
* Stop synchronization process.
* type: 0 - synchronization finished
* 1 - synchronization stopped
*/
static void
g_raid3_sync_stop(struct g_raid3_softc *sc, int type)
{
struct g_raid3_disk *disk;
struct g_consumer *cp;
g_topology_assert_not();
sx_assert(&sc->sc_lock, SX_LOCKED);
KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED,
("Device not in DEGRADED state (%s, %u).", sc->sc_name,
sc->sc_state));
disk = sc->sc_syncdisk;
sc->sc_syncdisk = NULL;
KASSERT(disk != NULL, ("No disk was synchronized (%s).", sc->sc_name));
KASSERT(disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING,
("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
if (disk->d_sync.ds_consumer == NULL)
return;
if (type == 0) {
G_RAID3_DEBUG(0, "Device %s: rebuilding provider %s finished.",
sc->sc_name, g_raid3_get_diskname(disk));
} else /* if (type == 1) */ {
G_RAID3_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
sc->sc_name, g_raid3_get_diskname(disk));
}
free(disk->d_sync.ds_bios, M_RAID3);
disk->d_sync.ds_bios = NULL;
cp = disk->d_sync.ds_consumer;
disk->d_sync.ds_consumer = NULL;
disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
g_topology_lock();
g_raid3_kill_consumer(sc, cp);
g_topology_unlock();
sx_xlock(&sc->sc_lock);
}
static void
g_raid3_launch_provider(struct g_raid3_softc *sc)
{
struct g_provider *pp;
struct g_raid3_disk *disk;
int n;
sx_assert(&sc->sc_lock, SX_LOCKED);
g_topology_lock();
pp = g_new_providerf(sc->sc_geom, "raid3/%s", sc->sc_name);
pp->mediasize = sc->sc_mediasize;
pp->sectorsize = sc->sc_sectorsize;
pp->stripesize = 0;
pp->stripeoffset = 0;
for (n = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if (disk->d_consumer && disk->d_consumer->provider &&
disk->d_consumer->provider->stripesize > pp->stripesize) {
pp->stripesize = disk->d_consumer->provider->stripesize;
pp->stripeoffset = disk->d_consumer->provider->stripeoffset;
}
}
pp->stripesize *= sc->sc_ndisks - 1;
pp->stripeoffset *= sc->sc_ndisks - 1;
sc->sc_provider = pp;
g_error_provider(pp, 0);
g_topology_unlock();
G_RAID3_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE), sc->sc_ndisks);
if (sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED)
g_raid3_sync_start(sc);
}
static void
g_raid3_destroy_provider(struct g_raid3_softc *sc)
{
struct bio *bp;
g_topology_assert_not();
KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
sc->sc_name));
g_topology_lock();
g_error_provider(sc->sc_provider, ENXIO);
mtx_lock(&sc->sc_queue_mtx);
while ((bp = bioq_first(&sc->sc_queue)) != NULL) {
bioq_remove(&sc->sc_queue, bp);
g_io_deliver(bp, ENXIO);
}
mtx_unlock(&sc->sc_queue_mtx);
G_RAID3_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name,
sc->sc_provider->name);
sc->sc_provider->flags |= G_PF_WITHER;
g_orphan_provider(sc->sc_provider, ENXIO);
g_topology_unlock();
sc->sc_provider = NULL;
if (sc->sc_syncdisk != NULL)
g_raid3_sync_stop(sc, 1);
}
static void
g_raid3_go(void *arg)
{
struct g_raid3_softc *sc;
sc = arg;
G_RAID3_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
g_raid3_event_send(sc, 0,
G_RAID3_EVENT_DONTWAIT | G_RAID3_EVENT_DEVICE);
}
static u_int
g_raid3_determine_state(struct g_raid3_disk *disk)
{
struct g_raid3_softc *sc;
u_int state;
sc = disk->d_softc;
if (sc->sc_syncid == disk->d_sync.ds_syncid) {
if ((disk->d_flags &
G_RAID3_DISK_FLAG_SYNCHRONIZING) == 0) {
/* Disk does not need synchronization. */
state = G_RAID3_DISK_STATE_ACTIVE;
} else {
if ((sc->sc_flags &
G_RAID3_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
(disk->d_flags &
G_RAID3_DISK_FLAG_FORCE_SYNC) != 0) {
/*
* We can start synchronization from
* the stored offset.
*/
state = G_RAID3_DISK_STATE_SYNCHRONIZING;
} else {
state = G_RAID3_DISK_STATE_STALE;
}
}
} else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
/*
* Reset all synchronization data for this disk,
* because if it even was synchronized, it was
* synchronized to disks with different syncid.
*/
disk->d_flags |= G_RAID3_DISK_FLAG_SYNCHRONIZING;
disk->d_sync.ds_offset = 0;
disk->d_sync.ds_offset_done = 0;
disk->d_sync.ds_syncid = sc->sc_syncid;
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
(disk->d_flags & G_RAID3_DISK_FLAG_FORCE_SYNC) != 0) {
state = G_RAID3_DISK_STATE_SYNCHRONIZING;
} else {
state = G_RAID3_DISK_STATE_STALE;
}
} else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
/*
* Not good, NOT GOOD!
* It means that device was started on stale disks
* and more fresh disk just arrive.
* If there were writes, device is broken, sorry.
* I think the best choice here is don't touch
* this disk and inform the user loudly.
*/
G_RAID3_DEBUG(0, "Device %s was started before the freshest "
"disk (%s) arrives!! It will not be connected to the "
"running device.", sc->sc_name,
g_raid3_get_diskname(disk));
g_raid3_destroy_disk(disk);
state = G_RAID3_DISK_STATE_NONE;
/* Return immediately, because disk was destroyed. */
return (state);
}
G_RAID3_DEBUG(3, "State for %s disk: %s.",
g_raid3_get_diskname(disk), g_raid3_disk_state2str(state));
return (state);
}
/*
* Update device state.
*/
static void
g_raid3_update_device(struct g_raid3_softc *sc, boolean_t force)
{
struct g_raid3_disk *disk;
u_int state;
sx_assert(&sc->sc_lock, SX_XLOCKED);
switch (sc->sc_state) {
case G_RAID3_DEVICE_STATE_STARTING:
{
u_int n, ndirty, ndisks, genid, syncid;
KASSERT(sc->sc_provider == NULL,
("Non-NULL provider in STARTING state (%s).", sc->sc_name));
/*
* Are we ready? We are, if all disks are connected or
* one disk is missing and 'force' is true.
*/
if (g_raid3_ndisks(sc, -1) + force == sc->sc_ndisks) {
if (!force)
callout_drain(&sc->sc_callout);
} else {
if (force) {
/*
* Timeout expired, so destroy device.
*/
sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROY;
G_RAID3_DEBUG(1, "root_mount_rel[%u] %p",
__LINE__, sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
}
return;
}
/*
* Find the biggest genid.
*/
genid = 0;
for (n = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
continue;
if (disk->d_genid > genid)
genid = disk->d_genid;
}
sc->sc_genid = genid;
/*
* Remove all disks without the biggest genid.
*/
for (n = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
continue;
if (disk->d_genid < genid) {
G_RAID3_DEBUG(0,
"Component %s (device %s) broken, skipping.",
g_raid3_get_diskname(disk), sc->sc_name);
g_raid3_destroy_disk(disk);
}
}
/*
* There must be at least 'sc->sc_ndisks - 1' components
* with the same syncid and without SYNCHRONIZING flag.
*/
/*
* Find the biggest syncid, number of valid components and
* number of dirty components.
*/
ndirty = ndisks = syncid = 0;
for (n = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
continue;
if ((disk->d_flags & G_RAID3_DISK_FLAG_DIRTY) != 0)
ndirty++;
if (disk->d_sync.ds_syncid > syncid) {
syncid = disk->d_sync.ds_syncid;
ndisks = 0;
} else if (disk->d_sync.ds_syncid < syncid) {
continue;
}
if ((disk->d_flags &
G_RAID3_DISK_FLAG_SYNCHRONIZING) != 0) {
continue;
}
ndisks++;
}
/*
* Do we have enough valid components?
*/
if (ndisks + 1 < sc->sc_ndisks) {
G_RAID3_DEBUG(0,
"Device %s is broken, too few valid components.",
sc->sc_name);
sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROY;
return;
}
/*
* If there is one DIRTY component and all disks are present,
* mark it for synchronization. If there is more than one DIRTY
* component, mark parity component for synchronization.
*/
if (ndisks == sc->sc_ndisks && ndirty == 1) {
for (n = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if ((disk->d_flags &
G_RAID3_DISK_FLAG_DIRTY) == 0) {
continue;
}
disk->d_flags |=
G_RAID3_DISK_FLAG_SYNCHRONIZING;
}
} else if (ndisks == sc->sc_ndisks && ndirty > 1) {
disk = &sc->sc_disks[sc->sc_ndisks - 1];
disk->d_flags |= G_RAID3_DISK_FLAG_SYNCHRONIZING;
}
sc->sc_syncid = syncid;
if (force) {
/* Remember to bump syncid on first write. */
sc->sc_bump_id |= G_RAID3_BUMP_SYNCID;
}
if (ndisks == sc->sc_ndisks)
state = G_RAID3_DEVICE_STATE_COMPLETE;
else /* if (ndisks == sc->sc_ndisks - 1) */
state = G_RAID3_DEVICE_STATE_DEGRADED;
G_RAID3_DEBUG(1, "Device %s state changed from %s to %s.",
sc->sc_name, g_raid3_device_state2str(sc->sc_state),
g_raid3_device_state2str(state));
sc->sc_state = state;
for (n = 0; n < sc->sc_ndisks; n++) {
disk = &sc->sc_disks[n];
if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
continue;
state = g_raid3_determine_state(disk);
g_raid3_event_send(disk, state, G_RAID3_EVENT_DONTWAIT);
if (state == G_RAID3_DISK_STATE_STALE)
sc->sc_bump_id |= G_RAID3_BUMP_SYNCID;
}
break;
}
case G_RAID3_DEVICE_STATE_DEGRADED:
/*
* Genid need to be bumped immediately, so do it here.
*/
if ((sc->sc_bump_id & G_RAID3_BUMP_GENID) != 0) {
sc->sc_bump_id &= ~G_RAID3_BUMP_GENID;
g_raid3_bump_genid(sc);
}
if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NEW) > 0)
return;
if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) <
sc->sc_ndisks - 1) {
if (sc->sc_provider != NULL)
g_raid3_destroy_provider(sc);
sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROY;
return;
}
if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) ==
sc->sc_ndisks) {
state = G_RAID3_DEVICE_STATE_COMPLETE;
G_RAID3_DEBUG(1,
"Device %s state changed from %s to %s.",
sc->sc_name, g_raid3_device_state2str(sc->sc_state),
g_raid3_device_state2str(state));
sc->sc_state = state;
}
if (sc->sc_provider == NULL)
g_raid3_launch_provider(sc);
if (sc->sc_rootmount != NULL) {
G_RAID3_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
}
break;
case G_RAID3_DEVICE_STATE_COMPLETE:
/*
* Genid need to be bumped immediately, so do it here.
*/
if ((sc->sc_bump_id & G_RAID3_BUMP_GENID) != 0) {
sc->sc_bump_id &= ~G_RAID3_BUMP_GENID;
g_raid3_bump_genid(sc);
}
if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NEW) > 0)
return;
KASSERT(g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) >=
sc->sc_ndisks - 1,
("Too few ACTIVE components in COMPLETE state (device %s).",
sc->sc_name));
if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) ==
sc->sc_ndisks - 1) {
state = G_RAID3_DEVICE_STATE_DEGRADED;
G_RAID3_DEBUG(1,
"Device %s state changed from %s to %s.",
sc->sc_name, g_raid3_device_state2str(sc->sc_state),
g_raid3_device_state2str(state));
sc->sc_state = state;
}
if (sc->sc_provider == NULL)
g_raid3_launch_provider(sc);
if (sc->sc_rootmount != NULL) {
G_RAID3_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
sc->sc_rootmount);
root_mount_rel(sc->sc_rootmount);
sc->sc_rootmount = NULL;
}
break;
default:
KASSERT(1 == 0, ("Wrong device state (%s, %s).", sc->sc_name,
g_raid3_device_state2str(sc->sc_state)));
break;
}
}
/*
* Update disk state and device state if needed.
*/
#define DISK_STATE_CHANGED() G_RAID3_DEBUG(1, \
"Disk %s state changed from %s to %s (device %s).", \
g_raid3_get_diskname(disk), \
g_raid3_disk_state2str(disk->d_state), \
g_raid3_disk_state2str(state), sc->sc_name)
static int
g_raid3_update_disk(struct g_raid3_disk *disk, u_int state)
{
struct g_raid3_softc *sc;
sc = disk->d_softc;
sx_assert(&sc->sc_lock, SX_XLOCKED);
again:
G_RAID3_DEBUG(3, "Changing disk %s state from %s to %s.",
g_raid3_get_diskname(disk), g_raid3_disk_state2str(disk->d_state),
g_raid3_disk_state2str(state));
switch (state) {
case G_RAID3_DISK_STATE_NEW:
/*
* Possible scenarios:
* 1. New disk arrive.
*/
/* Previous state should be NONE. */
KASSERT(disk->d_state == G_RAID3_DISK_STATE_NONE,
("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
DISK_STATE_CHANGED();
disk->d_state = state;
G_RAID3_DEBUG(1, "Device %s: provider %s detected.",
sc->sc_name, g_raid3_get_diskname(disk));
if (sc->sc_state == G_RAID3_DEVICE_STATE_STARTING)
break;
KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_raid3_device_state2str(sc->sc_state),
g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
state = g_raid3_determine_state(disk);
if (state != G_RAID3_DISK_STATE_NONE)
goto again;
break;
case G_RAID3_DISK_STATE_ACTIVE:
/*
* Possible scenarios:
* 1. New disk does not need synchronization.
* 2. Synchronization process finished successfully.
*/
KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_raid3_device_state2str(sc->sc_state),
g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
/* Previous state should be NEW or SYNCHRONIZING. */
KASSERT(disk->d_state == G_RAID3_DISK_STATE_NEW ||
disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING,
("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
DISK_STATE_CHANGED();
if (disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
disk->d_flags &= ~G_RAID3_DISK_FLAG_SYNCHRONIZING;
disk->d_flags &= ~G_RAID3_DISK_FLAG_FORCE_SYNC;
g_raid3_sync_stop(sc, 0);
}
disk->d_state = state;
disk->d_sync.ds_offset = 0;
disk->d_sync.ds_offset_done = 0;
g_raid3_update_idle(sc, disk);
g_raid3_update_metadata(disk);
G_RAID3_DEBUG(1, "Device %s: provider %s activated.",
sc->sc_name, g_raid3_get_diskname(disk));
break;
case G_RAID3_DISK_STATE_STALE:
/*
* Possible scenarios:
* 1. Stale disk was connected.
*/
/* Previous state should be NEW. */
KASSERT(disk->d_state == G_RAID3_DISK_STATE_NEW,
("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_raid3_device_state2str(sc->sc_state),
g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
/*
* STALE state is only possible if device is marked
* NOAUTOSYNC.
*/
KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOAUTOSYNC) != 0,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_raid3_device_state2str(sc->sc_state),
g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
DISK_STATE_CHANGED();
disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
disk->d_state = state;
g_raid3_update_metadata(disk);
G_RAID3_DEBUG(0, "Device %s: provider %s is stale.",
sc->sc_name, g_raid3_get_diskname(disk));
break;
case G_RAID3_DISK_STATE_SYNCHRONIZING:
/*
* Possible scenarios:
* 1. Disk which needs synchronization was connected.
*/
/* Previous state should be NEW. */
KASSERT(disk->d_state == G_RAID3_DISK_STATE_NEW,
("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE,
("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
g_raid3_device_state2str(sc->sc_state),
g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
DISK_STATE_CHANGED();
if (disk->d_state == G_RAID3_DISK_STATE_NEW)
disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
disk->d_state = state;
if (sc->sc_provider != NULL) {
g_raid3_sync_start(sc);
g_raid3_update_metadata(disk);
}
break;
case G_RAID3_DISK_STATE_DISCONNECTED:
/*
* Possible scenarios:
* 1. Device wasn't running yet, but disk disappear.
* 2. Disk was active and disapppear.
* 3. Disk disappear during synchronization process.
*/
if (sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
/*
* Previous state should be ACTIVE, STALE or
* SYNCHRONIZING.
*/
KASSERT(disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
disk->d_state == G_RAID3_DISK_STATE_STALE ||
disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING,
("Wrong disk state (%s, %s).",
g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
} else if (sc->sc_state == G_RAID3_DEVICE_STATE_STARTING) {
/* Previous state should be NEW. */
KASSERT(disk->d_state == G_RAID3_DISK_STATE_NEW,
("Wrong disk state (%s, %s).",
g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
/*
* Reset bumping syncid if disk disappeared in STARTING
* state.
*/
if ((sc->sc_bump_id & G_RAID3_BUMP_SYNCID) != 0)
sc->sc_bump_id &= ~G_RAID3_BUMP_SYNCID;
#ifdef INVARIANTS
} else {
KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
sc->sc_name,
g_raid3_device_state2str(sc->sc_state),
g_raid3_get_diskname(disk),
g_raid3_disk_state2str(disk->d_state)));
#endif
}
DISK_STATE_CHANGED();
G_RAID3_DEBUG(0, "Device %s: provider %s disconnected.",
sc->sc_name, g_raid3_get_diskname(disk));
g_raid3_destroy_disk(disk);
break;
default:
KASSERT(1 == 0, ("Unknown state (%u).", state));
break;
}
return (0);
}
#undef DISK_STATE_CHANGED
int
g_raid3_read_metadata(struct g_consumer *cp, struct g_raid3_metadata *md)
{
struct g_provider *pp;
u_char *buf;
int error;
g_topology_assert();
error = g_access(cp, 1, 0, 0);
if (error != 0)
return (error);
pp = cp->provider;
g_topology_unlock();
/* Metadata are stored on last sector. */
buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
&error);
g_topology_lock();
g_access(cp, -1, 0, 0);
if (buf == NULL) {
G_RAID3_DEBUG(1, "Cannot read metadata from %s (error=%d).",
cp->provider->name, error);
return (error);
}
/* Decode metadata. */
error = raid3_metadata_decode(buf, md);
g_free(buf);
if (strcmp(md->md_magic, G_RAID3_MAGIC) != 0)
return (EINVAL);
if (md->md_version > G_RAID3_VERSION) {
G_RAID3_DEBUG(0,
"Kernel module is too old to handle metadata from %s.",
cp->provider->name);
return (EINVAL);
}
if (error != 0) {
G_RAID3_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
cp->provider->name);
return (error);
}
if (md->md_sectorsize > MAXPHYS) {
G_RAID3_DEBUG(0, "The blocksize is too big.");
return (EINVAL);
}
return (0);
}
static int
g_raid3_check_metadata(struct g_raid3_softc *sc, struct g_provider *pp,
struct g_raid3_metadata *md)
{
if (md->md_no >= sc->sc_ndisks) {
G_RAID3_DEBUG(1, "Invalid disk %s number (no=%u), skipping.",
pp->name, md->md_no);
return (EINVAL);
}
if (sc->sc_disks[md->md_no].d_state != G_RAID3_DISK_STATE_NODISK) {
G_RAID3_DEBUG(1, "Disk %s (no=%u) already exists, skipping.",
pp->name, md->md_no);
return (EEXIST);
}
if (md->md_all != sc->sc_ndisks) {
G_RAID3_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_all", pp->name, sc->sc_name);
return (EINVAL);
}
if ((md->md_mediasize % md->md_sectorsize) != 0) {
G_RAID3_DEBUG(1, "Invalid metadata (mediasize %% sectorsize != "
"0) on disk %s (device %s), skipping.", pp->name,
sc->sc_name);
return (EINVAL);
}
if (md->md_mediasize != sc->sc_mediasize) {
G_RAID3_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_mediasize", pp->name, sc->sc_name);
return (EINVAL);
}
if ((md->md_mediasize % (sc->sc_ndisks - 1)) != 0) {
G_RAID3_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_mediasize", pp->name, sc->sc_name);
return (EINVAL);
}
if ((sc->sc_mediasize / (sc->sc_ndisks - 1)) > pp->mediasize) {
G_RAID3_DEBUG(1,
"Invalid size of disk %s (device %s), skipping.", pp->name,
sc->sc_name);
return (EINVAL);
}
if ((md->md_sectorsize / pp->sectorsize) < sc->sc_ndisks - 1) {
G_RAID3_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_sectorsize", pp->name, sc->sc_name);
return (EINVAL);
}
if (md->md_sectorsize != sc->sc_sectorsize) {
G_RAID3_DEBUG(1,
"Invalid '%s' field on disk %s (device %s), skipping.",
"md_sectorsize", pp->name, sc->sc_name);
return (EINVAL);
}
if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
G_RAID3_DEBUG(1,
"Invalid sector size of disk %s (device %s), skipping.",
pp->name, sc->sc_name);
return (EINVAL);
}
if ((md->md_mflags & ~G_RAID3_DEVICE_FLAG_MASK) != 0) {
G_RAID3_DEBUG(1,
"Invalid device flags on disk %s (device %s), skipping.",
pp->name, sc->sc_name);
return (EINVAL);
}
if ((md->md_mflags & G_RAID3_DEVICE_FLAG_VERIFY) != 0 &&
(md->md_mflags & G_RAID3_DEVICE_FLAG_ROUND_ROBIN) != 0) {
/*
* VERIFY and ROUND-ROBIN options are mutally exclusive.
*/
G_RAID3_DEBUG(1, "Both VERIFY and ROUND-ROBIN flags exist on "
"disk %s (device %s), skipping.", pp->name, sc->sc_name);
return (EINVAL);
}
if ((md->md_dflags & ~G_RAID3_DISK_FLAG_MASK) != 0) {
G_RAID3_DEBUG(1,
"Invalid disk flags on disk %s (device %s), skipping.",
pp->name, sc->sc_name);
return (EINVAL);
}
return (0);
}
int
g_raid3_add_disk(struct g_raid3_softc *sc, struct g_provider *pp,
struct g_raid3_metadata *md)
{
struct g_raid3_disk *disk;
int error;
g_topology_assert_not();
G_RAID3_DEBUG(2, "Adding disk %s.", pp->name);
error = g_raid3_check_metadata(sc, pp, md);
if (error != 0)
return (error);
if (sc->sc_state != G_RAID3_DEVICE_STATE_STARTING &&
md->md_genid < sc->sc_genid) {
G_RAID3_DEBUG(0, "Component %s (device %s) broken, skipping.",
pp->name, sc->sc_name);
return (EINVAL);
}
disk = g_raid3_init_disk(sc, pp, md, &error);
if (disk == NULL)
return (error);
error = g_raid3_event_send(disk, G_RAID3_DISK_STATE_NEW,
G_RAID3_EVENT_WAIT);
if (error != 0)
return (error);
if (md->md_version < G_RAID3_VERSION) {
G_RAID3_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
pp->name, md->md_version, G_RAID3_VERSION);
g_raid3_update_metadata(disk);
}
return (0);
}
static void
g_raid3_destroy_delayed(void *arg, int flag)
{
struct g_raid3_softc *sc;
int error;
if (flag == EV_CANCEL) {
G_RAID3_DEBUG(1, "Destroying canceled.");
return;
}
sc = arg;
g_topology_unlock();
sx_xlock(&sc->sc_lock);
KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) == 0,
("DESTROY flag set on %s.", sc->sc_name));
KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROYING) != 0,
("DESTROYING flag not set on %s.", sc->sc_name));
G_RAID3_DEBUG(0, "Destroying %s (delayed).", sc->sc_name);
error = g_raid3_destroy(sc, G_RAID3_DESTROY_SOFT);
if (error != 0) {
G_RAID3_DEBUG(0, "Cannot destroy %s.", sc->sc_name);
sx_xunlock(&sc->sc_lock);
}
g_topology_lock();
}
static int
g_raid3_access(struct g_provider *pp, int acr, int acw, int ace)
{
struct g_raid3_softc *sc;
int dcr, dcw, dce, error = 0;
g_topology_assert();
G_RAID3_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
acw, ace);
sc = pp->geom->softc;
if (sc == NULL && acr <= 0 && acw <= 0 && ace <= 0)
return (0);
KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
dcr = pp->acr + acr;
dcw = pp->acw + acw;
dce = pp->ace + ace;
g_topology_unlock();
sx_xlock(&sc->sc_lock);
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0 ||
g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) < sc->sc_ndisks - 1) {
if (acr > 0 || acw > 0 || ace > 0)
error = ENXIO;
goto end;
}
if (dcw == 0)
g_raid3_idle(sc, dcw);
if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROYING) != 0) {
if (acr > 0 || acw > 0 || ace > 0) {
error = ENXIO;
goto end;
}
if (dcr == 0 && dcw == 0 && dce == 0) {
g_post_event(g_raid3_destroy_delayed, sc, M_WAITOK,
sc, NULL);
}
}
end:
sx_xunlock(&sc->sc_lock);
g_topology_lock();
return (error);
}
static struct g_geom *
g_raid3_create(struct g_class *mp, const struct g_raid3_metadata *md)
{
struct g_raid3_softc *sc;
struct g_geom *gp;
int error, timeout;
u_int n;
g_topology_assert();
G_RAID3_DEBUG(1, "Creating device %s (id=%u).", md->md_name, md->md_id);
/* One disk is minimum. */
if (md->md_all < 1)
return (NULL);
/*
* Action geom.
*/
gp = g_new_geomf(mp, "%s", md->md_name);
sc = malloc(sizeof(*sc), M_RAID3, M_WAITOK | M_ZERO);
sc->sc_disks = malloc(sizeof(struct g_raid3_disk) * md->md_all, M_RAID3,
M_WAITOK | M_ZERO);
gp->start = g_raid3_start;
gp->orphan = g_raid3_orphan;
gp->access = g_raid3_access;
gp->dumpconf = g_raid3_dumpconf;
sc->sc_id = md->md_id;
sc->sc_mediasize = md->md_mediasize;
sc->sc_sectorsize = md->md_sectorsize;
sc->sc_ndisks = md->md_all;
sc->sc_round_robin = 0;
sc->sc_flags = md->md_mflags;
sc->sc_bump_id = 0;
sc->sc_idle = 1;
sc->sc_last_write = time_uptime;
sc->sc_writes = 0;
for (n = 0; n < sc->sc_ndisks; n++) {
sc->sc_disks[n].d_softc = sc;
sc->sc_disks[n].d_no = n;
sc->sc_disks[n].d_state = G_RAID3_DISK_STATE_NODISK;
}
sx_init(&sc->sc_lock, "graid3:lock");
bioq_init(&sc->sc_queue);
mtx_init(&sc->sc_queue_mtx, "graid3:queue", NULL, MTX_DEF);
bioq_init(&sc->sc_regular_delayed);
bioq_init(&sc->sc_inflight);
bioq_init(&sc->sc_sync_delayed);
TAILQ_INIT(&sc->sc_events);
mtx_init(&sc->sc_events_mtx, "graid3:events", NULL, MTX_DEF);
- callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_callout, 1);
sc->sc_state = G_RAID3_DEVICE_STATE_STARTING;
gp->softc = sc;
sc->sc_geom = gp;
sc->sc_provider = NULL;
/*
* Synchronization geom.
*/
gp = g_new_geomf(mp, "%s.sync", md->md_name);
gp->softc = sc;
gp->orphan = g_raid3_orphan;
sc->sc_sync.ds_geom = gp;
if (!g_raid3_use_malloc) {
sc->sc_zones[G_RAID3_ZONE_64K].sz_zone = uma_zcreate("gr3:64k",
65536, g_raid3_uma_ctor, g_raid3_uma_dtor, NULL, NULL,
UMA_ALIGN_PTR, 0);
sc->sc_zones[G_RAID3_ZONE_64K].sz_inuse = 0;
sc->sc_zones[G_RAID3_ZONE_64K].sz_max = g_raid3_n64k;
sc->sc_zones[G_RAID3_ZONE_64K].sz_requested =
sc->sc_zones[G_RAID3_ZONE_64K].sz_failed = 0;
sc->sc_zones[G_RAID3_ZONE_16K].sz_zone = uma_zcreate("gr3:16k",
16384, g_raid3_uma_ctor, g_raid3_uma_dtor, NULL, NULL,
UMA_ALIGN_PTR, 0);
sc->sc_zones[G_RAID3_ZONE_16K].sz_inuse = 0;
sc->sc_zones[G_RAID3_ZONE_16K].sz_max = g_raid3_n16k;
sc->sc_zones[G_RAID3_ZONE_16K].sz_requested =
sc->sc_zones[G_RAID3_ZONE_16K].sz_failed = 0;
sc->sc_zones[G_RAID3_ZONE_4K].sz_zone = uma_zcreate("gr3:4k",
4096, g_raid3_uma_ctor, g_raid3_uma_dtor, NULL, NULL,
UMA_ALIGN_PTR, 0);
sc->sc_zones[G_RAID3_ZONE_4K].sz_inuse = 0;
sc->sc_zones[G_RAID3_ZONE_4K].sz_max = g_raid3_n4k;
sc->sc_zones[G_RAID3_ZONE_4K].sz_requested =
sc->sc_zones[G_RAID3_ZONE_4K].sz_failed = 0;
}
error = kproc_create(g_raid3_worker, sc, &sc->sc_worker, 0, 0,
"g_raid3 %s", md->md_name);
if (error != 0) {
G_RAID3_DEBUG(1, "Cannot create kernel thread for %s.",
sc->sc_name);
if (!g_raid3_use_malloc) {
uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_64K].sz_zone);
uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_16K].sz_zone);
uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_4K].sz_zone);
}
g_destroy_geom(sc->sc_sync.ds_geom);
mtx_destroy(&sc->sc_events_mtx);
mtx_destroy(&sc->sc_queue_mtx);
sx_destroy(&sc->sc_lock);
g_destroy_geom(sc->sc_geom);
free(sc->sc_disks, M_RAID3);
free(sc, M_RAID3);
return (NULL);
}
G_RAID3_DEBUG(1, "Device %s created (%u components, id=%u).",
sc->sc_name, sc->sc_ndisks, sc->sc_id);
sc->sc_rootmount = root_mount_hold("GRAID3");
G_RAID3_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
/*
* Run timeout.
*/
timeout = atomic_load_acq_int(&g_raid3_timeout);
callout_reset(&sc->sc_callout, timeout * hz, g_raid3_go, sc);
return (sc->sc_geom);
}
int
g_raid3_destroy(struct g_raid3_softc *sc, int how)
{
struct g_provider *pp;
g_topology_assert_not();
if (sc == NULL)
return (ENXIO);
sx_assert(&sc->sc_lock, SX_XLOCKED);
pp = sc->sc_provider;
if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
switch (how) {
case G_RAID3_DESTROY_SOFT:
G_RAID3_DEBUG(1,
"Device %s is still open (r%dw%de%d).", pp->name,
pp->acr, pp->acw, pp->ace);
return (EBUSY);
case G_RAID3_DESTROY_DELAYED:
G_RAID3_DEBUG(1,
"Device %s will be destroyed on last close.",
pp->name);
if (sc->sc_syncdisk != NULL)
g_raid3_sync_stop(sc, 1);
sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROYING;
return (EBUSY);
case G_RAID3_DESTROY_HARD:
G_RAID3_DEBUG(1, "Device %s is still open, so it "
"can't be definitely removed.", pp->name);
break;
}
}
g_topology_lock();
if (sc->sc_geom->softc == NULL) {
g_topology_unlock();
return (0);
}
sc->sc_geom->softc = NULL;
sc->sc_sync.ds_geom->softc = NULL;
g_topology_unlock();
sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROY;
sc->sc_flags |= G_RAID3_DEVICE_FLAG_WAIT;
G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__, sc);
sx_xunlock(&sc->sc_lock);
mtx_lock(&sc->sc_queue_mtx);
wakeup(sc);
wakeup(&sc->sc_queue);
mtx_unlock(&sc->sc_queue_mtx);
G_RAID3_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
while (sc->sc_worker != NULL)
tsleep(&sc->sc_worker, PRIBIO, "r3:destroy", hz / 5);
G_RAID3_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
sx_xlock(&sc->sc_lock);
g_raid3_destroy_device(sc);
free(sc->sc_disks, M_RAID3);
free(sc, M_RAID3);
return (0);
}
static void
g_raid3_taste_orphan(struct g_consumer *cp)
{
KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
cp->provider->name));
}
static struct g_geom *
g_raid3_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
{
struct g_raid3_metadata md;
struct g_raid3_softc *sc;
struct g_consumer *cp;
struct g_geom *gp;
int error;
g_topology_assert();
g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
G_RAID3_DEBUG(2, "Tasting %s.", pp->name);
gp = g_new_geomf(mp, "raid3:taste");
/* This orphan function should be never called. */
gp->orphan = g_raid3_taste_orphan;
cp = g_new_consumer(gp);
g_attach(cp, pp);
error = g_raid3_read_metadata(cp, &md);
g_detach(cp);
g_destroy_consumer(cp);
g_destroy_geom(gp);
if (error != 0)
return (NULL);
gp = NULL;
if (md.md_provider[0] != '\0' &&
!g_compare_names(md.md_provider, pp->name))
return (NULL);
if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
return (NULL);
if (g_raid3_debug >= 2)
raid3_metadata_dump(&md);
/*
* Let's check if device already exists.
*/
sc = NULL;
LIST_FOREACH(gp, &mp->geom, geom) {
sc = gp->softc;
if (sc == NULL)
continue;
if (sc->sc_sync.ds_geom == gp)
continue;
if (strcmp(md.md_name, sc->sc_name) != 0)
continue;
if (md.md_id != sc->sc_id) {
G_RAID3_DEBUG(0, "Device %s already configured.",
sc->sc_name);
return (NULL);
}
break;
}
if (gp == NULL) {
gp = g_raid3_create(mp, &md);
if (gp == NULL) {
G_RAID3_DEBUG(0, "Cannot create device %s.",
md.md_name);
return (NULL);
}
sc = gp->softc;
}
G_RAID3_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
g_topology_unlock();
sx_xlock(&sc->sc_lock);
error = g_raid3_add_disk(sc, pp, &md);
if (error != 0) {
G_RAID3_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
pp->name, gp->name, error);
if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NODISK) ==
sc->sc_ndisks) {
g_cancel_event(sc);
g_raid3_destroy(sc, G_RAID3_DESTROY_HARD);
g_topology_lock();
return (NULL);
}
gp = NULL;
}
sx_xunlock(&sc->sc_lock);
g_topology_lock();
return (gp);
}
static int
g_raid3_destroy_geom(struct gctl_req *req __unused, struct g_class *mp __unused,
struct g_geom *gp)
{
struct g_raid3_softc *sc;
int error;
g_topology_unlock();
sc = gp->softc;
sx_xlock(&sc->sc_lock);
g_cancel_event(sc);
error = g_raid3_destroy(gp->softc, G_RAID3_DESTROY_SOFT);
if (error != 0)
sx_xunlock(&sc->sc_lock);
g_topology_lock();
return (error);
}
static void
g_raid3_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
struct g_consumer *cp, struct g_provider *pp)
{
struct g_raid3_softc *sc;
g_topology_assert();
sc = gp->softc;
if (sc == NULL)
return;
/* Skip synchronization geom. */
if (gp == sc->sc_sync.ds_geom)
return;
if (pp != NULL) {
/* Nothing here. */
} else if (cp != NULL) {
struct g_raid3_disk *disk;
disk = cp->private;
if (disk == NULL)
return;
g_topology_unlock();
sx_xlock(&sc->sc_lock);
sbuf_printf(sb, "%s<Type>", indent);
if (disk->d_no == sc->sc_ndisks - 1)
sbuf_printf(sb, "PARITY");
else
sbuf_printf(sb, "DATA");
sbuf_printf(sb, "</Type>\n");
sbuf_printf(sb, "%s<Number>%u</Number>\n", indent,
(u_int)disk->d_no);
if (disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
sbuf_printf(sb, "%s<Synchronized>", indent);
if (disk->d_sync.ds_offset == 0)
sbuf_printf(sb, "0%%");
else {
sbuf_printf(sb, "%u%%",
(u_int)((disk->d_sync.ds_offset * 100) /
(sc->sc_mediasize / (sc->sc_ndisks - 1))));
}
sbuf_printf(sb, "</Synchronized>\n");
if (disk->d_sync.ds_offset > 0) {
sbuf_printf(sb, "%s<BytesSynced>%jd"
"</BytesSynced>\n", indent,
(intmax_t)disk->d_sync.ds_offset);
}
}
sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
disk->d_sync.ds_syncid);
sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, disk->d_genid);
sbuf_printf(sb, "%s<Flags>", indent);
if (disk->d_flags == 0)
sbuf_printf(sb, "NONE");
else {
int first = 1;
#define ADD_FLAG(flag, name) do { \
if ((disk->d_flags & (flag)) != 0) { \
if (!first) \
sbuf_printf(sb, ", "); \
else \
first = 0; \
sbuf_printf(sb, name); \
} \
} while (0)
ADD_FLAG(G_RAID3_DISK_FLAG_DIRTY, "DIRTY");
ADD_FLAG(G_RAID3_DISK_FLAG_HARDCODED, "HARDCODED");
ADD_FLAG(G_RAID3_DISK_FLAG_SYNCHRONIZING,
"SYNCHRONIZING");
ADD_FLAG(G_RAID3_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
ADD_FLAG(G_RAID3_DISK_FLAG_BROKEN, "BROKEN");
#undef ADD_FLAG
}
sbuf_printf(sb, "</Flags>\n");
sbuf_printf(sb, "%s<State>%s</State>\n", indent,
g_raid3_disk_state2str(disk->d_state));
sx_xunlock(&sc->sc_lock);
g_topology_lock();
} else {
g_topology_unlock();
sx_xlock(&sc->sc_lock);
if (!g_raid3_use_malloc) {
sbuf_printf(sb,
"%s<Zone4kRequested>%u</Zone4kRequested>\n", indent,
sc->sc_zones[G_RAID3_ZONE_4K].sz_requested);
sbuf_printf(sb,
"%s<Zone4kFailed>%u</Zone4kFailed>\n", indent,
sc->sc_zones[G_RAID3_ZONE_4K].sz_failed);
sbuf_printf(sb,
"%s<Zone16kRequested>%u</Zone16kRequested>\n", indent,
sc->sc_zones[G_RAID3_ZONE_16K].sz_requested);
sbuf_printf(sb,
"%s<Zone16kFailed>%u</Zone16kFailed>\n", indent,
sc->sc_zones[G_RAID3_ZONE_16K].sz_failed);
sbuf_printf(sb,
"%s<Zone64kRequested>%u</Zone64kRequested>\n", indent,
sc->sc_zones[G_RAID3_ZONE_64K].sz_requested);
sbuf_printf(sb,
"%s<Zone64kFailed>%u</Zone64kFailed>\n", indent,
sc->sc_zones[G_RAID3_ZONE_64K].sz_failed);
}
sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
sbuf_printf(sb, "%s<Flags>", indent);
if (sc->sc_flags == 0)
sbuf_printf(sb, "NONE");
else {
int first = 1;
#define ADD_FLAG(flag, name) do { \
if ((sc->sc_flags & (flag)) != 0) { \
if (!first) \
sbuf_printf(sb, ", "); \
else \
first = 0; \
sbuf_printf(sb, name); \
} \
} while (0)
ADD_FLAG(G_RAID3_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
ADD_FLAG(G_RAID3_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
ADD_FLAG(G_RAID3_DEVICE_FLAG_ROUND_ROBIN,
"ROUND-ROBIN");
ADD_FLAG(G_RAID3_DEVICE_FLAG_VERIFY, "VERIFY");
#undef ADD_FLAG
}
sbuf_printf(sb, "</Flags>\n");
sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
sc->sc_ndisks);
sbuf_printf(sb, "%s<State>%s</State>\n", indent,
g_raid3_device_state2str(sc->sc_state));
sx_xunlock(&sc->sc_lock);
g_topology_lock();
}
}
static void
g_raid3_shutdown_post_sync(void *arg, int howto)
{
struct g_class *mp;
struct g_geom *gp, *gp2;
struct g_raid3_softc *sc;
int error;
mp = arg;
DROP_GIANT();
g_topology_lock();
g_raid3_shutdown = 1;
LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
if ((sc = gp->softc) == NULL)
continue;
/* Skip synchronization geom. */
if (gp == sc->sc_sync.ds_geom)
continue;
g_topology_unlock();
sx_xlock(&sc->sc_lock);
g_raid3_idle(sc, -1);
g_cancel_event(sc);
error = g_raid3_destroy(sc, G_RAID3_DESTROY_DELAYED);
if (error != 0)
sx_xunlock(&sc->sc_lock);
g_topology_lock();
}
g_topology_unlock();
PICKUP_GIANT();
}
static void
g_raid3_init(struct g_class *mp)
{
g_raid3_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
g_raid3_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
if (g_raid3_post_sync == NULL)
G_RAID3_DEBUG(0, "Warning! Cannot register shutdown event.");
}
static void
g_raid3_fini(struct g_class *mp)
{
if (g_raid3_post_sync != NULL)
EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_raid3_post_sync);
}
DECLARE_GEOM_CLASS(g_raid3_class, g_raid3);
Index: head/sys/geom/sched/gs_rr.c
===================================================================
--- head/sys/geom/sched/gs_rr.c (revision 283290)
+++ head/sys/geom/sched/gs_rr.c (revision 283291)
@@ -1,699 +1,699 @@
/*-
* Copyright (c) 2009-2010 Fabio Checconi
* Copyright (c) 2009-2010 Luigi Rizzo, Universita` di Pisa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* $Id$
* $FreeBSD$
*
* A round-robin (RR) anticipatory scheduler, with per-client queues.
*
* The goal of this implementation is to improve throughput compared
* to the pure elevator algorithm, and insure some fairness among
* clients.
*
* Requests coming from the same client are put in the same queue.
* We use anticipation to help reducing seeks, and each queue
* is never served continuously for more than a given amount of
* time or data. Queues are then served in a round-robin fashion.
*
* Each queue can be in any of the following states:
* READY immediately serve the first pending request;
* BUSY one request is under service, wait for completion;
* IDLING do not serve incoming requests immediately, unless
* they are "eligible" as defined later.
*
* Scheduling is made looking at the status of all queues,
* and the first one in round-robin order is privileged.
*/
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/bio.h>
#include <sys/callout.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include "gs_scheduler.h"
/* possible states of the scheduler */
enum g_rr_state {
G_QUEUE_READY = 0, /* Ready to dispatch. */
G_QUEUE_BUSY, /* Waiting for a completion. */
G_QUEUE_IDLING /* Waiting for a new request. */
};
/* possible queue flags */
enum g_rr_flags {
/* G_FLAG_COMPLETED means that the field q_slice_end is valid. */
G_FLAG_COMPLETED = 1, /* Completed a req. in the current budget. */
};
struct g_rr_softc;
/*
* Queue descriptor, containing reference count, scheduling
* state, a queue of pending requests, configuration parameters.
* Queues with pending request(s) and not under service are also
* stored in a Round Robin (RR) list.
*/
struct g_rr_queue {
struct g_rr_softc *q_sc; /* link to the parent */
enum g_rr_state q_status;
unsigned int q_service; /* service received so far */
int q_slice_end; /* actual slice end time, in ticks */
enum g_rr_flags q_flags; /* queue flags */
struct bio_queue_head q_bioq;
/* Scheduling parameters */
unsigned int q_budget; /* slice size in bytes */
unsigned int q_slice_duration; /* slice size in ticks */
unsigned int q_wait_ticks; /* wait time for anticipation */
/* Stats to drive the various heuristics. */
struct g_savg q_thinktime; /* Thinktime average. */
struct g_savg q_seekdist; /* Seek distance average. */
int q_bionum; /* Number of requests. */
off_t q_lastoff; /* Last submitted req. offset. */
int q_lastsub; /* Last submitted req. time. */
/* Expiration deadline for an empty queue. */
int q_expire;
TAILQ_ENTRY(g_rr_queue) q_tailq; /* RR list link field */
};
/* List types. */
TAILQ_HEAD(g_rr_tailq, g_rr_queue);
/* list of scheduler instances */
LIST_HEAD(g_scheds, g_rr_softc);
/* Default quantum for RR between queues. */
#define G_RR_DEFAULT_BUDGET 0x00800000
/*
* Per device descriptor, holding the Round Robin list of queues
* accessing the disk, a reference to the geom, and the timer.
*/
struct g_rr_softc {
struct g_geom *sc_geom;
/*
* sc_active is the queue we are anticipating for.
* It is set only in gs_rr_next(), and possibly cleared
* only in gs_rr_next() or on a timeout.
* The active queue is never in the Round Robin list
* even if it has requests queued.
*/
struct g_rr_queue *sc_active;
struct callout sc_wait; /* timer for sc_active */
struct g_rr_tailq sc_rr_tailq; /* the round-robin list */
int sc_nqueues; /* number of queues */
/* Statistics */
int sc_in_flight; /* requests in the driver */
LIST_ENTRY(g_rr_softc) sc_next;
};
/* Descriptor for bounded values, min and max are constant. */
struct x_bound {
const int x_min;
int x_cur;
const int x_max;
};
/*
* parameters, config and stats
*/
struct g_rr_params {
int queues; /* total number of queues */
int w_anticipate; /* anticipate writes */
int bypass; /* bypass scheduling writes */
int units; /* how many instances */
/* sc_head is used for debugging */
struct g_scheds sc_head; /* first scheduler instance */
struct x_bound queue_depth; /* max parallel requests */
struct x_bound wait_ms; /* wait time, milliseconds */
struct x_bound quantum_ms; /* quantum size, milliseconds */
struct x_bound quantum_kb; /* quantum size, Kb (1024 bytes) */
/* statistics */
int wait_hit; /* success in anticipation */
int wait_miss; /* failure in anticipation */
};
/*
* Default parameters for the scheduler. The quantum sizes target
* a 80MB/s disk; if the hw is faster or slower the minimum of the
* two will have effect: the clients will still be isolated but
* the fairness may be limited. A complete solution would involve
* the on-line measurement of the actual disk throughput to derive
* these parameters. Or we may just choose to ignore service domain
* fairness and accept what can be achieved with time-only budgets.
*/
static struct g_rr_params me = {
.sc_head = LIST_HEAD_INITIALIZER(&me.sc_head),
.w_anticipate = 1,
.queue_depth = { 1, 1, 50 },
.wait_ms = { 1, 10, 30 },
.quantum_ms = { 1, 100, 500 },
.quantum_kb = { 16, 8192, 65536 },
};
struct g_rr_params *gs_rr_me = &me;
SYSCTL_DECL(_kern_geom_sched);
static SYSCTL_NODE(_kern_geom_sched, OID_AUTO, rr, CTLFLAG_RW, 0,
"GEOM_SCHED ROUND ROBIN stuff");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, units, CTLFLAG_RD,
&me.units, 0, "Scheduler instances");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, queues, CTLFLAG_RD,
&me.queues, 0, "Total rr queues");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, wait_ms, CTLFLAG_RW,
&me.wait_ms.x_cur, 0, "Wait time milliseconds");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, quantum_ms, CTLFLAG_RW,
&me.quantum_ms.x_cur, 0, "Quantum size milliseconds");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, bypass, CTLFLAG_RW,
&me.bypass, 0, "Bypass scheduler");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, w_anticipate, CTLFLAG_RW,
&me.w_anticipate, 0, "Do anticipation on writes");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, quantum_kb, CTLFLAG_RW,
&me.quantum_kb.x_cur, 0, "Quantum size Kbytes");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, queue_depth, CTLFLAG_RW,
&me.queue_depth.x_cur, 0, "Maximum simultaneous requests");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, wait_hit, CTLFLAG_RW,
&me.wait_hit, 0, "Hits in anticipation");
SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, wait_miss, CTLFLAG_RW,
&me.wait_miss, 0, "Misses in anticipation");
#ifdef DEBUG_QUEUES
/* print the status of a queue */
static void
gs_rr_dump_q(struct g_rr_queue *qp, int index)
{
int l = 0;
struct bio *bp;
TAILQ_FOREACH(bp, &(qp->q_bioq.queue), bio_queue) {
l++;
}
printf("--- rr queue %d %p status %d len %d ---\n",
index, qp, qp->q_status, l);
}
/*
* Dump the scheduler status when writing to this sysctl variable.
* XXX right now we only dump the status of the last instance created.
* not a severe issue because this is only for debugging
*/
static int
gs_rr_sysctl_status(SYSCTL_HANDLER_ARGS)
{
int error, val = 0;
struct g_rr_softc *sc;
error = sysctl_handle_int(oidp, &val, 0, req);
if (error || !req->newptr )
return (error);
printf("called %s\n", __FUNCTION__);
LIST_FOREACH(sc, &me.sc_head, sc_next) {
int i, tot = 0;
printf("--- sc %p active %p nqueues %d "
"callout %d in_flight %d ---\n",
sc, sc->sc_active, sc->sc_nqueues,
callout_active(&sc->sc_wait),
sc->sc_in_flight);
for (i = 0; i < G_RR_HASH_SIZE; i++) {
struct g_rr_queue *qp;
LIST_FOREACH(qp, &sc->sc_hash[i], q_hash) {
gs_rr_dump_q(qp, tot);
tot++;
}
}
}
return (0);
}
SYSCTL_PROC(_kern_geom_sched_rr, OID_AUTO, status,
CTLTYPE_UINT | CTLFLAG_RW,
0, sizeof(int), gs_rr_sysctl_status, "I", "status");
#endif /* DEBUG_QUEUES */
/*
* Get a bounded value, optionally convert to a min of t_min ticks.
*/
static int
get_bounded(struct x_bound *v, int t_min)
{
int x;
x = v->x_cur;
if (x < v->x_min)
x = v->x_min;
else if (x > v->x_max)
x = v->x_max;
if (t_min) {
x = x * hz / 1000; /* convert to ticks */
if (x < t_min)
x = t_min;
}
return x;
}
/*
* Get a reference to the queue for bp, using the generic
* classification mechanism.
*/
static struct g_rr_queue *
g_rr_queue_get(struct g_rr_softc *sc, struct bio *bp)
{
return (g_sched_get_class(sc->sc_geom, bp));
}
static int
g_rr_init_class(void *data, void *priv)
{
struct g_rr_softc *sc = data;
struct g_rr_queue *qp = priv;
bioq_init(&qp->q_bioq);
/*
* Set the initial parameters for the client:
* slice size in bytes and ticks, and wait ticks.
* Right now these are constant, but we could have
* autoconfiguration code to adjust the values based on
* the actual workload.
*/
qp->q_budget = 1024 * get_bounded(&me.quantum_kb, 0);
qp->q_slice_duration = get_bounded(&me.quantum_ms, 2);
qp->q_wait_ticks = get_bounded(&me.wait_ms, 2);
qp->q_sc = sc; /* link to the parent */
qp->q_sc->sc_nqueues++;
me.queues++;
return (0);
}
/*
* Release a reference to the queue.
*/
static void
g_rr_queue_put(struct g_rr_queue *qp)
{
g_sched_put_class(qp->q_sc->sc_geom, qp);
}
static void
g_rr_fini_class(void *data, void *priv)
{
struct g_rr_queue *qp = priv;
KASSERT(bioq_first(&qp->q_bioq) == NULL,
("released nonempty queue"));
qp->q_sc->sc_nqueues--;
me.queues--;
}
static inline int
g_rr_queue_expired(struct g_rr_queue *qp)
{
if (qp->q_service >= qp->q_budget)
return (1);
if ((qp->q_flags & G_FLAG_COMPLETED) &&
ticks - qp->q_slice_end >= 0)
return (1);
return (0);
}
static inline int
g_rr_should_anticipate(struct g_rr_queue *qp, struct bio *bp)
{
int wait = get_bounded(&me.wait_ms, 2);
if (!me.w_anticipate && (bp->bio_cmd & BIO_WRITE))
return (0);
if (g_savg_valid(&qp->q_thinktime) &&
g_savg_read(&qp->q_thinktime) > wait)
return (0);
if (g_savg_valid(&qp->q_seekdist) &&
g_savg_read(&qp->q_seekdist) > 8192)
return (0);
return (1);
}
/*
* Called on a request arrival, timeout or completion.
* Try to serve a request among those queued.
*/
static struct bio *
g_rr_next(void *data, int force)
{
struct g_rr_softc *sc = data;
struct g_rr_queue *qp;
struct bio *bp, *next;
int expired;
qp = sc->sc_active;
if (me.bypass == 0 && !force) {
if (sc->sc_in_flight >= get_bounded(&me.queue_depth, 0))
return (NULL);
/* Try with the queue under service first. */
if (qp != NULL && qp->q_status != G_QUEUE_READY) {
/*
* Queue is anticipating, ignore request.
* We should check that we are not past
* the timeout, but in that case the timeout
* will fire immediately afterwards so we
* don't bother.
*/
return (NULL);
}
} else if (qp != NULL && qp->q_status != G_QUEUE_READY) {
g_rr_queue_put(qp);
sc->sc_active = qp = NULL;
}
/*
* No queue under service, look for the first in RR order.
* If we find it, select if as sc_active, clear service
* and record the end time of the slice.
*/
if (qp == NULL) {
qp = TAILQ_FIRST(&sc->sc_rr_tailq);
if (qp == NULL)
return (NULL); /* no queues at all, return */
/* otherwise select the new queue for service. */
TAILQ_REMOVE(&sc->sc_rr_tailq, qp, q_tailq);
sc->sc_active = qp;
qp->q_service = 0;
qp->q_flags &= ~G_FLAG_COMPLETED;
}
bp = bioq_takefirst(&qp->q_bioq); /* surely not NULL */
qp->q_service += bp->bio_length; /* charge the service */
/*
* The request at the head of the active queue is always
* dispatched, and gs_rr_next() will be called again
* immediately.
* We need to prepare for what to do next:
*
* 1. have we reached the end of the (time or service) slice ?
* If so, clear sc_active and possibly requeue the previous
* active queue if it has more requests pending;
* 2. do we have more requests in sc_active ?
* If yes, do not anticipate, as gs_rr_next() will run again;
* if no, decide whether or not to anticipate depending
* on read or writes (e.g., anticipate only on reads).
*/
expired = g_rr_queue_expired(qp); /* are we expired ? */
next = bioq_first(&qp->q_bioq); /* do we have one more ? */
if (expired) {
sc->sc_active = NULL;
/* Either requeue or release reference. */
if (next != NULL)
TAILQ_INSERT_TAIL(&sc->sc_rr_tailq, qp, q_tailq);
else
g_rr_queue_put(qp);
} else if (next != NULL) {
qp->q_status = G_QUEUE_READY;
} else {
if (!force && g_rr_should_anticipate(qp, bp)) {
/* anticipate */
qp->q_status = G_QUEUE_BUSY;
} else {
/* do not anticipate, release reference */
g_rr_queue_put(qp);
sc->sc_active = NULL;
}
}
/* If sc_active != NULL, its q_status is always correct. */
sc->sc_in_flight++;
return (bp);
}
static inline void
g_rr_update_thinktime(struct g_rr_queue *qp)
{
int delta = ticks - qp->q_lastsub, wait = get_bounded(&me.wait_ms, 2);
if (qp->q_sc->sc_active != qp)
return;
qp->q_lastsub = ticks;
delta = (delta > 2 * wait) ? 2 * wait : delta;
if (qp->q_bionum > 7)
g_savg_add_sample(&qp->q_thinktime, delta);
}
static inline void
g_rr_update_seekdist(struct g_rr_queue *qp, struct bio *bp)
{
off_t dist;
if (qp->q_lastoff > bp->bio_offset)
dist = qp->q_lastoff - bp->bio_offset;
else
dist = bp->bio_offset - qp->q_lastoff;
if (dist > (8192 * 8))
dist = 8192 * 8;
qp->q_lastoff = bp->bio_offset + bp->bio_length;
if (qp->q_bionum > 7)
g_savg_add_sample(&qp->q_seekdist, dist);
}
/*
* Called when a real request for disk I/O arrives.
* Locate the queue associated with the client.
* If the queue is the one we are anticipating for, reset its timeout;
* if the queue is not in the round robin list, insert it in the list.
* On any error, do not queue the request and return -1, the caller
* will take care of this request.
*/
static int
g_rr_start(void *data, struct bio *bp)
{
struct g_rr_softc *sc = data;
struct g_rr_queue *qp;
if (me.bypass)
return (-1); /* bypass the scheduler */
/* Get the queue for the request. */
qp = g_rr_queue_get(sc, bp);
if (qp == NULL)
return (-1); /* allocation failed, tell upstream */
if (bioq_first(&qp->q_bioq) == NULL) {
/*
* We are inserting into an empty queue.
* Reset its state if it is sc_active,
* otherwise insert it in the RR list.
*/
if (qp == sc->sc_active) {
qp->q_status = G_QUEUE_READY;
callout_stop(&sc->sc_wait);
} else {
g_sched_priv_ref(qp);
TAILQ_INSERT_TAIL(&sc->sc_rr_tailq, qp, q_tailq);
}
}
qp->q_bionum = 1 + qp->q_bionum - (qp->q_bionum >> 3);
g_rr_update_thinktime(qp);
g_rr_update_seekdist(qp, bp);
/* Inherit the reference returned by g_rr_queue_get(). */
bp->bio_caller1 = qp;
bioq_disksort(&qp->q_bioq, bp);
return (0);
}
/*
* Callout executed when a queue times out anticipating a new request.
*/
static void
g_rr_wait_timeout(void *data)
{
struct g_rr_softc *sc = data;
struct g_geom *geom = sc->sc_geom;
g_sched_lock(geom);
/*
* We can race with other events, so check if
* sc_active is still valid.
*/
if (sc->sc_active != NULL) {
/* Release the reference to the queue. */
g_rr_queue_put(sc->sc_active);
sc->sc_active = NULL;
me.wait_hit--;
me.wait_miss++; /* record the miss */
}
g_sched_dispatch(geom);
g_sched_unlock(geom);
}
/*
* Module glue: allocate descriptor, initialize its fields.
*/
static void *
g_rr_init(struct g_geom *geom)
{
struct g_rr_softc *sc;
/* XXX check whether we can sleep */
sc = malloc(sizeof *sc, M_GEOM_SCHED, M_NOWAIT | M_ZERO);
sc->sc_geom = geom;
TAILQ_INIT(&sc->sc_rr_tailq);
- callout_init(&sc->sc_wait, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_wait, 1);
LIST_INSERT_HEAD(&me.sc_head, sc, sc_next);
me.units++;
return (sc);
}
/*
* Module glue -- drain the callout structure, destroy the
* hash table and its element, and free the descriptor.
*/
static void
g_rr_fini(void *data)
{
struct g_rr_softc *sc = data;
callout_drain(&sc->sc_wait);
KASSERT(sc->sc_active == NULL, ("still a queue under service"));
KASSERT(TAILQ_EMPTY(&sc->sc_rr_tailq), ("still scheduled queues"));
LIST_REMOVE(sc, sc_next);
me.units--;
free(sc, M_GEOM_SCHED);
}
/*
* Called when the request under service terminates.
* Start the anticipation timer if needed.
*/
static void
g_rr_done(void *data, struct bio *bp)
{
struct g_rr_softc *sc = data;
struct g_rr_queue *qp;
sc->sc_in_flight--;
qp = bp->bio_caller1;
/*
* When the first request for this queue completes, update the
* duration and end of the slice. We do not do it when the
* slice starts to avoid charging to the queue the time for
* the first seek.
*/
if (!(qp->q_flags & G_FLAG_COMPLETED)) {
qp->q_flags |= G_FLAG_COMPLETED;
/*
* recompute the slice duration, in case we want
* to make it adaptive. This is not used right now.
* XXX should we do the same for q_quantum and q_wait_ticks ?
*/
qp->q_slice_duration = get_bounded(&me.quantum_ms, 2);
qp->q_slice_end = ticks + qp->q_slice_duration;
}
if (qp == sc->sc_active && qp->q_status == G_QUEUE_BUSY) {
/* The queue is trying anticipation, start the timer. */
qp->q_status = G_QUEUE_IDLING;
/* may make this adaptive */
qp->q_wait_ticks = get_bounded(&me.wait_ms, 2);
me.wait_hit++;
callout_reset(&sc->sc_wait, qp->q_wait_ticks,
g_rr_wait_timeout, sc);
} else
g_sched_dispatch(sc->sc_geom);
/* Release a reference to the queue. */
g_rr_queue_put(qp);
}
static void
g_rr_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
struct g_consumer *cp, struct g_provider *pp)
{
if (indent == NULL) { /* plaintext */
sbuf_printf(sb, " units %d queues %d",
me.units, me.queues);
}
}
static struct g_gsched g_rr = {
.gs_name = "rr",
.gs_priv_size = sizeof(struct g_rr_queue),
.gs_init = g_rr_init,
.gs_fini = g_rr_fini,
.gs_start = g_rr_start,
.gs_done = g_rr_done,
.gs_next = g_rr_next,
.gs_dumpconf = g_rr_dumpconf,
.gs_init_class = g_rr_init_class,
.gs_fini_class = g_rr_fini_class,
};
DECLARE_GSCHED_MODULE(rr, &g_rr);
Index: head/sys/i386/i386/mp_watchdog.c
===================================================================
--- head/sys/i386/i386/mp_watchdog.c (revision 283290)
+++ head/sys/i386/i386/mp_watchdog.c (revision 283291)
@@ -1,210 +1,210 @@
/*-
* Copyright (c) 2004 Robert N. M. Watson
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include "opt_mp_watchdog.h"
#include "opt_sched.h"
#ifdef SCHED_ULE
#error MP_WATCHDOG cannot currently be used with SCHED_ULE
#endif
#include <sys/param.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/pcpu.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <machine/smp.h>
#include <x86/apicreg.h>
#include <x86/apicvar.h>
#include <machine/mp_watchdog.h>
/*
* mp_watchdog hijacks the idle thread on a specified CPU, prevents new work
* from being scheduled there, and uses it as a "watchdog" to detect kernel
* failure on other CPUs. This is made reasonable by inclusion of logical
* processors in Xeon hardware. The watchdog is configured by setting the
* debug.watchdog sysctl/tunable to the CPU of interest. A callout will then
* begin executing reseting a timer that is gradually lowered by the watching
* thread. If the timer reaches 0, the watchdog fires by ether dropping
* directly to the debugger, or by sending an NMI IPI to the boot processor.
* This is a somewhat less efficient substitute for dedicated watchdog
* hardware, but can be quite an effective tool for debugging hangs.
*
* XXXRW: This should really use the watchdog(9)/watchdog(4) framework, but
* doesn't yet.
*/
static int watchdog_cpu = -1;
static int watchdog_dontfire = 1;
static int watchdog_timer = -1;
static int watchdog_nmi = 1;
SYSCTL_INT(_debug, OID_AUTO, watchdog_nmi, CTLFLAG_RWTUN, &watchdog_nmi, 0,
"IPI the boot processor with an NMI to enter the debugger");
static struct callout watchdog_callout;
static void watchdog_change(int wdcpu);
/*
* Number of seconds before the watchdog will fire if the callout fails to
* reset the timer.
*/
#define WATCHDOG_THRESHOLD 10
static void
watchdog_init(void *arg)
{
- callout_init(&watchdog_callout, CALLOUT_MPSAFE);
+ callout_init(&watchdog_callout, 1);
if (watchdog_cpu != -1)
watchdog_change(watchdog_cpu);
}
/*
* This callout resets a timer until the watchdog kicks in. It acquires some
* critical locks to make sure things haven't gotten wedged with hose locks
* held.
*/
static void
watchdog_function(void *arg)
{
/*
* Since the timer ran, we must not be wedged. Acquire some critical
* locks to make sure. Then reset the timer.
*/
mtx_lock(&Giant);
watchdog_timer = WATCHDOG_THRESHOLD;
mtx_unlock(&Giant);
callout_reset(&watchdog_callout, 1 * hz, watchdog_function, NULL);
}
SYSINIT(watchdog_init, SI_SUB_DRIVERS, SI_ORDER_ANY, watchdog_init, NULL);
static void
watchdog_change(int wdcpu)
{
if (wdcpu == -1 || wdcpu == 0xffffffff) {
/*
* Disable the watchdog.
*/
watchdog_cpu = -1;
watchdog_dontfire = 1;
callout_stop(&watchdog_callout);
printf("watchdog stopped\n");
} else {
watchdog_timer = WATCHDOG_THRESHOLD;
watchdog_dontfire = 0;
watchdog_cpu = wdcpu;
callout_reset(&watchdog_callout, 1 * hz, watchdog_function,
NULL);
}
}
/*
* This sysctl sets which CPU is the watchdog CPU. Set to -1 or 0xffffffff
* to disable the watchdog.
*/
static int
sysctl_watchdog(SYSCTL_HANDLER_ARGS)
{
int error, temp;
temp = watchdog_cpu;
error = sysctl_handle_int(oidp, &temp, 0, req);
if (error)
return (error);
if (req->newptr != NULL)
watchdog_change(temp);
return (0);
}
SYSCTL_PROC(_debug, OID_AUTO, watchdog, CTLTYPE_INT|CTLFLAG_RW, 0, 0,
sysctl_watchdog, "I", "");
/*
* Drop into the debugger by sending an IPI NMI to the boot processor.
*/
static void
watchdog_ipi_nmi(void)
{
/*
* Deliver NMI to the boot processor. Why not?
*/
lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_NMI,
boot_cpu_id);
lapic_ipi_wait(-1);
}
/*
* ap_watchdog() is called by the SMP idle loop code. It works on the same
* premise that the disabling of logical processors does: that if the cpu is
* idle, then it can ignore the world from then on, as nothing will be
* scheduled on it. Leaving aside multi-runqueue schedulers (SCHED_ULE) and
* explicit process migration (sched_bind()), this is not an unreasonable
* assumption.
*/
void
ap_watchdog(u_int cpuid)
{
char old_pcomm[MAXCOMLEN + 1];
struct proc *p;
if (watchdog_cpu != cpuid)
return;
printf("watchdog started on cpu %d\n", cpuid);
p = curproc;
bcopy(p->p_comm, old_pcomm, MAXCOMLEN + 1);
snprintf(p->p_comm, MAXCOMLEN + 1, "mp_watchdog cpu %d", cpuid);
while (1) {
DELAY(1000000); /* One second. */
if (watchdog_cpu != cpuid)
break;
atomic_subtract_int(&watchdog_timer, 1);
if (watchdog_timer < 4)
printf("Watchdog timer: %d\n", watchdog_timer);
if (watchdog_timer == 0 && watchdog_dontfire == 0) {
printf("Watchdog firing!\n");
watchdog_dontfire = 1;
if (watchdog_nmi)
watchdog_ipi_nmi();
else
kdb_enter(KDB_WHY_WATCHDOG, "mp_watchdog");
}
}
bcopy(old_pcomm, p->p_comm, MAXCOMLEN + 1);
printf("watchdog stopped on cpu %d\n", cpuid);
}
Index: head/sys/kern/init_main.c
===================================================================
--- head/sys/kern/init_main.c (revision 283290)
+++ head/sys/kern/init_main.c (revision 283291)
@@ -1,867 +1,867 @@
/*-
* Copyright (c) 1995 Terrence R. Lambert
* All rights reserved.
*
* Copyright (c) 1982, 1986, 1989, 1991, 1992, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the University of
* California, Berkeley and its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)init_main.c 8.9 (Berkeley) 1/21/94
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include "opt_init_path.h"
#include "opt_verbose_sysinit.h"
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/exec.h>
#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/jail.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/loginclass.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/proc.h>
#include <sys/racct.h>
#include <sys/resourcevar.h>
#include <sys/systm.h>
#include <sys/signalvar.h>
#include <sys/vnode.h>
#include <sys/sysent.h>
#include <sys/reboot.h>
#include <sys/sched.h>
#include <sys/sx.h>
#include <sys/sysproto.h>
#include <sys/vmmeter.h>
#include <sys/unistd.h>
#include <sys/malloc.h>
#include <sys/conf.h>
#include <sys/cpuset.h>
#include <machine/cpu.h>
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <sys/copyright.h>
#include <ddb/ddb.h>
#include <ddb/db_sym.h>
void mi_startup(void); /* Should be elsewhere */
/* Components of the first process -- never freed. */
static struct session session0;
static struct pgrp pgrp0;
struct proc proc0;
struct thread thread0 __aligned(16);
struct vmspace vmspace0;
struct proc *initproc;
#ifndef BOOTHOWTO
#define BOOTHOWTO 0
#endif
int boothowto = BOOTHOWTO; /* initialized so that it can be patched */
SYSCTL_INT(_debug, OID_AUTO, boothowto, CTLFLAG_RD, &boothowto, 0,
"Boot control flags, passed from loader");
#ifndef BOOTVERBOSE
#define BOOTVERBOSE 0
#endif
int bootverbose = BOOTVERBOSE;
SYSCTL_INT(_debug, OID_AUTO, bootverbose, CTLFLAG_RW, &bootverbose, 0,
"Control the output of verbose kernel messages");
/*
* This ensures that there is at least one entry so that the sysinit_set
* symbol is not undefined. A sybsystem ID of SI_SUB_DUMMY is never
* executed.
*/
SYSINIT(placeholder, SI_SUB_DUMMY, SI_ORDER_ANY, NULL, NULL);
/*
* The sysinit table itself. Items are checked off as the are run.
* If we want to register new sysinit types, add them to newsysinit.
*/
SET_DECLARE(sysinit_set, struct sysinit);
struct sysinit **sysinit, **sysinit_end;
struct sysinit **newsysinit, **newsysinit_end;
/*
* Merge a new sysinit set into the current set, reallocating it if
* necessary. This can only be called after malloc is running.
*/
void
sysinit_add(struct sysinit **set, struct sysinit **set_end)
{
struct sysinit **newset;
struct sysinit **sipp;
struct sysinit **xipp;
int count;
count = set_end - set;
if (newsysinit)
count += newsysinit_end - newsysinit;
else
count += sysinit_end - sysinit;
newset = malloc(count * sizeof(*sipp), M_TEMP, M_NOWAIT);
if (newset == NULL)
panic("cannot malloc for sysinit");
xipp = newset;
if (newsysinit)
for (sipp = newsysinit; sipp < newsysinit_end; sipp++)
*xipp++ = *sipp;
else
for (sipp = sysinit; sipp < sysinit_end; sipp++)
*xipp++ = *sipp;
for (sipp = set; sipp < set_end; sipp++)
*xipp++ = *sipp;
if (newsysinit)
free(newsysinit, M_TEMP);
newsysinit = newset;
newsysinit_end = newset + count;
}
#if defined (DDB) && defined(VERBOSE_SYSINIT)
static const char *
symbol_name(vm_offset_t va, db_strategy_t strategy)
{
const char *name;
c_db_sym_t sym;
db_expr_t offset;
if (va == 0)
return (NULL);
sym = db_search_symbol(va, strategy, &offset);
if (offset != 0)
return (NULL);
db_symbol_values(sym, &name, NULL);
return (name);
}
#endif
/*
* System startup; initialize the world, create process 0, mount root
* filesystem, and fork to create init and pagedaemon. Most of the
* hard work is done in the lower-level initialization routines including
* startup(), which does memory initialization and autoconfiguration.
*
* This allows simple addition of new kernel subsystems that require
* boot time initialization. It also allows substitution of subsystem
* (for instance, a scheduler, kernel profiler, or VM system) by object
* module. Finally, it allows for optional "kernel threads".
*/
void
mi_startup(void)
{
register struct sysinit **sipp; /* system initialization*/
register struct sysinit **xipp; /* interior loop of sort*/
register struct sysinit *save; /* bubble*/
#if defined(VERBOSE_SYSINIT)
int last;
int verbose;
#endif
if (boothowto & RB_VERBOSE)
bootverbose++;
if (sysinit == NULL) {
sysinit = SET_BEGIN(sysinit_set);
sysinit_end = SET_LIMIT(sysinit_set);
}
restart:
/*
* Perform a bubble sort of the system initialization objects by
* their subsystem (primary key) and order (secondary key).
*/
for (sipp = sysinit; sipp < sysinit_end; sipp++) {
for (xipp = sipp + 1; xipp < sysinit_end; xipp++) {
if ((*sipp)->subsystem < (*xipp)->subsystem ||
((*sipp)->subsystem == (*xipp)->subsystem &&
(*sipp)->order <= (*xipp)->order))
continue; /* skip*/
save = *sipp;
*sipp = *xipp;
*xipp = save;
}
}
#if defined(VERBOSE_SYSINIT)
last = SI_SUB_COPYRIGHT;
verbose = 0;
#if !defined(DDB)
printf("VERBOSE_SYSINIT: DDB not enabled, symbol lookups disabled.\n");
#endif
#endif
/*
* Traverse the (now) ordered list of system initialization tasks.
* Perform each task, and continue on to the next task.
*/
for (sipp = sysinit; sipp < sysinit_end; sipp++) {
if ((*sipp)->subsystem == SI_SUB_DUMMY)
continue; /* skip dummy task(s)*/
if ((*sipp)->subsystem == SI_SUB_DONE)
continue;
#if defined(VERBOSE_SYSINIT)
if ((*sipp)->subsystem > last) {
verbose = 1;
last = (*sipp)->subsystem;
printf("subsystem %x\n", last);
}
if (verbose) {
#if defined(DDB)
const char *func, *data;
func = symbol_name((vm_offset_t)(*sipp)->func,
DB_STGY_PROC);
data = symbol_name((vm_offset_t)(*sipp)->udata,
DB_STGY_ANY);
if (func != NULL && data != NULL)
printf(" %s(&%s)... ", func, data);
else if (func != NULL)
printf(" %s(%p)... ", func, (*sipp)->udata);
else
#endif
printf(" %p(%p)... ", (*sipp)->func,
(*sipp)->udata);
}
#endif
/* Call function */
(*((*sipp)->func))((*sipp)->udata);
#if defined(VERBOSE_SYSINIT)
if (verbose)
printf("done.\n");
#endif
/* Check off the one we're just done */
(*sipp)->subsystem = SI_SUB_DONE;
/* Check if we've installed more sysinit items via KLD */
if (newsysinit != NULL) {
if (sysinit != SET_BEGIN(sysinit_set))
free(sysinit, M_TEMP);
sysinit = newsysinit;
sysinit_end = newsysinit_end;
newsysinit = NULL;
newsysinit_end = NULL;
goto restart;
}
}
mtx_assert(&Giant, MA_OWNED | MA_NOTRECURSED);
mtx_unlock(&Giant);
/*
* Now hand over this thread to swapper.
*/
swapper();
/* NOTREACHED*/
}
/*
***************************************************************************
****
**** The following SYSINIT's belong elsewhere, but have not yet
**** been moved.
****
***************************************************************************
*/
static void
print_caddr_t(void *data)
{
printf("%s", (char *)data);
}
static void
print_version(void *data __unused)
{
int len;
/* Strip a trailing newline from version. */
len = strlen(version);
while (len > 0 && version[len - 1] == '\n')
len--;
printf("%.*s %s\n", len, version, machine);
printf("%s\n", compiler_version);
}
SYSINIT(announce, SI_SUB_COPYRIGHT, SI_ORDER_FIRST, print_caddr_t,
copyright);
SYSINIT(trademark, SI_SUB_COPYRIGHT, SI_ORDER_SECOND, print_caddr_t,
trademark);
SYSINIT(version, SI_SUB_COPYRIGHT, SI_ORDER_THIRD, print_version, NULL);
#ifdef WITNESS
static char wit_warn[] =
"WARNING: WITNESS option enabled, expect reduced performance.\n";
SYSINIT(witwarn, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 1,
print_caddr_t, wit_warn);
SYSINIT(witwarn2, SI_SUB_LAST, SI_ORDER_THIRD + 1,
print_caddr_t, wit_warn);
#endif
#ifdef DIAGNOSTIC
static char diag_warn[] =
"WARNING: DIAGNOSTIC option enabled, expect reduced performance.\n";
SYSINIT(diagwarn, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 2,
print_caddr_t, diag_warn);
SYSINIT(diagwarn2, SI_SUB_LAST, SI_ORDER_THIRD + 2,
print_caddr_t, diag_warn);
#endif
static int
null_fetch_syscall_args(struct thread *td __unused,
struct syscall_args *sa __unused)
{
panic("null_fetch_syscall_args");
}
static void
null_set_syscall_retval(struct thread *td __unused, int error __unused)
{
panic("null_set_syscall_retval");
}
struct sysentvec null_sysvec = {
.sv_size = 0,
.sv_table = NULL,
.sv_mask = 0,
.sv_sigsize = 0,
.sv_sigtbl = NULL,
.sv_errsize = 0,
.sv_errtbl = NULL,
.sv_transtrap = NULL,
.sv_fixup = NULL,
.sv_sendsig = NULL,
.sv_sigcode = NULL,
.sv_szsigcode = NULL,
.sv_prepsyscall = NULL,
.sv_name = "null",
.sv_coredump = NULL,
.sv_imgact_try = NULL,
.sv_minsigstksz = 0,
.sv_pagesize = PAGE_SIZE,
.sv_minuser = VM_MIN_ADDRESS,
.sv_maxuser = VM_MAXUSER_ADDRESS,
.sv_usrstack = USRSTACK,
.sv_psstrings = PS_STRINGS,
.sv_stackprot = VM_PROT_ALL,
.sv_copyout_strings = NULL,
.sv_setregs = NULL,
.sv_fixlimit = NULL,
.sv_maxssiz = NULL,
.sv_flags = 0,
.sv_set_syscall_retval = null_set_syscall_retval,
.sv_fetch_syscall_args = null_fetch_syscall_args,
.sv_syscallnames = NULL,
.sv_schedtail = NULL,
};
/*
***************************************************************************
****
**** The two following SYSINIT's are proc0 specific glue code. I am not
**** convinced that they can not be safely combined, but their order of
**** operation has been maintained as the same as the original init_main.c
**** for right now.
****
**** These probably belong in init_proc.c or kern_proc.c, since they
**** deal with proc0 (the fork template process).
****
***************************************************************************
*/
/* ARGSUSED*/
static void
proc0_init(void *dummy __unused)
{
struct proc *p;
struct thread *td;
struct ucred *newcred;
vm_paddr_t pageablemem;
int i;
GIANT_REQUIRED;
p = &proc0;
td = &thread0;
/*
* Initialize magic number and osrel.
*/
p->p_magic = P_MAGIC;
p->p_osrel = osreldate;
/*
* Initialize thread and process structures.
*/
procinit(); /* set up proc zone */
threadinit(); /* set up UMA zones */
/*
* Initialise scheduler resources.
* Add scheduler specific parts to proc, thread as needed.
*/
schedinit(); /* scheduler gets its house in order */
/*
* Create process 0 (the swapper).
*/
LIST_INSERT_HEAD(&allproc, p, p_list);
LIST_INSERT_HEAD(PIDHASH(0), p, p_hash);
mtx_init(&pgrp0.pg_mtx, "process group", NULL, MTX_DEF | MTX_DUPOK);
p->p_pgrp = &pgrp0;
LIST_INSERT_HEAD(PGRPHASH(0), &pgrp0, pg_hash);
LIST_INIT(&pgrp0.pg_members);
LIST_INSERT_HEAD(&pgrp0.pg_members, p, p_pglist);
pgrp0.pg_session = &session0;
mtx_init(&session0.s_mtx, "session", NULL, MTX_DEF);
refcount_init(&session0.s_count, 1);
session0.s_leader = p;
p->p_sysent = &null_sysvec;
p->p_flag = P_SYSTEM | P_INMEM;
p->p_flag2 = 0;
p->p_state = PRS_NORMAL;
knlist_init_mtx(&p->p_klist, &p->p_mtx);
STAILQ_INIT(&p->p_ktr);
p->p_nice = NZERO;
/* pid_max cannot be greater than PID_MAX */
td->td_tid = PID_MAX + 1;
LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash);
td->td_state = TDS_RUNNING;
td->td_pri_class = PRI_TIMESHARE;
td->td_user_pri = PUSER;
td->td_base_user_pri = PUSER;
td->td_lend_user_pri = PRI_MAX;
td->td_priority = PVM;
td->td_base_pri = PVM;
td->td_oncpu = 0;
td->td_flags = TDF_INMEM;
td->td_pflags = TDP_KTHREAD;
td->td_cpuset = cpuset_thread0();
prison0_init();
p->p_peers = 0;
p->p_leader = p;
p->p_reaper = p;
LIST_INIT(&p->p_reaplist);
strncpy(p->p_comm, "kernel", sizeof (p->p_comm));
strncpy(td->td_name, "swapper", sizeof (td->td_name));
callout_init_mtx(&p->p_itcallout, &p->p_mtx, 0);
callout_init_mtx(&p->p_limco, &p->p_mtx, 0);
- callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
+ callout_init(&td->td_slpcallout, 1);
/* Create credentials. */
newcred = crget();
newcred->cr_ngroups = 1; /* group 0 */
newcred->cr_uidinfo = uifind(0);
newcred->cr_ruidinfo = uifind(0);
newcred->cr_prison = &prison0;
newcred->cr_loginclass = loginclass_find("default");
proc_set_cred_init(p, newcred);
#ifdef AUDIT
audit_cred_kproc0(newcred);
#endif
#ifdef MAC
mac_cred_create_swapper(newcred);
#endif
td->td_ucred = crhold(newcred);
/* Create sigacts. */
p->p_sigacts = sigacts_alloc();
/* Initialize signal state for process 0. */
siginit(&proc0);
/* Create the file descriptor table. */
p->p_fd = fdinit(NULL, false);
p->p_fdtol = NULL;
/* Create the limits structures. */
p->p_limit = lim_alloc();
for (i = 0; i < RLIM_NLIMITS; i++)
p->p_limit->pl_rlimit[i].rlim_cur =
p->p_limit->pl_rlimit[i].rlim_max = RLIM_INFINITY;
p->p_limit->pl_rlimit[RLIMIT_NOFILE].rlim_cur =
p->p_limit->pl_rlimit[RLIMIT_NOFILE].rlim_max = maxfiles;
p->p_limit->pl_rlimit[RLIMIT_NPROC].rlim_cur =
p->p_limit->pl_rlimit[RLIMIT_NPROC].rlim_max = maxproc;
p->p_limit->pl_rlimit[RLIMIT_DATA].rlim_cur = dfldsiz;
p->p_limit->pl_rlimit[RLIMIT_DATA].rlim_max = maxdsiz;
p->p_limit->pl_rlimit[RLIMIT_STACK].rlim_cur = dflssiz;
p->p_limit->pl_rlimit[RLIMIT_STACK].rlim_max = maxssiz;
/* Cast to avoid overflow on i386/PAE. */
pageablemem = ptoa((vm_paddr_t)vm_cnt.v_free_count);
p->p_limit->pl_rlimit[RLIMIT_RSS].rlim_cur =
p->p_limit->pl_rlimit[RLIMIT_RSS].rlim_max = pageablemem;
p->p_limit->pl_rlimit[RLIMIT_MEMLOCK].rlim_cur = pageablemem / 3;
p->p_limit->pl_rlimit[RLIMIT_MEMLOCK].rlim_max = pageablemem;
p->p_cpulimit = RLIM_INFINITY;
/* Initialize resource accounting structures. */
racct_create(&p->p_racct);
p->p_stats = pstats_alloc();
/* Allocate a prototype map so we have something to fork. */
p->p_vmspace = &vmspace0;
vmspace0.vm_refcnt = 1;
pmap_pinit0(vmspace_pmap(&vmspace0));
/*
* proc0 is not expected to enter usermode, so there is no special
* handling for sv_minuser here, like is done for exec_new_vmspace().
*/
vm_map_init(&vmspace0.vm_map, vmspace_pmap(&vmspace0),
p->p_sysent->sv_minuser, p->p_sysent->sv_maxuser);
/*
* Call the init and ctor for the new thread and proc. We wait
* to do this until all other structures are fairly sane.
*/
EVENTHANDLER_INVOKE(process_init, p);
EVENTHANDLER_INVOKE(thread_init, td);
EVENTHANDLER_INVOKE(process_ctor, p);
EVENTHANDLER_INVOKE(thread_ctor, td);
/*
* Charge root for one process.
*/
(void)chgproccnt(p->p_ucred->cr_ruidinfo, 1, 0);
PROC_LOCK(p);
racct_add_force(p, RACCT_NPROC, 1);
PROC_UNLOCK(p);
}
SYSINIT(p0init, SI_SUB_INTRINSIC, SI_ORDER_FIRST, proc0_init, NULL);
/* ARGSUSED*/
static void
proc0_post(void *dummy __unused)
{
struct timespec ts;
struct proc *p;
struct rusage ru;
struct thread *td;
/*
* Now we can look at the time, having had a chance to verify the
* time from the filesystem. Pretend that proc0 started now.
*/
sx_slock(&allproc_lock);
FOREACH_PROC_IN_SYSTEM(p) {
microuptime(&p->p_stats->p_start);
PROC_STATLOCK(p);
rufetch(p, &ru); /* Clears thread stats */
PROC_STATUNLOCK(p);
p->p_rux.rux_runtime = 0;
p->p_rux.rux_uticks = 0;
p->p_rux.rux_sticks = 0;
p->p_rux.rux_iticks = 0;
FOREACH_THREAD_IN_PROC(p, td) {
td->td_runtime = 0;
}
}
sx_sunlock(&allproc_lock);
PCPU_SET(switchtime, cpu_ticks());
PCPU_SET(switchticks, ticks);
/*
* Give the ``random'' number generator a thump.
*/
nanotime(&ts);
srandom(ts.tv_sec ^ ts.tv_nsec);
}
SYSINIT(p0post, SI_SUB_INTRINSIC_POST, SI_ORDER_FIRST, proc0_post, NULL);
static void
random_init(void *dummy __unused)
{
/*
* After CPU has been started we have some randomness on most
* platforms via get_cyclecount(). For platforms that don't
* we will reseed random(9) in proc0_post() as well.
*/
srandom(get_cyclecount());
}
SYSINIT(random, SI_SUB_RANDOM, SI_ORDER_FIRST, random_init, NULL);
/*
***************************************************************************
****
**** The following SYSINIT's and glue code should be moved to the
**** respective files on a per subsystem basis.
****
***************************************************************************
*/
/*
***************************************************************************
****
**** The following code probably belongs in another file, like
**** kern/init_init.c.
****
***************************************************************************
*/
/*
* List of paths to try when searching for "init".
*/
static char init_path[MAXPATHLEN] =
#ifdef INIT_PATH
__XSTRING(INIT_PATH);
#else
"/sbin/init:/sbin/oinit:/sbin/init.bak:/rescue/init";
#endif
SYSCTL_STRING(_kern, OID_AUTO, init_path, CTLFLAG_RD, init_path, 0,
"Path used to search the init process");
/*
* Shutdown timeout of init(8).
* Unused within kernel, but used to control init(8), hence do not remove.
*/
#ifndef INIT_SHUTDOWN_TIMEOUT
#define INIT_SHUTDOWN_TIMEOUT 120
#endif
static int init_shutdown_timeout = INIT_SHUTDOWN_TIMEOUT;
SYSCTL_INT(_kern, OID_AUTO, init_shutdown_timeout,
CTLFLAG_RW, &init_shutdown_timeout, 0, "Shutdown timeout of init(8). "
"Unused within kernel, but used to control init(8)");
/*
* Start the initial user process; try exec'ing each pathname in init_path.
* The program is invoked with one argument containing the boot flags.
*/
static void
start_init(void *dummy)
{
vm_offset_t addr;
struct execve_args args;
int options, error;
char *var, *path, *next, *s;
char *ucp, **uap, *arg0, *arg1;
struct thread *td;
struct proc *p;
mtx_lock(&Giant);
GIANT_REQUIRED;
td = curthread;
p = td->td_proc;
vfs_mountroot();
/* Wipe GELI passphrase from the environment. */
kern_unsetenv("kern.geom.eli.passphrase");
/*
* Need just enough stack to hold the faked-up "execve()" arguments.
*/
addr = p->p_sysent->sv_usrstack - PAGE_SIZE;
if (vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr, PAGE_SIZE, 0,
VMFS_NO_SPACE, VM_PROT_ALL, VM_PROT_ALL, 0) != 0)
panic("init: couldn't allocate argument space");
p->p_vmspace->vm_maxsaddr = (caddr_t)addr;
p->p_vmspace->vm_ssize = 1;
if ((var = kern_getenv("init_path")) != NULL) {
strlcpy(init_path, var, sizeof(init_path));
freeenv(var);
}
for (path = init_path; *path != '\0'; path = next) {
while (*path == ':')
path++;
if (*path == '\0')
break;
for (next = path; *next != '\0' && *next != ':'; next++)
/* nothing */ ;
if (bootverbose)
printf("start_init: trying %.*s\n", (int)(next - path),
path);
/*
* Move out the boot flag argument.
*/
options = 0;
ucp = (char *)p->p_sysent->sv_usrstack;
(void)subyte(--ucp, 0); /* trailing zero */
if (boothowto & RB_SINGLE) {
(void)subyte(--ucp, 's');
options = 1;
}
#ifdef notyet
if (boothowto & RB_FASTBOOT) {
(void)subyte(--ucp, 'f');
options = 1;
}
#endif
#ifdef BOOTCDROM
(void)subyte(--ucp, 'C');
options = 1;
#endif
if (options == 0)
(void)subyte(--ucp, '-');
(void)subyte(--ucp, '-'); /* leading hyphen */
arg1 = ucp;
/*
* Move out the file name (also arg 0).
*/
(void)subyte(--ucp, 0);
for (s = next - 1; s >= path; s--)
(void)subyte(--ucp, *s);
arg0 = ucp;
/*
* Move out the arg pointers.
*/
uap = (char **)((intptr_t)ucp & ~(sizeof(intptr_t)-1));
(void)suword((caddr_t)--uap, (long)0); /* terminator */
(void)suword((caddr_t)--uap, (long)(intptr_t)arg1);
(void)suword((caddr_t)--uap, (long)(intptr_t)arg0);
/*
* Point at the arguments.
*/
args.fname = arg0;
args.argv = uap;
args.envv = NULL;
/*
* Now try to exec the program. If can't for any reason
* other than it doesn't exist, complain.
*
* Otherwise, return via fork_trampoline() all the way
* to user mode as init!
*/
if ((error = sys_execve(td, &args)) == 0) {
mtx_unlock(&Giant);
return;
}
if (error != ENOENT)
printf("exec %.*s: error %d\n", (int)(next - path),
path, error);
}
printf("init: not found in path %s\n", init_path);
panic("no init");
}
/*
* Like kproc_create(), but runs in it's own address space.
* We do this early to reserve pid 1.
*
* Note special case - do not make it runnable yet. Other work
* in progress will change this more.
*/
static void
create_init(const void *udata __unused)
{
struct ucred *newcred, *oldcred;
int error;
error = fork1(&thread0, RFFDG | RFPROC | RFSTOPPED, 0, &initproc,
NULL, 0);
if (error)
panic("cannot fork init: %d\n", error);
KASSERT(initproc->p_pid == 1, ("create_init: initproc->p_pid != 1"));
/* divorce init's credentials from the kernel's */
newcred = crget();
sx_xlock(&proctree_lock);
PROC_LOCK(initproc);
initproc->p_flag |= P_SYSTEM | P_INMEM;
initproc->p_treeflag |= P_TREE_REAPER;
LIST_INSERT_HEAD(&initproc->p_reaplist, &proc0, p_reapsibling);
oldcred = initproc->p_ucred;
crcopy(newcred, oldcred);
#ifdef MAC
mac_cred_create_init(newcred);
#endif
#ifdef AUDIT
audit_cred_proc1(newcred);
#endif
proc_set_cred(initproc, newcred);
PROC_UNLOCK(initproc);
sx_xunlock(&proctree_lock);
crfree(oldcred);
cred_update_thread(FIRST_THREAD_IN_PROC(initproc));
cpu_set_fork_handler(FIRST_THREAD_IN_PROC(initproc), start_init, NULL);
}
SYSINIT(init, SI_SUB_CREATE_INIT, SI_ORDER_FIRST, create_init, NULL);
/*
* Make it runnable now.
*/
static void
kick_init(const void *udata __unused)
{
struct thread *td;
td = FIRST_THREAD_IN_PROC(initproc);
thread_lock(td);
TD_SET_CAN_RUN(td);
sched_add(td, SRQ_BORING);
thread_unlock(td);
}
SYSINIT(kickinit, SI_SUB_KTHREAD_INIT, SI_ORDER_MIDDLE, kick_init, NULL);
Index: head/sys/kern/kern_cons.c
===================================================================
--- head/sys/kern/kern_cons.c (revision 283290)
+++ head/sys/kern/kern_cons.c (revision 283291)
@@ -1,734 +1,734 @@
/*-
* Copyright (c) 1988 University of Utah.
* Copyright (c) 1991 The Regents of the University of California.
* Copyright (c) 1999 Michael Smith
* Copyright (c) 2005 Pawel Jakub Dawidek <pjd@FreeBSD.org>
*
* All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* the Systems Programming Group of the University of Utah Computer
* Science Department.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* from: @(#)cons.c 7.2 (Berkeley) 5/9/91
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include "opt_syscons.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/conf.h>
#include <sys/cons.h>
#include <sys/fcntl.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/msgbuf.h>
#include <sys/namei.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/reboot.h>
#include <sys/sysctl.h>
#include <sys/sbuf.h>
#include <sys/tty.h>
#include <sys/uio.h>
#include <sys/vnode.h>
#include <ddb/ddb.h>
#include <machine/cpu.h>
#include <machine/clock.h>
static MALLOC_DEFINE(M_TTYCONS, "tty console", "tty console handling");
struct cn_device {
STAILQ_ENTRY(cn_device) cnd_next;
struct consdev *cnd_cn;
};
#define CNDEVPATHMAX 32
#define CNDEVTAB_SIZE 4
static struct cn_device cn_devtab[CNDEVTAB_SIZE];
static STAILQ_HEAD(, cn_device) cn_devlist =
STAILQ_HEAD_INITIALIZER(cn_devlist);
int cons_avail_mask = 0; /* Bit mask. Each registered low level console
* which is currently unavailable for inpit
* (i.e., if it is in graphics mode) will have
* this bit cleared.
*/
static int cn_mute;
static char *consbuf; /* buffer used by `consmsgbuf' */
static struct callout conscallout; /* callout for outputting to constty */
struct msgbuf consmsgbuf; /* message buffer for console tty */
static u_char console_pausing; /* pause after each line during probe */
static char *console_pausestr=
"<pause; press any key to proceed to next line or '.' to end pause mode>";
struct tty *constty; /* pointer to console "window" tty */
static struct mtx cnputs_mtx; /* Mutex for cnputs(). */
static int use_cnputs_mtx = 0; /* != 0 if cnputs_mtx locking reqd. */
static void constty_timeout(void *arg);
static struct consdev cons_consdev;
DATA_SET(cons_set, cons_consdev);
SET_DECLARE(cons_set, struct consdev);
void
cninit(void)
{
struct consdev *best_cn, *cn, **list;
/*
* Check if we should mute the console (for security reasons perhaps)
* It can be changes dynamically using sysctl kern.consmute
* once we are up and going.
*
*/
cn_mute = ((boothowto & (RB_MUTE
|RB_SINGLE
|RB_VERBOSE
|RB_ASKNAME)) == RB_MUTE);
/*
* Find the first console with the highest priority.
*/
best_cn = NULL;
SET_FOREACH(list, cons_set) {
cn = *list;
cnremove(cn);
/* Skip cons_consdev. */
if (cn->cn_ops == NULL)
continue;
cn->cn_ops->cn_probe(cn);
if (cn->cn_pri == CN_DEAD)
continue;
if (best_cn == NULL || cn->cn_pri > best_cn->cn_pri)
best_cn = cn;
if (boothowto & RB_MULTIPLE) {
/*
* Initialize console, and attach to it.
*/
cn->cn_ops->cn_init(cn);
cnadd(cn);
}
}
if (best_cn == NULL)
return;
if ((boothowto & RB_MULTIPLE) == 0) {
best_cn->cn_ops->cn_init(best_cn);
cnadd(best_cn);
}
if (boothowto & RB_PAUSE)
console_pausing = 1;
/*
* Make the best console the preferred console.
*/
cnselect(best_cn);
#ifdef EARLY_PRINTF
/*
* Release early console.
*/
early_putc = NULL;
#endif
}
void
cninit_finish()
{
console_pausing = 0;
}
/* add a new physical console to back the virtual console */
int
cnadd(struct consdev *cn)
{
struct cn_device *cnd;
int i;
STAILQ_FOREACH(cnd, &cn_devlist, cnd_next)
if (cnd->cnd_cn == cn)
return (0);
for (i = 0; i < CNDEVTAB_SIZE; i++) {
cnd = &cn_devtab[i];
if (cnd->cnd_cn == NULL)
break;
}
if (cnd->cnd_cn != NULL)
return (ENOMEM);
cnd->cnd_cn = cn;
if (cn->cn_name[0] == '\0') {
/* XXX: it is unclear if/where this print might output */
printf("WARNING: console at %p has no name\n", cn);
}
STAILQ_INSERT_TAIL(&cn_devlist, cnd, cnd_next);
if (STAILQ_FIRST(&cn_devlist) == cnd)
ttyconsdev_select(cnd->cnd_cn->cn_name);
/* Add device to the active mask. */
cnavailable(cn, (cn->cn_flags & CN_FLAG_NOAVAIL) == 0);
return (0);
}
void
cnremove(struct consdev *cn)
{
struct cn_device *cnd;
int i;
STAILQ_FOREACH(cnd, &cn_devlist, cnd_next) {
if (cnd->cnd_cn != cn)
continue;
if (STAILQ_FIRST(&cn_devlist) == cnd)
ttyconsdev_select(NULL);
STAILQ_REMOVE(&cn_devlist, cnd, cn_device, cnd_next);
cnd->cnd_cn = NULL;
/* Remove this device from available mask. */
for (i = 0; i < CNDEVTAB_SIZE; i++)
if (cnd == &cn_devtab[i]) {
cons_avail_mask &= ~(1 << i);
break;
}
#if 0
/*
* XXX
* syscons gets really confused if console resources are
* freed after the system has initialized.
*/
if (cn->cn_term != NULL)
cn->cn_ops->cn_term(cn);
#endif
return;
}
}
void
cnselect(struct consdev *cn)
{
struct cn_device *cnd;
STAILQ_FOREACH(cnd, &cn_devlist, cnd_next) {
if (cnd->cnd_cn != cn)
continue;
if (cnd == STAILQ_FIRST(&cn_devlist))
return;
STAILQ_REMOVE(&cn_devlist, cnd, cn_device, cnd_next);
STAILQ_INSERT_HEAD(&cn_devlist, cnd, cnd_next);
ttyconsdev_select(cnd->cnd_cn->cn_name);
return;
}
}
void
cnavailable(struct consdev *cn, int available)
{
int i;
for (i = 0; i < CNDEVTAB_SIZE; i++) {
if (cn_devtab[i].cnd_cn == cn)
break;
}
if (available) {
if (i < CNDEVTAB_SIZE)
cons_avail_mask |= (1 << i);
cn->cn_flags &= ~CN_FLAG_NOAVAIL;
} else {
if (i < CNDEVTAB_SIZE)
cons_avail_mask &= ~(1 << i);
cn->cn_flags |= CN_FLAG_NOAVAIL;
}
}
int
cnunavailable(void)
{
return (cons_avail_mask == 0);
}
/*
* sysctl_kern_console() provides output parseable in conscontrol(1).
*/
static int
sysctl_kern_console(SYSCTL_HANDLER_ARGS)
{
struct cn_device *cnd;
struct consdev *cp, **list;
char *p;
int delete, error;
struct sbuf *sb;
sb = sbuf_new(NULL, NULL, CNDEVPATHMAX * 2, SBUF_AUTOEXTEND |
SBUF_INCLUDENUL);
if (sb == NULL)
return (ENOMEM);
sbuf_clear(sb);
STAILQ_FOREACH(cnd, &cn_devlist, cnd_next)
sbuf_printf(sb, "%s,", cnd->cnd_cn->cn_name);
sbuf_printf(sb, "/");
SET_FOREACH(list, cons_set) {
cp = *list;
if (cp->cn_name[0] != '\0')
sbuf_printf(sb, "%s,", cp->cn_name);
}
sbuf_finish(sb);
error = sysctl_handle_string(oidp, sbuf_data(sb), sbuf_len(sb), req);
if (error == 0 && req->newptr != NULL) {
p = sbuf_data(sb);
error = ENXIO;
delete = 0;
if (*p == '-') {
delete = 1;
p++;
}
SET_FOREACH(list, cons_set) {
cp = *list;
if (strcmp(p, cp->cn_name) != 0)
continue;
if (delete) {
cnremove(cp);
error = 0;
} else {
error = cnadd(cp);
if (error == 0)
cnselect(cp);
}
break;
}
}
sbuf_delete(sb);
return (error);
}
SYSCTL_PROC(_kern, OID_AUTO, console, CTLTYPE_STRING|CTLFLAG_RW,
0, 0, sysctl_kern_console, "A", "Console device control");
/*
* User has changed the state of the console muting.
* This may require us to open or close the device in question.
*/
static int
sysctl_kern_consmute(SYSCTL_HANDLER_ARGS)
{
int error;
error = sysctl_handle_int(oidp, &cn_mute, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
return (error);
}
SYSCTL_PROC(_kern, OID_AUTO, consmute, CTLTYPE_INT|CTLFLAG_RW,
0, sizeof(cn_mute), sysctl_kern_consmute, "I",
"State of the console muting");
void
cngrab()
{
struct cn_device *cnd;
struct consdev *cn;
STAILQ_FOREACH(cnd, &cn_devlist, cnd_next) {
cn = cnd->cnd_cn;
if (!kdb_active || !(cn->cn_flags & CN_FLAG_NODEBUG))
cn->cn_ops->cn_grab(cn);
}
}
void
cnungrab()
{
struct cn_device *cnd;
struct consdev *cn;
STAILQ_FOREACH(cnd, &cn_devlist, cnd_next) {
cn = cnd->cnd_cn;
if (!kdb_active || !(cn->cn_flags & CN_FLAG_NODEBUG))
cn->cn_ops->cn_ungrab(cn);
}
}
/*
* Low level console routines.
*/
int
cngetc(void)
{
int c;
if (cn_mute)
return (-1);
while ((c = cncheckc()) == -1)
cpu_spinwait();
if (c == '\r')
c = '\n'; /* console input is always ICRNL */
return (c);
}
int
cncheckc(void)
{
struct cn_device *cnd;
struct consdev *cn;
int c;
if (cn_mute)
return (-1);
STAILQ_FOREACH(cnd, &cn_devlist, cnd_next) {
cn = cnd->cnd_cn;
if (!kdb_active || !(cn->cn_flags & CN_FLAG_NODEBUG)) {
c = cn->cn_ops->cn_getc(cn);
if (c != -1)
return (c);
}
}
return (-1);
}
void
cngets(char *cp, size_t size, int visible)
{
char *lp, *end;
int c;
cngrab();
lp = cp;
end = cp + size - 1;
for (;;) {
c = cngetc() & 0177;
switch (c) {
case '\n':
case '\r':
cnputc(c);
*lp = '\0';
cnungrab();
return;
case '\b':
case '\177':
if (lp > cp) {
if (visible)
cnputs("\b \b");
lp--;
}
continue;
case '\0':
continue;
default:
if (lp < end) {
switch (visible) {
case GETS_NOECHO:
break;
case GETS_ECHOPASS:
cnputc('*');
break;
default:
cnputc(c);
break;
}
*lp++ = c;
}
}
}
}
void
cnputc(int c)
{
struct cn_device *cnd;
struct consdev *cn;
char *cp;
#ifdef EARLY_PRINTF
if (early_putc != NULL) {
if (c == '\n')
early_putc('\r');
early_putc(c);
return;
}
#endif
if (cn_mute || c == '\0')
return;
STAILQ_FOREACH(cnd, &cn_devlist, cnd_next) {
cn = cnd->cnd_cn;
if (!kdb_active || !(cn->cn_flags & CN_FLAG_NODEBUG)) {
if (c == '\n')
cn->cn_ops->cn_putc(cn, '\r');
cn->cn_ops->cn_putc(cn, c);
}
}
if (console_pausing && c == '\n' && !kdb_active) {
for (cp = console_pausestr; *cp != '\0'; cp++)
cnputc(*cp);
cngrab();
if (cngetc() == '.')
console_pausing = 0;
cnungrab();
cnputc('\r');
for (cp = console_pausestr; *cp != '\0'; cp++)
cnputc(' ');
cnputc('\r');
}
}
void
cnputs(char *p)
{
int c;
int unlock_reqd = 0;
if (use_cnputs_mtx) {
/*
* NOTE: Debug prints and/or witness printouts in
* console driver clients can cause the "cnputs_mtx"
* mutex to recurse. Simply return if that happens.
*/
if (mtx_owned(&cnputs_mtx))
return;
mtx_lock_spin(&cnputs_mtx);
unlock_reqd = 1;
}
while ((c = *p++) != '\0')
cnputc(c);
if (unlock_reqd)
mtx_unlock_spin(&cnputs_mtx);
}
static int consmsgbuf_size = 8192;
SYSCTL_INT(_kern, OID_AUTO, consmsgbuf_size, CTLFLAG_RW, &consmsgbuf_size, 0,
"Console tty buffer size");
/*
* Redirect console output to a tty.
*/
void
constty_set(struct tty *tp)
{
int size;
KASSERT(tp != NULL, ("constty_set: NULL tp"));
if (consbuf == NULL) {
size = consmsgbuf_size;
consbuf = malloc(size, M_TTYCONS, M_WAITOK);
msgbuf_init(&consmsgbuf, consbuf, size);
callout_init(&conscallout, 0);
}
constty = tp;
constty_timeout(NULL);
}
/*
* Disable console redirection to a tty.
*/
void
constty_clear(void)
{
int c;
constty = NULL;
if (consbuf == NULL)
return;
callout_stop(&conscallout);
while ((c = msgbuf_getchar(&consmsgbuf)) != -1)
cnputc(c);
free(consbuf, M_TTYCONS);
consbuf = NULL;
}
/* Times per second to check for pending console tty messages. */
static int constty_wakeups_per_second = 5;
SYSCTL_INT(_kern, OID_AUTO, constty_wakeups_per_second, CTLFLAG_RW,
&constty_wakeups_per_second, 0,
"Times per second to check for pending console tty messages");
static void
constty_timeout(void *arg)
{
int c;
if (constty != NULL) {
tty_lock(constty);
while ((c = msgbuf_getchar(&consmsgbuf)) != -1) {
if (tty_putchar(constty, c) < 0) {
tty_unlock(constty);
constty = NULL;
break;
}
}
if (constty != NULL)
tty_unlock(constty);
}
if (constty != NULL) {
callout_reset(&conscallout, hz / constty_wakeups_per_second,
constty_timeout, NULL);
} else {
/* Deallocate the constty buffer memory. */
constty_clear();
}
}
static void
cn_drvinit(void *unused)
{
mtx_init(&cnputs_mtx, "cnputs_mtx", NULL, MTX_SPIN | MTX_NOWITNESS);
use_cnputs_mtx = 1;
}
SYSINIT(cndev, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, cn_drvinit, NULL);
/*
* Sysbeep(), if we have hardware for it
*/
#ifdef HAS_TIMER_SPKR
static int beeping;
static struct callout beeping_timer;
static void
sysbeepstop(void *chan)
{
timer_spkr_release();
beeping = 0;
}
int
sysbeep(int pitch, int period)
{
if (timer_spkr_acquire()) {
if (!beeping) {
/* Something else owns it. */
return (EBUSY);
}
}
timer_spkr_setfreq(pitch);
if (!beeping) {
beeping = period;
callout_reset(&beeping_timer, period, sysbeepstop, NULL);
}
return (0);
}
static void
sysbeep_init(void *unused)
{
- callout_init(&beeping_timer, CALLOUT_MPSAFE);
+ callout_init(&beeping_timer, 1);
}
SYSINIT(sysbeep, SI_SUB_SOFTINTR, SI_ORDER_ANY, sysbeep_init, NULL);
#else
/*
* No hardware, no sound
*/
int
sysbeep(int pitch __unused, int period __unused)
{
return (ENODEV);
}
#endif
/*
* Temporary support for sc(4) to vt(4) transition.
*/
static unsigned vty_prefer;
static char vty_name[16];
SYSCTL_STRING(_kern, OID_AUTO, vty, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, vty_name,
0, "Console vty driver");
int
vty_enabled(unsigned vty)
{
static unsigned vty_selected = 0;
if (vty_selected == 0) {
TUNABLE_STR_FETCH("kern.vty", vty_name, sizeof(vty_name));
do {
#if defined(DEV_SC)
if (strcmp(vty_name, "sc") == 0) {
vty_selected = VTY_SC;
break;
}
#endif
#if defined(DEV_VT)
if (strcmp(vty_name, "vt") == 0) {
vty_selected = VTY_VT;
break;
}
#endif
if (vty_prefer != 0) {
vty_selected = vty_prefer;
break;
}
#if defined(DEV_VT)
vty_selected = VTY_VT;
#elif defined(DEV_SC)
vty_selected = VTY_SC;
#endif
} while (0);
if (vty_selected == VTY_VT)
strcpy(vty_name, "vt");
else if (vty_selected == VTY_SC)
strcpy(vty_name, "sc");
}
return ((vty_selected & vty) != 0);
}
void
vty_set_preferred(unsigned vty)
{
vty_prefer = vty;
#if !defined(DEV_SC)
vty_prefer &= ~VTY_SC;
#endif
#if !defined(DEV_VT)
vty_prefer &= ~VTY_VT;
#endif
}
Index: head/sys/kern/kern_event.c
===================================================================
--- head/sys/kern/kern_event.c (revision 283290)
+++ head/sys/kern/kern_event.c (revision 283291)
@@ -1,2367 +1,2367 @@
/*-
* Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
* Copyright 2004 John-Mark Gurney <jmg@FreeBSD.org>
* Copyright (c) 2009 Apple, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ktrace.h"
#include "opt_kqueue.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/capsicum.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/rwlock.h>
#include <sys/proc.h>
#include <sys/malloc.h>
#include <sys/unistd.h>
#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/filio.h>
#include <sys/fcntl.h>
#include <sys/kthread.h>
#include <sys/selinfo.h>
#include <sys/stdatomic.h>
#include <sys/queue.h>
#include <sys/event.h>
#include <sys/eventvar.h>
#include <sys/poll.h>
#include <sys/protosw.h>
#include <sys/resourcevar.h>
#include <sys/sigio.h>
#include <sys/signalvar.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/syscallsubr.h>
#include <sys/taskqueue.h>
#include <sys/uio.h>
#include <sys/user.h>
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
#include <vm/uma.h>
static MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system");
/*
* This lock is used if multiple kq locks are required. This possibly
* should be made into a per proc lock.
*/
static struct mtx kq_global;
MTX_SYSINIT(kq_global, &kq_global, "kqueue order", MTX_DEF);
#define KQ_GLOBAL_LOCK(lck, haslck) do { \
if (!haslck) \
mtx_lock(lck); \
haslck = 1; \
} while (0)
#define KQ_GLOBAL_UNLOCK(lck, haslck) do { \
if (haslck) \
mtx_unlock(lck); \
haslck = 0; \
} while (0)
TASKQUEUE_DEFINE_THREAD(kqueue);
static int kevent_copyout(void *arg, struct kevent *kevp, int count);
static int kevent_copyin(void *arg, struct kevent *kevp, int count);
static int kqueue_register(struct kqueue *kq, struct kevent *kev,
struct thread *td, int waitok);
static int kqueue_acquire(struct file *fp, struct kqueue **kqp);
static void kqueue_release(struct kqueue *kq, int locked);
static int kqueue_expand(struct kqueue *kq, struct filterops *fops,
uintptr_t ident, int waitok);
static void kqueue_task(void *arg, int pending);
static int kqueue_scan(struct kqueue *kq, int maxevents,
struct kevent_copyops *k_ops,
const struct timespec *timeout,
struct kevent *keva, struct thread *td);
static void kqueue_wakeup(struct kqueue *kq);
static struct filterops *kqueue_fo_find(int filt);
static void kqueue_fo_release(int filt);
static fo_ioctl_t kqueue_ioctl;
static fo_poll_t kqueue_poll;
static fo_kqfilter_t kqueue_kqfilter;
static fo_stat_t kqueue_stat;
static fo_close_t kqueue_close;
static fo_fill_kinfo_t kqueue_fill_kinfo;
static struct fileops kqueueops = {
.fo_read = invfo_rdwr,
.fo_write = invfo_rdwr,
.fo_truncate = invfo_truncate,
.fo_ioctl = kqueue_ioctl,
.fo_poll = kqueue_poll,
.fo_kqfilter = kqueue_kqfilter,
.fo_stat = kqueue_stat,
.fo_close = kqueue_close,
.fo_chmod = invfo_chmod,
.fo_chown = invfo_chown,
.fo_sendfile = invfo_sendfile,
.fo_fill_kinfo = kqueue_fill_kinfo,
};
static int knote_attach(struct knote *kn, struct kqueue *kq);
static void knote_drop(struct knote *kn, struct thread *td);
static void knote_enqueue(struct knote *kn);
static void knote_dequeue(struct knote *kn);
static void knote_init(void);
static struct knote *knote_alloc(int waitok);
static void knote_free(struct knote *kn);
static void filt_kqdetach(struct knote *kn);
static int filt_kqueue(struct knote *kn, long hint);
static int filt_procattach(struct knote *kn);
static void filt_procdetach(struct knote *kn);
static int filt_proc(struct knote *kn, long hint);
static int filt_fileattach(struct knote *kn);
static void filt_timerexpire(void *knx);
static int filt_timerattach(struct knote *kn);
static void filt_timerdetach(struct knote *kn);
static int filt_timer(struct knote *kn, long hint);
static int filt_userattach(struct knote *kn);
static void filt_userdetach(struct knote *kn);
static int filt_user(struct knote *kn, long hint);
static void filt_usertouch(struct knote *kn, struct kevent *kev,
u_long type);
static struct filterops file_filtops = {
.f_isfd = 1,
.f_attach = filt_fileattach,
};
static struct filterops kqread_filtops = {
.f_isfd = 1,
.f_detach = filt_kqdetach,
.f_event = filt_kqueue,
};
/* XXX - move to kern_proc.c? */
static struct filterops proc_filtops = {
.f_isfd = 0,
.f_attach = filt_procattach,
.f_detach = filt_procdetach,
.f_event = filt_proc,
};
static struct filterops timer_filtops = {
.f_isfd = 0,
.f_attach = filt_timerattach,
.f_detach = filt_timerdetach,
.f_event = filt_timer,
};
static struct filterops user_filtops = {
.f_attach = filt_userattach,
.f_detach = filt_userdetach,
.f_event = filt_user,
.f_touch = filt_usertouch,
};
static uma_zone_t knote_zone;
static atomic_uint kq_ncallouts = ATOMIC_VAR_INIT(0);
static unsigned int kq_calloutmax = 4 * 1024;
SYSCTL_UINT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW,
&kq_calloutmax, 0, "Maximum number of callouts allocated for kqueue");
/* XXX - ensure not KN_INFLUX?? */
#define KNOTE_ACTIVATE(kn, islock) do { \
if ((islock)) \
mtx_assert(&(kn)->kn_kq->kq_lock, MA_OWNED); \
else \
KQ_LOCK((kn)->kn_kq); \
(kn)->kn_status |= KN_ACTIVE; \
if (((kn)->kn_status & (KN_QUEUED | KN_DISABLED)) == 0) \
knote_enqueue((kn)); \
if (!(islock)) \
KQ_UNLOCK((kn)->kn_kq); \
} while(0)
#define KQ_LOCK(kq) do { \
mtx_lock(&(kq)->kq_lock); \
} while (0)
#define KQ_FLUX_WAKEUP(kq) do { \
if (((kq)->kq_state & KQ_FLUXWAIT) == KQ_FLUXWAIT) { \
(kq)->kq_state &= ~KQ_FLUXWAIT; \
wakeup((kq)); \
} \
} while (0)
#define KQ_UNLOCK_FLUX(kq) do { \
KQ_FLUX_WAKEUP(kq); \
mtx_unlock(&(kq)->kq_lock); \
} while (0)
#define KQ_UNLOCK(kq) do { \
mtx_unlock(&(kq)->kq_lock); \
} while (0)
#define KQ_OWNED(kq) do { \
mtx_assert(&(kq)->kq_lock, MA_OWNED); \
} while (0)
#define KQ_NOTOWNED(kq) do { \
mtx_assert(&(kq)->kq_lock, MA_NOTOWNED); \
} while (0)
#define KN_LIST_LOCK(kn) do { \
if (kn->kn_knlist != NULL) \
kn->kn_knlist->kl_lock(kn->kn_knlist->kl_lockarg); \
} while (0)
#define KN_LIST_UNLOCK(kn) do { \
if (kn->kn_knlist != NULL) \
kn->kn_knlist->kl_unlock(kn->kn_knlist->kl_lockarg); \
} while (0)
#define KNL_ASSERT_LOCK(knl, islocked) do { \
if (islocked) \
KNL_ASSERT_LOCKED(knl); \
else \
KNL_ASSERT_UNLOCKED(knl); \
} while (0)
#ifdef INVARIANTS
#define KNL_ASSERT_LOCKED(knl) do { \
knl->kl_assert_locked((knl)->kl_lockarg); \
} while (0)
#define KNL_ASSERT_UNLOCKED(knl) do { \
knl->kl_assert_unlocked((knl)->kl_lockarg); \
} while (0)
#else /* !INVARIANTS */
#define KNL_ASSERT_LOCKED(knl) do {} while(0)
#define KNL_ASSERT_UNLOCKED(knl) do {} while (0)
#endif /* INVARIANTS */
#ifndef KN_HASHSIZE
#define KN_HASHSIZE 64 /* XXX should be tunable */
#endif
#define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
static int
filt_nullattach(struct knote *kn)
{
return (ENXIO);
};
struct filterops null_filtops = {
.f_isfd = 0,
.f_attach = filt_nullattach,
};
/* XXX - make SYSINIT to add these, and move into respective modules. */
extern struct filterops sig_filtops;
extern struct filterops fs_filtops;
/*
* Table for for all system-defined filters.
*/
static struct mtx filterops_lock;
MTX_SYSINIT(kqueue_filterops, &filterops_lock, "protect sysfilt_ops",
MTX_DEF);
static struct {
struct filterops *for_fop;
int for_nolock;
int for_refcnt;
} sysfilt_ops[EVFILT_SYSCOUNT] = {
{ &file_filtops, 1 }, /* EVFILT_READ */
{ &file_filtops, 1 }, /* EVFILT_WRITE */
{ &null_filtops }, /* EVFILT_AIO */
{ &file_filtops, 1 }, /* EVFILT_VNODE */
{ &proc_filtops, 1 }, /* EVFILT_PROC */
{ &sig_filtops, 1 }, /* EVFILT_SIGNAL */
{ &timer_filtops, 1 }, /* EVFILT_TIMER */
{ &file_filtops, 1 }, /* EVFILT_PROCDESC */
{ &fs_filtops, 1 }, /* EVFILT_FS */
{ &null_filtops }, /* EVFILT_LIO */
{ &user_filtops, 1 }, /* EVFILT_USER */
{ &null_filtops }, /* EVFILT_SENDFILE */
};
/*
* Simple redirection for all cdevsw style objects to call their fo_kqfilter
* method.
*/
static int
filt_fileattach(struct knote *kn)
{
return (fo_kqfilter(kn->kn_fp, kn));
}
/*ARGSUSED*/
static int
kqueue_kqfilter(struct file *fp, struct knote *kn)
{
struct kqueue *kq = kn->kn_fp->f_data;
if (kn->kn_filter != EVFILT_READ)
return (EINVAL);
kn->kn_status |= KN_KQUEUE;
kn->kn_fop = &kqread_filtops;
knlist_add(&kq->kq_sel.si_note, kn, 0);
return (0);
}
static void
filt_kqdetach(struct knote *kn)
{
struct kqueue *kq = kn->kn_fp->f_data;
knlist_remove(&kq->kq_sel.si_note, kn, 0);
}
/*ARGSUSED*/
static int
filt_kqueue(struct knote *kn, long hint)
{
struct kqueue *kq = kn->kn_fp->f_data;
kn->kn_data = kq->kq_count;
return (kn->kn_data > 0);
}
/* XXX - move to kern_proc.c? */
static int
filt_procattach(struct knote *kn)
{
struct proc *p;
int immediate;
int error;
immediate = 0;
p = pfind(kn->kn_id);
if (p == NULL && (kn->kn_sfflags & NOTE_EXIT)) {
p = zpfind(kn->kn_id);
immediate = 1;
} else if (p != NULL && (p->p_flag & P_WEXIT)) {
immediate = 1;
}
if (p == NULL)
return (ESRCH);
if ((error = p_cansee(curthread, p))) {
PROC_UNLOCK(p);
return (error);
}
kn->kn_ptr.p_proc = p;
kn->kn_flags |= EV_CLEAR; /* automatically set */
/*
* internal flag indicating registration done by kernel
*/
if (kn->kn_flags & EV_FLAG1) {
kn->kn_data = kn->kn_sdata; /* ppid */
kn->kn_fflags = NOTE_CHILD;
kn->kn_flags &= ~EV_FLAG1;
}
if (immediate == 0)
knlist_add(&p->p_klist, kn, 1);
/*
* Immediately activate any exit notes if the target process is a
* zombie. This is necessary to handle the case where the target
* process, e.g. a child, dies before the kevent is registered.
*/
if (immediate && filt_proc(kn, NOTE_EXIT))
KNOTE_ACTIVATE(kn, 0);
PROC_UNLOCK(p);
return (0);
}
/*
* The knote may be attached to a different process, which may exit,
* leaving nothing for the knote to be attached to. So when the process
* exits, the knote is marked as DETACHED and also flagged as ONESHOT so
* it will be deleted when read out. However, as part of the knote deletion,
* this routine is called, so a check is needed to avoid actually performing
* a detach, because the original process does not exist any more.
*/
/* XXX - move to kern_proc.c? */
static void
filt_procdetach(struct knote *kn)
{
struct proc *p;
p = kn->kn_ptr.p_proc;
knlist_remove(&p->p_klist, kn, 0);
kn->kn_ptr.p_proc = NULL;
}
/* XXX - move to kern_proc.c? */
static int
filt_proc(struct knote *kn, long hint)
{
struct proc *p;
u_int event;
p = kn->kn_ptr.p_proc;
/* Mask off extra data. */
event = (u_int)hint & NOTE_PCTRLMASK;
/* If the user is interested in this event, record it. */
if (kn->kn_sfflags & event)
kn->kn_fflags |= event;
/* Process is gone, so flag the event as finished. */
if (event == NOTE_EXIT) {
if (!(kn->kn_status & KN_DETACHED))
knlist_remove_inevent(&p->p_klist, kn);
kn->kn_flags |= EV_EOF | EV_ONESHOT;
kn->kn_ptr.p_proc = NULL;
if (kn->kn_fflags & NOTE_EXIT)
kn->kn_data = p->p_xstat;
if (kn->kn_fflags == 0)
kn->kn_flags |= EV_DROP;
return (1);
}
return (kn->kn_fflags != 0);
}
/*
* Called when the process forked. It mostly does the same as the
* knote(), activating all knotes registered to be activated when the
* process forked. Additionally, for each knote attached to the
* parent, check whether user wants to track the new process. If so
* attach a new knote to it, and immediately report an event with the
* child's pid.
*/
void
knote_fork(struct knlist *list, int pid)
{
struct kqueue *kq;
struct knote *kn;
struct kevent kev;
int error;
if (list == NULL)
return;
list->kl_lock(list->kl_lockarg);
SLIST_FOREACH(kn, &list->kl_list, kn_selnext) {
/*
* XXX - Why do we skip the kn if it is _INFLUX? Does this
* mean we will not properly wake up some notes?
*/
if ((kn->kn_status & KN_INFLUX) == KN_INFLUX)
continue;
kq = kn->kn_kq;
KQ_LOCK(kq);
if ((kn->kn_status & (KN_INFLUX | KN_SCAN)) == KN_INFLUX) {
KQ_UNLOCK(kq);
continue;
}
/*
* The same as knote(), activate the event.
*/
if ((kn->kn_sfflags & NOTE_TRACK) == 0) {
kn->kn_status |= KN_HASKQLOCK;
if (kn->kn_fop->f_event(kn, NOTE_FORK))
KNOTE_ACTIVATE(kn, 1);
kn->kn_status &= ~KN_HASKQLOCK;
KQ_UNLOCK(kq);
continue;
}
/*
* The NOTE_TRACK case. In addition to the activation
* of the event, we need to register new event to
* track the child. Drop the locks in preparation for
* the call to kqueue_register().
*/
kn->kn_status |= KN_INFLUX;
KQ_UNLOCK(kq);
list->kl_unlock(list->kl_lockarg);
/*
* Activate existing knote and register a knote with
* new process.
*/
kev.ident = pid;
kev.filter = kn->kn_filter;
kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1;
kev.fflags = kn->kn_sfflags;
kev.data = kn->kn_id; /* parent */
kev.udata = kn->kn_kevent.udata;/* preserve udata */
error = kqueue_register(kq, &kev, NULL, 0);
if (error)
kn->kn_fflags |= NOTE_TRACKERR;
if (kn->kn_fop->f_event(kn, NOTE_FORK))
KNOTE_ACTIVATE(kn, 0);
KQ_LOCK(kq);
kn->kn_status &= ~KN_INFLUX;
KQ_UNLOCK_FLUX(kq);
list->kl_lock(list->kl_lockarg);
}
list->kl_unlock(list->kl_lockarg);
}
/*
* XXX: EVFILT_TIMER should perhaps live in kern_time.c beside the
* interval timer support code.
*/
#define NOTE_TIMER_PRECMASK (NOTE_SECONDS|NOTE_MSECONDS|NOTE_USECONDS| \
NOTE_NSECONDS)
static __inline sbintime_t
timer2sbintime(intptr_t data, int flags)
{
sbintime_t modifier;
switch (flags & NOTE_TIMER_PRECMASK) {
case NOTE_SECONDS:
modifier = SBT_1S;
break;
case NOTE_MSECONDS: /* FALLTHROUGH */
case 0:
modifier = SBT_1MS;
break;
case NOTE_USECONDS:
modifier = SBT_1US;
break;
case NOTE_NSECONDS:
modifier = SBT_1NS;
break;
default:
return (-1);
}
#ifdef __LP64__
if (data > SBT_MAX / modifier)
return (SBT_MAX);
#endif
return (modifier * data);
}
static void
filt_timerexpire(void *knx)
{
struct callout *calloutp;
struct knote *kn;
kn = knx;
kn->kn_data++;
KNOTE_ACTIVATE(kn, 0); /* XXX - handle locking */
if ((kn->kn_flags & EV_ONESHOT) != EV_ONESHOT) {
calloutp = (struct callout *)kn->kn_hook;
*kn->kn_ptr.p_nexttime += timer2sbintime(kn->kn_sdata,
kn->kn_sfflags);
callout_reset_sbt_on(calloutp, *kn->kn_ptr.p_nexttime, 0,
filt_timerexpire, kn, PCPU_GET(cpuid), C_ABSOLUTE);
}
}
/*
* data contains amount of time to sleep
*/
static int
filt_timerattach(struct knote *kn)
{
struct callout *calloutp;
sbintime_t to;
unsigned int ncallouts;
if ((intptr_t)kn->kn_sdata < 0)
return (EINVAL);
if ((intptr_t)kn->kn_sdata == 0 && (kn->kn_flags & EV_ONESHOT) == 0)
kn->kn_sdata = 1;
/* Only precision unit are supported in flags so far */
if (kn->kn_sfflags & ~NOTE_TIMER_PRECMASK)
return (EINVAL);
to = timer2sbintime(kn->kn_sdata, kn->kn_sfflags);
if (to < 0)
return (EINVAL);
ncallouts = atomic_load_explicit(&kq_ncallouts, memory_order_relaxed);
do {
if (ncallouts >= kq_calloutmax)
return (ENOMEM);
} while (!atomic_compare_exchange_weak_explicit(&kq_ncallouts,
&ncallouts, ncallouts + 1, memory_order_relaxed,
memory_order_relaxed));
kn->kn_flags |= EV_CLEAR; /* automatically set */
kn->kn_status &= ~KN_DETACHED; /* knlist_add clears it */
kn->kn_ptr.p_nexttime = malloc(sizeof(sbintime_t), M_KQUEUE, M_WAITOK);
calloutp = malloc(sizeof(*calloutp), M_KQUEUE, M_WAITOK);
- callout_init(calloutp, CALLOUT_MPSAFE);
+ callout_init(calloutp, 1);
kn->kn_hook = calloutp;
*kn->kn_ptr.p_nexttime = to + sbinuptime();
callout_reset_sbt_on(calloutp, *kn->kn_ptr.p_nexttime, 0,
filt_timerexpire, kn, PCPU_GET(cpuid), C_ABSOLUTE);
return (0);
}
static void
filt_timerdetach(struct knote *kn)
{
struct callout *calloutp;
unsigned int old;
calloutp = (struct callout *)kn->kn_hook;
callout_drain(calloutp);
free(calloutp, M_KQUEUE);
free(kn->kn_ptr.p_nexttime, M_KQUEUE);
old = atomic_fetch_sub_explicit(&kq_ncallouts, 1, memory_order_relaxed);
KASSERT(old > 0, ("Number of callouts cannot become negative"));
kn->kn_status |= KN_DETACHED; /* knlist_remove sets it */
}
static int
filt_timer(struct knote *kn, long hint)
{
return (kn->kn_data != 0);
}
static int
filt_userattach(struct knote *kn)
{
/*
* EVFILT_USER knotes are not attached to anything in the kernel.
*/
kn->kn_hook = NULL;
if (kn->kn_fflags & NOTE_TRIGGER)
kn->kn_hookid = 1;
else
kn->kn_hookid = 0;
return (0);
}
static void
filt_userdetach(__unused struct knote *kn)
{
/*
* EVFILT_USER knotes are not attached to anything in the kernel.
*/
}
static int
filt_user(struct knote *kn, __unused long hint)
{
return (kn->kn_hookid);
}
static void
filt_usertouch(struct knote *kn, struct kevent *kev, u_long type)
{
u_int ffctrl;
switch (type) {
case EVENT_REGISTER:
if (kev->fflags & NOTE_TRIGGER)
kn->kn_hookid = 1;
ffctrl = kev->fflags & NOTE_FFCTRLMASK;
kev->fflags &= NOTE_FFLAGSMASK;
switch (ffctrl) {
case NOTE_FFNOP:
break;
case NOTE_FFAND:
kn->kn_sfflags &= kev->fflags;
break;
case NOTE_FFOR:
kn->kn_sfflags |= kev->fflags;
break;
case NOTE_FFCOPY:
kn->kn_sfflags = kev->fflags;
break;
default:
/* XXX Return error? */
break;
}
kn->kn_sdata = kev->data;
if (kev->flags & EV_CLEAR) {
kn->kn_hookid = 0;
kn->kn_data = 0;
kn->kn_fflags = 0;
}
break;
case EVENT_PROCESS:
*kev = kn->kn_kevent;
kev->fflags = kn->kn_sfflags;
kev->data = kn->kn_sdata;
if (kn->kn_flags & EV_CLEAR) {
kn->kn_hookid = 0;
kn->kn_data = 0;
kn->kn_fflags = 0;
}
break;
default:
panic("filt_usertouch() - invalid type (%ld)", type);
break;
}
}
int
sys_kqueue(struct thread *td, struct kqueue_args *uap)
{
struct filedesc *fdp;
struct kqueue *kq;
struct file *fp;
struct proc *p;
struct ucred *cred;
int fd, error;
p = td->td_proc;
cred = td->td_ucred;
crhold(cred);
PROC_LOCK(p);
if (!chgkqcnt(cred->cr_ruidinfo, 1, lim_cur(td->td_proc,
RLIMIT_KQUEUES))) {
PROC_UNLOCK(p);
crfree(cred);
return (ENOMEM);
}
PROC_UNLOCK(p);
fdp = p->p_fd;
error = falloc(td, &fp, &fd, 0);
if (error)
goto done2;
/* An extra reference on `fp' has been held for us by falloc(). */
kq = malloc(sizeof *kq, M_KQUEUE, M_WAITOK | M_ZERO);
mtx_init(&kq->kq_lock, "kqueue", NULL, MTX_DEF|MTX_DUPOK);
TAILQ_INIT(&kq->kq_head);
kq->kq_fdp = fdp;
kq->kq_cred = cred;
knlist_init_mtx(&kq->kq_sel.si_note, &kq->kq_lock);
TASK_INIT(&kq->kq_task, 0, kqueue_task, kq);
FILEDESC_XLOCK(fdp);
TAILQ_INSERT_HEAD(&fdp->fd_kqlist, kq, kq_list);
FILEDESC_XUNLOCK(fdp);
finit(fp, FREAD | FWRITE, DTYPE_KQUEUE, kq, &kqueueops);
fdrop(fp, td);
td->td_retval[0] = fd;
done2:
if (error != 0) {
chgkqcnt(cred->cr_ruidinfo, -1, 0);
crfree(cred);
}
return (error);
}
#ifndef _SYS_SYSPROTO_H_
struct kevent_args {
int fd;
const struct kevent *changelist;
int nchanges;
struct kevent *eventlist;
int nevents;
const struct timespec *timeout;
};
#endif
int
sys_kevent(struct thread *td, struct kevent_args *uap)
{
struct timespec ts, *tsp;
struct kevent_copyops k_ops = { uap,
kevent_copyout,
kevent_copyin};
int error;
#ifdef KTRACE
struct uio ktruio;
struct iovec ktriov;
struct uio *ktruioin = NULL;
struct uio *ktruioout = NULL;
#endif
if (uap->timeout != NULL) {
error = copyin(uap->timeout, &ts, sizeof(ts));
if (error)
return (error);
tsp = &ts;
} else
tsp = NULL;
#ifdef KTRACE
if (KTRPOINT(td, KTR_GENIO)) {
ktriov.iov_base = uap->changelist;
ktriov.iov_len = uap->nchanges * sizeof(struct kevent);
ktruio = (struct uio){ .uio_iov = &ktriov, .uio_iovcnt = 1,
.uio_segflg = UIO_USERSPACE, .uio_rw = UIO_READ,
.uio_td = td };
ktruioin = cloneuio(&ktruio);
ktriov.iov_base = uap->eventlist;
ktriov.iov_len = uap->nevents * sizeof(struct kevent);
ktruioout = cloneuio(&ktruio);
}
#endif
error = kern_kevent(td, uap->fd, uap->nchanges, uap->nevents,
&k_ops, tsp);
#ifdef KTRACE
if (ktruioin != NULL) {
ktruioin->uio_resid = uap->nchanges * sizeof(struct kevent);
ktrgenio(uap->fd, UIO_WRITE, ktruioin, 0);
ktruioout->uio_resid = td->td_retval[0] * sizeof(struct kevent);
ktrgenio(uap->fd, UIO_READ, ktruioout, error);
}
#endif
return (error);
}
/*
* Copy 'count' items into the destination list pointed to by uap->eventlist.
*/
static int
kevent_copyout(void *arg, struct kevent *kevp, int count)
{
struct kevent_args *uap;
int error;
KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
uap = (struct kevent_args *)arg;
error = copyout(kevp, uap->eventlist, count * sizeof *kevp);
if (error == 0)
uap->eventlist += count;
return (error);
}
/*
* Copy 'count' items from the list pointed to by uap->changelist.
*/
static int
kevent_copyin(void *arg, struct kevent *kevp, int count)
{
struct kevent_args *uap;
int error;
KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
uap = (struct kevent_args *)arg;
error = copyin(uap->changelist, kevp, count * sizeof *kevp);
if (error == 0)
uap->changelist += count;
return (error);
}
int
kern_kevent(struct thread *td, int fd, int nchanges, int nevents,
struct kevent_copyops *k_ops, const struct timespec *timeout)
{
struct kevent keva[KQ_NEVENTS];
struct kevent *kevp, *changes;
struct kqueue *kq;
struct file *fp;
cap_rights_t rights;
int i, n, nerrors, error;
cap_rights_init(&rights);
if (nchanges > 0)
cap_rights_set(&rights, CAP_KQUEUE_CHANGE);
if (nevents > 0)
cap_rights_set(&rights, CAP_KQUEUE_EVENT);
error = fget(td, fd, &rights, &fp);
if (error != 0)
return (error);
error = kqueue_acquire(fp, &kq);
if (error != 0)
goto done_norel;
nerrors = 0;
while (nchanges > 0) {
n = nchanges > KQ_NEVENTS ? KQ_NEVENTS : nchanges;
error = k_ops->k_copyin(k_ops->arg, keva, n);
if (error)
goto done;
changes = keva;
for (i = 0; i < n; i++) {
kevp = &changes[i];
if (!kevp->filter)
continue;
kevp->flags &= ~EV_SYSFLAGS;
error = kqueue_register(kq, kevp, td, 1);
if (error || (kevp->flags & EV_RECEIPT)) {
if (nevents != 0) {
kevp->flags = EV_ERROR;
kevp->data = error;
(void) k_ops->k_copyout(k_ops->arg,
kevp, 1);
nevents--;
nerrors++;
} else {
goto done;
}
}
}
nchanges -= n;
}
if (nerrors) {
td->td_retval[0] = nerrors;
error = 0;
goto done;
}
error = kqueue_scan(kq, nevents, k_ops, timeout, keva, td);
done:
kqueue_release(kq, 0);
done_norel:
fdrop(fp, td);
return (error);
}
int
kqueue_add_filteropts(int filt, struct filterops *filtops)
{
int error;
error = 0;
if (filt > 0 || filt + EVFILT_SYSCOUNT < 0) {
printf(
"trying to add a filterop that is out of range: %d is beyond %d\n",
~filt, EVFILT_SYSCOUNT);
return EINVAL;
}
mtx_lock(&filterops_lock);
if (sysfilt_ops[~filt].for_fop != &null_filtops &&
sysfilt_ops[~filt].for_fop != NULL)
error = EEXIST;
else {
sysfilt_ops[~filt].for_fop = filtops;
sysfilt_ops[~filt].for_refcnt = 0;
}
mtx_unlock(&filterops_lock);
return (error);
}
int
kqueue_del_filteropts(int filt)
{
int error;
error = 0;
if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
return EINVAL;
mtx_lock(&filterops_lock);
if (sysfilt_ops[~filt].for_fop == &null_filtops ||
sysfilt_ops[~filt].for_fop == NULL)
error = EINVAL;
else if (sysfilt_ops[~filt].for_refcnt != 0)
error = EBUSY;
else {
sysfilt_ops[~filt].for_fop = &null_filtops;
sysfilt_ops[~filt].for_refcnt = 0;
}
mtx_unlock(&filterops_lock);
return error;
}
static struct filterops *
kqueue_fo_find(int filt)
{
if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
return NULL;
if (sysfilt_ops[~filt].for_nolock)
return sysfilt_ops[~filt].for_fop;
mtx_lock(&filterops_lock);
sysfilt_ops[~filt].for_refcnt++;
if (sysfilt_ops[~filt].for_fop == NULL)
sysfilt_ops[~filt].for_fop = &null_filtops;
mtx_unlock(&filterops_lock);
return sysfilt_ops[~filt].for_fop;
}
static void
kqueue_fo_release(int filt)
{
if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
return;
if (sysfilt_ops[~filt].for_nolock)
return;
mtx_lock(&filterops_lock);
KASSERT(sysfilt_ops[~filt].for_refcnt > 0,
("filter object refcount not valid on release"));
sysfilt_ops[~filt].for_refcnt--;
mtx_unlock(&filterops_lock);
}
/*
* A ref to kq (obtained via kqueue_acquire) must be held. waitok will
* influence if memory allocation should wait. Make sure it is 0 if you
* hold any mutexes.
*/
static int
kqueue_register(struct kqueue *kq, struct kevent *kev, struct thread *td, int waitok)
{
struct filterops *fops;
struct file *fp;
struct knote *kn, *tkn;
cap_rights_t rights;
int error, filt, event;
int haskqglobal, filedesc_unlock;
fp = NULL;
kn = NULL;
error = 0;
haskqglobal = 0;
filedesc_unlock = 0;
filt = kev->filter;
fops = kqueue_fo_find(filt);
if (fops == NULL)
return EINVAL;
if (kev->flags & EV_ADD)
tkn = knote_alloc(waitok); /* prevent waiting with locks */
else
tkn = NULL;
findkn:
if (fops->f_isfd) {
KASSERT(td != NULL, ("td is NULL"));
error = fget(td, kev->ident,
cap_rights_init(&rights, CAP_EVENT), &fp);
if (error)
goto done;
if ((kev->flags & EV_ADD) == EV_ADD && kqueue_expand(kq, fops,
kev->ident, 0) != 0) {
/* try again */
fdrop(fp, td);
fp = NULL;
error = kqueue_expand(kq, fops, kev->ident, waitok);
if (error)
goto done;
goto findkn;
}
if (fp->f_type == DTYPE_KQUEUE) {
/*
* if we add some inteligence about what we are doing,
* we should be able to support events on ourselves.
* We need to know when we are doing this to prevent
* getting both the knlist lock and the kq lock since
* they are the same thing.
*/
if (fp->f_data == kq) {
error = EINVAL;
goto done;
}
/*
* Pre-lock the filedesc before the global
* lock mutex, see the comment in
* kqueue_close().
*/
FILEDESC_XLOCK(td->td_proc->p_fd);
filedesc_unlock = 1;
KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
}
KQ_LOCK(kq);
if (kev->ident < kq->kq_knlistsize) {
SLIST_FOREACH(kn, &kq->kq_knlist[kev->ident], kn_link)
if (kev->filter == kn->kn_filter)
break;
}
} else {
if ((kev->flags & EV_ADD) == EV_ADD)
kqueue_expand(kq, fops, kev->ident, waitok);
KQ_LOCK(kq);
if (kq->kq_knhashmask != 0) {
struct klist *list;
list = &kq->kq_knhash[
KN_HASH((u_long)kev->ident, kq->kq_knhashmask)];
SLIST_FOREACH(kn, list, kn_link)
if (kev->ident == kn->kn_id &&
kev->filter == kn->kn_filter)
break;
}
}
/* knote is in the process of changing, wait for it to stablize. */
if (kn != NULL && (kn->kn_status & KN_INFLUX) == KN_INFLUX) {
KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
if (filedesc_unlock) {
FILEDESC_XUNLOCK(td->td_proc->p_fd);
filedesc_unlock = 0;
}
kq->kq_state |= KQ_FLUXWAIT;
msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqflxwt", 0);
if (fp != NULL) {
fdrop(fp, td);
fp = NULL;
}
goto findkn;
}
/*
* kn now contains the matching knote, or NULL if no match
*/
if (kn == NULL) {
if (kev->flags & EV_ADD) {
kn = tkn;
tkn = NULL;
if (kn == NULL) {
KQ_UNLOCK(kq);
error = ENOMEM;
goto done;
}
kn->kn_fp = fp;
kn->kn_kq = kq;
kn->kn_fop = fops;
/*
* apply reference counts to knote structure, and
* do not release it at the end of this routine.
*/
fops = NULL;
fp = NULL;
kn->kn_sfflags = kev->fflags;
kn->kn_sdata = kev->data;
kev->fflags = 0;
kev->data = 0;
kn->kn_kevent = *kev;
kn->kn_kevent.flags &= ~(EV_ADD | EV_DELETE |
EV_ENABLE | EV_DISABLE | EV_FORCEONESHOT);
kn->kn_status = KN_INFLUX|KN_DETACHED;
error = knote_attach(kn, kq);
KQ_UNLOCK(kq);
if (error != 0) {
tkn = kn;
goto done;
}
if ((error = kn->kn_fop->f_attach(kn)) != 0) {
knote_drop(kn, td);
goto done;
}
KN_LIST_LOCK(kn);
goto done_ev_add;
} else {
/* No matching knote and the EV_ADD flag is not set. */
KQ_UNLOCK(kq);
error = ENOENT;
goto done;
}
}
if (kev->flags & EV_DELETE) {
kn->kn_status |= KN_INFLUX;
KQ_UNLOCK(kq);
if (!(kn->kn_status & KN_DETACHED))
kn->kn_fop->f_detach(kn);
knote_drop(kn, td);
goto done;
}
if (kev->flags & EV_FORCEONESHOT) {
kn->kn_flags |= EV_ONESHOT;
KNOTE_ACTIVATE(kn, 1);
}
/*
* The user may change some filter values after the initial EV_ADD,
* but doing so will not reset any filter which has already been
* triggered.
*/
kn->kn_status |= KN_INFLUX | KN_SCAN;
KQ_UNLOCK(kq);
KN_LIST_LOCK(kn);
kn->kn_kevent.udata = kev->udata;
if (!fops->f_isfd && fops->f_touch != NULL) {
fops->f_touch(kn, kev, EVENT_REGISTER);
} else {
kn->kn_sfflags = kev->fflags;
kn->kn_sdata = kev->data;
}
/*
* We can get here with kn->kn_knlist == NULL. This can happen when
* the initial attach event decides that the event is "completed"
* already. i.e. filt_procattach is called on a zombie process. It
* will call filt_proc which will remove it from the list, and NULL
* kn_knlist.
*/
done_ev_add:
if ((kev->flags & EV_DISABLE) &&
((kn->kn_status & KN_DISABLED) == 0)) {
kn->kn_status |= KN_DISABLED;
}
if ((kn->kn_status & KN_DISABLED) == 0)
event = kn->kn_fop->f_event(kn, 0);
else
event = 0;
KQ_LOCK(kq);
if (event)
KNOTE_ACTIVATE(kn, 1);
kn->kn_status &= ~(KN_INFLUX | KN_SCAN);
KN_LIST_UNLOCK(kn);
if ((kev->flags & EV_ENABLE) && (kn->kn_status & KN_DISABLED)) {
kn->kn_status &= ~KN_DISABLED;
if ((kn->kn_status & KN_ACTIVE) &&
((kn->kn_status & KN_QUEUED) == 0))
knote_enqueue(kn);
}
KQ_UNLOCK_FLUX(kq);
done:
KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
if (filedesc_unlock)
FILEDESC_XUNLOCK(td->td_proc->p_fd);
if (fp != NULL)
fdrop(fp, td);
if (tkn != NULL)
knote_free(tkn);
if (fops != NULL)
kqueue_fo_release(filt);
return (error);
}
static int
kqueue_acquire(struct file *fp, struct kqueue **kqp)
{
int error;
struct kqueue *kq;
error = 0;
kq = fp->f_data;
if (fp->f_type != DTYPE_KQUEUE || kq == NULL)
return (EBADF);
*kqp = kq;
KQ_LOCK(kq);
if ((kq->kq_state & KQ_CLOSING) == KQ_CLOSING) {
KQ_UNLOCK(kq);
return (EBADF);
}
kq->kq_refcnt++;
KQ_UNLOCK(kq);
return error;
}
static void
kqueue_release(struct kqueue *kq, int locked)
{
if (locked)
KQ_OWNED(kq);
else
KQ_LOCK(kq);
kq->kq_refcnt--;
if (kq->kq_refcnt == 1)
wakeup(&kq->kq_refcnt);
if (!locked)
KQ_UNLOCK(kq);
}
static void
kqueue_schedtask(struct kqueue *kq)
{
KQ_OWNED(kq);
KASSERT(((kq->kq_state & KQ_TASKDRAIN) != KQ_TASKDRAIN),
("scheduling kqueue task while draining"));
if ((kq->kq_state & KQ_TASKSCHED) != KQ_TASKSCHED) {
taskqueue_enqueue(taskqueue_kqueue, &kq->kq_task);
kq->kq_state |= KQ_TASKSCHED;
}
}
/*
* Expand the kq to make sure we have storage for fops/ident pair.
*
* Return 0 on success (or no work necessary), return errno on failure.
*
* Not calling hashinit w/ waitok (proper malloc flag) should be safe.
* If kqueue_register is called from a non-fd context, there usually/should
* be no locks held.
*/
static int
kqueue_expand(struct kqueue *kq, struct filterops *fops, uintptr_t ident,
int waitok)
{
struct klist *list, *tmp_knhash, *to_free;
u_long tmp_knhashmask;
int size;
int fd;
int mflag = waitok ? M_WAITOK : M_NOWAIT;
KQ_NOTOWNED(kq);
to_free = NULL;
if (fops->f_isfd) {
fd = ident;
if (kq->kq_knlistsize <= fd) {
size = kq->kq_knlistsize;
while (size <= fd)
size += KQEXTENT;
list = malloc(size * sizeof(*list), M_KQUEUE, mflag);
if (list == NULL)
return ENOMEM;
KQ_LOCK(kq);
if (kq->kq_knlistsize > fd) {
to_free = list;
list = NULL;
} else {
if (kq->kq_knlist != NULL) {
bcopy(kq->kq_knlist, list,
kq->kq_knlistsize * sizeof(*list));
to_free = kq->kq_knlist;
kq->kq_knlist = NULL;
}
bzero((caddr_t)list +
kq->kq_knlistsize * sizeof(*list),
(size - kq->kq_knlistsize) * sizeof(*list));
kq->kq_knlistsize = size;
kq->kq_knlist = list;
}
KQ_UNLOCK(kq);
}
} else {
if (kq->kq_knhashmask == 0) {
tmp_knhash = hashinit(KN_HASHSIZE, M_KQUEUE,
&tmp_knhashmask);
if (tmp_knhash == NULL)
return ENOMEM;
KQ_LOCK(kq);
if (kq->kq_knhashmask == 0) {
kq->kq_knhash = tmp_knhash;
kq->kq_knhashmask = tmp_knhashmask;
} else {
to_free = tmp_knhash;
}
KQ_UNLOCK(kq);
}
}
free(to_free, M_KQUEUE);
KQ_NOTOWNED(kq);
return 0;
}
static void
kqueue_task(void *arg, int pending)
{
struct kqueue *kq;
int haskqglobal;
haskqglobal = 0;
kq = arg;
KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
KQ_LOCK(kq);
KNOTE_LOCKED(&kq->kq_sel.si_note, 0);
kq->kq_state &= ~KQ_TASKSCHED;
if ((kq->kq_state & KQ_TASKDRAIN) == KQ_TASKDRAIN) {
wakeup(&kq->kq_state);
}
KQ_UNLOCK(kq);
KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
}
/*
* Scan, update kn_data (if not ONESHOT), and copyout triggered events.
* We treat KN_MARKER knotes as if they are INFLUX.
*/
static int
kqueue_scan(struct kqueue *kq, int maxevents, struct kevent_copyops *k_ops,
const struct timespec *tsp, struct kevent *keva, struct thread *td)
{
struct kevent *kevp;
struct knote *kn, *marker;
sbintime_t asbt, rsbt;
int count, error, haskqglobal, influx, nkev, touch;
count = maxevents;
nkev = 0;
error = 0;
haskqglobal = 0;
if (maxevents == 0)
goto done_nl;
rsbt = 0;
if (tsp != NULL) {
if (tsp->tv_sec < 0 || tsp->tv_nsec < 0 ||
tsp->tv_nsec >= 1000000000) {
error = EINVAL;
goto done_nl;
}
if (timespecisset(tsp)) {
if (tsp->tv_sec <= INT32_MAX) {
rsbt = tstosbt(*tsp);
if (TIMESEL(&asbt, rsbt))
asbt += tc_tick_sbt;
if (asbt <= SBT_MAX - rsbt)
asbt += rsbt;
else
asbt = 0;
rsbt >>= tc_precexp;
} else
asbt = 0;
} else
asbt = -1;
} else
asbt = 0;
marker = knote_alloc(1);
if (marker == NULL) {
error = ENOMEM;
goto done_nl;
}
marker->kn_status = KN_MARKER;
KQ_LOCK(kq);
retry:
kevp = keva;
if (kq->kq_count == 0) {
if (asbt == -1) {
error = EWOULDBLOCK;
} else {
kq->kq_state |= KQ_SLEEP;
error = msleep_sbt(kq, &kq->kq_lock, PSOCK | PCATCH,
"kqread", asbt, rsbt, C_ABSOLUTE);
}
if (error == 0)
goto retry;
/* don't restart after signals... */
if (error == ERESTART)
error = EINTR;
else if (error == EWOULDBLOCK)
error = 0;
goto done;
}
TAILQ_INSERT_TAIL(&kq->kq_head, marker, kn_tqe);
influx = 0;
while (count) {
KQ_OWNED(kq);
kn = TAILQ_FIRST(&kq->kq_head);
if ((kn->kn_status == KN_MARKER && kn != marker) ||
(kn->kn_status & KN_INFLUX) == KN_INFLUX) {
if (influx) {
influx = 0;
KQ_FLUX_WAKEUP(kq);
}
kq->kq_state |= KQ_FLUXWAIT;
error = msleep(kq, &kq->kq_lock, PSOCK,
"kqflxwt", 0);
continue;
}
TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
if ((kn->kn_status & KN_DISABLED) == KN_DISABLED) {
kn->kn_status &= ~KN_QUEUED;
kq->kq_count--;
continue;
}
if (kn == marker) {
KQ_FLUX_WAKEUP(kq);
if (count == maxevents)
goto retry;
goto done;
}
KASSERT((kn->kn_status & KN_INFLUX) == 0,
("KN_INFLUX set when not suppose to be"));
if ((kn->kn_flags & EV_DROP) == EV_DROP) {
kn->kn_status &= ~KN_QUEUED;
kn->kn_status |= KN_INFLUX;
kq->kq_count--;
KQ_UNLOCK(kq);
/*
* We don't need to lock the list since we've marked
* it _INFLUX.
*/
if (!(kn->kn_status & KN_DETACHED))
kn->kn_fop->f_detach(kn);
knote_drop(kn, td);
KQ_LOCK(kq);
continue;
} else if ((kn->kn_flags & EV_ONESHOT) == EV_ONESHOT) {
kn->kn_status &= ~KN_QUEUED;
kn->kn_status |= KN_INFLUX;
kq->kq_count--;
KQ_UNLOCK(kq);
/*
* We don't need to lock the list since we've marked
* it _INFLUX.
*/
*kevp = kn->kn_kevent;
if (!(kn->kn_status & KN_DETACHED))
kn->kn_fop->f_detach(kn);
knote_drop(kn, td);
KQ_LOCK(kq);
kn = NULL;
} else {
kn->kn_status |= KN_INFLUX | KN_SCAN;
KQ_UNLOCK(kq);
if ((kn->kn_status & KN_KQUEUE) == KN_KQUEUE)
KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
KN_LIST_LOCK(kn);
if (kn->kn_fop->f_event(kn, 0) == 0) {
KQ_LOCK(kq);
KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
kn->kn_status &=
~(KN_QUEUED | KN_ACTIVE | KN_INFLUX |
KN_SCAN);
kq->kq_count--;
KN_LIST_UNLOCK(kn);
influx = 1;
continue;
}
touch = (!kn->kn_fop->f_isfd &&
kn->kn_fop->f_touch != NULL);
if (touch)
kn->kn_fop->f_touch(kn, kevp, EVENT_PROCESS);
else
*kevp = kn->kn_kevent;
KQ_LOCK(kq);
KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
if (kn->kn_flags & (EV_CLEAR | EV_DISPATCH)) {
/*
* Manually clear knotes who weren't
* 'touch'ed.
*/
if (touch == 0 && kn->kn_flags & EV_CLEAR) {
kn->kn_data = 0;
kn->kn_fflags = 0;
}
if (kn->kn_flags & EV_DISPATCH)
kn->kn_status |= KN_DISABLED;
kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE);
kq->kq_count--;
} else
TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
kn->kn_status &= ~(KN_INFLUX | KN_SCAN);
KN_LIST_UNLOCK(kn);
influx = 1;
}
/* we are returning a copy to the user */
kevp++;
nkev++;
count--;
if (nkev == KQ_NEVENTS) {
influx = 0;
KQ_UNLOCK_FLUX(kq);
error = k_ops->k_copyout(k_ops->arg, keva, nkev);
nkev = 0;
kevp = keva;
KQ_LOCK(kq);
if (error)
break;
}
}
TAILQ_REMOVE(&kq->kq_head, marker, kn_tqe);
done:
KQ_OWNED(kq);
KQ_UNLOCK_FLUX(kq);
knote_free(marker);
done_nl:
KQ_NOTOWNED(kq);
if (nkev != 0)
error = k_ops->k_copyout(k_ops->arg, keva, nkev);
td->td_retval[0] = maxevents - count;
return (error);
}
/*ARGSUSED*/
static int
kqueue_ioctl(struct file *fp, u_long cmd, void *data,
struct ucred *active_cred, struct thread *td)
{
/*
* Enabling sigio causes two major problems:
* 1) infinite recursion:
* Synopsys: kevent is being used to track signals and have FIOASYNC
* set. On receipt of a signal this will cause a kqueue to recurse
* into itself over and over. Sending the sigio causes the kqueue
* to become ready, which in turn posts sigio again, forever.
* Solution: this can be solved by setting a flag in the kqueue that
* we have a SIGIO in progress.
* 2) locking problems:
* Synopsys: Kqueue is a leaf subsystem, but adding signalling puts
* us above the proc and pgrp locks.
* Solution: Post a signal using an async mechanism, being sure to
* record a generation count in the delivery so that we do not deliver
* a signal to the wrong process.
*
* Note, these two mechanisms are somewhat mutually exclusive!
*/
#if 0
struct kqueue *kq;
kq = fp->f_data;
switch (cmd) {
case FIOASYNC:
if (*(int *)data) {
kq->kq_state |= KQ_ASYNC;
} else {
kq->kq_state &= ~KQ_ASYNC;
}
return (0);
case FIOSETOWN:
return (fsetown(*(int *)data, &kq->kq_sigio));
case FIOGETOWN:
*(int *)data = fgetown(&kq->kq_sigio);
return (0);
}
#endif
return (ENOTTY);
}
/*ARGSUSED*/
static int
kqueue_poll(struct file *fp, int events, struct ucred *active_cred,
struct thread *td)
{
struct kqueue *kq;
int revents = 0;
int error;
if ((error = kqueue_acquire(fp, &kq)))
return POLLERR;
KQ_LOCK(kq);
if (events & (POLLIN | POLLRDNORM)) {
if (kq->kq_count) {
revents |= events & (POLLIN | POLLRDNORM);
} else {
selrecord(td, &kq->kq_sel);
if (SEL_WAITING(&kq->kq_sel))
kq->kq_state |= KQ_SEL;
}
}
kqueue_release(kq, 1);
KQ_UNLOCK(kq);
return (revents);
}
/*ARGSUSED*/
static int
kqueue_stat(struct file *fp, struct stat *st, struct ucred *active_cred,
struct thread *td)
{
bzero((void *)st, sizeof *st);
/*
* We no longer return kq_count because the unlocked value is useless.
* If you spent all this time getting the count, why not spend your
* syscall better by calling kevent?
*
* XXX - This is needed for libc_r.
*/
st->st_mode = S_IFIFO;
return (0);
}
/*ARGSUSED*/
static int
kqueue_close(struct file *fp, struct thread *td)
{
struct kqueue *kq = fp->f_data;
struct filedesc *fdp;
struct knote *kn;
int i;
int error;
int filedesc_unlock;
if ((error = kqueue_acquire(fp, &kq)))
return error;
filedesc_unlock = 0;
KQ_LOCK(kq);
KASSERT((kq->kq_state & KQ_CLOSING) != KQ_CLOSING,
("kqueue already closing"));
kq->kq_state |= KQ_CLOSING;
if (kq->kq_refcnt > 1)
msleep(&kq->kq_refcnt, &kq->kq_lock, PSOCK, "kqclose", 0);
KASSERT(kq->kq_refcnt == 1, ("other refs are out there!"));
fdp = kq->kq_fdp;
KASSERT(knlist_empty(&kq->kq_sel.si_note),
("kqueue's knlist not empty"));
for (i = 0; i < kq->kq_knlistsize; i++) {
while ((kn = SLIST_FIRST(&kq->kq_knlist[i])) != NULL) {
if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
kq->kq_state |= KQ_FLUXWAIT;
msleep(kq, &kq->kq_lock, PSOCK, "kqclo1", 0);
continue;
}
kn->kn_status |= KN_INFLUX;
KQ_UNLOCK(kq);
if (!(kn->kn_status & KN_DETACHED))
kn->kn_fop->f_detach(kn);
knote_drop(kn, td);
KQ_LOCK(kq);
}
}
if (kq->kq_knhashmask != 0) {
for (i = 0; i <= kq->kq_knhashmask; i++) {
while ((kn = SLIST_FIRST(&kq->kq_knhash[i])) != NULL) {
if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
kq->kq_state |= KQ_FLUXWAIT;
msleep(kq, &kq->kq_lock, PSOCK,
"kqclo2", 0);
continue;
}
kn->kn_status |= KN_INFLUX;
KQ_UNLOCK(kq);
if (!(kn->kn_status & KN_DETACHED))
kn->kn_fop->f_detach(kn);
knote_drop(kn, td);
KQ_LOCK(kq);
}
}
}
if ((kq->kq_state & KQ_TASKSCHED) == KQ_TASKSCHED) {
kq->kq_state |= KQ_TASKDRAIN;
msleep(&kq->kq_state, &kq->kq_lock, PSOCK, "kqtqdr", 0);
}
if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
selwakeuppri(&kq->kq_sel, PSOCK);
if (!SEL_WAITING(&kq->kq_sel))
kq->kq_state &= ~KQ_SEL;
}
KQ_UNLOCK(kq);
/*
* We could be called due to the knote_drop() doing fdrop(),
* called from kqueue_register(). In this case the global
* lock is owned, and filedesc sx is locked before, to not
* take the sleepable lock after non-sleepable.
*/
if (!sx_xlocked(FILEDESC_LOCK(fdp))) {
FILEDESC_XLOCK(fdp);
filedesc_unlock = 1;
} else
filedesc_unlock = 0;
TAILQ_REMOVE(&fdp->fd_kqlist, kq, kq_list);
if (filedesc_unlock)
FILEDESC_XUNLOCK(fdp);
seldrain(&kq->kq_sel);
knlist_destroy(&kq->kq_sel.si_note);
mtx_destroy(&kq->kq_lock);
kq->kq_fdp = NULL;
if (kq->kq_knhash != NULL)
free(kq->kq_knhash, M_KQUEUE);
if (kq->kq_knlist != NULL)
free(kq->kq_knlist, M_KQUEUE);
funsetown(&kq->kq_sigio);
chgkqcnt(kq->kq_cred->cr_ruidinfo, -1, 0);
crfree(kq->kq_cred);
free(kq, M_KQUEUE);
fp->f_data = NULL;
return (0);
}
static int
kqueue_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
{
kif->kf_type = KF_TYPE_KQUEUE;
return (0);
}
static void
kqueue_wakeup(struct kqueue *kq)
{
KQ_OWNED(kq);
if ((kq->kq_state & KQ_SLEEP) == KQ_SLEEP) {
kq->kq_state &= ~KQ_SLEEP;
wakeup(kq);
}
if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
selwakeuppri(&kq->kq_sel, PSOCK);
if (!SEL_WAITING(&kq->kq_sel))
kq->kq_state &= ~KQ_SEL;
}
if (!knlist_empty(&kq->kq_sel.si_note))
kqueue_schedtask(kq);
if ((kq->kq_state & KQ_ASYNC) == KQ_ASYNC) {
pgsigio(&kq->kq_sigio, SIGIO, 0);
}
}
/*
* Walk down a list of knotes, activating them if their event has triggered.
*
* There is a possibility to optimize in the case of one kq watching another.
* Instead of scheduling a task to wake it up, you could pass enough state
* down the chain to make up the parent kqueue. Make this code functional
* first.
*/
void
knote(struct knlist *list, long hint, int lockflags)
{
struct kqueue *kq;
struct knote *kn;
int error;
if (list == NULL)
return;
KNL_ASSERT_LOCK(list, lockflags & KNF_LISTLOCKED);
if ((lockflags & KNF_LISTLOCKED) == 0)
list->kl_lock(list->kl_lockarg);
/*
* If we unlock the list lock (and set KN_INFLUX), we can eliminate
* the kqueue scheduling, but this will introduce four
* lock/unlock's for each knote to test. If we do, continue to use
* SLIST_FOREACH, SLIST_FOREACH_SAFE is not safe in our case, it is
* only safe if you want to remove the current item, which we are
* not doing.
*/
SLIST_FOREACH(kn, &list->kl_list, kn_selnext) {
kq = kn->kn_kq;
KQ_LOCK(kq);
if ((kn->kn_status & (KN_INFLUX | KN_SCAN)) == KN_INFLUX) {
/*
* Do not process the influx notes, except for
* the influx coming from the kq unlock in the
* kqueue_scan(). In the later case, we do
* not interfere with the scan, since the code
* fragment in kqueue_scan() locks the knlist,
* and cannot proceed until we finished.
*/
KQ_UNLOCK(kq);
} else if ((lockflags & KNF_NOKQLOCK) != 0) {
kn->kn_status |= KN_INFLUX;
KQ_UNLOCK(kq);
error = kn->kn_fop->f_event(kn, hint);
KQ_LOCK(kq);
kn->kn_status &= ~KN_INFLUX;
if (error)
KNOTE_ACTIVATE(kn, 1);
KQ_UNLOCK_FLUX(kq);
} else {
kn->kn_status |= KN_HASKQLOCK;
if (kn->kn_fop->f_event(kn, hint))
KNOTE_ACTIVATE(kn, 1);
kn->kn_status &= ~KN_HASKQLOCK;
KQ_UNLOCK(kq);
}
}
if ((lockflags & KNF_LISTLOCKED) == 0)
list->kl_unlock(list->kl_lockarg);
}
/*
* add a knote to a knlist
*/
void
knlist_add(struct knlist *knl, struct knote *kn, int islocked)
{
KNL_ASSERT_LOCK(knl, islocked);
KQ_NOTOWNED(kn->kn_kq);
KASSERT((kn->kn_status & (KN_INFLUX|KN_DETACHED)) ==
(KN_INFLUX|KN_DETACHED), ("knote not KN_INFLUX and KN_DETACHED"));
if (!islocked)
knl->kl_lock(knl->kl_lockarg);
SLIST_INSERT_HEAD(&knl->kl_list, kn, kn_selnext);
if (!islocked)
knl->kl_unlock(knl->kl_lockarg);
KQ_LOCK(kn->kn_kq);
kn->kn_knlist = knl;
kn->kn_status &= ~KN_DETACHED;
KQ_UNLOCK(kn->kn_kq);
}
static void
knlist_remove_kq(struct knlist *knl, struct knote *kn, int knlislocked, int kqislocked)
{
KASSERT(!(!!kqislocked && !knlislocked), ("kq locked w/o knl locked"));
KNL_ASSERT_LOCK(knl, knlislocked);
mtx_assert(&kn->kn_kq->kq_lock, kqislocked ? MA_OWNED : MA_NOTOWNED);
if (!kqislocked)
KASSERT((kn->kn_status & (KN_INFLUX|KN_DETACHED)) == KN_INFLUX,
("knlist_remove called w/o knote being KN_INFLUX or already removed"));
if (!knlislocked)
knl->kl_lock(knl->kl_lockarg);
SLIST_REMOVE(&knl->kl_list, kn, knote, kn_selnext);
kn->kn_knlist = NULL;
if (!knlislocked)
knl->kl_unlock(knl->kl_lockarg);
if (!kqislocked)
KQ_LOCK(kn->kn_kq);
kn->kn_status |= KN_DETACHED;
if (!kqislocked)
KQ_UNLOCK(kn->kn_kq);
}
/*
* remove knote from the specified knlist
*/
void
knlist_remove(struct knlist *knl, struct knote *kn, int islocked)
{
knlist_remove_kq(knl, kn, islocked, 0);
}
/*
* remove knote from the specified knlist while in f_event handler.
*/
void
knlist_remove_inevent(struct knlist *knl, struct knote *kn)
{
knlist_remove_kq(knl, kn, 1,
(kn->kn_status & KN_HASKQLOCK) == KN_HASKQLOCK);
}
int
knlist_empty(struct knlist *knl)
{
KNL_ASSERT_LOCKED(knl);
return SLIST_EMPTY(&knl->kl_list);
}
static struct mtx knlist_lock;
MTX_SYSINIT(knlist_lock, &knlist_lock, "knlist lock for lockless objects",
MTX_DEF);
static void knlist_mtx_lock(void *arg);
static void knlist_mtx_unlock(void *arg);
static void
knlist_mtx_lock(void *arg)
{
mtx_lock((struct mtx *)arg);
}
static void
knlist_mtx_unlock(void *arg)
{
mtx_unlock((struct mtx *)arg);
}
static void
knlist_mtx_assert_locked(void *arg)
{
mtx_assert((struct mtx *)arg, MA_OWNED);
}
static void
knlist_mtx_assert_unlocked(void *arg)
{
mtx_assert((struct mtx *)arg, MA_NOTOWNED);
}
static void
knlist_rw_rlock(void *arg)
{
rw_rlock((struct rwlock *)arg);
}
static void
knlist_rw_runlock(void *arg)
{
rw_runlock((struct rwlock *)arg);
}
static void
knlist_rw_assert_locked(void *arg)
{
rw_assert((struct rwlock *)arg, RA_LOCKED);
}
static void
knlist_rw_assert_unlocked(void *arg)
{
rw_assert((struct rwlock *)arg, RA_UNLOCKED);
}
void
knlist_init(struct knlist *knl, void *lock, void (*kl_lock)(void *),
void (*kl_unlock)(void *),
void (*kl_assert_locked)(void *), void (*kl_assert_unlocked)(void *))
{
if (lock == NULL)
knl->kl_lockarg = &knlist_lock;
else
knl->kl_lockarg = lock;
if (kl_lock == NULL)
knl->kl_lock = knlist_mtx_lock;
else
knl->kl_lock = kl_lock;
if (kl_unlock == NULL)
knl->kl_unlock = knlist_mtx_unlock;
else
knl->kl_unlock = kl_unlock;
if (kl_assert_locked == NULL)
knl->kl_assert_locked = knlist_mtx_assert_locked;
else
knl->kl_assert_locked = kl_assert_locked;
if (kl_assert_unlocked == NULL)
knl->kl_assert_unlocked = knlist_mtx_assert_unlocked;
else
knl->kl_assert_unlocked = kl_assert_unlocked;
SLIST_INIT(&knl->kl_list);
}
void
knlist_init_mtx(struct knlist *knl, struct mtx *lock)
{
knlist_init(knl, lock, NULL, NULL, NULL, NULL);
}
void
knlist_init_rw_reader(struct knlist *knl, struct rwlock *lock)
{
knlist_init(knl, lock, knlist_rw_rlock, knlist_rw_runlock,
knlist_rw_assert_locked, knlist_rw_assert_unlocked);
}
void
knlist_destroy(struct knlist *knl)
{
#ifdef INVARIANTS
/*
* if we run across this error, we need to find the offending
* driver and have it call knlist_clear or knlist_delete.
*/
if (!SLIST_EMPTY(&knl->kl_list))
printf("WARNING: destroying knlist w/ knotes on it!\n");
#endif
knl->kl_lockarg = knl->kl_lock = knl->kl_unlock = NULL;
SLIST_INIT(&knl->kl_list);
}
/*
* Even if we are locked, we may need to drop the lock to allow any influx
* knotes time to "settle".
*/
void
knlist_cleardel(struct knlist *knl, struct thread *td, int islocked, int killkn)
{
struct knote *kn, *kn2;
struct kqueue *kq;
if (islocked)
KNL_ASSERT_LOCKED(knl);
else {
KNL_ASSERT_UNLOCKED(knl);
again: /* need to reacquire lock since we have dropped it */
knl->kl_lock(knl->kl_lockarg);
}
SLIST_FOREACH_SAFE(kn, &knl->kl_list, kn_selnext, kn2) {
kq = kn->kn_kq;
KQ_LOCK(kq);
if ((kn->kn_status & KN_INFLUX)) {
KQ_UNLOCK(kq);
continue;
}
knlist_remove_kq(knl, kn, 1, 1);
if (killkn) {
kn->kn_status |= KN_INFLUX | KN_DETACHED;
KQ_UNLOCK(kq);
knote_drop(kn, td);
} else {
/* Make sure cleared knotes disappear soon */
kn->kn_flags |= (EV_EOF | EV_ONESHOT);
KQ_UNLOCK(kq);
}
kq = NULL;
}
if (!SLIST_EMPTY(&knl->kl_list)) {
/* there are still KN_INFLUX remaining */
kn = SLIST_FIRST(&knl->kl_list);
kq = kn->kn_kq;
KQ_LOCK(kq);
KASSERT(kn->kn_status & KN_INFLUX,
("knote removed w/o list lock"));
knl->kl_unlock(knl->kl_lockarg);
kq->kq_state |= KQ_FLUXWAIT;
msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqkclr", 0);
kq = NULL;
goto again;
}
if (islocked)
KNL_ASSERT_LOCKED(knl);
else {
knl->kl_unlock(knl->kl_lockarg);
KNL_ASSERT_UNLOCKED(knl);
}
}
/*
* Remove all knotes referencing a specified fd must be called with FILEDESC
* lock. This prevents a race where a new fd comes along and occupies the
* entry and we attach a knote to the fd.
*/
void
knote_fdclose(struct thread *td, int fd)
{
struct filedesc *fdp = td->td_proc->p_fd;
struct kqueue *kq;
struct knote *kn;
int influx;
FILEDESC_XLOCK_ASSERT(fdp);
/*
* We shouldn't have to worry about new kevents appearing on fd
* since filedesc is locked.
*/
TAILQ_FOREACH(kq, &fdp->fd_kqlist, kq_list) {
KQ_LOCK(kq);
again:
influx = 0;
while (kq->kq_knlistsize > fd &&
(kn = SLIST_FIRST(&kq->kq_knlist[fd])) != NULL) {
if (kn->kn_status & KN_INFLUX) {
/* someone else might be waiting on our knote */
if (influx)
wakeup(kq);
kq->kq_state |= KQ_FLUXWAIT;
msleep(kq, &kq->kq_lock, PSOCK, "kqflxwt", 0);
goto again;
}
kn->kn_status |= KN_INFLUX;
KQ_UNLOCK(kq);
if (!(kn->kn_status & KN_DETACHED))
kn->kn_fop->f_detach(kn);
knote_drop(kn, td);
influx = 1;
KQ_LOCK(kq);
}
KQ_UNLOCK_FLUX(kq);
}
}
static int
knote_attach(struct knote *kn, struct kqueue *kq)
{
struct klist *list;
KASSERT(kn->kn_status & KN_INFLUX, ("knote not marked INFLUX"));
KQ_OWNED(kq);
if (kn->kn_fop->f_isfd) {
if (kn->kn_id >= kq->kq_knlistsize)
return ENOMEM;
list = &kq->kq_knlist[kn->kn_id];
} else {
if (kq->kq_knhash == NULL)
return ENOMEM;
list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
}
SLIST_INSERT_HEAD(list, kn, kn_link);
return 0;
}
/*
* knote must already have been detached using the f_detach method.
* no lock need to be held, it is assumed that the KN_INFLUX flag is set
* to prevent other removal.
*/
static void
knote_drop(struct knote *kn, struct thread *td)
{
struct kqueue *kq;
struct klist *list;
kq = kn->kn_kq;
KQ_NOTOWNED(kq);
KASSERT((kn->kn_status & KN_INFLUX) == KN_INFLUX,
("knote_drop called without KN_INFLUX set in kn_status"));
KQ_LOCK(kq);
if (kn->kn_fop->f_isfd)
list = &kq->kq_knlist[kn->kn_id];
else
list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
if (!SLIST_EMPTY(list))
SLIST_REMOVE(list, kn, knote, kn_link);
if (kn->kn_status & KN_QUEUED)
knote_dequeue(kn);
KQ_UNLOCK_FLUX(kq);
if (kn->kn_fop->f_isfd) {
fdrop(kn->kn_fp, td);
kn->kn_fp = NULL;
}
kqueue_fo_release(kn->kn_kevent.filter);
kn->kn_fop = NULL;
knote_free(kn);
}
static void
knote_enqueue(struct knote *kn)
{
struct kqueue *kq = kn->kn_kq;
KQ_OWNED(kn->kn_kq);
KASSERT((kn->kn_status & KN_QUEUED) == 0, ("knote already queued"));
TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
kn->kn_status |= KN_QUEUED;
kq->kq_count++;
kqueue_wakeup(kq);
}
static void
knote_dequeue(struct knote *kn)
{
struct kqueue *kq = kn->kn_kq;
KQ_OWNED(kn->kn_kq);
KASSERT(kn->kn_status & KN_QUEUED, ("knote not queued"));
TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
kn->kn_status &= ~KN_QUEUED;
kq->kq_count--;
}
static void
knote_init(void)
{
knote_zone = uma_zcreate("KNOTE", sizeof(struct knote), NULL, NULL,
NULL, NULL, UMA_ALIGN_PTR, 0);
}
SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL);
static struct knote *
knote_alloc(int waitok)
{
return ((struct knote *)uma_zalloc(knote_zone,
(waitok ? M_WAITOK : M_NOWAIT)|M_ZERO));
}
static void
knote_free(struct knote *kn)
{
if (kn != NULL)
uma_zfree(knote_zone, kn);
}
/*
* Register the kev w/ the kq specified by fd.
*/
int
kqfd_register(int fd, struct kevent *kev, struct thread *td, int waitok)
{
struct kqueue *kq;
struct file *fp;
cap_rights_t rights;
int error;
error = fget(td, fd, cap_rights_init(&rights, CAP_KQUEUE_CHANGE), &fp);
if (error != 0)
return (error);
if ((error = kqueue_acquire(fp, &kq)) != 0)
goto noacquire;
error = kqueue_register(kq, kev, td, waitok);
kqueue_release(kq, 0);
noacquire:
fdrop(fp, td);
return error;
}
Index: head/sys/kern/kern_synch.c
===================================================================
--- head/sys/kern/kern_synch.c (revision 283290)
+++ head/sys/kern/kern_synch.c (revision 283291)
@@ -1,611 +1,611 @@
/*-
* Copyright (c) 1982, 1986, 1990, 1991, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)kern_synch.c 8.9 (Berkeley) 5/19/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ktrace.h"
#include "opt_sched.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/condvar.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/signalvar.h>
#include <sys/sleepqueue.h>
#include <sys/smp.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/vmmeter.h>
#ifdef KTRACE
#include <sys/uio.h>
#include <sys/ktrace.h>
#endif
#include <machine/cpu.h>
#define KTDSTATE(td) \
(((td)->td_inhibitors & TDI_SLEEPING) != 0 ? "sleep" : \
((td)->td_inhibitors & TDI_SUSPENDED) != 0 ? "suspended" : \
((td)->td_inhibitors & TDI_SWAPPED) != 0 ? "swapped" : \
((td)->td_inhibitors & TDI_LOCK) != 0 ? "blocked" : \
((td)->td_inhibitors & TDI_IWAIT) != 0 ? "iwait" : "yielding")
static void synch_setup(void *dummy);
SYSINIT(synch_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, synch_setup,
NULL);
int hogticks;
static uint8_t pause_wchan[MAXCPU];
static struct callout loadav_callout;
struct loadavg averunnable =
{ {0, 0, 0}, FSCALE }; /* load average, of runnable procs */
/*
* Constants for averages over 1, 5, and 15 minutes
* when sampling at 5 second intervals.
*/
static fixpt_t cexp[3] = {
0.9200444146293232 * FSCALE, /* exp(-1/12) */
0.9834714538216174 * FSCALE, /* exp(-1/60) */
0.9944598480048967 * FSCALE, /* exp(-1/180) */
};
/* kernel uses `FSCALE', userland (SHOULD) use kern.fscale */
SYSCTL_INT(_kern, OID_AUTO, fscale, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, FSCALE, "");
static void loadav(void *arg);
SDT_PROVIDER_DECLARE(sched);
SDT_PROBE_DEFINE(sched, , , preempt);
static void
sleepinit(void *unused)
{
hogticks = (hz / 10) * 2; /* Default only. */
init_sleepqueues();
}
/*
* vmem tries to lock the sleepq mutexes when free'ing kva, so make sure
* it is available.
*/
SYSINIT(sleepinit, SI_SUB_KMEM, SI_ORDER_ANY, sleepinit, 0);
/*
* General sleep call. Suspends the current thread until a wakeup is
* performed on the specified identifier. The thread will then be made
* runnable with the specified priority. Sleeps at most sbt units of time
* (0 means no timeout). If pri includes the PCATCH flag, let signals
* interrupt the sleep, otherwise ignore them while sleeping. Returns 0 if
* awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a
* signal becomes pending, ERESTART is returned if the current system
* call should be restarted if possible, and EINTR is returned if the system
* call should be interrupted by the signal (return EINTR).
*
* The lock argument is unlocked before the caller is suspended, and
* re-locked before _sleep() returns. If priority includes the PDROP
* flag the lock is not re-locked before returning.
*/
int
_sleep(void *ident, struct lock_object *lock, int priority,
const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
{
struct thread *td;
struct proc *p;
struct lock_class *class;
uintptr_t lock_state;
int catch, pri, rval, sleepq_flags;
WITNESS_SAVE_DECL(lock_witness);
td = curthread;
p = td->td_proc;
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW))
ktrcsw(1, 0, wmesg);
#endif
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
"Sleeping on \"%s\"", wmesg);
KASSERT(sbt != 0 || mtx_owned(&Giant) || lock != NULL,
("sleeping without a lock"));
KASSERT(p != NULL, ("msleep1"));
KASSERT(ident != NULL && TD_IS_RUNNING(td), ("msleep"));
if (priority & PDROP)
KASSERT(lock != NULL && lock != &Giant.lock_object,
("PDROP requires a non-Giant lock"));
if (lock != NULL)
class = LOCK_CLASS(lock);
else
class = NULL;
if (cold || SCHEDULER_STOPPED()) {
/*
* During autoconfiguration, just return;
* don't run any other threads or panic below,
* in case this is the idle thread and already asleep.
* XXX: this used to do "s = splhigh(); splx(safepri);
* splx(s);" to give interrupts a chance, but there is
* no way to give interrupts a chance now.
*/
if (lock != NULL && priority & PDROP)
class->lc_unlock(lock);
return (0);
}
catch = priority & PCATCH;
pri = priority & PRIMASK;
/*
* If we are already on a sleep queue, then remove us from that
* sleep queue first. We have to do this to handle recursive
* sleeps.
*/
if (TD_ON_SLEEPQ(td))
sleepq_remove(td, td->td_wchan);
if ((uint8_t *)ident >= &pause_wchan[0] &&
(uint8_t *)ident <= &pause_wchan[MAXCPU - 1])
sleepq_flags = SLEEPQ_PAUSE;
else
sleepq_flags = SLEEPQ_SLEEP;
if (catch)
sleepq_flags |= SLEEPQ_INTERRUPTIBLE;
sleepq_lock(ident);
CTR5(KTR_PROC, "sleep: thread %ld (pid %ld, %s) on %s (%p)",
td->td_tid, p->p_pid, td->td_name, wmesg, ident);
if (lock == &Giant.lock_object)
mtx_assert(&Giant, MA_OWNED);
DROP_GIANT();
if (lock != NULL && lock != &Giant.lock_object &&
!(class->lc_flags & LC_SLEEPABLE)) {
WITNESS_SAVE(lock, lock_witness);
lock_state = class->lc_unlock(lock);
} else
/* GCC needs to follow the Yellow Brick Road */
lock_state = -1;
/*
* We put ourselves on the sleep queue and start our timeout
* before calling thread_suspend_check, as we could stop there,
* and a wakeup or a SIGCONT (or both) could occur while we were
* stopped without resuming us. Thus, we must be ready for sleep
* when cursig() is called. If the wakeup happens while we're
* stopped, then td will no longer be on a sleep queue upon
* return from cursig().
*/
sleepq_add(ident, lock, wmesg, sleepq_flags, 0);
if (sbt != 0)
sleepq_set_timeout_sbt(ident, sbt, pr, flags);
if (lock != NULL && class->lc_flags & LC_SLEEPABLE) {
sleepq_release(ident);
WITNESS_SAVE(lock, lock_witness);
lock_state = class->lc_unlock(lock);
sleepq_lock(ident);
}
if (sbt != 0 && catch)
rval = sleepq_timedwait_sig(ident, pri);
else if (sbt != 0)
rval = sleepq_timedwait(ident, pri);
else if (catch)
rval = sleepq_wait_sig(ident, pri);
else {
sleepq_wait(ident, pri);
rval = 0;
}
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW))
ktrcsw(0, 0, wmesg);
#endif
PICKUP_GIANT();
if (lock != NULL && lock != &Giant.lock_object && !(priority & PDROP)) {
class->lc_lock(lock, lock_state);
WITNESS_RESTORE(lock, lock_witness);
}
return (rval);
}
int
msleep_spin_sbt(void *ident, struct mtx *mtx, const char *wmesg,
sbintime_t sbt, sbintime_t pr, int flags)
{
struct thread *td;
struct proc *p;
int rval;
WITNESS_SAVE_DECL(mtx);
td = curthread;
p = td->td_proc;
KASSERT(mtx != NULL, ("sleeping without a mutex"));
KASSERT(p != NULL, ("msleep1"));
KASSERT(ident != NULL && TD_IS_RUNNING(td), ("msleep"));
if (cold || SCHEDULER_STOPPED()) {
/*
* During autoconfiguration, just return;
* don't run any other threads or panic below,
* in case this is the idle thread and already asleep.
* XXX: this used to do "s = splhigh(); splx(safepri);
* splx(s);" to give interrupts a chance, but there is
* no way to give interrupts a chance now.
*/
return (0);
}
sleepq_lock(ident);
CTR5(KTR_PROC, "msleep_spin: thread %ld (pid %ld, %s) on %s (%p)",
td->td_tid, p->p_pid, td->td_name, wmesg, ident);
DROP_GIANT();
mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED);
WITNESS_SAVE(&mtx->lock_object, mtx);
mtx_unlock_spin(mtx);
/*
* We put ourselves on the sleep queue and start our timeout.
*/
sleepq_add(ident, &mtx->lock_object, wmesg, SLEEPQ_SLEEP, 0);
if (sbt != 0)
sleepq_set_timeout_sbt(ident, sbt, pr, flags);
/*
* Can't call ktrace with any spin locks held so it can lock the
* ktrace_mtx lock, and WITNESS_WARN considers it an error to hold
* any spin lock. Thus, we have to drop the sleepq spin lock while
* we handle those requests. This is safe since we have placed our
* thread on the sleep queue already.
*/
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW)) {
sleepq_release(ident);
ktrcsw(1, 0, wmesg);
sleepq_lock(ident);
}
#endif
#ifdef WITNESS
sleepq_release(ident);
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "Sleeping on \"%s\"",
wmesg);
sleepq_lock(ident);
#endif
if (sbt != 0)
rval = sleepq_timedwait(ident, 0);
else {
sleepq_wait(ident, 0);
rval = 0;
}
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW))
ktrcsw(0, 0, wmesg);
#endif
PICKUP_GIANT();
mtx_lock_spin(mtx);
WITNESS_RESTORE(&mtx->lock_object, mtx);
return (rval);
}
/*
* pause() delays the calling thread by the given number of system ticks.
* During cold bootup, pause() uses the DELAY() function instead of
* the tsleep() function to do the waiting. The "timo" argument must be
* greater than or equal to zero. A "timo" value of zero is equivalent
* to a "timo" value of one.
*/
int
pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
{
KASSERT(sbt >= 0, ("pause: timeout must be >= 0"));
/* silently convert invalid timeouts */
if (sbt == 0)
sbt = tick_sbt;
if (cold || kdb_active) {
/*
* We delay one second at a time to avoid overflowing the
* system specific DELAY() function(s):
*/
while (sbt >= SBT_1S) {
DELAY(1000000);
sbt -= SBT_1S;
}
/* Do the delay remainder, if any */
sbt = (sbt + SBT_1US - 1) / SBT_1US;
if (sbt > 0)
DELAY(sbt);
return (0);
}
return (_sleep(&pause_wchan[curcpu], NULL, 0, wmesg, sbt, pr, flags));
}
/*
* Make all threads sleeping on the specified identifier runnable.
*/
void
wakeup(void *ident)
{
int wakeup_swapper;
sleepq_lock(ident);
wakeup_swapper = sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0);
sleepq_release(ident);
if (wakeup_swapper) {
KASSERT(ident != &proc0,
("wakeup and wakeup_swapper and proc0"));
kick_proc0();
}
}
/*
* Make a thread sleeping on the specified identifier runnable.
* May wake more than one thread if a target thread is currently
* swapped out.
*/
void
wakeup_one(void *ident)
{
int wakeup_swapper;
sleepq_lock(ident);
wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0);
sleepq_release(ident);
if (wakeup_swapper)
kick_proc0();
}
static void
kdb_switch(void)
{
thread_unlock(curthread);
kdb_backtrace();
kdb_reenter();
panic("%s: did not reenter debugger", __func__);
}
/*
* The machine independent parts of context switching.
*/
void
mi_switch(int flags, struct thread *newtd)
{
uint64_t runtime, new_switchtime;
struct thread *td;
struct proc *p;
td = curthread; /* XXX */
THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED);
p = td->td_proc; /* XXX */
KASSERT(!TD_ON_RUNQ(td), ("mi_switch: called by old code"));
#ifdef INVARIANTS
if (!TD_ON_LOCK(td) && !TD_IS_RUNNING(td))
mtx_assert(&Giant, MA_NOTOWNED);
#endif
KASSERT(td->td_critnest == 1 || panicstr,
("mi_switch: switch in a critical section"));
KASSERT((flags & (SW_INVOL | SW_VOL)) != 0,
("mi_switch: switch must be voluntary or involuntary"));
KASSERT(newtd != curthread, ("mi_switch: preempting back to ourself"));
/*
* Don't perform context switches from the debugger.
*/
if (kdb_active)
kdb_switch();
if (SCHEDULER_STOPPED())
return;
if (flags & SW_VOL) {
td->td_ru.ru_nvcsw++;
td->td_swvoltick = ticks;
} else
td->td_ru.ru_nivcsw++;
#ifdef SCHED_STATS
SCHED_STAT_INC(sched_switch_stats[flags & SW_TYPE_MASK]);
#endif
/*
* Compute the amount of time during which the current
* thread was running, and add that to its total so far.
*/
new_switchtime = cpu_ticks();
runtime = new_switchtime - PCPU_GET(switchtime);
td->td_runtime += runtime;
td->td_incruntime += runtime;
PCPU_SET(switchtime, new_switchtime);
td->td_generation++; /* bump preempt-detect counter */
PCPU_INC(cnt.v_swtch);
PCPU_SET(switchticks, ticks);
CTR4(KTR_PROC, "mi_switch: old thread %ld (td_sched %p, pid %ld, %s)",
td->td_tid, td->td_sched, p->p_pid, td->td_name);
#if (KTR_COMPILE & KTR_SCHED) != 0
if (TD_IS_IDLETHREAD(td))
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "idle",
"prio:%d", td->td_priority);
else
KTR_STATE3(KTR_SCHED, "thread", sched_tdname(td), KTDSTATE(td),
"prio:%d", td->td_priority, "wmesg:\"%s\"", td->td_wmesg,
"lockname:\"%s\"", td->td_lockname);
#endif
SDT_PROBE0(sched, , , preempt);
sched_switch(td, newtd, flags);
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",
"prio:%d", td->td_priority);
CTR4(KTR_PROC, "mi_switch: new thread %ld (td_sched %p, pid %ld, %s)",
td->td_tid, td->td_sched, p->p_pid, td->td_name);
/*
* If the last thread was exiting, finish cleaning it up.
*/
if ((td = PCPU_GET(deadthread))) {
PCPU_SET(deadthread, NULL);
thread_stash(td);
}
}
/*
* Change thread state to be runnable, placing it on the run queue if
* it is in memory. If it is swapped out, return true so our caller
* will know to awaken the swapper.
*/
int
setrunnable(struct thread *td)
{
THREAD_LOCK_ASSERT(td, MA_OWNED);
KASSERT(td->td_proc->p_state != PRS_ZOMBIE,
("setrunnable: pid %d is a zombie", td->td_proc->p_pid));
switch (td->td_state) {
case TDS_RUNNING:
case TDS_RUNQ:
return (0);
case TDS_INHIBITED:
/*
* If we are only inhibited because we are swapped out
* then arange to swap in this process. Otherwise just return.
*/
if (td->td_inhibitors != TDI_SWAPPED)
return (0);
/* FALLTHROUGH */
case TDS_CAN_RUN:
break;
default:
printf("state is 0x%x", td->td_state);
panic("setrunnable(2)");
}
if ((td->td_flags & TDF_INMEM) == 0) {
if ((td->td_flags & TDF_SWAPINREQ) == 0) {
td->td_flags |= TDF_SWAPINREQ;
return (1);
}
} else
sched_wakeup(td);
return (0);
}
/*
* Compute a tenex style load average of a quantity on
* 1, 5 and 15 minute intervals.
*/
static void
loadav(void *arg)
{
int i, nrun;
struct loadavg *avg;
nrun = sched_load();
avg = &averunnable;
for (i = 0; i < 3; i++)
avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
/*
* Schedule the next update to occur after 5 seconds, but add a
* random variation to avoid synchronisation with processes that
* run at regular intervals.
*/
callout_reset_sbt(&loadav_callout,
SBT_1US * (4000000 + (int)(random() % 2000001)), SBT_1US,
loadav, NULL, C_DIRECT_EXEC | C_PREL(32));
}
/* ARGSUSED */
static void
synch_setup(void *dummy)
{
- callout_init(&loadav_callout, CALLOUT_MPSAFE);
+ callout_init(&loadav_callout, 1);
/* Kick off timeout driven events by calling first time. */
loadav(NULL);
}
int
should_yield(void)
{
return ((u_int)ticks - (u_int)curthread->td_swvoltick >= hogticks);
}
void
maybe_yield(void)
{
if (should_yield())
kern_yield(PRI_USER);
}
void
kern_yield(int prio)
{
struct thread *td;
td = curthread;
DROP_GIANT();
thread_lock(td);
if (prio == PRI_USER)
prio = td->td_user_pri;
if (prio >= 0)
sched_prio(td, prio);
mi_switch(SW_VOL | SWT_RELINQUISH, NULL);
thread_unlock(td);
PICKUP_GIANT();
}
/*
* General purpose yield system call.
*/
int
sys_yield(struct thread *td, struct yield_args *uap)
{
thread_lock(td);
if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
sched_prio(td, PRI_MAX_TIMESHARE);
mi_switch(SW_VOL | SWT_RELINQUISH, NULL);
thread_unlock(td);
td->td_retval[0] = 0;
return (0);
}
Index: head/sys/kern/kern_thread.c
===================================================================
--- head/sys/kern/kern_thread.c (revision 283290)
+++ head/sys/kern/kern_thread.c (revision 283291)
@@ -1,1146 +1,1146 @@
/*-
* Copyright (C) 2001 Julian Elischer <julian@freebsd.org>.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice(s), this list of conditions and the following disclaimer as
* the first lines of this file unmodified other than the possible
* addition of one or more copyright notices.
* 2. Redistributions in binary form must reproduce the above copyright
* notice(s), this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/
#include "opt_witness.h"
#include "opt_hwpmc_hooks.h"
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/rangelock.h>
#include <sys/resourcevar.h>
#include <sys/sdt.h>
#include <sys/smp.h>
#include <sys/sched.h>
#include <sys/sleepqueue.h>
#include <sys/selinfo.h>
#include <sys/turnstile.h>
#include <sys/ktr.h>
#include <sys/rwlock.h>
#include <sys/umtx.h>
#include <sys/cpuset.h>
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
#endif
#include <security/audit/audit.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/uma.h>
#include <sys/eventhandler.h>
SDT_PROVIDER_DECLARE(proc);
SDT_PROBE_DEFINE(proc, , , lwp__exit);
/*
* thread related storage.
*/
static uma_zone_t thread_zone;
TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
static struct mtx zombie_lock;
MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN);
static void thread_zombie(struct thread *);
static int thread_unsuspend_one(struct thread *td, struct proc *p,
bool boundary);
#define TID_BUFFER_SIZE 1024
struct mtx tid_lock;
static struct unrhdr *tid_unrhdr;
static lwpid_t tid_buffer[TID_BUFFER_SIZE];
static int tid_head, tid_tail;
static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash");
struct tidhashhead *tidhashtbl;
u_long tidhash;
struct rwlock tidhash_lock;
static lwpid_t
tid_alloc(void)
{
lwpid_t tid;
tid = alloc_unr(tid_unrhdr);
if (tid != -1)
return (tid);
mtx_lock(&tid_lock);
if (tid_head == tid_tail) {
mtx_unlock(&tid_lock);
return (-1);
}
tid = tid_buffer[tid_head];
tid_head = (tid_head + 1) % TID_BUFFER_SIZE;
mtx_unlock(&tid_lock);
return (tid);
}
static void
tid_free(lwpid_t tid)
{
lwpid_t tmp_tid = -1;
mtx_lock(&tid_lock);
if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) {
tmp_tid = tid_buffer[tid_head];
tid_head = (tid_head + 1) % TID_BUFFER_SIZE;
}
tid_buffer[tid_tail] = tid;
tid_tail = (tid_tail + 1) % TID_BUFFER_SIZE;
mtx_unlock(&tid_lock);
if (tmp_tid != -1)
free_unr(tid_unrhdr, tmp_tid);
}
/*
* Prepare a thread for use.
*/
static int
thread_ctor(void *mem, int size, void *arg, int flags)
{
struct thread *td;
td = (struct thread *)mem;
td->td_state = TDS_INACTIVE;
td->td_oncpu = NOCPU;
td->td_tid = tid_alloc();
/*
* Note that td_critnest begins life as 1 because the thread is not
* running and is thereby implicitly waiting to be on the receiving
* end of a context switch.
*/
td->td_critnest = 1;
td->td_lend_user_pri = PRI_MAX;
EVENTHANDLER_INVOKE(thread_ctor, td);
#ifdef AUDIT
audit_thread_alloc(td);
#endif
umtx_thread_alloc(td);
return (0);
}
/*
* Reclaim a thread after use.
*/
static void
thread_dtor(void *mem, int size, void *arg)
{
struct thread *td;
td = (struct thread *)mem;
#ifdef INVARIANTS
/* Verify that this thread is in a safe state to free. */
switch (td->td_state) {
case TDS_INHIBITED:
case TDS_RUNNING:
case TDS_CAN_RUN:
case TDS_RUNQ:
/*
* We must never unlink a thread that is in one of
* these states, because it is currently active.
*/
panic("bad state for thread unlinking");
/* NOTREACHED */
case TDS_INACTIVE:
break;
default:
panic("bad thread state");
/* NOTREACHED */
}
#endif
#ifdef AUDIT
audit_thread_free(td);
#endif
/* Free all OSD associated to this thread. */
osd_thread_exit(td);
EVENTHANDLER_INVOKE(thread_dtor, td);
tid_free(td->td_tid);
}
/*
* Initialize type-stable parts of a thread (when newly created).
*/
static int
thread_init(void *mem, int size, int flags)
{
struct thread *td;
td = (struct thread *)mem;
td->td_sleepqueue = sleepq_alloc();
td->td_turnstile = turnstile_alloc();
td->td_rlqe = NULL;
EVENTHANDLER_INVOKE(thread_init, td);
td->td_sched = (struct td_sched *)&td[1];
umtx_thread_init(td);
td->td_kstack = 0;
td->td_sel = NULL;
return (0);
}
/*
* Tear down type-stable parts of a thread (just before being discarded).
*/
static void
thread_fini(void *mem, int size)
{
struct thread *td;
td = (struct thread *)mem;
EVENTHANDLER_INVOKE(thread_fini, td);
rlqentry_free(td->td_rlqe);
turnstile_free(td->td_turnstile);
sleepq_free(td->td_sleepqueue);
umtx_thread_fini(td);
seltdfini(td);
}
/*
* For a newly created process,
* link up all the structures and its initial threads etc.
* called from:
* {arch}/{arch}/machdep.c {arch}_init(), init386() etc.
* proc_dtor() (should go away)
* proc_init()
*/
void
proc_linkup0(struct proc *p, struct thread *td)
{
TAILQ_INIT(&p->p_threads); /* all threads in proc */
proc_linkup(p, td);
}
void
proc_linkup(struct proc *p, struct thread *td)
{
sigqueue_init(&p->p_sigqueue, p);
p->p_ksi = ksiginfo_alloc(1);
if (p->p_ksi != NULL) {
/* XXX p_ksi may be null if ksiginfo zone is not ready */
p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
}
LIST_INIT(&p->p_mqnotifier);
p->p_numthreads = 0;
thread_link(td, p);
}
/*
* Initialize global thread allocation resources.
*/
void
threadinit(void)
{
mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF);
/*
* pid_max cannot be greater than PID_MAX.
* leave one number for thread0.
*/
tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock);
thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
thread_ctor, thread_dtor, thread_init, thread_fini,
16 - 1, 0);
tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash);
rw_init(&tidhash_lock, "tidhash");
}
/*
* Place an unused thread on the zombie list.
* Use the slpq as that must be unused by now.
*/
void
thread_zombie(struct thread *td)
{
mtx_lock_spin(&zombie_lock);
TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq);
mtx_unlock_spin(&zombie_lock);
}
/*
* Release a thread that has exited after cpu_throw().
*/
void
thread_stash(struct thread *td)
{
atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1);
thread_zombie(td);
}
/*
* Reap zombie resources.
*/
void
thread_reap(void)
{
struct thread *td_first, *td_next;
/*
* Don't even bother to lock if none at this instant,
* we really don't care about the next instant..
*/
if (!TAILQ_EMPTY(&zombie_threads)) {
mtx_lock_spin(&zombie_lock);
td_first = TAILQ_FIRST(&zombie_threads);
if (td_first)
TAILQ_INIT(&zombie_threads);
mtx_unlock_spin(&zombie_lock);
while (td_first) {
td_next = TAILQ_NEXT(td_first, td_slpq);
if (td_first->td_ucred)
crfree(td_first->td_ucred);
thread_free(td_first);
td_first = td_next;
}
}
}
/*
* Allocate a thread.
*/
struct thread *
thread_alloc(int pages)
{
struct thread *td;
thread_reap(); /* check if any zombies to get */
td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK);
KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack"));
if (!vm_thread_new(td, pages)) {
uma_zfree(thread_zone, td);
return (NULL);
}
cpu_thread_alloc(td);
return (td);
}
int
thread_alloc_stack(struct thread *td, int pages)
{
KASSERT(td->td_kstack == 0,
("thread_alloc_stack called on a thread with kstack"));
if (!vm_thread_new(td, pages))
return (0);
cpu_thread_alloc(td);
return (1);
}
/*
* Deallocate a thread.
*/
void
thread_free(struct thread *td)
{
lock_profile_thread_exit(td);
if (td->td_cpuset)
cpuset_rel(td->td_cpuset);
td->td_cpuset = NULL;
cpu_thread_free(td);
if (td->td_kstack != 0)
vm_thread_dispose(td);
uma_zfree(thread_zone, td);
}
/*
* Discard the current thread and exit from its context.
* Always called with scheduler locked.
*
* Because we can't free a thread while we're operating under its context,
* push the current thread into our CPU's deadthread holder. This means
* we needn't worry about someone else grabbing our context before we
* do a cpu_throw().
*/
void
thread_exit(void)
{
uint64_t runtime, new_switchtime;
struct thread *td;
struct thread *td2;
struct proc *p;
int wakeup_swapper;
td = curthread;
p = td->td_proc;
PROC_SLOCK_ASSERT(p, MA_OWNED);
mtx_assert(&Giant, MA_NOTOWNED);
PROC_LOCK_ASSERT(p, MA_OWNED);
KASSERT(p != NULL, ("thread exiting without a process"));
CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
(long)p->p_pid, td->td_name);
KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending"));
#ifdef AUDIT
AUDIT_SYSCALL_EXIT(0, td);
#endif
/*
* drop FPU & debug register state storage, or any other
* architecture specific resources that
* would not be on a new untouched process.
*/
cpu_thread_exit(td); /* XXXSMP */
/*
* The last thread is left attached to the process
* So that the whole bundle gets recycled. Skip
* all this stuff if we never had threads.
* EXIT clears all sign of other threads when
* it goes to single threading, so the last thread always
* takes the short path.
*/
if (p->p_flag & P_HADTHREADS) {
if (p->p_numthreads > 1) {
atomic_add_int(&td->td_proc->p_exitthreads, 1);
thread_unlink(td);
td2 = FIRST_THREAD_IN_PROC(p);
sched_exit_thread(td2, td);
/*
* The test below is NOT true if we are the
* sole exiting thread. P_STOPPED_SINGLE is unset
* in exit1() after it is the only survivor.
*/
if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
if (p->p_numthreads == p->p_suspcount) {
thread_lock(p->p_singlethread);
wakeup_swapper = thread_unsuspend_one(
p->p_singlethread, p, false);
thread_unlock(p->p_singlethread);
if (wakeup_swapper)
kick_proc0();
}
}
PCPU_SET(deadthread, td);
} else {
/*
* The last thread is exiting.. but not through exit()
*/
panic ("thread_exit: Last thread exiting on its own");
}
}
#ifdef HWPMC_HOOKS
/*
* If this thread is part of a process that is being tracked by hwpmc(4),
* inform the module of the thread's impending exit.
*/
if (PMC_PROC_IS_USING_PMCS(td->td_proc))
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
#endif
PROC_UNLOCK(p);
PROC_STATLOCK(p);
thread_lock(td);
PROC_SUNLOCK(p);
/* Do the same timestamp bookkeeping that mi_switch() would do. */
new_switchtime = cpu_ticks();
runtime = new_switchtime - PCPU_GET(switchtime);
td->td_runtime += runtime;
td->td_incruntime += runtime;
PCPU_SET(switchtime, new_switchtime);
PCPU_SET(switchticks, ticks);
PCPU_INC(cnt.v_swtch);
/* Save our resource usage in our process. */
td->td_ru.ru_nvcsw++;
ruxagg(p, td);
rucollect(&p->p_ru, &td->td_ru);
PROC_STATUNLOCK(p);
td->td_state = TDS_INACTIVE;
#ifdef WITNESS
witness_thread_exit(td);
#endif
CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
sched_throw(td);
panic("I'm a teapot!");
/* NOTREACHED */
}
/*
* Do any thread specific cleanups that may be needed in wait()
* called with Giant, proc and schedlock not held.
*/
void
thread_wait(struct proc *p)
{
struct thread *td;
mtx_assert(&Giant, MA_NOTOWNED);
KASSERT(p->p_numthreads == 1, ("multiple threads in thread_wait()"));
KASSERT(p->p_exitthreads == 0, ("p_exitthreads leaking"));
td = FIRST_THREAD_IN_PROC(p);
/* Lock the last thread so we spin until it exits cpu_throw(). */
thread_lock(td);
thread_unlock(td);
lock_profile_thread_exit(td);
cpuset_rel(td->td_cpuset);
td->td_cpuset = NULL;
cpu_thread_clean(td);
crfree(td->td_ucred);
thread_reap(); /* check for zombie threads etc. */
}
/*
* Link a thread to a process.
* set up anything that needs to be initialized for it to
* be used by the process.
*/
void
thread_link(struct thread *td, struct proc *p)
{
/*
* XXX This can't be enabled because it's called for proc0 before
* its lock has been created.
* PROC_LOCK_ASSERT(p, MA_OWNED);
*/
td->td_state = TDS_INACTIVE;
td->td_proc = p;
td->td_flags = TDF_INMEM;
LIST_INIT(&td->td_contested);
LIST_INIT(&td->td_lprof[0]);
LIST_INIT(&td->td_lprof[1]);
sigqueue_init(&td->td_sigqueue, p);
- callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
+ callout_init(&td->td_slpcallout, 1);
TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist);
p->p_numthreads++;
}
/*
* Called from:
* thread_exit()
*/
void
thread_unlink(struct thread *td)
{
struct proc *p = td->td_proc;
PROC_LOCK_ASSERT(p, MA_OWNED);
TAILQ_REMOVE(&p->p_threads, td, td_plist);
p->p_numthreads--;
/* could clear a few other things here */
/* Must NOT clear links to proc! */
}
static int
calc_remaining(struct proc *p, int mode)
{
int remaining;
PROC_LOCK_ASSERT(p, MA_OWNED);
PROC_SLOCK_ASSERT(p, MA_OWNED);
if (mode == SINGLE_EXIT)
remaining = p->p_numthreads;
else if (mode == SINGLE_BOUNDARY)
remaining = p->p_numthreads - p->p_boundary_count;
else if (mode == SINGLE_NO_EXIT || mode == SINGLE_ALLPROC)
remaining = p->p_numthreads - p->p_suspcount;
else
panic("calc_remaining: wrong mode %d", mode);
return (remaining);
}
static int
remain_for_mode(int mode)
{
return (mode == SINGLE_ALLPROC ? 0 : 1);
}
static int
weed_inhib(int mode, struct thread *td2, struct proc *p)
{
int wakeup_swapper;
PROC_LOCK_ASSERT(p, MA_OWNED);
PROC_SLOCK_ASSERT(p, MA_OWNED);
THREAD_LOCK_ASSERT(td2, MA_OWNED);
wakeup_swapper = 0;
switch (mode) {
case SINGLE_EXIT:
if (TD_IS_SUSPENDED(td2))
wakeup_swapper |= thread_unsuspend_one(td2, p, true);
if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0)
wakeup_swapper |= sleepq_abort(td2, EINTR);
break;
case SINGLE_BOUNDARY:
if (TD_IS_SUSPENDED(td2) && (td2->td_flags & TDF_BOUNDARY) == 0)
wakeup_swapper |= thread_unsuspend_one(td2, p, false);
if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0)
wakeup_swapper |= sleepq_abort(td2, ERESTART);
break;
case SINGLE_NO_EXIT:
if (TD_IS_SUSPENDED(td2) && (td2->td_flags & TDF_BOUNDARY) == 0)
wakeup_swapper |= thread_unsuspend_one(td2, p, false);
if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0)
wakeup_swapper |= sleepq_abort(td2, ERESTART);
break;
case SINGLE_ALLPROC:
/*
* ALLPROC suspend tries to avoid spurious EINTR for
* threads sleeping interruptable, by suspending the
* thread directly, similarly to sig_suspend_threads().
* Since such sleep is not performed at the user
* boundary, TDF_BOUNDARY flag is not set, and TDF_ALLPROCSUSP
* is used to avoid immediate un-suspend.
*/
if (TD_IS_SUSPENDED(td2) && (td2->td_flags & (TDF_BOUNDARY |
TDF_ALLPROCSUSP)) == 0)
wakeup_swapper |= thread_unsuspend_one(td2, p, false);
if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0) {
if ((td2->td_flags & TDF_SBDRY) == 0) {
thread_suspend_one(td2);
td2->td_flags |= TDF_ALLPROCSUSP;
} else {
wakeup_swapper |= sleepq_abort(td2, ERESTART);
}
}
break;
}
return (wakeup_swapper);
}
/*
* Enforce single-threading.
*
* Returns 1 if the caller must abort (another thread is waiting to
* exit the process or similar). Process is locked!
* Returns 0 when you are successfully the only thread running.
* A process has successfully single threaded in the suspend mode when
* There are no threads in user mode. Threads in the kernel must be
* allowed to continue until they get to the user boundary. They may even
* copy out their return values and data before suspending. They may however be
* accelerated in reaching the user boundary as we will wake up
* any sleeping threads that are interruptable. (PCATCH).
*/
int
thread_single(struct proc *p, int mode)
{
struct thread *td;
struct thread *td2;
int remaining, wakeup_swapper;
td = curthread;
KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY ||
mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT,
("invalid mode %d", mode));
/*
* If allowing non-ALLPROC singlethreading for non-curproc
* callers, calc_remaining() and remain_for_mode() should be
* adjusted to also account for td->td_proc != p. For now
* this is not implemented because it is not used.
*/
KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) ||
(mode != SINGLE_ALLPROC && td->td_proc == p),
("mode %d proc %p curproc %p", mode, p, td->td_proc));
mtx_assert(&Giant, MA_NOTOWNED);
PROC_LOCK_ASSERT(p, MA_OWNED);
if ((p->p_flag & P_HADTHREADS) == 0 && mode != SINGLE_ALLPROC)
return (0);
/* Is someone already single threading? */
if (p->p_singlethread != NULL && p->p_singlethread != td)
return (1);
if (mode == SINGLE_EXIT) {
p->p_flag |= P_SINGLE_EXIT;
p->p_flag &= ~P_SINGLE_BOUNDARY;
} else {
p->p_flag &= ~P_SINGLE_EXIT;
if (mode == SINGLE_BOUNDARY)
p->p_flag |= P_SINGLE_BOUNDARY;
else
p->p_flag &= ~P_SINGLE_BOUNDARY;
}
if (mode == SINGLE_ALLPROC)
p->p_flag |= P_TOTAL_STOP;
p->p_flag |= P_STOPPED_SINGLE;
PROC_SLOCK(p);
p->p_singlethread = td;
remaining = calc_remaining(p, mode);
while (remaining != remain_for_mode(mode)) {
if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
goto stopme;
wakeup_swapper = 0;
FOREACH_THREAD_IN_PROC(p, td2) {
if (td2 == td)
continue;
thread_lock(td2);
td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
if (TD_IS_INHIBITED(td2)) {
wakeup_swapper |= weed_inhib(mode, td2, p);
#ifdef SMP
} else if (TD_IS_RUNNING(td2) && td != td2) {
forward_signal(td2);
#endif
}
thread_unlock(td2);
}
if (wakeup_swapper)
kick_proc0();
remaining = calc_remaining(p, mode);
/*
* Maybe we suspended some threads.. was it enough?
*/
if (remaining == remain_for_mode(mode))
break;
stopme:
/*
* Wake us up when everyone else has suspended.
* In the mean time we suspend as well.
*/
thread_suspend_switch(td, p);
remaining = calc_remaining(p, mode);
}
if (mode == SINGLE_EXIT) {
/*
* Convert the process to an unthreaded process. The
* SINGLE_EXIT is called by exit1() or execve(), in
* both cases other threads must be retired.
*/
KASSERT(p->p_numthreads == 1, ("Unthreading with >1 threads"));
p->p_singlethread = NULL;
p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_HADTHREADS);
/*
* Wait for any remaining threads to exit cpu_throw().
*/
while (p->p_exitthreads != 0) {
PROC_SUNLOCK(p);
PROC_UNLOCK(p);
sched_relinquish(td);
PROC_LOCK(p);
PROC_SLOCK(p);
}
} else if (mode == SINGLE_BOUNDARY) {
/*
* Wait until all suspended threads are removed from
* the processors. The thread_suspend_check()
* increments p_boundary_count while it is still
* running, which makes it possible for the execve()
* to destroy vmspace while our other threads are
* still using the address space.
*
* We lock the thread, which is only allowed to
* succeed after context switch code finished using
* the address space.
*/
FOREACH_THREAD_IN_PROC(p, td2) {
if (td2 == td)
continue;
thread_lock(td2);
KASSERT((td2->td_flags & TDF_BOUNDARY) != 0,
("td %p not on boundary", td2));
KASSERT(TD_IS_SUSPENDED(td2),
("td %p is not suspended", td2));
thread_unlock(td2);
}
}
PROC_SUNLOCK(p);
return (0);
}
bool
thread_suspend_check_needed(void)
{
struct proc *p;
struct thread *td;
td = curthread;
p = td->td_proc;
PROC_LOCK_ASSERT(p, MA_OWNED);
return (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) != 0 &&
(td->td_dbgflags & TDB_SUSPEND) != 0));
}
/*
* Called in from locations that can safely check to see
* whether we have to suspend or at least throttle for a
* single-thread event (e.g. fork).
*
* Such locations include userret().
* If the "return_instead" argument is non zero, the thread must be able to
* accept 0 (caller may continue), or 1 (caller must abort) as a result.
*
* The 'return_instead' argument tells the function if it may do a
* thread_exit() or suspend, or whether the caller must abort and back
* out instead.
*
* If the thread that set the single_threading request has set the
* P_SINGLE_EXIT bit in the process flags then this call will never return
* if 'return_instead' is false, but will exit.
*
* P_SINGLE_EXIT | return_instead == 0| return_instead != 0
*---------------+--------------------+---------------------
* 0 | returns 0 | returns 0 or 1
* | when ST ends | immediately
*---------------+--------------------+---------------------
* 1 | thread exits | returns 1
* | | immediately
* 0 = thread_exit() or suspension ok,
* other = return error instead of stopping the thread.
*
* While a full suspension is under effect, even a single threading
* thread would be suspended if it made this call (but it shouldn't).
* This call should only be made from places where
* thread_exit() would be safe as that may be the outcome unless
* return_instead is set.
*/
int
thread_suspend_check(int return_instead)
{
struct thread *td;
struct proc *p;
int wakeup_swapper;
td = curthread;
p = td->td_proc;
mtx_assert(&Giant, MA_NOTOWNED);
PROC_LOCK_ASSERT(p, MA_OWNED);
while (thread_suspend_check_needed()) {
if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
KASSERT(p->p_singlethread != NULL,
("singlethread not set"));
/*
* The only suspension in action is a
* single-threading. Single threader need not stop.
* XXX Should be safe to access unlocked
* as it can only be set to be true by us.
*/
if (p->p_singlethread == td)
return (0); /* Exempt from stopping. */
}
if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
return (EINTR);
/* Should we goto user boundary if we didn't come from there? */
if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
(p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
return (ERESTART);
/*
* Ignore suspend requests for stop signals if they
* are deferred.
*/
if ((P_SHOULDSTOP(p) == P_STOPPED_SIG ||
(p->p_flag & P_TOTAL_STOP) != 0) &&
(td->td_flags & TDF_SBDRY) != 0) {
KASSERT(return_instead,
("TDF_SBDRY set for unsafe thread_suspend_check"));
return (0);
}
/*
* If the process is waiting for us to exit,
* this thread should just suicide.
* Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
*/
if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) {
PROC_UNLOCK(p);
tidhash_remove(td);
PROC_LOCK(p);
tdsigcleanup(td);
umtx_thread_exit(td);
PROC_SLOCK(p);
thread_stopped(p);
thread_exit();
}
PROC_SLOCK(p);
thread_stopped(p);
if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
if (p->p_numthreads == p->p_suspcount + 1) {
thread_lock(p->p_singlethread);
wakeup_swapper = thread_unsuspend_one(
p->p_singlethread, p, false);
thread_unlock(p->p_singlethread);
if (wakeup_swapper)
kick_proc0();
}
}
PROC_UNLOCK(p);
thread_lock(td);
/*
* When a thread suspends, it just
* gets taken off all queues.
*/
thread_suspend_one(td);
if (return_instead == 0) {
p->p_boundary_count++;
td->td_flags |= TDF_BOUNDARY;
}
PROC_SUNLOCK(p);
mi_switch(SW_INVOL | SWT_SUSPEND, NULL);
thread_unlock(td);
PROC_LOCK(p);
}
return (0);
}
void
thread_suspend_switch(struct thread *td, struct proc *p)
{
KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
PROC_LOCK_ASSERT(p, MA_OWNED);
PROC_SLOCK_ASSERT(p, MA_OWNED);
/*
* We implement thread_suspend_one in stages here to avoid
* dropping the proc lock while the thread lock is owned.
*/
if (p == td->td_proc) {
thread_stopped(p);
p->p_suspcount++;
}
PROC_UNLOCK(p);
thread_lock(td);
td->td_flags &= ~TDF_NEEDSUSPCHK;
TD_SET_SUSPENDED(td);
sched_sleep(td, 0);
PROC_SUNLOCK(p);
DROP_GIANT();
mi_switch(SW_VOL | SWT_SUSPEND, NULL);
thread_unlock(td);
PICKUP_GIANT();
PROC_LOCK(p);
PROC_SLOCK(p);
}
void
thread_suspend_one(struct thread *td)
{
struct proc *p;
p = td->td_proc;
PROC_SLOCK_ASSERT(p, MA_OWNED);
THREAD_LOCK_ASSERT(td, MA_OWNED);
KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
p->p_suspcount++;
td->td_flags &= ~TDF_NEEDSUSPCHK;
TD_SET_SUSPENDED(td);
sched_sleep(td, 0);
}
static int
thread_unsuspend_one(struct thread *td, struct proc *p, bool boundary)
{
THREAD_LOCK_ASSERT(td, MA_OWNED);
KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
TD_CLR_SUSPENDED(td);
td->td_flags &= ~TDF_ALLPROCSUSP;
if (td->td_proc == p) {
PROC_SLOCK_ASSERT(p, MA_OWNED);
p->p_suspcount--;
if (boundary && (td->td_flags & TDF_BOUNDARY) != 0) {
td->td_flags &= ~TDF_BOUNDARY;
p->p_boundary_count--;
}
}
return (setrunnable(td));
}
/*
* Allow all threads blocked by single threading to continue running.
*/
void
thread_unsuspend(struct proc *p)
{
struct thread *td;
int wakeup_swapper;
PROC_LOCK_ASSERT(p, MA_OWNED);
PROC_SLOCK_ASSERT(p, MA_OWNED);
wakeup_swapper = 0;
if (!P_SHOULDSTOP(p)) {
FOREACH_THREAD_IN_PROC(p, td) {
thread_lock(td);
if (TD_IS_SUSPENDED(td)) {
wakeup_swapper |= thread_unsuspend_one(td, p,
true);
}
thread_unlock(td);
}
} else if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
p->p_numthreads == p->p_suspcount) {
/*
* Stopping everything also did the job for the single
* threading request. Now we've downgraded to single-threaded,
* let it continue.
*/
if (p->p_singlethread->td_proc == p) {
thread_lock(p->p_singlethread);
wakeup_swapper = thread_unsuspend_one(
p->p_singlethread, p, false);
thread_unlock(p->p_singlethread);
}
}
if (wakeup_swapper)
kick_proc0();
}
/*
* End the single threading mode..
*/
void
thread_single_end(struct proc *p, int mode)
{
struct thread *td;
int wakeup_swapper;
KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY ||
mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT,
("invalid mode %d", mode));
PROC_LOCK_ASSERT(p, MA_OWNED);
KASSERT((mode == SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) != 0) ||
(mode != SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) == 0),
("mode %d does not match P_TOTAL_STOP", mode));
KASSERT(mode == SINGLE_ALLPROC || p->p_singlethread == curthread,
("thread_single_end from other thread %p %p",
curthread, p->p_singlethread));
KASSERT(mode != SINGLE_BOUNDARY ||
(p->p_flag & P_SINGLE_BOUNDARY) != 0,
("mis-matched SINGLE_BOUNDARY flags %x", p->p_flag));
p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY |
P_TOTAL_STOP);
PROC_SLOCK(p);
p->p_singlethread = NULL;
wakeup_swapper = 0;
/*
* If there are other threads they may now run,
* unless of course there is a blanket 'stop order'
* on the process. The single threader must be allowed
* to continue however as this is a bad place to stop.
*/
if (p->p_numthreads != remain_for_mode(mode) && !P_SHOULDSTOP(p)) {
FOREACH_THREAD_IN_PROC(p, td) {
thread_lock(td);
if (TD_IS_SUSPENDED(td)) {
wakeup_swapper |= thread_unsuspend_one(td, p,
mode == SINGLE_BOUNDARY);
}
thread_unlock(td);
}
}
KASSERT(mode != SINGLE_BOUNDARY || p->p_boundary_count == 0,
("inconsistent boundary count %d", p->p_boundary_count));
PROC_SUNLOCK(p);
if (wakeup_swapper)
kick_proc0();
}
struct thread *
thread_find(struct proc *p, lwpid_t tid)
{
struct thread *td;
PROC_LOCK_ASSERT(p, MA_OWNED);
FOREACH_THREAD_IN_PROC(p, td) {
if (td->td_tid == tid)
break;
}
return (td);
}
/* Locate a thread by number; return with proc lock held. */
struct thread *
tdfind(lwpid_t tid, pid_t pid)
{
#define RUN_THRESH 16
struct thread *td;
int run = 0;
rw_rlock(&tidhash_lock);
LIST_FOREACH(td, TIDHASH(tid), td_hash) {
if (td->td_tid == tid) {
if (pid != -1 && td->td_proc->p_pid != pid) {
td = NULL;
break;
}
PROC_LOCK(td->td_proc);
if (td->td_proc->p_state == PRS_NEW) {
PROC_UNLOCK(td->td_proc);
td = NULL;
break;
}
if (run > RUN_THRESH) {
if (rw_try_upgrade(&tidhash_lock)) {
LIST_REMOVE(td, td_hash);
LIST_INSERT_HEAD(TIDHASH(td->td_tid),
td, td_hash);
rw_wunlock(&tidhash_lock);
return (td);
}
}
break;
}
run++;
}
rw_runlock(&tidhash_lock);
return (td);
}
void
tidhash_add(struct thread *td)
{
rw_wlock(&tidhash_lock);
LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash);
rw_wunlock(&tidhash_lock);
}
void
tidhash_remove(struct thread *td)
{
rw_wlock(&tidhash_lock);
LIST_REMOVE(td, td_hash);
rw_wunlock(&tidhash_lock);
}
Index: head/sys/kern/subr_vmem.c
===================================================================
--- head/sys/kern/subr_vmem.c (revision 283290)
+++ head/sys/kern/subr_vmem.c (revision 283291)
@@ -1,1588 +1,1588 @@
/*-
* Copyright (c)2006,2007,2008,2009 YAMAMOTO Takashi,
* Copyright (c) 2013 EMC Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* From:
* $NetBSD: vmem_impl.h,v 1.2 2013/01/29 21:26:24 para Exp $
* $NetBSD: subr_vmem.c,v 1.83 2013/03/06 11:20:10 yamt Exp $
*/
/*
* reference:
* - Magazines and Vmem: Extending the Slab Allocator
* to Many CPUs and Arbitrary Resources
* http://www.usenix.org/event/usenix01/bonwick.html
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/queue.h>
#include <sys/callout.h>
#include <sys/hash.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/condvar.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/vmem.h>
#include "opt_vm.h"
#include <vm/uma.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
#include <vm/vm_param.h>
#include <vm/vm_pageout.h>
#define VMEM_OPTORDER 5
#define VMEM_OPTVALUE (1 << VMEM_OPTORDER)
#define VMEM_MAXORDER \
(VMEM_OPTVALUE - 1 + sizeof(vmem_size_t) * NBBY - VMEM_OPTORDER)
#define VMEM_HASHSIZE_MIN 16
#define VMEM_HASHSIZE_MAX 131072
#define VMEM_QCACHE_IDX_MAX 16
#define VMEM_FITMASK (M_BESTFIT | M_FIRSTFIT)
#define VMEM_FLAGS \
(M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | M_BESTFIT | M_FIRSTFIT)
#define BT_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM)
#define QC_NAME_MAX 16
/*
* Data structures private to vmem.
*/
MALLOC_DEFINE(M_VMEM, "vmem", "vmem internal structures");
typedef struct vmem_btag bt_t;
TAILQ_HEAD(vmem_seglist, vmem_btag);
LIST_HEAD(vmem_freelist, vmem_btag);
LIST_HEAD(vmem_hashlist, vmem_btag);
struct qcache {
uma_zone_t qc_cache;
vmem_t *qc_vmem;
vmem_size_t qc_size;
char qc_name[QC_NAME_MAX];
};
typedef struct qcache qcache_t;
#define QC_POOL_TO_QCACHE(pool) ((qcache_t *)(pool->pr_qcache))
#define VMEM_NAME_MAX 16
/* vmem arena */
struct vmem {
struct mtx_padalign vm_lock;
struct cv vm_cv;
char vm_name[VMEM_NAME_MAX+1];
LIST_ENTRY(vmem) vm_alllist;
struct vmem_hashlist vm_hash0[VMEM_HASHSIZE_MIN];
struct vmem_freelist vm_freelist[VMEM_MAXORDER];
struct vmem_seglist vm_seglist;
struct vmem_hashlist *vm_hashlist;
vmem_size_t vm_hashsize;
/* Constant after init */
vmem_size_t vm_qcache_max;
vmem_size_t vm_quantum_mask;
vmem_size_t vm_import_quantum;
int vm_quantum_shift;
/* Written on alloc/free */
LIST_HEAD(, vmem_btag) vm_freetags;
int vm_nfreetags;
int vm_nbusytag;
vmem_size_t vm_inuse;
vmem_size_t vm_size;
/* Used on import. */
vmem_import_t *vm_importfn;
vmem_release_t *vm_releasefn;
void *vm_arg;
/* Space exhaustion callback. */
vmem_reclaim_t *vm_reclaimfn;
/* quantum cache */
qcache_t vm_qcache[VMEM_QCACHE_IDX_MAX];
};
/* boundary tag */
struct vmem_btag {
TAILQ_ENTRY(vmem_btag) bt_seglist;
union {
LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */
LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */
} bt_u;
#define bt_hashlist bt_u.u_hashlist
#define bt_freelist bt_u.u_freelist
vmem_addr_t bt_start;
vmem_size_t bt_size;
int bt_type;
};
#define BT_TYPE_SPAN 1 /* Allocated from importfn */
#define BT_TYPE_SPAN_STATIC 2 /* vmem_add() or create. */
#define BT_TYPE_FREE 3 /* Available space. */
#define BT_TYPE_BUSY 4 /* Used space. */
#define BT_ISSPAN_P(bt) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC)
#define BT_END(bt) ((bt)->bt_start + (bt)->bt_size - 1)
#if defined(DIAGNOSTIC)
static int enable_vmem_check = 1;
SYSCTL_INT(_debug, OID_AUTO, vmem_check, CTLFLAG_RWTUN,
&enable_vmem_check, 0, "Enable vmem check");
static void vmem_check(vmem_t *);
#endif
static struct callout vmem_periodic_ch;
static int vmem_periodic_interval;
static struct task vmem_periodic_wk;
static struct mtx_padalign vmem_list_lock;
static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list);
/* ---- misc */
#define VMEM_CONDVAR_INIT(vm, wchan) cv_init(&vm->vm_cv, wchan)
#define VMEM_CONDVAR_DESTROY(vm) cv_destroy(&vm->vm_cv)
#define VMEM_CONDVAR_WAIT(vm) cv_wait(&vm->vm_cv, &vm->vm_lock)
#define VMEM_CONDVAR_BROADCAST(vm) cv_broadcast(&vm->vm_cv)
#define VMEM_LOCK(vm) mtx_lock(&vm->vm_lock)
#define VMEM_TRYLOCK(vm) mtx_trylock(&vm->vm_lock)
#define VMEM_UNLOCK(vm) mtx_unlock(&vm->vm_lock)
#define VMEM_LOCK_INIT(vm, name) mtx_init(&vm->vm_lock, (name), NULL, MTX_DEF)
#define VMEM_LOCK_DESTROY(vm) mtx_destroy(&vm->vm_lock)
#define VMEM_ASSERT_LOCKED(vm) mtx_assert(&vm->vm_lock, MA_OWNED);
#define VMEM_ALIGNUP(addr, align) (-(-(addr) & -(align)))
#define VMEM_CROSS_P(addr1, addr2, boundary) \
((((addr1) ^ (addr2)) & -(boundary)) != 0)
#define ORDER2SIZE(order) ((order) < VMEM_OPTVALUE ? ((order) + 1) : \
(vmem_size_t)1 << ((order) - (VMEM_OPTVALUE - VMEM_OPTORDER - 1)))
#define SIZE2ORDER(size) ((size) <= VMEM_OPTVALUE ? ((size) - 1) : \
(flsl(size) + (VMEM_OPTVALUE - VMEM_OPTORDER - 2)))
/*
* Maximum number of boundary tags that may be required to satisfy an
* allocation. Two may be required to import. Another two may be
* required to clip edges.
*/
#define BT_MAXALLOC 4
/*
* Max free limits the number of locally cached boundary tags. We
* just want to avoid hitting the zone allocator for every call.
*/
#define BT_MAXFREE (BT_MAXALLOC * 8)
/* Allocator for boundary tags. */
static uma_zone_t vmem_bt_zone;
/* boot time arena storage. */
static struct vmem kernel_arena_storage;
static struct vmem kmem_arena_storage;
static struct vmem buffer_arena_storage;
static struct vmem transient_arena_storage;
vmem_t *kernel_arena = &kernel_arena_storage;
vmem_t *kmem_arena = &kmem_arena_storage;
vmem_t *buffer_arena = &buffer_arena_storage;
vmem_t *transient_arena = &transient_arena_storage;
#ifdef DEBUG_MEMGUARD
static struct vmem memguard_arena_storage;
vmem_t *memguard_arena = &memguard_arena_storage;
#endif
/*
* Fill the vmem's boundary tag cache. We guarantee that boundary tag
* allocation will not fail once bt_fill() passes. To do so we cache
* at least the maximum possible tag allocations in the arena.
*/
static int
bt_fill(vmem_t *vm, int flags)
{
bt_t *bt;
VMEM_ASSERT_LOCKED(vm);
/*
* Only allow the kmem arena to dip into reserve tags. It is the
* vmem where new tags come from.
*/
flags &= BT_FLAGS;
if (vm != kmem_arena)
flags &= ~M_USE_RESERVE;
/*
* Loop until we meet the reserve. To minimize the lock shuffle
* and prevent simultaneous fills we first try a NOWAIT regardless
* of the caller's flags. Specify M_NOVM so we don't recurse while
* holding a vmem lock.
*/
while (vm->vm_nfreetags < BT_MAXALLOC) {
bt = uma_zalloc(vmem_bt_zone,
(flags & M_USE_RESERVE) | M_NOWAIT | M_NOVM);
if (bt == NULL) {
VMEM_UNLOCK(vm);
bt = uma_zalloc(vmem_bt_zone, flags);
VMEM_LOCK(vm);
if (bt == NULL && (flags & M_NOWAIT) != 0)
break;
}
LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
vm->vm_nfreetags++;
}
if (vm->vm_nfreetags < BT_MAXALLOC)
return ENOMEM;
return 0;
}
/*
* Pop a tag off of the freetag stack.
*/
static bt_t *
bt_alloc(vmem_t *vm)
{
bt_t *bt;
VMEM_ASSERT_LOCKED(vm);
bt = LIST_FIRST(&vm->vm_freetags);
MPASS(bt != NULL);
LIST_REMOVE(bt, bt_freelist);
vm->vm_nfreetags--;
return bt;
}
/*
* Trim the per-vmem free list. Returns with the lock released to
* avoid allocator recursions.
*/
static void
bt_freetrim(vmem_t *vm, int freelimit)
{
LIST_HEAD(, vmem_btag) freetags;
bt_t *bt;
LIST_INIT(&freetags);
VMEM_ASSERT_LOCKED(vm);
while (vm->vm_nfreetags > freelimit) {
bt = LIST_FIRST(&vm->vm_freetags);
LIST_REMOVE(bt, bt_freelist);
vm->vm_nfreetags--;
LIST_INSERT_HEAD(&freetags, bt, bt_freelist);
}
VMEM_UNLOCK(vm);
while ((bt = LIST_FIRST(&freetags)) != NULL) {
LIST_REMOVE(bt, bt_freelist);
uma_zfree(vmem_bt_zone, bt);
}
}
static inline void
bt_free(vmem_t *vm, bt_t *bt)
{
VMEM_ASSERT_LOCKED(vm);
MPASS(LIST_FIRST(&vm->vm_freetags) != bt);
LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
vm->vm_nfreetags++;
}
/*
* freelist[0] ... [1, 1]
* freelist[1] ... [2, 2]
* :
* freelist[29] ... [30, 30]
* freelist[30] ... [31, 31]
* freelist[31] ... [32, 63]
* freelist[33] ... [64, 127]
* :
* freelist[n] ... [(1 << (n - 26)), (1 << (n - 25)) - 1]
* :
*/
static struct vmem_freelist *
bt_freehead_tofree(vmem_t *vm, vmem_size_t size)
{
const vmem_size_t qsize = size >> vm->vm_quantum_shift;
const int idx = SIZE2ORDER(qsize);
MPASS(size != 0 && qsize != 0);
MPASS((size & vm->vm_quantum_mask) == 0);
MPASS(idx >= 0);
MPASS(idx < VMEM_MAXORDER);
return &vm->vm_freelist[idx];
}
/*
* bt_freehead_toalloc: return the freelist for the given size and allocation
* strategy.
*
* For M_FIRSTFIT, return the list in which any blocks are large enough
* for the requested size. otherwise, return the list which can have blocks
* large enough for the requested size.
*/
static struct vmem_freelist *
bt_freehead_toalloc(vmem_t *vm, vmem_size_t size, int strat)
{
const vmem_size_t qsize = size >> vm->vm_quantum_shift;
int idx = SIZE2ORDER(qsize);
MPASS(size != 0 && qsize != 0);
MPASS((size & vm->vm_quantum_mask) == 0);
if (strat == M_FIRSTFIT && ORDER2SIZE(idx) != qsize) {
idx++;
/* check too large request? */
}
MPASS(idx >= 0);
MPASS(idx < VMEM_MAXORDER);
return &vm->vm_freelist[idx];
}
/* ---- boundary tag hash */
static struct vmem_hashlist *
bt_hashhead(vmem_t *vm, vmem_addr_t addr)
{
struct vmem_hashlist *list;
unsigned int hash;
hash = hash32_buf(&addr, sizeof(addr), 0);
list = &vm->vm_hashlist[hash % vm->vm_hashsize];
return list;
}
static bt_t *
bt_lookupbusy(vmem_t *vm, vmem_addr_t addr)
{
struct vmem_hashlist *list;
bt_t *bt;
VMEM_ASSERT_LOCKED(vm);
list = bt_hashhead(vm, addr);
LIST_FOREACH(bt, list, bt_hashlist) {
if (bt->bt_start == addr) {
break;
}
}
return bt;
}
static void
bt_rembusy(vmem_t *vm, bt_t *bt)
{
VMEM_ASSERT_LOCKED(vm);
MPASS(vm->vm_nbusytag > 0);
vm->vm_inuse -= bt->bt_size;
vm->vm_nbusytag--;
LIST_REMOVE(bt, bt_hashlist);
}
static void
bt_insbusy(vmem_t *vm, bt_t *bt)
{
struct vmem_hashlist *list;
VMEM_ASSERT_LOCKED(vm);
MPASS(bt->bt_type == BT_TYPE_BUSY);
list = bt_hashhead(vm, bt->bt_start);
LIST_INSERT_HEAD(list, bt, bt_hashlist);
vm->vm_nbusytag++;
vm->vm_inuse += bt->bt_size;
}
/* ---- boundary tag list */
static void
bt_remseg(vmem_t *vm, bt_t *bt)
{
TAILQ_REMOVE(&vm->vm_seglist, bt, bt_seglist);
bt_free(vm, bt);
}
static void
bt_insseg(vmem_t *vm, bt_t *bt, bt_t *prev)
{
TAILQ_INSERT_AFTER(&vm->vm_seglist, prev, bt, bt_seglist);
}
static void
bt_insseg_tail(vmem_t *vm, bt_t *bt)
{
TAILQ_INSERT_TAIL(&vm->vm_seglist, bt, bt_seglist);
}
static void
bt_remfree(vmem_t *vm, bt_t *bt)
{
MPASS(bt->bt_type == BT_TYPE_FREE);
LIST_REMOVE(bt, bt_freelist);
}
static void
bt_insfree(vmem_t *vm, bt_t *bt)
{
struct vmem_freelist *list;
list = bt_freehead_tofree(vm, bt->bt_size);
LIST_INSERT_HEAD(list, bt, bt_freelist);
}
/* ---- vmem internal functions */
/*
* Import from the arena into the quantum cache in UMA.
*/
static int
qc_import(void *arg, void **store, int cnt, int flags)
{
qcache_t *qc;
vmem_addr_t addr;
int i;
qc = arg;
if ((flags & VMEM_FITMASK) == 0)
flags |= M_BESTFIT;
for (i = 0; i < cnt; i++) {
if (vmem_xalloc(qc->qc_vmem, qc->qc_size, 0, 0, 0,
VMEM_ADDR_MIN, VMEM_ADDR_MAX, flags, &addr) != 0)
break;
store[i] = (void *)addr;
/* Only guarantee one allocation. */
flags &= ~M_WAITOK;
flags |= M_NOWAIT;
}
return i;
}
/*
* Release memory from the UMA cache to the arena.
*/
static void
qc_release(void *arg, void **store, int cnt)
{
qcache_t *qc;
int i;
qc = arg;
for (i = 0; i < cnt; i++)
vmem_xfree(qc->qc_vmem, (vmem_addr_t)store[i], qc->qc_size);
}
static void
qc_init(vmem_t *vm, vmem_size_t qcache_max)
{
qcache_t *qc;
vmem_size_t size;
int qcache_idx_max;
int i;
MPASS((qcache_max & vm->vm_quantum_mask) == 0);
qcache_idx_max = MIN(qcache_max >> vm->vm_quantum_shift,
VMEM_QCACHE_IDX_MAX);
vm->vm_qcache_max = qcache_idx_max << vm->vm_quantum_shift;
for (i = 0; i < qcache_idx_max; i++) {
qc = &vm->vm_qcache[i];
size = (i + 1) << vm->vm_quantum_shift;
snprintf(qc->qc_name, sizeof(qc->qc_name), "%s-%zu",
vm->vm_name, size);
qc->qc_vmem = vm;
qc->qc_size = size;
qc->qc_cache = uma_zcache_create(qc->qc_name, size,
NULL, NULL, NULL, NULL, qc_import, qc_release, qc,
UMA_ZONE_VM);
MPASS(qc->qc_cache);
}
}
static void
qc_destroy(vmem_t *vm)
{
int qcache_idx_max;
int i;
qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift;
for (i = 0; i < qcache_idx_max; i++)
uma_zdestroy(vm->vm_qcache[i].qc_cache);
}
static void
qc_drain(vmem_t *vm)
{
int qcache_idx_max;
int i;
qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift;
for (i = 0; i < qcache_idx_max; i++)
zone_drain(vm->vm_qcache[i].qc_cache);
}
#ifndef UMA_MD_SMALL_ALLOC
static struct mtx_padalign vmem_bt_lock;
/*
* vmem_bt_alloc: Allocate a new page of boundary tags.
*
* On architectures with uma_small_alloc there is no recursion; no address
* space need be allocated to allocate boundary tags. For the others, we
* must handle recursion. Boundary tags are necessary to allocate new
* boundary tags.
*
* UMA guarantees that enough tags are held in reserve to allocate a new
* page of kva. We dip into this reserve by specifying M_USE_RESERVE only
* when allocating the page to hold new boundary tags. In this way the
* reserve is automatically filled by the allocation that uses the reserve.
*
* We still have to guarantee that the new tags are allocated atomically since
* many threads may try concurrently. The bt_lock provides this guarantee.
* We convert WAITOK allocations to NOWAIT and then handle the blocking here
* on failure. It's ok to return NULL for a WAITOK allocation as UMA will
* loop again after checking to see if we lost the race to allocate.
*
* There is a small race between vmem_bt_alloc() returning the page and the
* zone lock being acquired to add the page to the zone. For WAITOK
* allocations we just pause briefly. NOWAIT may experience a transient
* failure. To alleviate this we permit a small number of simultaneous
* fills to proceed concurrently so NOWAIT is less likely to fail unless
* we are really out of KVA.
*/
static void *
vmem_bt_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
{
vmem_addr_t addr;
*pflag = UMA_SLAB_KMEM;
/*
* Single thread boundary tag allocation so that the address space
* and memory are added in one atomic operation.
*/
mtx_lock(&vmem_bt_lock);
if (vmem_xalloc(kmem_arena, bytes, 0, 0, 0, VMEM_ADDR_MIN,
VMEM_ADDR_MAX, M_NOWAIT | M_NOVM | M_USE_RESERVE | M_BESTFIT,
&addr) == 0) {
if (kmem_back(kmem_object, addr, bytes,
M_NOWAIT | M_USE_RESERVE) == 0) {
mtx_unlock(&vmem_bt_lock);
return ((void *)addr);
}
vmem_xfree(kmem_arena, addr, bytes);
mtx_unlock(&vmem_bt_lock);
/*
* Out of memory, not address space. This may not even be
* possible due to M_USE_RESERVE page allocation.
*/
if (wait & M_WAITOK)
VM_WAIT;
return (NULL);
}
mtx_unlock(&vmem_bt_lock);
/*
* We're either out of address space or lost a fill race.
*/
if (wait & M_WAITOK)
pause("btalloc", 1);
return (NULL);
}
#endif
void
vmem_startup(void)
{
mtx_init(&vmem_list_lock, "vmem list lock", NULL, MTX_DEF);
vmem_bt_zone = uma_zcreate("vmem btag",
sizeof(struct vmem_btag), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, UMA_ZONE_VM);
#ifndef UMA_MD_SMALL_ALLOC
mtx_init(&vmem_bt_lock, "btag lock", NULL, MTX_DEF);
uma_prealloc(vmem_bt_zone, BT_MAXALLOC);
/*
* Reserve enough tags to allocate new tags. We allow multiple
* CPUs to attempt to allocate new tags concurrently to limit
* false restarts in UMA.
*/
uma_zone_reserve(vmem_bt_zone, BT_MAXALLOC * (mp_ncpus + 1) / 2);
uma_zone_set_allocf(vmem_bt_zone, vmem_bt_alloc);
#endif
}
/* ---- rehash */
static int
vmem_rehash(vmem_t *vm, vmem_size_t newhashsize)
{
bt_t *bt;
int i;
struct vmem_hashlist *newhashlist;
struct vmem_hashlist *oldhashlist;
vmem_size_t oldhashsize;
MPASS(newhashsize > 0);
newhashlist = malloc(sizeof(struct vmem_hashlist) * newhashsize,
M_VMEM, M_NOWAIT);
if (newhashlist == NULL)
return ENOMEM;
for (i = 0; i < newhashsize; i++) {
LIST_INIT(&newhashlist[i]);
}
VMEM_LOCK(vm);
oldhashlist = vm->vm_hashlist;
oldhashsize = vm->vm_hashsize;
vm->vm_hashlist = newhashlist;
vm->vm_hashsize = newhashsize;
if (oldhashlist == NULL) {
VMEM_UNLOCK(vm);
return 0;
}
for (i = 0; i < oldhashsize; i++) {
while ((bt = LIST_FIRST(&oldhashlist[i])) != NULL) {
bt_rembusy(vm, bt);
bt_insbusy(vm, bt);
}
}
VMEM_UNLOCK(vm);
if (oldhashlist != vm->vm_hash0) {
free(oldhashlist, M_VMEM);
}
return 0;
}
static void
vmem_periodic_kick(void *dummy)
{
taskqueue_enqueue(taskqueue_thread, &vmem_periodic_wk);
}
static void
vmem_periodic(void *unused, int pending)
{
vmem_t *vm;
vmem_size_t desired;
vmem_size_t current;
mtx_lock(&vmem_list_lock);
LIST_FOREACH(vm, &vmem_list, vm_alllist) {
#ifdef DIAGNOSTIC
/* Convenient time to verify vmem state. */
if (enable_vmem_check == 1) {
VMEM_LOCK(vm);
vmem_check(vm);
VMEM_UNLOCK(vm);
}
#endif
desired = 1 << flsl(vm->vm_nbusytag);
desired = MIN(MAX(desired, VMEM_HASHSIZE_MIN),
VMEM_HASHSIZE_MAX);
current = vm->vm_hashsize;
/* Grow in powers of two. Shrink less aggressively. */
if (desired >= current * 2 || desired * 4 <= current)
vmem_rehash(vm, desired);
/*
* Periodically wake up threads waiting for resources,
* so they could ask for reclamation again.
*/
VMEM_CONDVAR_BROADCAST(vm);
}
mtx_unlock(&vmem_list_lock);
callout_reset(&vmem_periodic_ch, vmem_periodic_interval,
vmem_periodic_kick, NULL);
}
static void
vmem_start_callout(void *unused)
{
TASK_INIT(&vmem_periodic_wk, 0, vmem_periodic, NULL);
vmem_periodic_interval = hz * 10;
- callout_init(&vmem_periodic_ch, CALLOUT_MPSAFE);
+ callout_init(&vmem_periodic_ch, 1);
callout_reset(&vmem_periodic_ch, vmem_periodic_interval,
vmem_periodic_kick, NULL);
}
SYSINIT(vfs, SI_SUB_CONFIGURE, SI_ORDER_ANY, vmem_start_callout, NULL);
static void
vmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int type)
{
bt_t *btspan;
bt_t *btfree;
MPASS(type == BT_TYPE_SPAN || type == BT_TYPE_SPAN_STATIC);
MPASS((size & vm->vm_quantum_mask) == 0);
btspan = bt_alloc(vm);
btspan->bt_type = type;
btspan->bt_start = addr;
btspan->bt_size = size;
bt_insseg_tail(vm, btspan);
btfree = bt_alloc(vm);
btfree->bt_type = BT_TYPE_FREE;
btfree->bt_start = addr;
btfree->bt_size = size;
bt_insseg(vm, btfree, btspan);
bt_insfree(vm, btfree);
vm->vm_size += size;
}
static void
vmem_destroy1(vmem_t *vm)
{
bt_t *bt;
/*
* Drain per-cpu quantum caches.
*/
qc_destroy(vm);
/*
* The vmem should now only contain empty segments.
*/
VMEM_LOCK(vm);
MPASS(vm->vm_nbusytag == 0);
while ((bt = TAILQ_FIRST(&vm->vm_seglist)) != NULL)
bt_remseg(vm, bt);
if (vm->vm_hashlist != NULL && vm->vm_hashlist != vm->vm_hash0)
free(vm->vm_hashlist, M_VMEM);
bt_freetrim(vm, 0);
VMEM_CONDVAR_DESTROY(vm);
VMEM_LOCK_DESTROY(vm);
free(vm, M_VMEM);
}
static int
vmem_import(vmem_t *vm, vmem_size_t size, vmem_size_t align, int flags)
{
vmem_addr_t addr;
int error;
if (vm->vm_importfn == NULL)
return EINVAL;
/*
* To make sure we get a span that meets the alignment we double it
* and add the size to the tail. This slightly overestimates.
*/
if (align != vm->vm_quantum_mask + 1)
size = (align * 2) + size;
size = roundup(size, vm->vm_import_quantum);
/*
* Hide MAXALLOC tags so we're guaranteed to be able to add this
* span and the tag we want to allocate from it.
*/
MPASS(vm->vm_nfreetags >= BT_MAXALLOC);
vm->vm_nfreetags -= BT_MAXALLOC;
VMEM_UNLOCK(vm);
error = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr);
VMEM_LOCK(vm);
vm->vm_nfreetags += BT_MAXALLOC;
if (error)
return ENOMEM;
vmem_add1(vm, addr, size, BT_TYPE_SPAN);
return 0;
}
/*
* vmem_fit: check if a bt can satisfy the given restrictions.
*
* it's a caller's responsibility to ensure the region is big enough
* before calling us.
*/
static int
vmem_fit(const bt_t *bt, vmem_size_t size, vmem_size_t align,
vmem_size_t phase, vmem_size_t nocross, vmem_addr_t minaddr,
vmem_addr_t maxaddr, vmem_addr_t *addrp)
{
vmem_addr_t start;
vmem_addr_t end;
MPASS(size > 0);
MPASS(bt->bt_size >= size); /* caller's responsibility */
/*
* XXX assumption: vmem_addr_t and vmem_size_t are
* unsigned integer of the same size.
*/
start = bt->bt_start;
if (start < minaddr) {
start = minaddr;
}
end = BT_END(bt);
if (end > maxaddr)
end = maxaddr;
if (start > end)
return (ENOMEM);
start = VMEM_ALIGNUP(start - phase, align) + phase;
if (start < bt->bt_start)
start += align;
if (VMEM_CROSS_P(start, start + size - 1, nocross)) {
MPASS(align < nocross);
start = VMEM_ALIGNUP(start - phase, nocross) + phase;
}
if (start <= end && end - start >= size - 1) {
MPASS((start & (align - 1)) == phase);
MPASS(!VMEM_CROSS_P(start, start + size - 1, nocross));
MPASS(minaddr <= start);
MPASS(maxaddr == 0 || start + size - 1 <= maxaddr);
MPASS(bt->bt_start <= start);
MPASS(BT_END(bt) - start >= size - 1);
*addrp = start;
return (0);
}
return (ENOMEM);
}
/*
* vmem_clip: Trim the boundary tag edges to the requested start and size.
*/
static void
vmem_clip(vmem_t *vm, bt_t *bt, vmem_addr_t start, vmem_size_t size)
{
bt_t *btnew;
bt_t *btprev;
VMEM_ASSERT_LOCKED(vm);
MPASS(bt->bt_type == BT_TYPE_FREE);
MPASS(bt->bt_size >= size);
bt_remfree(vm, bt);
if (bt->bt_start != start) {
btprev = bt_alloc(vm);
btprev->bt_type = BT_TYPE_FREE;
btprev->bt_start = bt->bt_start;
btprev->bt_size = start - bt->bt_start;
bt->bt_start = start;
bt->bt_size -= btprev->bt_size;
bt_insfree(vm, btprev);
bt_insseg(vm, btprev,
TAILQ_PREV(bt, vmem_seglist, bt_seglist));
}
MPASS(bt->bt_start == start);
if (bt->bt_size != size && bt->bt_size - size > vm->vm_quantum_mask) {
/* split */
btnew = bt_alloc(vm);
btnew->bt_type = BT_TYPE_BUSY;
btnew->bt_start = bt->bt_start;
btnew->bt_size = size;
bt->bt_start = bt->bt_start + size;
bt->bt_size -= size;
bt_insfree(vm, bt);
bt_insseg(vm, btnew,
TAILQ_PREV(bt, vmem_seglist, bt_seglist));
bt_insbusy(vm, btnew);
bt = btnew;
} else {
bt->bt_type = BT_TYPE_BUSY;
bt_insbusy(vm, bt);
}
MPASS(bt->bt_size >= size);
bt->bt_type = BT_TYPE_BUSY;
}
/* ---- vmem API */
void
vmem_set_import(vmem_t *vm, vmem_import_t *importfn,
vmem_release_t *releasefn, void *arg, vmem_size_t import_quantum)
{
VMEM_LOCK(vm);
vm->vm_importfn = importfn;
vm->vm_releasefn = releasefn;
vm->vm_arg = arg;
vm->vm_import_quantum = import_quantum;
VMEM_UNLOCK(vm);
}
void
vmem_set_reclaim(vmem_t *vm, vmem_reclaim_t *reclaimfn)
{
VMEM_LOCK(vm);
vm->vm_reclaimfn = reclaimfn;
VMEM_UNLOCK(vm);
}
/*
* vmem_init: Initializes vmem arena.
*/
vmem_t *
vmem_init(vmem_t *vm, const char *name, vmem_addr_t base, vmem_size_t size,
vmem_size_t quantum, vmem_size_t qcache_max, int flags)
{
int i;
MPASS(quantum > 0);
MPASS((quantum & (quantum - 1)) == 0);
bzero(vm, sizeof(*vm));
VMEM_CONDVAR_INIT(vm, name);
VMEM_LOCK_INIT(vm, name);
vm->vm_nfreetags = 0;
LIST_INIT(&vm->vm_freetags);
strlcpy(vm->vm_name, name, sizeof(vm->vm_name));
vm->vm_quantum_mask = quantum - 1;
vm->vm_quantum_shift = flsl(quantum) - 1;
vm->vm_nbusytag = 0;
vm->vm_size = 0;
vm->vm_inuse = 0;
qc_init(vm, qcache_max);
TAILQ_INIT(&vm->vm_seglist);
for (i = 0; i < VMEM_MAXORDER; i++) {
LIST_INIT(&vm->vm_freelist[i]);
}
memset(&vm->vm_hash0, 0, sizeof(vm->vm_hash0));
vm->vm_hashsize = VMEM_HASHSIZE_MIN;
vm->vm_hashlist = vm->vm_hash0;
if (size != 0) {
if (vmem_add(vm, base, size, flags) != 0) {
vmem_destroy1(vm);
return NULL;
}
}
mtx_lock(&vmem_list_lock);
LIST_INSERT_HEAD(&vmem_list, vm, vm_alllist);
mtx_unlock(&vmem_list_lock);
return vm;
}
/*
* vmem_create: create an arena.
*/
vmem_t *
vmem_create(const char *name, vmem_addr_t base, vmem_size_t size,
vmem_size_t quantum, vmem_size_t qcache_max, int flags)
{
vmem_t *vm;
vm = malloc(sizeof(*vm), M_VMEM, flags & (M_WAITOK|M_NOWAIT));
if (vm == NULL)
return (NULL);
if (vmem_init(vm, name, base, size, quantum, qcache_max,
flags) == NULL) {
free(vm, M_VMEM);
return (NULL);
}
return (vm);
}
void
vmem_destroy(vmem_t *vm)
{
mtx_lock(&vmem_list_lock);
LIST_REMOVE(vm, vm_alllist);
mtx_unlock(&vmem_list_lock);
vmem_destroy1(vm);
}
vmem_size_t
vmem_roundup_size(vmem_t *vm, vmem_size_t size)
{
return (size + vm->vm_quantum_mask) & ~vm->vm_quantum_mask;
}
/*
* vmem_alloc: allocate resource from the arena.
*/
int
vmem_alloc(vmem_t *vm, vmem_size_t size, int flags, vmem_addr_t *addrp)
{
const int strat __unused = flags & VMEM_FITMASK;
qcache_t *qc;
flags &= VMEM_FLAGS;
MPASS(size > 0);
MPASS(strat == M_BESTFIT || strat == M_FIRSTFIT);
if ((flags & M_NOWAIT) == 0)
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "vmem_alloc");
if (size <= vm->vm_qcache_max) {
qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift];
*addrp = (vmem_addr_t)uma_zalloc(qc->qc_cache, flags);
if (*addrp == 0)
return (ENOMEM);
return (0);
}
return vmem_xalloc(vm, size, 0, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX,
flags, addrp);
}
int
vmem_xalloc(vmem_t *vm, const vmem_size_t size0, vmem_size_t align,
const vmem_size_t phase, const vmem_size_t nocross,
const vmem_addr_t minaddr, const vmem_addr_t maxaddr, int flags,
vmem_addr_t *addrp)
{
const vmem_size_t size = vmem_roundup_size(vm, size0);
struct vmem_freelist *list;
struct vmem_freelist *first;
struct vmem_freelist *end;
vmem_size_t avail;
bt_t *bt;
int error;
int strat;
flags &= VMEM_FLAGS;
strat = flags & VMEM_FITMASK;
MPASS(size0 > 0);
MPASS(size > 0);
MPASS(strat == M_BESTFIT || strat == M_FIRSTFIT);
MPASS((flags & (M_NOWAIT|M_WAITOK)) != (M_NOWAIT|M_WAITOK));
if ((flags & M_NOWAIT) == 0)
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "vmem_xalloc");
MPASS((align & vm->vm_quantum_mask) == 0);
MPASS((align & (align - 1)) == 0);
MPASS((phase & vm->vm_quantum_mask) == 0);
MPASS((nocross & vm->vm_quantum_mask) == 0);
MPASS((nocross & (nocross - 1)) == 0);
MPASS((align == 0 && phase == 0) || phase < align);
MPASS(nocross == 0 || nocross >= size);
MPASS(minaddr <= maxaddr);
MPASS(!VMEM_CROSS_P(phase, phase + size - 1, nocross));
if (align == 0)
align = vm->vm_quantum_mask + 1;
*addrp = 0;
end = &vm->vm_freelist[VMEM_MAXORDER];
/*
* choose a free block from which we allocate.
*/
first = bt_freehead_toalloc(vm, size, strat);
VMEM_LOCK(vm);
for (;;) {
/*
* Make sure we have enough tags to complete the
* operation.
*/
if (vm->vm_nfreetags < BT_MAXALLOC &&
bt_fill(vm, flags) != 0) {
error = ENOMEM;
break;
}
/*
* Scan freelists looking for a tag that satisfies the
* allocation. If we're doing BESTFIT we may encounter
* sizes below the request. If we're doing FIRSTFIT we
* inspect only the first element from each list.
*/
for (list = first; list < end; list++) {
LIST_FOREACH(bt, list, bt_freelist) {
if (bt->bt_size >= size) {
error = vmem_fit(bt, size, align, phase,
nocross, minaddr, maxaddr, addrp);
if (error == 0) {
vmem_clip(vm, bt, *addrp, size);
goto out;
}
}
/* FIRST skips to the next list. */
if (strat == M_FIRSTFIT)
break;
}
}
/*
* Retry if the fast algorithm failed.
*/
if (strat == M_FIRSTFIT) {
strat = M_BESTFIT;
first = bt_freehead_toalloc(vm, size, strat);
continue;
}
/*
* XXX it is possible to fail to meet restrictions with the
* imported region. It is up to the user to specify the
* import quantum such that it can satisfy any allocation.
*/
if (vmem_import(vm, size, align, flags) == 0)
continue;
/*
* Try to free some space from the quantum cache or reclaim
* functions if available.
*/
if (vm->vm_qcache_max != 0 || vm->vm_reclaimfn != NULL) {
avail = vm->vm_size - vm->vm_inuse;
VMEM_UNLOCK(vm);
if (vm->vm_qcache_max != 0)
qc_drain(vm);
if (vm->vm_reclaimfn != NULL)
vm->vm_reclaimfn(vm, flags);
VMEM_LOCK(vm);
/* If we were successful retry even NOWAIT. */
if (vm->vm_size - vm->vm_inuse > avail)
continue;
}
if ((flags & M_NOWAIT) != 0) {
error = ENOMEM;
break;
}
VMEM_CONDVAR_WAIT(vm);
}
out:
VMEM_UNLOCK(vm);
if (error != 0 && (flags & M_NOWAIT) == 0)
panic("failed to allocate waiting allocation\n");
return (error);
}
/*
* vmem_free: free the resource to the arena.
*/
void
vmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size)
{
qcache_t *qc;
MPASS(size > 0);
if (size <= vm->vm_qcache_max) {
qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift];
uma_zfree(qc->qc_cache, (void *)addr);
} else
vmem_xfree(vm, addr, size);
}
void
vmem_xfree(vmem_t *vm, vmem_addr_t addr, vmem_size_t size)
{
bt_t *bt;
bt_t *t;
MPASS(size > 0);
VMEM_LOCK(vm);
bt = bt_lookupbusy(vm, addr);
MPASS(bt != NULL);
MPASS(bt->bt_start == addr);
MPASS(bt->bt_size == vmem_roundup_size(vm, size) ||
bt->bt_size - vmem_roundup_size(vm, size) <= vm->vm_quantum_mask);
MPASS(bt->bt_type == BT_TYPE_BUSY);
bt_rembusy(vm, bt);
bt->bt_type = BT_TYPE_FREE;
/* coalesce */
t = TAILQ_NEXT(bt, bt_seglist);
if (t != NULL && t->bt_type == BT_TYPE_FREE) {
MPASS(BT_END(bt) < t->bt_start); /* YYY */
bt->bt_size += t->bt_size;
bt_remfree(vm, t);
bt_remseg(vm, t);
}
t = TAILQ_PREV(bt, vmem_seglist, bt_seglist);
if (t != NULL && t->bt_type == BT_TYPE_FREE) {
MPASS(BT_END(t) < bt->bt_start); /* YYY */
bt->bt_size += t->bt_size;
bt->bt_start = t->bt_start;
bt_remfree(vm, t);
bt_remseg(vm, t);
}
t = TAILQ_PREV(bt, vmem_seglist, bt_seglist);
MPASS(t != NULL);
MPASS(BT_ISSPAN_P(t) || t->bt_type == BT_TYPE_BUSY);
if (vm->vm_releasefn != NULL && t->bt_type == BT_TYPE_SPAN &&
t->bt_size == bt->bt_size) {
vmem_addr_t spanaddr;
vmem_size_t spansize;
MPASS(t->bt_start == bt->bt_start);
spanaddr = bt->bt_start;
spansize = bt->bt_size;
bt_remseg(vm, bt);
bt_remseg(vm, t);
vm->vm_size -= spansize;
VMEM_CONDVAR_BROADCAST(vm);
bt_freetrim(vm, BT_MAXFREE);
(*vm->vm_releasefn)(vm->vm_arg, spanaddr, spansize);
} else {
bt_insfree(vm, bt);
VMEM_CONDVAR_BROADCAST(vm);
bt_freetrim(vm, BT_MAXFREE);
}
}
/*
* vmem_add:
*
*/
int
vmem_add(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int flags)
{
int error;
error = 0;
flags &= VMEM_FLAGS;
VMEM_LOCK(vm);
if (vm->vm_nfreetags >= BT_MAXALLOC || bt_fill(vm, flags) == 0)
vmem_add1(vm, addr, size, BT_TYPE_SPAN_STATIC);
else
error = ENOMEM;
VMEM_UNLOCK(vm);
return (error);
}
/*
* vmem_size: information about arenas size
*/
vmem_size_t
vmem_size(vmem_t *vm, int typemask)
{
int i;
switch (typemask) {
case VMEM_ALLOC:
return vm->vm_inuse;
case VMEM_FREE:
return vm->vm_size - vm->vm_inuse;
case VMEM_FREE|VMEM_ALLOC:
return vm->vm_size;
case VMEM_MAXFREE:
VMEM_LOCK(vm);
for (i = VMEM_MAXORDER - 1; i >= 0; i--) {
if (LIST_EMPTY(&vm->vm_freelist[i]))
continue;
VMEM_UNLOCK(vm);
return ((vmem_size_t)ORDER2SIZE(i) <<
vm->vm_quantum_shift);
}
VMEM_UNLOCK(vm);
return (0);
default:
panic("vmem_size");
}
}
/* ---- debug */
#if defined(DDB) || defined(DIAGNOSTIC)
static void bt_dump(const bt_t *, int (*)(const char *, ...)
__printflike(1, 2));
static const char *
bt_type_string(int type)
{
switch (type) {
case BT_TYPE_BUSY:
return "busy";
case BT_TYPE_FREE:
return "free";
case BT_TYPE_SPAN:
return "span";
case BT_TYPE_SPAN_STATIC:
return "static span";
default:
break;
}
return "BOGUS";
}
static void
bt_dump(const bt_t *bt, int (*pr)(const char *, ...))
{
(*pr)("\t%p: %jx %jx, %d(%s)\n",
bt, (intmax_t)bt->bt_start, (intmax_t)bt->bt_size,
bt->bt_type, bt_type_string(bt->bt_type));
}
static void
vmem_dump(const vmem_t *vm , int (*pr)(const char *, ...) __printflike(1, 2))
{
const bt_t *bt;
int i;
(*pr)("vmem %p '%s'\n", vm, vm->vm_name);
TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
bt_dump(bt, pr);
}
for (i = 0; i < VMEM_MAXORDER; i++) {
const struct vmem_freelist *fl = &vm->vm_freelist[i];
if (LIST_EMPTY(fl)) {
continue;
}
(*pr)("freelist[%d]\n", i);
LIST_FOREACH(bt, fl, bt_freelist) {
bt_dump(bt, pr);
}
}
}
#endif /* defined(DDB) || defined(DIAGNOSTIC) */
#if defined(DDB)
#include <ddb/ddb.h>
static bt_t *
vmem_whatis_lookup(vmem_t *vm, vmem_addr_t addr)
{
bt_t *bt;
TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
if (BT_ISSPAN_P(bt)) {
continue;
}
if (bt->bt_start <= addr && addr <= BT_END(bt)) {
return bt;
}
}
return NULL;
}
void
vmem_whatis(vmem_addr_t addr, int (*pr)(const char *, ...))
{
vmem_t *vm;
LIST_FOREACH(vm, &vmem_list, vm_alllist) {
bt_t *bt;
bt = vmem_whatis_lookup(vm, addr);
if (bt == NULL) {
continue;
}
(*pr)("%p is %p+%zu in VMEM '%s' (%s)\n",
(void *)addr, (void *)bt->bt_start,
(vmem_size_t)(addr - bt->bt_start), vm->vm_name,
(bt->bt_type == BT_TYPE_BUSY) ? "allocated" : "free");
}
}
void
vmem_printall(const char *modif, int (*pr)(const char *, ...))
{
const vmem_t *vm;
LIST_FOREACH(vm, &vmem_list, vm_alllist) {
vmem_dump(vm, pr);
}
}
void
vmem_print(vmem_addr_t addr, const char *modif, int (*pr)(const char *, ...))
{
const vmem_t *vm = (const void *)addr;
vmem_dump(vm, pr);
}
DB_SHOW_COMMAND(vmemdump, vmemdump)
{
if (!have_addr) {
db_printf("usage: show vmemdump <addr>\n");
return;
}
vmem_dump((const vmem_t *)addr, db_printf);
}
DB_SHOW_ALL_COMMAND(vmemdump, vmemdumpall)
{
const vmem_t *vm;
LIST_FOREACH(vm, &vmem_list, vm_alllist)
vmem_dump(vm, db_printf);
}
DB_SHOW_COMMAND(vmem, vmem_summ)
{
const vmem_t *vm = (const void *)addr;
const bt_t *bt;
size_t ft[VMEM_MAXORDER], ut[VMEM_MAXORDER];
size_t fs[VMEM_MAXORDER], us[VMEM_MAXORDER];
int ord;
if (!have_addr) {
db_printf("usage: show vmem <addr>\n");
return;
}
db_printf("vmem %p '%s'\n", vm, vm->vm_name);
db_printf("\tquantum:\t%zu\n", vm->vm_quantum_mask + 1);
db_printf("\tsize:\t%zu\n", vm->vm_size);
db_printf("\tinuse:\t%zu\n", vm->vm_inuse);
db_printf("\tfree:\t%zu\n", vm->vm_size - vm->vm_inuse);
db_printf("\tbusy tags:\t%d\n", vm->vm_nbusytag);
db_printf("\tfree tags:\t%d\n", vm->vm_nfreetags);
memset(&ft, 0, sizeof(ft));
memset(&ut, 0, sizeof(ut));
memset(&fs, 0, sizeof(fs));
memset(&us, 0, sizeof(us));
TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
ord = SIZE2ORDER(bt->bt_size >> vm->vm_quantum_shift);
if (bt->bt_type == BT_TYPE_BUSY) {
ut[ord]++;
us[ord] += bt->bt_size;
} else if (bt->bt_type == BT_TYPE_FREE) {
ft[ord]++;
fs[ord] += bt->bt_size;
}
}
db_printf("\t\t\tinuse\tsize\t\tfree\tsize\n");
for (ord = 0; ord < VMEM_MAXORDER; ord++) {
if (ut[ord] == 0 && ft[ord] == 0)
continue;
db_printf("\t%-15zu %zu\t%-15zu %zu\t%-16zu\n",
ORDER2SIZE(ord) << vm->vm_quantum_shift,
ut[ord], us[ord], ft[ord], fs[ord]);
}
}
DB_SHOW_ALL_COMMAND(vmem, vmem_summall)
{
const vmem_t *vm;
LIST_FOREACH(vm, &vmem_list, vm_alllist)
vmem_summ((db_expr_t)vm, TRUE, count, modif);
}
#endif /* defined(DDB) */
#define vmem_printf printf
#if defined(DIAGNOSTIC)
static bool
vmem_check_sanity(vmem_t *vm)
{
const bt_t *bt, *bt2;
MPASS(vm != NULL);
TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
if (bt->bt_start > BT_END(bt)) {
printf("corrupted tag\n");
bt_dump(bt, vmem_printf);
return false;
}
}
TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
TAILQ_FOREACH(bt2, &vm->vm_seglist, bt_seglist) {
if (bt == bt2) {
continue;
}
if (BT_ISSPAN_P(bt) != BT_ISSPAN_P(bt2)) {
continue;
}
if (bt->bt_start <= BT_END(bt2) &&
bt2->bt_start <= BT_END(bt)) {
printf("overwrapped tags\n");
bt_dump(bt, vmem_printf);
bt_dump(bt2, vmem_printf);
return false;
}
}
}
return true;
}
static void
vmem_check(vmem_t *vm)
{
if (!vmem_check_sanity(vm)) {
panic("insanity vmem %p", vm);
}
}
#endif /* defined(DIAGNOSTIC) */
Index: head/sys/kern/uipc_domain.c
===================================================================
--- head/sys/kern/uipc_domain.c (revision 283290)
+++ head/sys/kern/uipc_domain.c (revision 283291)
@@ -1,524 +1,524 @@
/*-
* Copyright (c) 1982, 1986, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)uipc_domain.c 8.2 (Berkeley) 10/18/93
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/socket.h>
#include <sys/protosw.h>
#include <sys/domain.h>
#include <sys/eventhandler.h>
#include <sys/mbuf.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/socketvar.h>
#include <sys/systm.h>
#include <net/vnet.h>
/*
* System initialization
*
* Note: domain initialization takes place on a per domain basis
* as a result of traversing a SYSINIT linker set. Most likely,
* each domain would want to call DOMAIN_SET(9) itself, which
* would cause the domain to be added just after domaininit()
* is called during startup.
*
* See DOMAIN_SET(9) for details on its use.
*/
static void domaininit(void *);
SYSINIT(domain, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, domaininit, NULL);
static void domainfinalize(void *);
SYSINIT(domainfin, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_FIRST, domainfinalize,
NULL);
static struct callout pffast_callout;
static struct callout pfslow_callout;
static void pffasttimo(void *);
static void pfslowtimo(void *);
struct domain *domains; /* registered protocol domains */
int domain_init_status = 0;
static struct mtx dom_mtx; /* domain list lock */
MTX_SYSINIT(domain, &dom_mtx, "domain list", MTX_DEF);
/*
* Dummy protocol specific user requests function pointer array.
* All functions return EOPNOTSUPP.
*/
struct pr_usrreqs nousrreqs = {
.pru_accept = pru_accept_notsupp,
.pru_attach = pru_attach_notsupp,
.pru_bind = pru_bind_notsupp,
.pru_connect = pru_connect_notsupp,
.pru_connect2 = pru_connect2_notsupp,
.pru_control = pru_control_notsupp,
.pru_disconnect = pru_disconnect_notsupp,
.pru_listen = pru_listen_notsupp,
.pru_peeraddr = pru_peeraddr_notsupp,
.pru_rcvd = pru_rcvd_notsupp,
.pru_rcvoob = pru_rcvoob_notsupp,
.pru_send = pru_send_notsupp,
.pru_sense = pru_sense_null,
.pru_shutdown = pru_shutdown_notsupp,
.pru_sockaddr = pru_sockaddr_notsupp,
.pru_sosend = pru_sosend_notsupp,
.pru_soreceive = pru_soreceive_notsupp,
.pru_sopoll = pru_sopoll_notsupp,
};
static void
protosw_init(struct protosw *pr)
{
struct pr_usrreqs *pu;
pu = pr->pr_usrreqs;
KASSERT(pu != NULL, ("protosw_init: %ssw[%d] has no usrreqs!",
pr->pr_domain->dom_name,
(int)(pr - pr->pr_domain->dom_protosw)));
/*
* Protocol switch methods fall into three categories: mandatory,
* mandatory but protosw_init() provides a default, and optional.
*
* For true protocols (i.e., pru_attach != NULL), KASSERT truly
* mandatory methods with no defaults, and initialize defaults for
* other mandatory methods if the protocol hasn't defined an
* implementation (NULL function pointer).
*/
#if 0
if (pu->pru_attach != NULL) {
KASSERT(pu->pru_abort != NULL,
("protosw_init: %ssw[%d] pru_abort NULL",
pr->pr_domain->dom_name,
(int)(pr - pr->pr_domain->dom_protosw)));
KASSERT(pu->pru_send != NULL,
("protosw_init: %ssw[%d] pru_send NULL",
pr->pr_domain->dom_name,
(int)(pr - pr->pr_domain->dom_protosw)));
}
#endif
#define DEFAULT(foo, bar) if ((foo) == NULL) (foo) = (bar)
DEFAULT(pu->pru_accept, pru_accept_notsupp);
DEFAULT(pu->pru_bind, pru_bind_notsupp);
DEFAULT(pu->pru_bindat, pru_bindat_notsupp);
DEFAULT(pu->pru_connect, pru_connect_notsupp);
DEFAULT(pu->pru_connect2, pru_connect2_notsupp);
DEFAULT(pu->pru_connectat, pru_connectat_notsupp);
DEFAULT(pu->pru_control, pru_control_notsupp);
DEFAULT(pu->pru_disconnect, pru_disconnect_notsupp);
DEFAULT(pu->pru_listen, pru_listen_notsupp);
DEFAULT(pu->pru_peeraddr, pru_peeraddr_notsupp);
DEFAULT(pu->pru_rcvd, pru_rcvd_notsupp);
DEFAULT(pu->pru_rcvoob, pru_rcvoob_notsupp);
DEFAULT(pu->pru_sense, pru_sense_null);
DEFAULT(pu->pru_shutdown, pru_shutdown_notsupp);
DEFAULT(pu->pru_sockaddr, pru_sockaddr_notsupp);
DEFAULT(pu->pru_sosend, sosend_generic);
DEFAULT(pu->pru_soreceive, soreceive_generic);
DEFAULT(pu->pru_sopoll, sopoll_generic);
DEFAULT(pu->pru_ready, pru_ready_notsupp);
#undef DEFAULT
if (pr->pr_init)
(*pr->pr_init)();
}
/*
* Add a new protocol domain to the list of supported domains
* Note: you cant unload it again because a socket may be using it.
* XXX can't fail at this time.
*/
void
domain_init(void *arg)
{
struct domain *dp = arg;
struct protosw *pr;
if (dp->dom_init)
(*dp->dom_init)();
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
protosw_init(pr);
/*
* update global information about maximums
*/
max_hdr = max_linkhdr + max_protohdr;
max_datalen = MHLEN - max_hdr;
if (max_datalen < 1)
panic("%s: max_datalen < 1", __func__);
}
#ifdef VIMAGE
void
vnet_domain_init(void *arg)
{
/* Virtualized case is no different -- call init functions. */
domain_init(arg);
}
void
vnet_domain_uninit(void *arg)
{
struct domain *dp = arg;
struct protosw *pr;
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
if (pr->pr_destroy)
(*pr->pr_destroy)();
if (dp->dom_destroy)
(*dp->dom_destroy)();
}
#endif
/*
* Add a new protocol domain to the list of supported domains
* Note: you cant unload it again because a socket may be using it.
* XXX can't fail at this time.
*/
void
domain_add(void *data)
{
struct domain *dp;
dp = (struct domain *)data;
mtx_lock(&dom_mtx);
dp->dom_next = domains;
domains = dp;
KASSERT(domain_init_status >= 1,
("attempt to domain_add(%s) before domaininit()",
dp->dom_name));
#ifndef INVARIANTS
if (domain_init_status < 1)
printf("WARNING: attempt to domain_add(%s) before "
"domaininit()\n", dp->dom_name);
#endif
#ifdef notyet
KASSERT(domain_init_status < 2,
("attempt to domain_add(%s) after domainfinalize()",
dp->dom_name));
#else
if (domain_init_status >= 2)
printf("WARNING: attempt to domain_add(%s) after "
"domainfinalize()\n", dp->dom_name);
#endif
mtx_unlock(&dom_mtx);
}
/* ARGSUSED*/
static void
domaininit(void *dummy)
{
if (max_linkhdr < 16) /* XXX */
max_linkhdr = 16;
- callout_init(&pffast_callout, CALLOUT_MPSAFE);
- callout_init(&pfslow_callout, CALLOUT_MPSAFE);
+ callout_init(&pffast_callout, 1);
+ callout_init(&pfslow_callout, 1);
mtx_lock(&dom_mtx);
KASSERT(domain_init_status == 0, ("domaininit called too late!"));
domain_init_status = 1;
mtx_unlock(&dom_mtx);
}
/* ARGSUSED*/
static void
domainfinalize(void *dummy)
{
mtx_lock(&dom_mtx);
KASSERT(domain_init_status == 1, ("domainfinalize called too late!"));
domain_init_status = 2;
mtx_unlock(&dom_mtx);
callout_reset(&pffast_callout, 1, pffasttimo, NULL);
callout_reset(&pfslow_callout, 1, pfslowtimo, NULL);
}
struct domain *
pffinddomain(int family)
{
struct domain *dp;
for (dp = domains; dp != NULL; dp = dp->dom_next)
if (dp->dom_family == family)
return (dp);
return (NULL);
}
struct protosw *
pffindtype(int family, int type)
{
struct domain *dp;
struct protosw *pr;
dp = pffinddomain(family);
if (dp == NULL)
return (NULL);
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
if (pr->pr_type && pr->pr_type == type)
return (pr);
return (NULL);
}
struct protosw *
pffindproto(int family, int protocol, int type)
{
struct domain *dp;
struct protosw *pr;
struct protosw *maybe;
maybe = NULL;
if (family == 0)
return (NULL);
dp = pffinddomain(family);
if (dp == NULL)
return (NULL);
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) {
if ((pr->pr_protocol == protocol) && (pr->pr_type == type))
return (pr);
if (type == SOCK_RAW && pr->pr_type == SOCK_RAW &&
pr->pr_protocol == 0 && maybe == NULL)
maybe = pr;
}
return (maybe);
}
/*
* The caller must make sure that the new protocol is fully set up and ready to
* accept requests before it is registered.
*/
int
pf_proto_register(int family, struct protosw *npr)
{
VNET_ITERATOR_DECL(vnet_iter);
struct domain *dp;
struct protosw *pr, *fpr;
/* Sanity checks. */
if (family == 0)
return (EPFNOSUPPORT);
if (npr->pr_type == 0)
return (EPROTOTYPE);
if (npr->pr_protocol == 0)
return (EPROTONOSUPPORT);
if (npr->pr_usrreqs == NULL)
return (ENXIO);
/* Try to find the specified domain based on the family. */
dp = pffinddomain(family);
if (dp == NULL)
return (EPFNOSUPPORT);
/* Initialize backpointer to struct domain. */
npr->pr_domain = dp;
fpr = NULL;
/*
* Protect us against races when two protocol registrations for
* the same protocol happen at the same time.
*/
mtx_lock(&dom_mtx);
/* The new protocol must not yet exist. */
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) {
if ((pr->pr_type == npr->pr_type) &&
(pr->pr_protocol == npr->pr_protocol)) {
mtx_unlock(&dom_mtx);
return (EEXIST); /* XXX: Check only protocol? */
}
/* While here, remember the first free spacer. */
if ((fpr == NULL) && (pr->pr_protocol == PROTO_SPACER))
fpr = pr;
}
/* If no free spacer is found we can't add the new protocol. */
if (fpr == NULL) {
mtx_unlock(&dom_mtx);
return (ENOMEM);
}
/* Copy the new struct protosw over the spacer. */
bcopy(npr, fpr, sizeof(*fpr));
/* Job is done, no more protection required. */
mtx_unlock(&dom_mtx);
/* Initialize and activate the protocol. */
VNET_LIST_RLOCK();
VNET_FOREACH(vnet_iter) {
CURVNET_SET_QUIET(vnet_iter);
protosw_init(fpr);
CURVNET_RESTORE();
}
VNET_LIST_RUNLOCK();
return (0);
}
/*
* The caller must make sure the protocol and its functions correctly shut down
* all sockets and release all locks and memory references.
*/
int
pf_proto_unregister(int family, int protocol, int type)
{
struct domain *dp;
struct protosw *pr, *dpr;
/* Sanity checks. */
if (family == 0)
return (EPFNOSUPPORT);
if (protocol == 0)
return (EPROTONOSUPPORT);
if (type == 0)
return (EPROTOTYPE);
/* Try to find the specified domain based on the family type. */
dp = pffinddomain(family);
if (dp == NULL)
return (EPFNOSUPPORT);
dpr = NULL;
/* Lock out everyone else while we are manipulating the protosw. */
mtx_lock(&dom_mtx);
/* The protocol must exist and only once. */
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) {
if ((pr->pr_type == type) && (pr->pr_protocol == protocol)) {
if (dpr != NULL) {
mtx_unlock(&dom_mtx);
return (EMLINK); /* Should not happen! */
} else
dpr = pr;
}
}
/* Protocol does not exist. */
if (dpr == NULL) {
mtx_unlock(&dom_mtx);
return (EPROTONOSUPPORT);
}
/* De-orbit the protocol and make the slot available again. */
dpr->pr_type = 0;
dpr->pr_domain = dp;
dpr->pr_protocol = PROTO_SPACER;
dpr->pr_flags = 0;
dpr->pr_input = NULL;
dpr->pr_output = NULL;
dpr->pr_ctlinput = NULL;
dpr->pr_ctloutput = NULL;
dpr->pr_init = NULL;
dpr->pr_fasttimo = NULL;
dpr->pr_slowtimo = NULL;
dpr->pr_drain = NULL;
dpr->pr_usrreqs = &nousrreqs;
/* Job is done, not more protection required. */
mtx_unlock(&dom_mtx);
return (0);
}
void
pfctlinput(int cmd, struct sockaddr *sa)
{
struct domain *dp;
struct protosw *pr;
for (dp = domains; dp; dp = dp->dom_next)
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
if (pr->pr_ctlinput)
(*pr->pr_ctlinput)(cmd, sa, (void *)0);
}
void
pfctlinput2(int cmd, struct sockaddr *sa, void *ctlparam)
{
struct domain *dp;
struct protosw *pr;
if (!sa)
return;
for (dp = domains; dp; dp = dp->dom_next) {
/*
* the check must be made by xx_ctlinput() anyways, to
* make sure we use data item pointed to by ctlparam in
* correct way. the following check is made just for safety.
*/
if (dp->dom_family != sa->sa_family)
continue;
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
if (pr->pr_ctlinput)
(*pr->pr_ctlinput)(cmd, sa, ctlparam);
}
}
static void
pfslowtimo(void *arg)
{
struct domain *dp;
struct protosw *pr;
for (dp = domains; dp; dp = dp->dom_next)
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
if (pr->pr_slowtimo)
(*pr->pr_slowtimo)();
callout_reset(&pfslow_callout, hz/2, pfslowtimo, NULL);
}
static void
pffasttimo(void *arg)
{
struct domain *dp;
struct protosw *pr;
for (dp = domains; dp; dp = dp->dom_next)
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
if (pr->pr_fasttimo)
(*pr->pr_fasttimo)();
callout_reset(&pffast_callout, hz/5, pffasttimo, NULL);
}
Index: head/sys/mips/cavium/octe/ethernet.c
===================================================================
--- head/sys/mips/cavium/octe/ethernet.c (revision 283290)
+++ head/sys/mips/cavium/octe/ethernet.c (revision 283291)
@@ -1,508 +1,508 @@
/*************************************************************************
Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
* Neither the name of Cavium Networks nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written
permission.
This Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries.
TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
*************************************************************************/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/endian.h>
#include <sys/kernel.h>
#include <sys/rman.h>
#include <sys/mbuf.h>
#include <sys/socket.h>
#include <sys/module.h>
#include <sys/smp.h>
#include <sys/taskqueue.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_types.h>
#include "wrapper-cvmx-includes.h"
#include "ethernet-headers.h"
#include "octebusvar.h"
/*
* XXX/juli
* Convert 0444 to tunables, 0644 to sysctls.
*/
#if defined(CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS) && CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS
int num_packet_buffers = CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS;
#else
int num_packet_buffers = 1024;
#endif
TUNABLE_INT("hw.octe.num_packet_buffers", &num_packet_buffers);
/*
"\t\tNumber of packet buffers to allocate and store in the\n"
"\t\tFPA. By default, 1024 packet buffers are used unless\n"
"\t\tCONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS is defined." */
int pow_receive_group = 15;
TUNABLE_INT("hw.octe.pow_receive_group", &pow_receive_group);
/*
"\t\tPOW group to receive packets from. All ethernet hardware\n"
"\t\twill be configured to send incomming packets to this POW\n"
"\t\tgroup. Also any other software can submit packets to this\n"
"\t\tgroup for the kernel to process." */
/**
* Periodic timer to check auto negotiation
*/
static struct callout cvm_oct_poll_timer;
/**
* Array of every ethernet device owned by this driver indexed by
* the ipd input port number.
*/
struct ifnet *cvm_oct_device[TOTAL_NUMBER_OF_PORTS];
/**
* Task to handle link status changes.
*/
static struct taskqueue *cvm_oct_link_taskq;
/*
* Number of buffers in output buffer pool.
*/
static int cvm_oct_num_output_buffers;
/**
* Function to update link status.
*/
static void cvm_oct_update_link(void *context, int pending)
{
cvm_oct_private_t *priv = (cvm_oct_private_t *)context;
struct ifnet *ifp = priv->ifp;
cvmx_helper_link_info_t link_info;
link_info.u64 = priv->link_info;
if (link_info.s.link_up) {
if_link_state_change(ifp, LINK_STATE_UP);
DEBUGPRINT("%s: %u Mbps %s duplex, port %2d, queue %2d\n",
if_name(ifp), link_info.s.speed,
(link_info.s.full_duplex) ? "Full" : "Half",
priv->port, priv->queue);
} else {
if_link_state_change(ifp, LINK_STATE_DOWN);
DEBUGPRINT("%s: Link down\n", if_name(ifp));
}
priv->need_link_update = 0;
}
/**
* Periodic timer tick for slow management operations
*
* @param arg Device to check
*/
static void cvm_do_timer(void *arg)
{
static int port;
static int updated;
if (port < CVMX_PIP_NUM_INPUT_PORTS) {
if (cvm_oct_device[port]) {
int queues_per_port;
int qos;
cvm_oct_private_t *priv = (cvm_oct_private_t *)cvm_oct_device[port]->if_softc;
cvm_oct_common_poll(priv->ifp);
if (priv->need_link_update) {
updated++;
taskqueue_enqueue(cvm_oct_link_taskq, &priv->link_task);
}
queues_per_port = cvmx_pko_get_num_queues(port);
/* Drain any pending packets in the free list */
for (qos = 0; qos < queues_per_port; qos++) {
if (_IF_QLEN(&priv->tx_free_queue[qos]) > 0) {
IF_LOCK(&priv->tx_free_queue[qos]);
while (_IF_QLEN(&priv->tx_free_queue[qos]) > cvmx_fau_fetch_and_add32(priv->fau+qos*4, 0)) {
struct mbuf *m;
_IF_DEQUEUE(&priv->tx_free_queue[qos], m);
m_freem(m);
}
IF_UNLOCK(&priv->tx_free_queue[qos]);
/*
* XXX locking!
*/
priv->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
}
}
}
port++;
/* Poll the next port in a 50th of a second.
This spreads the polling of ports out a little bit */
callout_reset(&cvm_oct_poll_timer, hz / 50, cvm_do_timer, NULL);
} else {
port = 0;
/* If any updates were made in this run, continue iterating at
* 1/50th of a second, so that if a link has merely gone down
* temporarily (e.g. because of interface reinitialization) it
* will not be forced to stay down for an entire second.
*/
if (updated > 0) {
updated = 0;
callout_reset(&cvm_oct_poll_timer, hz / 50, cvm_do_timer, NULL);
} else {
/* All ports have been polled. Start the next iteration through
the ports in one second */
callout_reset(&cvm_oct_poll_timer, hz, cvm_do_timer, NULL);
}
}
}
/**
* Configure common hardware for all interfaces
*/
static void cvm_oct_configure_common_hw(device_t bus)
{
struct octebus_softc *sc;
int pko_queues;
int error;
int rid;
sc = device_get_softc(bus);
/* Setup the FPA */
cvmx_fpa_enable();
cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE,
num_packet_buffers);
cvm_oct_mem_fill_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE,
num_packet_buffers);
if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL) {
/*
* If the FPA uses different pools for output buffers and
* packets, size the output buffer pool based on the number
* of PKO queues.
*/
if (OCTEON_IS_MODEL(OCTEON_CN38XX))
pko_queues = 128;
else if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
pko_queues = 32;
else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
pko_queues = 32;
else
pko_queues = 256;
cvm_oct_num_output_buffers = 4 * pko_queues;
cvm_oct_mem_fill_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL,
CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE,
cvm_oct_num_output_buffers);
}
if (USE_RED)
cvmx_helper_setup_red(num_packet_buffers/4,
num_packet_buffers/8);
/* Enable the MII interface */
if (cvmx_sysinfo_get()->board_type != CVMX_BOARD_TYPE_SIM)
cvmx_write_csr(CVMX_SMI_EN, 1);
/* Register an IRQ hander for to receive POW interrupts */
rid = 0;
sc->sc_rx_irq = bus_alloc_resource(bus, SYS_RES_IRQ, &rid,
OCTEON_IRQ_WORKQ0 + pow_receive_group,
OCTEON_IRQ_WORKQ0 + pow_receive_group,
1, RF_ACTIVE);
if (sc->sc_rx_irq == NULL) {
device_printf(bus, "could not allocate workq irq");
return;
}
error = bus_setup_intr(bus, sc->sc_rx_irq, INTR_TYPE_NET | INTR_MPSAFE,
cvm_oct_do_interrupt, NULL, cvm_oct_device,
&sc->sc_rx_intr_cookie);
if (error != 0) {
device_printf(bus, "could not setup workq irq");
return;
}
#ifdef SMP
{
cvmx_ciu_intx0_t en;
int core;
CPU_FOREACH(core) {
if (core == PCPU_GET(cpuid))
continue;
en.u64 = cvmx_read_csr(CVMX_CIU_INTX_EN0(core*2));
en.s.workq |= (1<<pow_receive_group);
cvmx_write_csr(CVMX_CIU_INTX_EN0(core*2), en.u64);
}
}
#endif
}
/**
* Free a work queue entry received in a intercept callback.
*
* @param work_queue_entry
* Work queue entry to free
* @return Zero on success, Negative on failure.
*/
int cvm_oct_free_work(void *work_queue_entry)
{
cvmx_wqe_t *work = work_queue_entry;
int segments = work->word2.s.bufs;
cvmx_buf_ptr_t segment_ptr = work->packet_ptr;
while (segments--) {
cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8);
if (__predict_false(!segment_ptr.s.i))
cvmx_fpa_free(cvm_oct_get_buffer_ptr(segment_ptr), segment_ptr.s.pool, DONT_WRITEBACK(CVMX_FPA_PACKET_POOL_SIZE/128));
segment_ptr = next_ptr;
}
cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, DONT_WRITEBACK(1));
return 0;
}
/**
* Module/ driver initialization. Creates the linux network
* devices.
*
* @return Zero on success
*/
int cvm_oct_init_module(device_t bus)
{
device_t dev;
int ifnum;
int num_interfaces;
int interface;
int fau = FAU_NUM_PACKET_BUFFERS_TO_FREE;
int qos;
cvm_oct_rx_initialize();
cvm_oct_configure_common_hw(bus);
cvmx_helper_initialize_packet_io_global();
/* Change the input group for all ports before input is enabled */
num_interfaces = cvmx_helper_get_number_of_interfaces();
for (interface = 0; interface < num_interfaces; interface++) {
int num_ports = cvmx_helper_ports_on_interface(interface);
int port;
for (port = 0; port < num_ports; port++) {
cvmx_pip_prt_tagx_t pip_prt_tagx;
int pkind = cvmx_helper_get_ipd_port(interface, port);
pip_prt_tagx.u64 = cvmx_read_csr(CVMX_PIP_PRT_TAGX(pkind));
pip_prt_tagx.s.grp = pow_receive_group;
cvmx_write_csr(CVMX_PIP_PRT_TAGX(pkind), pip_prt_tagx.u64);
}
}
cvmx_helper_ipd_and_packet_input_enable();
memset(cvm_oct_device, 0, sizeof(cvm_oct_device));
cvm_oct_link_taskq = taskqueue_create("octe link", M_NOWAIT,
taskqueue_thread_enqueue, &cvm_oct_link_taskq);
taskqueue_start_threads(&cvm_oct_link_taskq, 1, PI_NET,
"octe link taskq");
/* Initialize the FAU used for counting packet buffers that need to be freed */
cvmx_fau_atomic_write32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
ifnum = 0;
num_interfaces = cvmx_helper_get_number_of_interfaces();
for (interface = 0; interface < num_interfaces; interface++) {
cvmx_helper_interface_mode_t imode = cvmx_helper_interface_get_mode(interface);
int num_ports = cvmx_helper_ports_on_interface(interface);
int port;
for (port = cvmx_helper_get_ipd_port(interface, 0);
port < cvmx_helper_get_ipd_port(interface, num_ports);
ifnum++, port++) {
cvm_oct_private_t *priv;
struct ifnet *ifp;
dev = BUS_ADD_CHILD(bus, 0, "octe", ifnum);
if (dev != NULL)
ifp = if_alloc(IFT_ETHER);
if (dev == NULL || ifp == NULL) {
printf("Failed to allocate ethernet device for interface %d port %d\n", interface, port);
continue;
}
/* Initialize the device private structure. */
device_probe(dev);
priv = device_get_softc(dev);
priv->dev = dev;
priv->ifp = ifp;
priv->imode = imode;
priv->port = port;
priv->queue = cvmx_pko_get_base_queue(priv->port);
priv->fau = fau - cvmx_pko_get_num_queues(port) * 4;
for (qos = 0; qos < cvmx_pko_get_num_queues(port); qos++)
cvmx_fau_atomic_write32(priv->fau+qos*4, 0);
TASK_INIT(&priv->link_task, 0, cvm_oct_update_link, priv);
switch (priv->imode) {
/* These types don't support ports to IPD/PKO */
case CVMX_HELPER_INTERFACE_MODE_DISABLED:
case CVMX_HELPER_INTERFACE_MODE_PCIE:
case CVMX_HELPER_INTERFACE_MODE_PICMG:
break;
case CVMX_HELPER_INTERFACE_MODE_NPI:
priv->init = cvm_oct_common_init;
priv->uninit = cvm_oct_common_uninit;
device_set_desc(dev, "Cavium Octeon NPI Ethernet");
break;
case CVMX_HELPER_INTERFACE_MODE_XAUI:
priv->init = cvm_oct_xaui_init;
priv->uninit = cvm_oct_common_uninit;
device_set_desc(dev, "Cavium Octeon XAUI Ethernet");
break;
case CVMX_HELPER_INTERFACE_MODE_LOOP:
priv->init = cvm_oct_common_init;
priv->uninit = cvm_oct_common_uninit;
device_set_desc(dev, "Cavium Octeon LOOP Ethernet");
break;
case CVMX_HELPER_INTERFACE_MODE_SGMII:
priv->init = cvm_oct_sgmii_init;
priv->uninit = cvm_oct_common_uninit;
device_set_desc(dev, "Cavium Octeon SGMII Ethernet");
break;
case CVMX_HELPER_INTERFACE_MODE_SPI:
priv->init = cvm_oct_spi_init;
priv->uninit = cvm_oct_spi_uninit;
device_set_desc(dev, "Cavium Octeon SPI Ethernet");
break;
case CVMX_HELPER_INTERFACE_MODE_RGMII:
priv->init = cvm_oct_rgmii_init;
priv->uninit = cvm_oct_rgmii_uninit;
device_set_desc(dev, "Cavium Octeon RGMII Ethernet");
break;
case CVMX_HELPER_INTERFACE_MODE_GMII:
priv->init = cvm_oct_rgmii_init;
priv->uninit = cvm_oct_rgmii_uninit;
device_set_desc(dev, "Cavium Octeon GMII Ethernet");
break;
}
ifp->if_softc = priv;
if (!priv->init) {
printf("octe%d: unsupported device type interface %d, port %d\n",
ifnum, interface, priv->port);
if_free(ifp);
} else if (priv->init(ifp) != 0) {
printf("octe%d: failed to register device for interface %d, port %d\n",
ifnum, interface, priv->port);
if_free(ifp);
} else {
cvm_oct_device[priv->port] = ifp;
fau -= cvmx_pko_get_num_queues(priv->port) * sizeof(uint32_t);
}
}
}
if (INTERRUPT_LIMIT) {
/* Set the POW timer rate to give an interrupt at most INTERRUPT_LIMIT times per second */
cvmx_write_csr(CVMX_POW_WQ_INT_PC, cvmx_clock_get_rate(CVMX_CLOCK_CORE)/(INTERRUPT_LIMIT*16*256)<<8);
/* Enable POW timer interrupt. It will count when there are packets available */
cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0x1ful<<24);
} else {
/* Enable POW interrupt when our port has at least one packet */
cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0x1001);
}
- callout_init(&cvm_oct_poll_timer, CALLOUT_MPSAFE);
+ callout_init(&cvm_oct_poll_timer, 1);
callout_reset(&cvm_oct_poll_timer, hz, cvm_do_timer, NULL);
return 0;
}
/**
* Module / driver shutdown
*
* @return Zero on success
*/
void cvm_oct_cleanup_module(device_t bus)
{
int port;
struct octebus_softc *sc = device_get_softc(bus);
/* Disable POW interrupt */
cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0);
/* Free the interrupt handler */
bus_teardown_intr(bus, sc->sc_rx_irq, sc->sc_rx_intr_cookie);
callout_stop(&cvm_oct_poll_timer);
cvm_oct_rx_shutdown();
cvmx_helper_shutdown_packet_io_global();
/* Free the ethernet devices */
for (port = 0; port < TOTAL_NUMBER_OF_PORTS; port++) {
if (cvm_oct_device[port]) {
cvm_oct_tx_shutdown(cvm_oct_device[port]);
#if 0
unregister_netdev(cvm_oct_device[port]);
kfree(cvm_oct_device[port]);
#else
panic("%s: need to detach and free interface.", __func__);
#endif
cvm_oct_device[port] = NULL;
}
}
/* Free the HW pools */
cvm_oct_mem_empty_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, num_packet_buffers);
cvm_oct_mem_empty_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE, num_packet_buffers);
if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL)
cvm_oct_mem_empty_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL, CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, cvm_oct_num_output_buffers);
/* Disable FPA, all buffers are free, not done by helper shutdown. */
cvmx_fpa_disable();
}
Index: head/sys/mips/cavium/octeon_rnd.c
===================================================================
--- head/sys/mips/cavium/octeon_rnd.c (revision 283290)
+++ head/sys/mips/cavium/octeon_rnd.c (revision 283291)
@@ -1,132 +1,132 @@
/*-
* Copyright (c) 2010 Juli Mallett <jmallett@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/clock.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/module.h>
#include <sys/random.h>
#include <contrib/octeon-sdk/cvmx.h>
#include <contrib/octeon-sdk/cvmx-rng.h>
#define OCTEON_RND_WORDS 2
struct octeon_rnd_softc {
uint64_t sc_entropy[OCTEON_RND_WORDS];
struct callout sc_callout;
};
static void octeon_rnd_identify(driver_t *drv, device_t parent);
static int octeon_rnd_attach(device_t dev);
static int octeon_rnd_probe(device_t dev);
static int octeon_rnd_detach(device_t dev);
static void octeon_rnd_harvest(void *);
static device_method_t octeon_rnd_methods[] = {
/* Device interface */
DEVMETHOD(device_identify, octeon_rnd_identify),
DEVMETHOD(device_probe, octeon_rnd_probe),
DEVMETHOD(device_attach, octeon_rnd_attach),
DEVMETHOD(device_detach, octeon_rnd_detach),
{ 0, 0 }
};
static driver_t octeon_rnd_driver = {
"rnd",
octeon_rnd_methods,
sizeof (struct octeon_rnd_softc)
};
static devclass_t octeon_rnd_devclass;
DRIVER_MODULE(rnd, nexus, octeon_rnd_driver, octeon_rnd_devclass, 0, 0);
static void
octeon_rnd_identify(driver_t *drv, device_t parent)
{
BUS_ADD_CHILD(parent, 0, "rnd", 0);
}
static int
octeon_rnd_probe(device_t dev)
{
if (device_get_unit(dev) != 0)
return (ENXIO);
device_set_desc(dev, "Cavium Octeon Random Number Generator");
return (BUS_PROBE_NOWILDCARD);
}
static int
octeon_rnd_attach(device_t dev)
{
struct octeon_rnd_softc *sc;
sc = device_get_softc(dev);
- callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_callout, 1);
callout_reset(&sc->sc_callout, hz * 5, octeon_rnd_harvest, sc);
cvmx_rng_enable();
return (0);
}
static int
octeon_rnd_detach(device_t dev)
{
struct octeon_rnd_softc *sc;
sc = device_get_softc(dev);
callout_stop(&sc->sc_callout);
return (0);
}
static void
octeon_rnd_harvest(void *arg)
{
struct octeon_rnd_softc *sc;
unsigned i;
sc = arg;
for (i = 0; i < OCTEON_RND_WORDS; i++)
sc->sc_entropy[i] = cvmx_rng_get_random64();
random_harvest(sc->sc_entropy, sizeof sc->sc_entropy,
(sizeof(sc->sc_entropy)*8)/2, RANDOM_PURE_OCTEON);
callout_reset(&sc->sc_callout, hz * 5, octeon_rnd_harvest, sc);
}
Index: head/sys/mips/nlm/dev/net/xlpge.c
===================================================================
--- head/sys/mips/nlm/dev/net/xlpge.c (revision 283290)
+++ head/sys/mips/nlm/dev/net/xlpge.c (revision 283291)
@@ -1,1542 +1,1542 @@
/*-
* Copyright (c) 2003-2012 Broadcom Corporation
* All Rights Reserved
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/endian.h>
#include <sys/systm.h>
#include <sys/sockio.h>
#include <sys/param.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/limits.h>
#include <sys/bus.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/socket.h>
#define __RMAN_RESOURCE_VISIBLE
#include <sys/rman.h>
#include <sys/taskqueue.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_arp.h>
#include <net/ethernet.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/bpf.h>
#include <net/if_types.h>
#include <net/if_vlan_var.h>
#include <dev/pci/pcivar.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/uma.h>
#include <machine/reg.h>
#include <machine/cpu.h>
#include <machine/mips_opcode.h>
#include <machine/asm.h>
#include <machine/cpuregs.h>
#include <machine/param.h>
#include <machine/intr_machdep.h>
#include <machine/clock.h> /* for DELAY */
#include <machine/bus.h>
#include <machine/resource.h>
#include <mips/nlm/hal/haldefs.h>
#include <mips/nlm/hal/iomap.h>
#include <mips/nlm/hal/mips-extns.h>
#include <mips/nlm/hal/cop2.h>
#include <mips/nlm/hal/fmn.h>
#include <mips/nlm/hal/sys.h>
#include <mips/nlm/hal/nae.h>
#include <mips/nlm/hal/mdio.h>
#include <mips/nlm/hal/sgmii.h>
#include <mips/nlm/hal/xaui.h>
#include <mips/nlm/hal/poe.h>
#include <ucore_app_bin.h>
#include <mips/nlm/hal/ucore_loader.h>
#include <mips/nlm/xlp.h>
#include <mips/nlm/board.h>
#include <mips/nlm/msgring.h>
#include <dev/mii/mii.h>
#include <dev/mii/miivar.h>
#include "miidevs.h"
#include <dev/mii/brgphyreg.h>
#include "miibus_if.h"
#include <sys/sysctl.h>
#include <mips/nlm/dev/net/xlpge.h>
/*#define XLP_DRIVER_LOOPBACK*/
static struct nae_port_config nae_port_config[64];
int poe_cl_tbl[MAX_POE_CLASSES] = {
0x0, 0x249249,
0x492492, 0x6db6db,
0x924924, 0xb6db6d,
0xdb6db6, 0xffffff
};
/* #define DUMP_PACKET */
static uint64_t
nlm_paddr_ld(uint64_t paddr)
{
uint64_t xkaddr = 0x9800000000000000 | paddr;
return (nlm_load_dword_daddr(xkaddr));
}
struct nlm_xlp_portdata ifp_ports[64];
static uma_zone_t nl_tx_desc_zone;
/* This implementation will register the following tree of device
* registration:
* pcibus
* |
* xlpnae (1 instance - virtual entity)
* |
* xlpge
* (18 sgmii / 4 xaui / 2 interlaken instances)
* |
* miibus
*/
static int nlm_xlpnae_probe(device_t);
static int nlm_xlpnae_attach(device_t);
static int nlm_xlpnae_detach(device_t);
static int nlm_xlpnae_suspend(device_t);
static int nlm_xlpnae_resume(device_t);
static int nlm_xlpnae_shutdown(device_t);
static device_method_t nlm_xlpnae_methods[] = {
/* Methods from the device interface */
DEVMETHOD(device_probe, nlm_xlpnae_probe),
DEVMETHOD(device_attach, nlm_xlpnae_attach),
DEVMETHOD(device_detach, nlm_xlpnae_detach),
DEVMETHOD(device_suspend, nlm_xlpnae_suspend),
DEVMETHOD(device_resume, nlm_xlpnae_resume),
DEVMETHOD(device_shutdown, nlm_xlpnae_shutdown),
DEVMETHOD(bus_driver_added, bus_generic_driver_added),
DEVMETHOD_END
};
static driver_t nlm_xlpnae_driver = {
"xlpnae",
nlm_xlpnae_methods,
sizeof(struct nlm_xlpnae_softc)
};
static devclass_t nlm_xlpnae_devclass;
static int nlm_xlpge_probe(device_t);
static int nlm_xlpge_attach(device_t);
static int nlm_xlpge_detach(device_t);
static int nlm_xlpge_suspend(device_t);
static int nlm_xlpge_resume(device_t);
static int nlm_xlpge_shutdown(device_t);
/* mii override functions */
static int nlm_xlpge_mii_read(struct device *, int, int);
static int nlm_xlpge_mii_write(struct device *, int, int, int);
static void nlm_xlpge_mii_statchg(device_t);
static device_method_t nlm_xlpge_methods[] = {
/* Methods from the device interface */
DEVMETHOD(device_probe, nlm_xlpge_probe),
DEVMETHOD(device_attach, nlm_xlpge_attach),
DEVMETHOD(device_detach, nlm_xlpge_detach),
DEVMETHOD(device_suspend, nlm_xlpge_suspend),
DEVMETHOD(device_resume, nlm_xlpge_resume),
DEVMETHOD(device_shutdown, nlm_xlpge_shutdown),
/* Methods from the nexus bus needed for explicitly
* probing children when driver is loaded as a kernel module
*/
DEVMETHOD(miibus_readreg, nlm_xlpge_mii_read),
DEVMETHOD(miibus_writereg, nlm_xlpge_mii_write),
DEVMETHOD(miibus_statchg, nlm_xlpge_mii_statchg),
/* Terminate method list */
DEVMETHOD_END
};
static driver_t nlm_xlpge_driver = {
"xlpge",
nlm_xlpge_methods,
sizeof(struct nlm_xlpge_softc)
};
static devclass_t nlm_xlpge_devclass;
DRIVER_MODULE(xlpnae, pci, nlm_xlpnae_driver, nlm_xlpnae_devclass, 0, 0);
DRIVER_MODULE(xlpge, xlpnae, nlm_xlpge_driver, nlm_xlpge_devclass, 0, 0);
DRIVER_MODULE(miibus, xlpge, miibus_driver, miibus_devclass, 0, 0);
MODULE_DEPEND(pci, xlpnae, 1, 1, 1);
MODULE_DEPEND(xlpnae, xlpge, 1, 1, 1);
MODULE_DEPEND(xlpge, ether, 1, 1, 1);
MODULE_DEPEND(xlpge, miibus, 1, 1, 1);
#define SGMII_RCV_CONTEXT_WIDTH 8
/* prototypes */
static void nlm_xlpge_msgring_handler(int vc, int size,
int code, int srcid, struct nlm_fmn_msg *msg, void *data);
static void nlm_xlpge_submit_rx_free_desc(struct nlm_xlpge_softc *sc, int num);
static void nlm_xlpge_init(void *addr);
static void nlm_xlpge_port_disable(struct nlm_xlpge_softc *sc);
static void nlm_xlpge_port_enable(struct nlm_xlpge_softc *sc);
/* globals */
int dbg_on = 1;
int cntx2port[524];
static __inline void
atomic_incr_long(unsigned long *addr)
{
atomic_add_long(addr, 1);
}
/*
* xlpnae driver implementation
*/
static int
nlm_xlpnae_probe(device_t dev)
{
if (pci_get_vendor(dev) != PCI_VENDOR_NETLOGIC ||
pci_get_device(dev) != PCI_DEVICE_ID_NLM_NAE)
return (ENXIO);
return (BUS_PROBE_DEFAULT);
}
static void
nlm_xlpnae_print_frin_desc_carving(struct nlm_xlpnae_softc *sc)
{
int intf;
uint32_t value;
int start, size;
/* XXXJC: use max_ports instead of 20 ? */
for (intf = 0; intf < 20; intf++) {
nlm_write_nae_reg(sc->base, NAE_FREE_IN_FIFO_CFG,
(0x80000000 | intf));
value = nlm_read_nae_reg(sc->base, NAE_FREE_IN_FIFO_CFG);
size = 2 * ((value >> 20) & 0x3ff);
start = 2 * ((value >> 8) & 0x1ff);
}
}
static void
nlm_config_egress(struct nlm_xlpnae_softc *sc, int nblock,
int context_base, int hwport, int max_channels)
{
int offset, num_channels;
uint32_t data;
num_channels = sc->portcfg[hwport].num_channels;
data = (2048 << 12) | (hwport << 4) | 1;
nlm_write_nae_reg(sc->base, NAE_TX_IF_BURSTMAX_CMD, data);
data = ((context_base + num_channels - 1) << 22) |
(context_base << 12) | (hwport << 4) | 1;
nlm_write_nae_reg(sc->base, NAE_TX_DDR_ACTVLIST_CMD, data);
config_egress_fifo_carvings(sc->base, hwport,
context_base, num_channels, max_channels, sc->portcfg);
config_egress_fifo_credits(sc->base, hwport,
context_base, num_channels, max_channels, sc->portcfg);
data = nlm_read_nae_reg(sc->base, NAE_DMA_TX_CREDIT_TH);
data |= (1 << 25) | (1 << 24);
nlm_write_nae_reg(sc->base, NAE_DMA_TX_CREDIT_TH, data);
for (offset = 0; offset < num_channels; offset++) {
nlm_write_nae_reg(sc->base, NAE_TX_SCHED_MAP_CMD1,
NAE_DRR_QUANTA);
data = (hwport << 15) | ((context_base + offset) << 5);
if (sc->cmplx_type[nblock] == ILC)
data |= (offset << 20);
nlm_write_nae_reg(sc->base, NAE_TX_SCHED_MAP_CMD0, data | 1);
nlm_write_nae_reg(sc->base, NAE_TX_SCHED_MAP_CMD0, data);
}
}
static int
xlpnae_get_maxchannels(struct nlm_xlpnae_softc *sc)
{
int maxchans = 0;
int i;
for (i = 0; i < sc->max_ports; i++) {
if (sc->portcfg[i].type == UNKNOWN)
continue;
maxchans += sc->portcfg[i].num_channels;
}
return (maxchans);
}
static void
nlm_setup_interface(struct nlm_xlpnae_softc *sc, int nblock,
int port, uint32_t cur_flow_base, uint32_t flow_mask,
int max_channels, int context)
{
uint64_t nae_base = sc->base;
int mtu = 1536; /* XXXJC: don't hard code */
uint32_t ucore_mask;
if (sc->cmplx_type[nblock] == XAUIC)
nlm_config_xaui(nae_base, nblock, mtu,
mtu, sc->portcfg[port].vlan_pri_en);
nlm_config_freein_fifo_uniq_cfg(nae_base,
port, sc->portcfg[port].free_desc_sizes);
nlm_config_ucore_iface_mask_cfg(nae_base,
port, sc->portcfg[port].ucore_mask);
nlm_program_flow_cfg(nae_base, port, cur_flow_base, flow_mask);
if (sc->cmplx_type[nblock] == SGMIIC)
nlm_configure_sgmii_interface(nae_base, nblock, port, mtu, 0);
nlm_config_egress(sc, nblock, context, port, max_channels);
nlm_nae_init_netior(nae_base, sc->nblocks);
nlm_nae_open_if(nae_base, nblock, sc->cmplx_type[nblock], port,
sc->portcfg[port].free_desc_sizes);
/* XXXJC: check mask calculation */
ucore_mask = (1 << sc->nucores) - 1;
nlm_nae_init_ucore(nae_base, port, ucore_mask);
}
static void
nlm_setup_interfaces(struct nlm_xlpnae_softc *sc)
{
uint64_t nae_base;
uint32_t cur_slot, cur_slot_base;
uint32_t cur_flow_base, port, flow_mask;
int max_channels;
int i, context;
cur_slot = 0;
cur_slot_base = 0;
cur_flow_base = 0;
nae_base = sc->base;
flow_mask = nlm_get_flow_mask(sc->total_num_ports);
/* calculate max_channels */
max_channels = xlpnae_get_maxchannels(sc);
port = 0;
context = 0;
for (i = 0; i < sc->max_ports; i++) {
if (sc->portcfg[i].type == UNKNOWN)
continue;
nlm_setup_interface(sc, sc->portcfg[i].block, i, cur_flow_base,
flow_mask, max_channels, context);
cur_flow_base += sc->per_port_num_flows;
context += sc->portcfg[i].num_channels;
}
}
static void
nlm_xlpnae_init(int node, struct nlm_xlpnae_softc *sc)
{
uint64_t nae_base;
uint32_t ucoremask = 0;
uint32_t val;
int i;
nae_base = sc->base;
nlm_nae_flush_free_fifo(nae_base, sc->nblocks);
nlm_deflate_frin_fifo_carving(nae_base, sc->max_ports);
nlm_reset_nae(node);
for (i = 0; i < sc->nucores; i++) /* XXXJC: code repeated below */
ucoremask |= (0x1 << i);
printf("Loading 0x%x ucores with microcode\n", ucoremask);
nlm_ucore_load_all(nae_base, ucoremask, 1);
val = nlm_set_device_frequency(node, DFS_DEVICE_NAE, sc->freq);
printf("Setup NAE frequency to %dMHz\n", val);
nlm_mdio_reset_all(nae_base);
printf("Initialze SGMII PCS for blocks 0x%x\n", sc->sgmiimask);
nlm_sgmii_pcs_init(nae_base, sc->sgmiimask);
printf("Initialze XAUI PCS for blocks 0x%x\n", sc->xauimask);
nlm_xaui_pcs_init(nae_base, sc->xauimask);
/* clear NETIOR soft reset */
nlm_write_nae_reg(nae_base, NAE_LANE_CFG_SOFTRESET, 0x0);
/* Disable RX enable bit in RX_CONFIG */
val = nlm_read_nae_reg(nae_base, NAE_RX_CONFIG);
val &= 0xfffffffe;
nlm_write_nae_reg(nae_base, NAE_RX_CONFIG, val);
if (nlm_is_xlp8xx_ax() == 0) {
val = nlm_read_nae_reg(nae_base, NAE_TX_CONFIG);
val &= ~(1 << 3);
nlm_write_nae_reg(nae_base, NAE_TX_CONFIG, val);
}
nlm_setup_poe_class_config(nae_base, MAX_POE_CLASSES,
sc->ncontexts, poe_cl_tbl);
nlm_setup_vfbid_mapping(nae_base);
nlm_setup_flow_crc_poly(nae_base, sc->flow_crc_poly);
nlm_setup_rx_cal_cfg(nae_base, sc->max_ports, sc->portcfg);
/* note: xlp8xx Ax does not have Tx Calendering */
if (!nlm_is_xlp8xx_ax())
nlm_setup_tx_cal_cfg(nae_base, sc->max_ports, sc->portcfg);
nlm_setup_interfaces(sc);
nlm_config_poe(sc->poe_base, sc->poedv_base);
if (sc->hw_parser_en)
nlm_enable_hardware_parser(nae_base);
if (sc->prepad_en)
nlm_prepad_enable(nae_base, sc->prepad_size);
if (sc->ieee_1588_en)
nlm_setup_1588_timer(sc->base, sc->portcfg);
}
static void
nlm_xlpnae_update_pde(void *dummy __unused)
{
struct nlm_xlpnae_softc *sc;
uint32_t dv[NUM_WORDS_PER_DV];
device_t dev;
int vec;
dev = devclass_get_device(devclass_find("xlpnae"), 0);
sc = device_get_softc(dev);
nlm_write_poe_reg(sc->poe_base, POE_DISTR_EN, 0);
for (vec = 0; vec < NUM_DIST_VEC; vec++) {
if (nlm_get_poe_distvec(vec, dv) != 0)
continue;
nlm_write_poe_distvec(sc->poedv_base, vec, dv);
}
nlm_write_poe_reg(sc->poe_base, POE_DISTR_EN, 1);
}
SYSINIT(nlm_xlpnae_update_pde, SI_SUB_SMP, SI_ORDER_ANY,
nlm_xlpnae_update_pde, NULL);
/* configuration common for sgmii, xaui, ilaken goes here */
static void
nlm_setup_portcfg(struct nlm_xlpnae_softc *sc, struct xlp_nae_ivars *naep,
int block, int port)
{
int i;
uint32_t ucore_mask = 0;
struct xlp_block_ivars *bp;
struct xlp_port_ivars *p;
bp = &(naep->block_ivars[block]);
p = &(bp->port_ivars[port & 0x3]);
sc->portcfg[port].node = p->node;
sc->portcfg[port].block = p->block;
sc->portcfg[port].port = p->port;
sc->portcfg[port].type = p->type;
sc->portcfg[port].mdio_bus = p->mdio_bus;
sc->portcfg[port].phy_addr = p->phy_addr;
sc->portcfg[port].loopback_mode = p->loopback_mode;
sc->portcfg[port].num_channels = p->num_channels;
if (p->free_desc_sizes != MCLBYTES) {
printf("[%d, %d] Error: free_desc_sizes %d != %d\n",
block, port, p->free_desc_sizes, MCLBYTES);
return;
}
sc->portcfg[port].free_desc_sizes = p->free_desc_sizes;
for (i = 0; i < sc->nucores; i++) /* XXXJC: configure this */
ucore_mask |= (0x1 << i);
sc->portcfg[port].ucore_mask = ucore_mask;
sc->portcfg[port].vlan_pri_en = p->vlan_pri_en;
sc->portcfg[port].num_free_descs = p->num_free_descs;
sc->portcfg[port].iface_fifo_size = p->iface_fifo_size;
sc->portcfg[port].rxbuf_size = p->rxbuf_size;
sc->portcfg[port].rx_slots_reqd = p->rx_slots_reqd;
sc->portcfg[port].tx_slots_reqd = p->tx_slots_reqd;
sc->portcfg[port].pseq_fifo_size = p->pseq_fifo_size;
sc->portcfg[port].stg2_fifo_size = p->stg2_fifo_size;
sc->portcfg[port].eh_fifo_size = p->eh_fifo_size;
sc->portcfg[port].frout_fifo_size = p->frout_fifo_size;
sc->portcfg[port].ms_fifo_size = p->ms_fifo_size;
sc->portcfg[port].pkt_fifo_size = p->pkt_fifo_size;
sc->portcfg[port].pktlen_fifo_size = p->pktlen_fifo_size;
sc->portcfg[port].max_stg2_offset = p->max_stg2_offset;
sc->portcfg[port].max_eh_offset = p->max_eh_offset;
sc->portcfg[port].max_frout_offset = p->max_frout_offset;
sc->portcfg[port].max_ms_offset = p->max_ms_offset;
sc->portcfg[port].max_pmem_offset = p->max_pmem_offset;
sc->portcfg[port].stg1_2_credit = p->stg1_2_credit;
sc->portcfg[port].stg2_eh_credit = p->stg2_eh_credit;
sc->portcfg[port].stg2_frout_credit = p->stg2_frout_credit;
sc->portcfg[port].stg2_ms_credit = p->stg2_ms_credit;
sc->portcfg[port].ieee1588_inc_intg = p->ieee1588_inc_intg;
sc->portcfg[port].ieee1588_inc_den = p->ieee1588_inc_den;
sc->portcfg[port].ieee1588_inc_num = p->ieee1588_inc_num;
sc->portcfg[port].ieee1588_userval = p->ieee1588_userval;
sc->portcfg[port].ieee1588_ptpoff = p->ieee1588_ptpoff;
sc->portcfg[port].ieee1588_tmr1 = p->ieee1588_tmr1;
sc->portcfg[port].ieee1588_tmr2 = p->ieee1588_tmr2;
sc->portcfg[port].ieee1588_tmr3 = p->ieee1588_tmr3;
sc->total_free_desc += sc->portcfg[port].free_desc_sizes;
sc->total_num_ports++;
}
static int
nlm_xlpnae_attach(device_t dev)
{
struct xlp_nae_ivars *nae_ivars;
struct nlm_xlpnae_softc *sc;
device_t tmpd;
uint32_t dv[NUM_WORDS_PER_DV];
int port, i, j, nchan, nblock, node, qstart, qnum;
int offset, context, txq_base, rxvcbase;
uint64_t poe_pcibase, nae_pcibase;
node = pci_get_slot(dev) / 8;
nae_ivars = &xlp_board_info.nodes[node].nae_ivars;
sc = device_get_softc(dev);
sc->xlpnae_dev = dev;
sc->node = nae_ivars->node;
sc->base = nlm_get_nae_regbase(sc->node);
sc->poe_base = nlm_get_poe_regbase(sc->node);
sc->poedv_base = nlm_get_poedv_regbase(sc->node);
sc->portcfg = nae_port_config;
sc->blockmask = nae_ivars->blockmask;
sc->ilmask = nae_ivars->ilmask;
sc->xauimask = nae_ivars->xauimask;
sc->sgmiimask = nae_ivars->sgmiimask;
sc->nblocks = nae_ivars->nblocks;
sc->freq = nae_ivars->freq;
/* flow table generation is done by CRC16 polynomial */
sc->flow_crc_poly = nae_ivars->flow_crc_poly;
sc->hw_parser_en = nae_ivars->hw_parser_en;
sc->prepad_en = nae_ivars->prepad_en;
sc->prepad_size = nae_ivars->prepad_size;
sc->ieee_1588_en = nae_ivars->ieee_1588_en;
nae_pcibase = nlm_get_nae_pcibase(sc->node);
sc->ncontexts = nlm_read_reg(nae_pcibase, XLP_PCI_DEVINFO_REG5);
sc->nucores = nlm_num_uengines(nae_pcibase);
for (nblock = 0; nblock < sc->nblocks; nblock++) {
sc->cmplx_type[nblock] = nae_ivars->block_ivars[nblock].type;
sc->portmask[nblock] = nae_ivars->block_ivars[nblock].portmask;
}
for (i = 0; i < sc->ncontexts; i++)
cntx2port[i] = 18; /* 18 is an invalid port */
if (sc->nblocks == 5)
sc->max_ports = 18; /* 8xx has a block 4 with 2 ports */
else
sc->max_ports = sc->nblocks * PORTS_PER_CMPLX;
for (i = 0; i < sc->max_ports; i++)
sc->portcfg[i].type = UNKNOWN; /* Port Not Present */
/*
* Now setup all internal fifo carvings based on
* total number of ports in the system
*/
sc->total_free_desc = 0;
sc->total_num_ports = 0;
port = 0;
context = 0;
txq_base = nlm_qidstart(nae_pcibase);
rxvcbase = txq_base + sc->ncontexts;
for (i = 0; i < sc->nblocks; i++) {
uint32_t portmask;
if ((nae_ivars->blockmask & (1 << i)) == 0) {
port += 4;
continue;
}
portmask = nae_ivars->block_ivars[i].portmask;
for (j = 0; j < PORTS_PER_CMPLX; j++, port++) {
if ((portmask & (1 << j)) == 0)
continue;
nlm_setup_portcfg(sc, nae_ivars, i, port);
nchan = sc->portcfg[port].num_channels;
for (offset = 0; offset < nchan; offset++)
cntx2port[context + offset] = port;
sc->portcfg[port].txq = txq_base + context;
sc->portcfg[port].rxfreeq = rxvcbase + port;
context += nchan;
}
}
poe_pcibase = nlm_get_poe_pcibase(sc->node);
sc->per_port_num_flows =
nlm_poe_max_flows(poe_pcibase) / sc->total_num_ports;
/* zone for P2P descriptors */
nl_tx_desc_zone = uma_zcreate("NL Tx Desc",
sizeof(struct xlpge_tx_desc), NULL, NULL, NULL, NULL,
NAE_CACHELINE_SIZE, 0);
/* NAE FMN messages have CMS src station id's in the
* range of qstart to qnum.
*/
qstart = nlm_qidstart(nae_pcibase);
qnum = nlm_qnum(nae_pcibase);
if (register_msgring_handler(qstart, qstart + qnum - 1,
nlm_xlpge_msgring_handler, sc)) {
panic("Couldn't register NAE msgring handler\n");
}
/* POE FMN messages have CMS src station id's in the
* range of qstart to qnum.
*/
qstart = nlm_qidstart(poe_pcibase);
qnum = nlm_qnum(poe_pcibase);
if (register_msgring_handler(qstart, qstart + qnum - 1,
nlm_xlpge_msgring_handler, sc)) {
panic("Couldn't register POE msgring handler\n");
}
nlm_xlpnae_init(node, sc);
for (i = 0; i < sc->max_ports; i++) {
char desc[32];
int block, port;
if (sc->portcfg[i].type == UNKNOWN)
continue;
block = sc->portcfg[i].block;
port = sc->portcfg[i].port;
tmpd = device_add_child(dev, "xlpge", i);
device_set_ivars(tmpd,
&(nae_ivars->block_ivars[block].port_ivars[port]));
sprintf(desc, "XLP NAE Port %d,%d", block, port);
device_set_desc_copy(tmpd, desc);
}
nlm_setup_iface_fifo_cfg(sc->base, sc->max_ports, sc->portcfg);
nlm_setup_rx_base_config(sc->base, sc->max_ports, sc->portcfg);
nlm_setup_rx_buf_config(sc->base, sc->max_ports, sc->portcfg);
nlm_setup_freein_fifo_cfg(sc->base, sc->portcfg);
nlm_program_nae_parser_seq_fifo(sc->base, sc->max_ports, sc->portcfg);
nlm_xlpnae_print_frin_desc_carving(sc);
bus_generic_probe(dev);
bus_generic_attach(dev);
/*
* Enable only boot cpu at this point, full distribution comes
* only after SMP is started
*/
nlm_write_poe_reg(sc->poe_base, POE_DISTR_EN, 0);
nlm_calc_poe_distvec(0x1, 0, 0, 0, 0x1 << XLPGE_RX_VC, dv);
nlm_write_poe_distvec(sc->poedv_base, 0, dv);
nlm_write_poe_reg(sc->poe_base, POE_DISTR_EN, 1);
return (0);
}
static int
nlm_xlpnae_detach(device_t dev)
{
/* TODO - free zone here */
return (0);
}
static int
nlm_xlpnae_suspend(device_t dev)
{
return (0);
}
static int
nlm_xlpnae_resume(device_t dev)
{
return (0);
}
static int
nlm_xlpnae_shutdown(device_t dev)
{
return (0);
}
/*
* xlpge driver implementation
*/
static void
nlm_xlpge_mac_set_rx_mode(struct nlm_xlpge_softc *sc)
{
if (sc->if_flags & IFF_PROMISC) {
if (sc->type == SGMIIC)
nlm_nae_setup_rx_mode_sgmii(sc->base_addr,
sc->block, sc->port, sc->type, 1 /* broadcast */,
1/* multicast */, 0 /* pause */, 1 /* promisc */);
else
nlm_nae_setup_rx_mode_xaui(sc->base_addr,
sc->block, sc->port, sc->type, 1 /* broadcast */,
1/* multicast */, 0 /* pause */, 1 /* promisc */);
} else {
if (sc->type == SGMIIC)
nlm_nae_setup_rx_mode_sgmii(sc->base_addr,
sc->block, sc->port, sc->type, 1 /* broadcast */,
1/* multicast */, 0 /* pause */, 0 /* promisc */);
else
nlm_nae_setup_rx_mode_xaui(sc->base_addr,
sc->block, sc->port, sc->type, 1 /* broadcast */,
1/* multicast */, 0 /* pause */, 0 /* promisc */);
}
}
static int
nlm_xlpge_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
{
struct mii_data *mii;
struct nlm_xlpge_softc *sc;
struct ifreq *ifr;
int error;
sc = ifp->if_softc;
error = 0;
ifr = (struct ifreq *)data;
switch (command) {
case SIOCSIFFLAGS:
XLPGE_LOCK(sc);
sc->if_flags = ifp->if_flags;
if (ifp->if_flags & IFF_UP) {
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
nlm_xlpge_init(sc);
else
nlm_xlpge_port_enable(sc);
nlm_xlpge_mac_set_rx_mode(sc);
sc->link = NLM_LINK_UP;
} else {
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
nlm_xlpge_port_disable(sc);
sc->link = NLM_LINK_DOWN;
}
XLPGE_UNLOCK(sc);
error = 0;
break;
case SIOCGIFMEDIA:
case SIOCSIFMEDIA:
if (sc->mii_bus != NULL) {
mii = device_get_softc(sc->mii_bus);
error = ifmedia_ioctl(ifp, ifr, &mii->mii_media,
command);
}
break;
default:
error = ether_ioctl(ifp, command, data);
break;
}
return (error);
}
static int
xlpge_tx(struct ifnet *ifp, struct mbuf *mbuf_chain)
{
struct nlm_fmn_msg msg;
struct xlpge_tx_desc *p2p;
struct nlm_xlpge_softc *sc;
struct mbuf *m;
vm_paddr_t paddr;
int fbid, dst, pos, err;
int ret = 0, tx_msgstatus, retries;
err = 0;
if (mbuf_chain == NULL)
return (0);
sc = ifp->if_softc;
p2p = NULL;
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING) ||
ifp->if_drv_flags & IFF_DRV_OACTIVE) {
err = ENXIO;
goto fail;
}
/* free a few in coming messages on the fb vc */
xlp_handle_msg_vc(1 << XLPGE_FB_VC, 2);
/* vfb id table is setup to map cpu to vc 3 of the cpu */
fbid = nlm_cpuid();
dst = sc->txq;
pos = 0;
p2p = uma_zalloc(nl_tx_desc_zone, M_NOWAIT);
if (p2p == NULL) {
printf("alloc fail\n");
err = ENOBUFS;
goto fail;
}
for (m = mbuf_chain; m != NULL; m = m->m_next) {
vm_offset_t buf = (vm_offset_t) m->m_data;
int len = m->m_len;
int frag_sz;
uint64_t desc;
/*printf("m_data = %p len %d\n", m->m_data, len); */
while (len) {
if (pos == XLP_NTXFRAGS - 3) {
device_printf(sc->xlpge_dev,
"packet defrag %d\n",
m_length(mbuf_chain, NULL));
err = ENOBUFS; /* TODO fix error */
goto fail;
}
paddr = vtophys(buf);
frag_sz = PAGE_SIZE - (buf & PAGE_MASK);
if (len < frag_sz)
frag_sz = len;
desc = nae_tx_desc(P2D_NEOP, 0, 127,
frag_sz, paddr);
p2p->frag[pos] = htobe64(desc);
pos++;
len -= frag_sz;
buf += frag_sz;
}
}
KASSERT(pos != 0, ("Zero-length mbuf chain?\n"));
/* Make the last one P2D EOP */
p2p->frag[pos-1] |= htobe64((uint64_t)P2D_EOP << 62);
/* stash useful pointers in the desc */
p2p->frag[XLP_NTXFRAGS-3] = 0xf00bad;
p2p->frag[XLP_NTXFRAGS-2] = (uintptr_t)p2p;
p2p->frag[XLP_NTXFRAGS-1] = (uintptr_t)mbuf_chain;
paddr = vtophys(p2p);
msg.msg[0] = nae_tx_desc(P2P, 0, fbid, pos, paddr);
for (retries = 16; retries > 0; retries--) {
ret = nlm_fmn_msgsend(dst, 1, FMN_SWCODE_NAE, &msg);
if (ret == 0)
return (0);
}
fail:
if (ret != 0) {
tx_msgstatus = nlm_read_c2_txmsgstatus();
if ((tx_msgstatus >> 24) & 0x1)
device_printf(sc->xlpge_dev, "Transmit queue full - ");
if ((tx_msgstatus >> 3) & 0x1)
device_printf(sc->xlpge_dev, "ECC error - ");
if ((tx_msgstatus >> 2) & 0x1)
device_printf(sc->xlpge_dev, "Pending Sync - ");
if ((tx_msgstatus >> 1) & 0x1)
device_printf(sc->xlpge_dev,
"Insufficient input queue credits - ");
if (tx_msgstatus & 0x1)
device_printf(sc->xlpge_dev,
"Insufficient output queue credits - ");
}
device_printf(sc->xlpge_dev, "Send failed! err = %d\n", err);
if (p2p)
uma_zfree(nl_tx_desc_zone, p2p);
m_freem(mbuf_chain);
if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
return (err);
}
static int
nlm_xlpge_gmac_config_speed(struct nlm_xlpge_softc *sc)
{
struct mii_data *mii;
if (sc->type == XAUIC || sc->type == ILC)
return (0);
if (sc->mii_bus) {
mii = device_get_softc(sc->mii_bus);
mii_pollstat(mii);
}
return (0);
}
static void
nlm_xlpge_port_disable(struct nlm_xlpge_softc *sc)
{
struct ifnet *ifp;
ifp = sc->xlpge_if;
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
callout_stop(&sc->xlpge_callout);
nlm_mac_disable(sc->base_addr, sc->block, sc->type, sc->port);
}
static void
nlm_mii_pollstat(void *arg)
{
struct nlm_xlpge_softc *sc = (struct nlm_xlpge_softc *)arg;
struct mii_data *mii = NULL;
if (sc->mii_bus) {
mii = device_get_softc(sc->mii_bus);
KASSERT(mii != NULL, ("mii ptr is NULL"));
mii_pollstat(mii);
callout_reset(&sc->xlpge_callout, hz,
nlm_mii_pollstat, sc);
}
}
static void
nlm_xlpge_port_enable(struct nlm_xlpge_softc *sc)
{
if ((sc->type != SGMIIC) && (sc->type != XAUIC))
return;
nlm_mac_enable(sc->base_addr, sc->block, sc->type, sc->port);
nlm_mii_pollstat((void *)sc);
}
static void
nlm_xlpge_init(void *addr)
{
struct nlm_xlpge_softc *sc;
struct ifnet *ifp;
struct mii_data *mii = NULL;
sc = (struct nlm_xlpge_softc *)addr;
ifp = sc->xlpge_if;
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
return;
if (sc->mii_bus) {
mii = device_get_softc(sc->mii_bus);
mii_mediachg(mii);
}
nlm_xlpge_gmac_config_speed(sc);
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
nlm_xlpge_port_enable(sc);
/* start the callout */
callout_reset(&sc->xlpge_callout, hz, nlm_mii_pollstat, sc);
}
/*
* Read the MAC address from FDT or board eeprom.
*/
static void
xlpge_read_mac_addr(struct nlm_xlpge_softc *sc)
{
xlpge_get_macaddr(sc->dev_addr);
/* last octet is port specific */
sc->dev_addr[5] += (sc->block * 4) + sc->port;
if (sc->type == SGMIIC)
nlm_nae_setup_mac_addr_sgmii(sc->base_addr, sc->block,
sc->port, sc->type, sc->dev_addr);
else if (sc->type == XAUIC)
nlm_nae_setup_mac_addr_xaui(sc->base_addr, sc->block,
sc->port, sc->type, sc->dev_addr);
}
static int
xlpge_mediachange(struct ifnet *ifp)
{
return (0);
}
static void
xlpge_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
{
struct nlm_xlpge_softc *sc;
struct mii_data *md;
md = NULL;
sc = ifp->if_softc;
if (sc->mii_bus)
md = device_get_softc(sc->mii_bus);
ifmr->ifm_status = IFM_AVALID;
ifmr->ifm_active = IFM_ETHER;
if (sc->link == NLM_LINK_DOWN)
return;
if (md != NULL)
ifmr->ifm_active = md->mii_media.ifm_cur->ifm_media;
ifmr->ifm_status |= IFM_ACTIVE;
}
static int
nlm_xlpge_ifinit(struct nlm_xlpge_softc *sc)
{
struct ifnet *ifp;
device_t dev;
int port = sc->block * 4 + sc->port;
dev = sc->xlpge_dev;
ifp = sc->xlpge_if = if_alloc(IFT_ETHER);
/*(sc->network_sc)->ifp_ports[port].xlpge_if = ifp;*/
ifp_ports[port].xlpge_if = ifp;
if (ifp == NULL) {
device_printf(dev, "cannot if_alloc()\n");
return (ENOSPC);
}
ifp->if_softc = sc;
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
sc->if_flags = ifp->if_flags;
/*ifp->if_capabilities = IFCAP_TXCSUM | IFCAP_VLAN_HWTAGGING;*/
ifp->if_capabilities = 0;
ifp->if_capenable = ifp->if_capabilities;
ifp->if_ioctl = nlm_xlpge_ioctl;
ifp->if_init = nlm_xlpge_init ;
ifp->if_hwassist = 0;
ifp->if_snd.ifq_drv_maxlen = NLM_XLPGE_TXQ_SIZE; /* TODO: make this a sysint */
IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
IFQ_SET_READY(&ifp->if_snd);
ifmedia_init(&sc->xlpge_mii.mii_media, 0, xlpge_mediachange,
xlpge_mediastatus);
ifmedia_add(&sc->xlpge_mii.mii_media, IFM_ETHER | IFM_AUTO, 0, NULL);
ifmedia_set(&sc->xlpge_mii.mii_media, IFM_ETHER | IFM_AUTO);
sc->xlpge_mii.mii_media.ifm_media =
sc->xlpge_mii.mii_media.ifm_cur->ifm_media;
xlpge_read_mac_addr(sc);
ether_ifattach(ifp, sc->dev_addr);
/* override if_transmit : per ifnet(9), do it after if_attach */
ifp->if_transmit = xlpge_tx;
return (0);
}
static int
nlm_xlpge_probe(device_t dev)
{
return (BUS_PROBE_DEFAULT);
}
static void *
get_buf(void)
{
struct mbuf *m_new;
uint64_t *md;
#ifdef INVARIANTS
vm_paddr_t temp1, temp2;
#endif
if ((m_new = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR)) == NULL)
return (NULL);
m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
KASSERT(((uintptr_t)m_new->m_data & (NAE_CACHELINE_SIZE - 1)) == 0,
("m_new->m_data is not cacheline aligned"));
md = (uint64_t *)m_new->m_data;
md[0] = (intptr_t)m_new; /* Back Ptr */
md[1] = 0xf00bad;
m_adj(m_new, NAE_CACHELINE_SIZE);
#ifdef INVARIANTS
temp1 = vtophys((vm_offset_t) m_new->m_data);
temp2 = vtophys((vm_offset_t) m_new->m_data + 1536);
KASSERT((temp1 + 1536) == temp2,
("Alloced buffer is not contiguous"));
#endif
return ((void *)m_new->m_data);
}
static void
nlm_xlpge_mii_init(device_t dev, struct nlm_xlpge_softc *sc)
{
int error;
error = mii_attach(dev, &sc->mii_bus, sc->xlpge_if,
xlpge_mediachange, xlpge_mediastatus,
BMSR_DEFCAPMASK, sc->phy_addr, MII_OFFSET_ANY, 0);
if (error) {
device_printf(dev, "attaching PHYs failed\n");
sc->mii_bus = NULL;
}
if (sc->mii_bus != NULL) {
/* enable MDIO interrupts in the PHY */
/* XXXJC: TODO */
}
}
static int
xlpge_stats_sysctl(SYSCTL_HANDLER_ARGS)
{
struct nlm_xlpge_softc *sc;
uint32_t val;
int reg, field;
sc = arg1;
field = arg2;
reg = SGMII_STATS_MLR(sc->block, sc->port) + field;
val = nlm_read_nae_reg(sc->base_addr, reg);
return (sysctl_handle_int(oidp, &val, 0, req));
}
static void
nlm_xlpge_setup_stats_sysctl(device_t dev, struct nlm_xlpge_softc *sc)
{
struct sysctl_ctx_list *ctx;
struct sysctl_oid_list *child;
struct sysctl_oid *tree;
ctx = device_get_sysctl_ctx(dev);
tree = device_get_sysctl_tree(dev);
child = SYSCTL_CHILDREN(tree);
#define XLPGE_STAT(name, offset, desc) \
SYSCTL_ADD_PROC(ctx, child, OID_AUTO, name, \
CTLTYPE_UINT | CTLFLAG_RD, sc, offset, \
xlpge_stats_sysctl, "IU", desc)
XLPGE_STAT("tr127", nlm_sgmii_stats_tr127, "TxRx 64 - 127 Bytes");
XLPGE_STAT("tr255", nlm_sgmii_stats_tr255, "TxRx 128 - 255 Bytes");
XLPGE_STAT("tr511", nlm_sgmii_stats_tr511, "TxRx 256 - 511 Bytes");
XLPGE_STAT("tr1k", nlm_sgmii_stats_tr1k, "TxRx 512 - 1023 Bytes");
XLPGE_STAT("trmax", nlm_sgmii_stats_trmax, "TxRx 1024 - 1518 Bytes");
XLPGE_STAT("trmgv", nlm_sgmii_stats_trmgv, "TxRx 1519 - 1522 Bytes");
XLPGE_STAT("rbyt", nlm_sgmii_stats_rbyt, "Rx Bytes");
XLPGE_STAT("rpkt", nlm_sgmii_stats_rpkt, "Rx Packets");
XLPGE_STAT("rfcs", nlm_sgmii_stats_rfcs, "Rx FCS Error");
XLPGE_STAT("rmca", nlm_sgmii_stats_rmca, "Rx Multicast Packets");
XLPGE_STAT("rbca", nlm_sgmii_stats_rbca, "Rx Broadcast Packets");
XLPGE_STAT("rxcf", nlm_sgmii_stats_rxcf, "Rx Control Frames");
XLPGE_STAT("rxpf", nlm_sgmii_stats_rxpf, "Rx Pause Frames");
XLPGE_STAT("rxuo", nlm_sgmii_stats_rxuo, "Rx Unknown Opcode");
XLPGE_STAT("raln", nlm_sgmii_stats_raln, "Rx Alignment Errors");
XLPGE_STAT("rflr", nlm_sgmii_stats_rflr, "Rx Framelength Errors");
XLPGE_STAT("rcde", nlm_sgmii_stats_rcde, "Rx Code Errors");
XLPGE_STAT("rcse", nlm_sgmii_stats_rcse, "Rx Carrier Sense Errors");
XLPGE_STAT("rund", nlm_sgmii_stats_rund, "Rx Undersize Packet Errors");
XLPGE_STAT("rovr", nlm_sgmii_stats_rovr, "Rx Oversize Packet Errors");
XLPGE_STAT("rfrg", nlm_sgmii_stats_rfrg, "Rx Fragments");
XLPGE_STAT("rjbr", nlm_sgmii_stats_rjbr, "Rx Jabber");
XLPGE_STAT("tbyt", nlm_sgmii_stats_tbyt, "Tx Bytes");
XLPGE_STAT("tpkt", nlm_sgmii_stats_tpkt, "Tx Packets");
XLPGE_STAT("tmca", nlm_sgmii_stats_tmca, "Tx Multicast Packets");
XLPGE_STAT("tbca", nlm_sgmii_stats_tbca, "Tx Broadcast Packets");
XLPGE_STAT("txpf", nlm_sgmii_stats_txpf, "Tx Pause Frame");
XLPGE_STAT("tdfr", nlm_sgmii_stats_tdfr, "Tx Deferral Packets");
XLPGE_STAT("tedf", nlm_sgmii_stats_tedf, "Tx Excessive Deferral Pkts");
XLPGE_STAT("tscl", nlm_sgmii_stats_tscl, "Tx Single Collisions");
XLPGE_STAT("tmcl", nlm_sgmii_stats_tmcl, "Tx Multiple Collisions");
XLPGE_STAT("tlcl", nlm_sgmii_stats_tlcl, "Tx Late Collision Pkts");
XLPGE_STAT("txcl", nlm_sgmii_stats_txcl, "Tx Excessive Collisions");
XLPGE_STAT("tncl", nlm_sgmii_stats_tncl, "Tx Total Collisions");
XLPGE_STAT("tjbr", nlm_sgmii_stats_tjbr, "Tx Jabber Frames");
XLPGE_STAT("tfcs", nlm_sgmii_stats_tfcs, "Tx FCS Errors");
XLPGE_STAT("txcf", nlm_sgmii_stats_txcf, "Tx Control Frames");
XLPGE_STAT("tovr", nlm_sgmii_stats_tovr, "Tx Oversize Frames");
XLPGE_STAT("tund", nlm_sgmii_stats_tund, "Tx Undersize Frames");
XLPGE_STAT("tfrg", nlm_sgmii_stats_tfrg, "Tx Fragments");
#undef XLPGE_STAT
}
static int
nlm_xlpge_attach(device_t dev)
{
struct xlp_port_ivars *pv;
struct nlm_xlpge_softc *sc;
int port;
pv = device_get_ivars(dev);
sc = device_get_softc(dev);
sc->xlpge_dev = dev;
sc->mii_bus = NULL;
sc->block = pv->block;
sc->node = pv->node;
sc->port = pv->port;
sc->type = pv->type;
sc->xlpge_if = NULL;
sc->phy_addr = pv->phy_addr;
sc->mdio_bus = pv->mdio_bus;
sc->portcfg = nae_port_config;
sc->hw_parser_en = pv->hw_parser_en;
/* default settings */
sc->speed = NLM_SGMII_SPEED_10;
sc->duplexity = NLM_SGMII_DUPLEX_FULL;
sc->link = NLM_LINK_DOWN;
sc->flowctrl = NLM_FLOWCTRL_DISABLED;
sc->network_sc = device_get_softc(device_get_parent(dev));
sc->base_addr = sc->network_sc->base;
sc->prepad_en = sc->network_sc->prepad_en;
sc->prepad_size = sc->network_sc->prepad_size;
- callout_init(&sc->xlpge_callout, CALLOUT_MPSAFE);
+ callout_init(&sc->xlpge_callout, 1);
XLPGE_LOCK_INIT(sc, device_get_nameunit(dev));
port = (sc->block*4)+sc->port;
sc->nfree_desc = nae_port_config[port].num_free_descs;
sc->txq = nae_port_config[port].txq;
sc->rxfreeq = nae_port_config[port].rxfreeq;
nlm_xlpge_submit_rx_free_desc(sc, sc->nfree_desc);
if (sc->hw_parser_en)
nlm_enable_hardware_parser_per_port(sc->base_addr,
sc->block, sc->port);
nlm_xlpge_ifinit(sc);
ifp_ports[port].xlpge_sc = sc;
nlm_xlpge_mii_init(dev, sc);
nlm_xlpge_setup_stats_sysctl(dev, sc);
return (0);
}
static int
nlm_xlpge_detach(device_t dev)
{
return (0);
}
static int
nlm_xlpge_suspend(device_t dev)
{
return (0);
}
static int
nlm_xlpge_resume(device_t dev)
{
return (0);
}
static int
nlm_xlpge_shutdown(device_t dev)
{
return (0);
}
/*
* miibus function with custom implementation
*/
static int
nlm_xlpge_mii_read(struct device *dev, int phyaddr, int regidx)
{
struct nlm_xlpge_softc *sc;
int val;
sc = device_get_softc(dev);
if (sc->type == SGMIIC)
val = nlm_gmac_mdio_read(sc->base_addr, sc->mdio_bus,
BLOCK_7, LANE_CFG, phyaddr, regidx);
else
val = 0xffff;
return (val);
}
static int
nlm_xlpge_mii_write(struct device *dev, int phyaddr, int regidx, int val)
{
struct nlm_xlpge_softc *sc;
sc = device_get_softc(dev);
if (sc->type == SGMIIC)
nlm_gmac_mdio_write(sc->base_addr, sc->mdio_bus, BLOCK_7,
LANE_CFG, phyaddr, regidx, val);
return (0);
}
static void
nlm_xlpge_mii_statchg(device_t dev)
{
struct nlm_xlpge_softc *sc;
struct mii_data *mii;
char *speed, *duplexity;
sc = device_get_softc(dev);
if (sc->mii_bus == NULL)
return;
mii = device_get_softc(sc->mii_bus);
if (mii->mii_media_status & IFM_ACTIVE) {
if (IFM_SUBTYPE(mii->mii_media_active) == IFM_10_T) {
sc->speed = NLM_SGMII_SPEED_10;
speed = "10Mbps";
} else if (IFM_SUBTYPE(mii->mii_media_active) == IFM_100_TX) {
sc->speed = NLM_SGMII_SPEED_100;
speed = "100Mbps";
} else { /* default to 1G */
sc->speed = NLM_SGMII_SPEED_1000;
speed = "1Gbps";
}
if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) {
sc->duplexity = NLM_SGMII_DUPLEX_FULL;
duplexity = "full";
} else {
sc->duplexity = NLM_SGMII_DUPLEX_HALF;
duplexity = "half";
}
printf("Port [%d, %d] setup with speed=%s duplex=%s\n",
sc->block, sc->port, speed, duplexity);
nlm_nae_setup_mac(sc->base_addr, sc->block, sc->port, 0, 1, 1,
sc->speed, sc->duplexity);
}
}
/*
* xlpge support function implementations
*/
static void
nlm_xlpge_release_mbuf(uint64_t paddr)
{
uint64_t mag, desc, mbuf;
paddr += (XLP_NTXFRAGS - 3) * sizeof(uint64_t);
mag = nlm_paddr_ld(paddr);
desc = nlm_paddr_ld(paddr + sizeof(uint64_t));
mbuf = nlm_paddr_ld(paddr + 2 * sizeof(uint64_t));
if (mag != 0xf00bad) {
/* somebody else packet Error - FIXME in intialization */
printf("cpu %d: ERR Tx packet paddr %jx, mag %jx, desc %jx mbuf %jx\n",
nlm_cpuid(), (uintmax_t)paddr, (uintmax_t)mag,
(intmax_t)desc, (uintmax_t)mbuf);
return;
}
m_freem((struct mbuf *)(uintptr_t)mbuf);
uma_zfree(nl_tx_desc_zone, (void *)(uintptr_t)desc);
}
static void
nlm_xlpge_rx(struct nlm_xlpge_softc *sc, int port, vm_paddr_t paddr, int len)
{
struct ifnet *ifp;
struct mbuf *m;
vm_offset_t temp;
unsigned long mag;
int prepad_size;
ifp = sc->xlpge_if;
temp = nlm_paddr_ld(paddr - NAE_CACHELINE_SIZE);
mag = nlm_paddr_ld(paddr - NAE_CACHELINE_SIZE + sizeof(uint64_t));
m = (struct mbuf *)(intptr_t)temp;
if (mag != 0xf00bad) {
/* somebody else packet Error - FIXME in intialization */
printf("cpu %d: ERR Rx packet paddr %jx, temp %p, mag %lx\n",
nlm_cpuid(), (uintmax_t)paddr, (void *)temp, mag);
return;
}
m->m_pkthdr.rcvif = ifp;
#ifdef DUMP_PACKET
{
int i = 0, j = 64;
unsigned char *buf = (char *)m->m_data;
printf("(cpu_%d: nlge_rx, !RX_COPY) Rx Packet: length=%d\n",
nlm_cpuid(), len);
if (len < j)
j = len;
if (sc->prepad_en)
j += ((sc->prepad_size + 1) * 16);
for (i = 0; i < j; i++) {
if (i && (i % 16) == 0)
printf("\n");
printf("%02x ", buf[i]);
}
printf("\n");
}
#endif
if (sc->prepad_en) {
prepad_size = ((sc->prepad_size + 1) * 16);
m->m_data += prepad_size;
m->m_pkthdr.len = m->m_len = (len - prepad_size);
} else
m->m_pkthdr.len = m->m_len = len;
if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
#ifdef XLP_DRIVER_LOOPBACK
if (port == 16 || port == 17)
(*ifp->if_input)(ifp, m);
else
xlpge_tx(ifp, m);
#else
(*ifp->if_input)(ifp, m);
#endif
}
void
nlm_xlpge_submit_rx_free_desc(struct nlm_xlpge_softc *sc, int num)
{
int i, size, ret, n;
struct nlm_fmn_msg msg;
void *ptr;
for(i = 0; i < num; i++) {
memset(&msg, 0, sizeof(msg));
ptr = get_buf();
if (!ptr) {
device_printf(sc->xlpge_dev, "Cannot allocate mbuf\n");
break;
}
msg.msg[0] = vtophys(ptr);
if (msg.msg[0] == 0) {
printf("Bad ptr for %p\n", ptr);
break;
}
size = 1;
n = 0;
while (1) {
/* on success returns 1, else 0 */
ret = nlm_fmn_msgsend(sc->rxfreeq, size, 0, &msg);
if (ret == 0)
break;
if (n++ > 10000) {
printf("Too many credit fails for send free desc\n");
break;
}
}
}
}
void
nlm_xlpge_msgring_handler(int vc, int size, int code, int src_id,
struct nlm_fmn_msg *msg, void *data)
{
uint64_t phys_addr;
struct nlm_xlpnae_softc *sc;
struct nlm_xlpge_softc *xlpge_sc;
struct ifnet *ifp;
uint32_t context;
uint32_t port = 0;
uint32_t length;
sc = (struct nlm_xlpnae_softc *)data;
KASSERT(sc != NULL, ("Null sc in msgring handler"));
if (size == 1) { /* process transmit complete */
phys_addr = msg->msg[0] & 0xffffffffffULL;
/* context is SGMII_RCV_CONTEXT_NUM + three bit vlan type
* or vlan priority
*/
context = (msg->msg[0] >> 40) & 0x3fff;
port = cntx2port[context];
if (port >= XLP_MAX_PORTS) {
printf("%s:%d Bad port %d (context=%d)\n",
__func__, __LINE__, port, context);
return;
}
ifp = ifp_ports[port].xlpge_if;
xlpge_sc = ifp_ports[port].xlpge_sc;
nlm_xlpge_release_mbuf(phys_addr);
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
} else if (size > 1) { /* Recieve packet */
phys_addr = msg->msg[1] & 0xffffffffc0ULL;
length = (msg->msg[1] >> 40) & 0x3fff;
length -= MAC_CRC_LEN;
/* context is SGMII_RCV_CONTEXT_NUM + three bit vlan type
* or vlan priority
*/
context = (msg->msg[1] >> 54) & 0x3ff;
port = cntx2port[context];
if (port >= XLP_MAX_PORTS) {
printf("%s:%d Bad port %d (context=%d)\n",
__func__, __LINE__, port, context);
return;
}
ifp = ifp_ports[port].xlpge_if;
xlpge_sc = ifp_ports[port].xlpge_sc;
nlm_xlpge_rx(xlpge_sc, port, phys_addr, length);
/* return back a free descriptor to NA */
nlm_xlpge_submit_rx_free_desc(xlpge_sc, 1);
}
}
Index: head/sys/mips/rmi/dev/xlr/rge.c
===================================================================
--- head/sys/mips/rmi/dev/xlr/rge.c (revision 283290)
+++ head/sys/mips/rmi/dev/xlr/rge.c (revision 283291)
@@ -1,2564 +1,2564 @@
/*-
* Copyright (c) 2003-2009 RMI Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of RMI Corporation, nor the names of its contributors,
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* RMI_BSD
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#ifdef HAVE_KERNEL_OPTION_HEADERS
#include "opt_device_polling.h"
#endif
#include <sys/types.h>
#include <sys/endian.h>
#include <sys/systm.h>
#include <sys/sockio.h>
#include <sys/param.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/limits.h>
#include <sys/bus.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/socket.h>
#define __RMAN_RESOURCE_VISIBLE
#include <sys/rman.h>
#include <sys/taskqueue.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <net/if.h>
#include <net/if_arp.h>
#include <net/ethernet.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/bpf.h>
#include <net/if_types.h>
#include <net/if_vlan_var.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/reg.h>
#include <machine/cpu.h>
#include <machine/mips_opcode.h>
#include <machine/asm.h>
#include <machine/param.h>
#include <machine/intr_machdep.h>
#include <machine/clock.h> /* for DELAY */
#include <machine/cpuregs.h>
#include <machine/bus.h> /* */
#include <machine/resource.h>
#include <dev/mii/mii.h>
#include <dev/mii/miivar.h>
#include <dev/mii/brgphyreg.h>
#include <mips/rmi/interrupt.h>
#include <mips/rmi/msgring.h>
#include <mips/rmi/iomap.h>
#include <mips/rmi/pic.h>
#include <mips/rmi/rmi_mips_exts.h>
#include <mips/rmi/rmi_boot_info.h>
#include <mips/rmi/board.h>
#include <mips/rmi/dev/xlr/debug.h>
#include <mips/rmi/dev/xlr/atx_cpld.h>
#include <mips/rmi/dev/xlr/xgmac_mdio.h>
#include <mips/rmi/dev/xlr/rge.h>
#include "miibus_if.h"
MODULE_DEPEND(rge, ether, 1, 1, 1);
MODULE_DEPEND(rge, miibus, 1, 1, 1);
/* #define DEBUG */
#define RGE_TX_THRESHOLD 1024
#define RGE_TX_Q_SIZE 1024
#ifdef DEBUG
#undef dbg_msg
int mac_debug = 1;
#define dbg_msg(fmt, args...) \
do {\
if (mac_debug) {\
printf("[%s@%d|%s]: cpu_%d: " fmt, \
__FILE__, __LINE__, __FUNCTION__, xlr_cpu_id(), ##args);\
}\
} while(0);
#define DUMP_PACKETS
#else
#undef dbg_msg
#define dbg_msg(fmt, args...)
int mac_debug = 0;
#endif
#define MAC_B2B_IPG 88
/* frame sizes need to be cacheline aligned */
#define MAX_FRAME_SIZE 1536
#define MAX_FRAME_SIZE_JUMBO 9216
#define MAC_SKB_BACK_PTR_SIZE SMP_CACHE_BYTES
#define MAC_PREPAD 0
#define BYTE_OFFSET 2
#define XLR_RX_BUF_SIZE (MAX_FRAME_SIZE+BYTE_OFFSET+MAC_PREPAD+MAC_SKB_BACK_PTR_SIZE+SMP_CACHE_BYTES)
#define MAC_CRC_LEN 4
#define MAX_NUM_MSGRNG_STN_CC 128
#define MAX_NUM_DESC 1024
#define MAX_SPILL_SIZE (MAX_NUM_DESC + 128)
#define MAC_FRIN_TO_BE_SENT_THRESHOLD 16
#define MAX_FRIN_SPILL (MAX_SPILL_SIZE << 2)
#define MAX_FROUT_SPILL (MAX_SPILL_SIZE << 2)
#define MAX_CLASS_0_SPILL (MAX_SPILL_SIZE << 2)
#define MAX_CLASS_1_SPILL (MAX_SPILL_SIZE << 2)
#define MAX_CLASS_2_SPILL (MAX_SPILL_SIZE << 2)
#define MAX_CLASS_3_SPILL (MAX_SPILL_SIZE << 2)
/*****************************************************************
* Phoenix Generic Mac driver
*****************************************************************/
extern uint32_t cpu_ltop_map[32];
#ifdef ENABLED_DEBUG
static int port_counters[4][8] __aligned(XLR_CACHELINE_SIZE);
#define port_inc_counter(port, counter) atomic_add_int(&port_counters[port][(counter)], 1)
#else
#define port_inc_counter(port, counter) /* Nothing */
#endif
int xlr_rge_tx_prepend[MAXCPU];
int xlr_rge_tx_done[MAXCPU];
int xlr_rge_get_p2d_failed[MAXCPU];
int xlr_rge_msg_snd_failed[MAXCPU];
int xlr_rge_tx_ok_done[MAXCPU];
int xlr_rge_rx_done[MAXCPU];
int xlr_rge_repl_done[MAXCPU];
/* #define mac_stats_add(x, val) ({(x) += (val);}) */
#define mac_stats_add(x, val) xlr_ldaddwu(val, &x)
#define XLR_MAX_CORE 8
#define RGE_LOCK_INIT(_sc, _name) \
mtx_init(&(_sc)->rge_mtx, _name, MTX_NETWORK_LOCK, MTX_DEF)
#define RGE_LOCK(_sc) mtx_lock(&(_sc)->rge_mtx)
#define RGE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->rge_mtx, MA_OWNED)
#define RGE_UNLOCK(_sc) mtx_unlock(&(_sc)->rge_mtx)
#define RGE_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rge_mtx)
#define XLR_MAX_MACS 8
#define XLR_MAX_TX_FRAGS 14
#define MAX_P2D_DESC_PER_PORT 512
struct p2d_tx_desc {
uint64_t frag[XLR_MAX_TX_FRAGS + 2];
};
#define MAX_TX_RING_SIZE (XLR_MAX_MACS * MAX_P2D_DESC_PER_PORT * sizeof(struct p2d_tx_desc))
struct rge_softc *dev_mac[XLR_MAX_MACS];
static int dev_mac_xgs0;
static int dev_mac_gmac0;
static int gmac_common_init_done;
static int rge_probe(device_t);
static int rge_attach(device_t);
static int rge_detach(device_t);
static int rge_suspend(device_t);
static int rge_resume(device_t);
static void rge_release_resources(struct rge_softc *);
static void rge_rx(struct rge_softc *, vm_paddr_t paddr, int);
static void rge_intr(void *);
static void rge_start_locked(struct ifnet *, int);
static void rge_start(struct ifnet *);
static int rge_ioctl(struct ifnet *, u_long, caddr_t);
static void rge_init(void *);
static void rge_stop(struct rge_softc *);
static int rge_shutdown(device_t);
static void rge_reset(struct rge_softc *);
static struct mbuf *get_mbuf(void);
static void free_buf(vm_paddr_t paddr);
static void *get_buf(void);
static void xlr_mac_get_hwaddr(struct rge_softc *);
static void xlr_mac_setup_hwaddr(struct driver_data *);
static void rmi_xlr_mac_set_enable(struct driver_data *priv, int flag);
static void rmi_xlr_xgmac_init(struct driver_data *priv);
static void rmi_xlr_gmac_init(struct driver_data *priv);
static void mac_common_init(void);
static int rge_mii_write(device_t, int, int, int);
static int rge_mii_read(device_t, int, int);
static void rmi_xlr_mac_mii_statchg(device_t);
static int rmi_xlr_mac_mediachange(struct ifnet *);
static void rmi_xlr_mac_mediastatus(struct ifnet *, struct ifmediareq *);
static void xlr_mac_set_rx_mode(struct rge_softc *sc);
void
rmi_xlr_mac_msgring_handler(int bucket, int size, int code,
int stid, struct msgrng_msg *msg,
void *data);
static void mac_frin_replenish(void *);
static int rmi_xlr_mac_open(struct rge_softc *);
static int rmi_xlr_mac_close(struct rge_softc *);
static int
mac_xmit(struct mbuf *, struct rge_softc *,
struct driver_data *, int, struct p2d_tx_desc *);
static int rmi_xlr_mac_xmit(struct mbuf *, struct rge_softc *, int, struct p2d_tx_desc *);
static struct rge_softc_stats *rmi_xlr_mac_get_stats(struct rge_softc *sc);
static void rmi_xlr_mac_set_multicast_list(struct rge_softc *sc);
static int rmi_xlr_mac_change_mtu(struct rge_softc *sc, int new_mtu);
static int rmi_xlr_mac_fill_rxfr(struct rge_softc *sc);
static void rmi_xlr_config_spill_area(struct driver_data *priv);
static int rmi_xlr_mac_set_speed(struct driver_data *s, xlr_mac_speed_t speed);
static int
rmi_xlr_mac_set_duplex(struct driver_data *s,
xlr_mac_duplex_t duplex, xlr_mac_fc_t fc);
static void serdes_regs_init(struct driver_data *priv);
static int rmi_xlr_gmac_reset(struct driver_data *priv);
/*Statistics...*/
static int get_p2d_desc_failed = 0;
static int msg_snd_failed = 0;
SYSCTL_INT(_hw, OID_AUTO, get_p2d_failed, CTLFLAG_RW,
&get_p2d_desc_failed, 0, "p2d desc failed");
SYSCTL_INT(_hw, OID_AUTO, msg_snd_failed, CTLFLAG_RW,
&msg_snd_failed, 0, "msg snd failed");
struct callout xlr_tx_stop_bkp;
static device_method_t rge_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, rge_probe),
DEVMETHOD(device_attach, rge_attach),
DEVMETHOD(device_detach, rge_detach),
DEVMETHOD(device_shutdown, rge_shutdown),
DEVMETHOD(device_suspend, rge_suspend),
DEVMETHOD(device_resume, rge_resume),
/* MII interface */
DEVMETHOD(miibus_readreg, rge_mii_read),
DEVMETHOD(miibus_statchg, rmi_xlr_mac_mii_statchg),
DEVMETHOD(miibus_writereg, rge_mii_write),
{0, 0}
};
static driver_t rge_driver = {
"rge",
rge_methods,
sizeof(struct rge_softc)
};
static devclass_t rge_devclass;
DRIVER_MODULE(rge, iodi, rge_driver, rge_devclass, 0, 0);
DRIVER_MODULE(miibus, rge, miibus_driver, miibus_devclass, 0, 0);
#ifndef __STR
#define __STR(x) #x
#endif
#ifndef STR
#define STR(x) __STR(x)
#endif
void *xlr_tx_ring_mem;
struct tx_desc_node {
struct p2d_tx_desc *ptr;
TAILQ_ENTRY(tx_desc_node) list;
};
#define XLR_MAX_TX_DESC_NODES (XLR_MAX_MACS * MAX_P2D_DESC_PER_PORT)
struct tx_desc_node tx_desc_nodes[XLR_MAX_TX_DESC_NODES];
static volatile int xlr_tot_avail_p2d[XLR_MAX_CORE];
static int xlr_total_active_core = 0;
/*
* This should contain the list of all free tx frag desc nodes pointing to tx
* p2d arrays
*/
static
TAILQ_HEAD(, tx_desc_node) tx_frag_desc[XLR_MAX_CORE] =
{
TAILQ_HEAD_INITIALIZER(tx_frag_desc[0]),
TAILQ_HEAD_INITIALIZER(tx_frag_desc[1]),
TAILQ_HEAD_INITIALIZER(tx_frag_desc[2]),
TAILQ_HEAD_INITIALIZER(tx_frag_desc[3]),
TAILQ_HEAD_INITIALIZER(tx_frag_desc[4]),
TAILQ_HEAD_INITIALIZER(tx_frag_desc[5]),
TAILQ_HEAD_INITIALIZER(tx_frag_desc[6]),
TAILQ_HEAD_INITIALIZER(tx_frag_desc[7]),
};
/* This contains a list of free tx frag node descriptors */
static
TAILQ_HEAD(, tx_desc_node) free_tx_frag_desc[XLR_MAX_CORE] =
{
TAILQ_HEAD_INITIALIZER(free_tx_frag_desc[0]),
TAILQ_HEAD_INITIALIZER(free_tx_frag_desc[1]),
TAILQ_HEAD_INITIALIZER(free_tx_frag_desc[2]),
TAILQ_HEAD_INITIALIZER(free_tx_frag_desc[3]),
TAILQ_HEAD_INITIALIZER(free_tx_frag_desc[4]),
TAILQ_HEAD_INITIALIZER(free_tx_frag_desc[5]),
TAILQ_HEAD_INITIALIZER(free_tx_frag_desc[6]),
TAILQ_HEAD_INITIALIZER(free_tx_frag_desc[7]),
};
static struct mtx tx_desc_lock[XLR_MAX_CORE];
static inline void
mac_make_desc_rfr(struct msgrng_msg *msg,
vm_paddr_t addr)
{
msg->msg0 = (uint64_t) addr & 0xffffffffe0ULL;
msg->msg1 = msg->msg2 = msg->msg3 = 0;
}
#define MAC_TX_DESC_ALIGNMENT (XLR_CACHELINE_SIZE - 1)
static void
init_p2d_allocation(void)
{
int active_core[8] = {0};
int i = 0;
uint32_t cpumask;
int cpu;
cpumask = xlr_hw_thread_mask;
for (i = 0; i < 32; i++) {
if (cpumask & (1 << i)) {
cpu = i;
if (!active_core[cpu / 4]) {
active_core[cpu / 4] = 1;
xlr_total_active_core++;
}
}
}
for (i = 0; i < XLR_MAX_CORE; i++) {
if (active_core[i])
xlr_tot_avail_p2d[i] = XLR_MAX_TX_DESC_NODES / xlr_total_active_core;
}
printf("Total Active Core %d\n", xlr_total_active_core);
}
static void
init_tx_ring(void)
{
int i;
int j = 0;
struct tx_desc_node *start, *node;
struct p2d_tx_desc *tx_desc;
vm_paddr_t paddr;
vm_offset_t unmapped_addr;
for (i = 0; i < XLR_MAX_CORE; i++)
mtx_init(&tx_desc_lock[i], "xlr tx_desc", NULL, MTX_SPIN);
start = &tx_desc_nodes[0];
/* TODO: try to get this from KSEG0 */
xlr_tx_ring_mem = contigmalloc((MAX_TX_RING_SIZE + XLR_CACHELINE_SIZE),
M_DEVBUF, M_NOWAIT | M_ZERO, 0,
0x10000000, XLR_CACHELINE_SIZE, 0);
if (xlr_tx_ring_mem == NULL) {
panic("TX ring memory allocation failed");
}
paddr = vtophys((vm_offset_t)xlr_tx_ring_mem);
unmapped_addr = MIPS_PHYS_TO_KSEG0(paddr);
tx_desc = (struct p2d_tx_desc *)unmapped_addr;
for (i = 0; i < XLR_MAX_TX_DESC_NODES; i++) {
node = start + i;
node->ptr = tx_desc;
tx_desc++;
TAILQ_INSERT_HEAD(&tx_frag_desc[j], node, list);
j = (i / (XLR_MAX_TX_DESC_NODES / xlr_total_active_core));
}
}
static inline struct p2d_tx_desc *
get_p2d_desc(void)
{
struct tx_desc_node *node;
struct p2d_tx_desc *tx_desc = NULL;
int cpu = xlr_core_id();
mtx_lock_spin(&tx_desc_lock[cpu]);
node = TAILQ_FIRST(&tx_frag_desc[cpu]);
if (node) {
xlr_tot_avail_p2d[cpu]--;
TAILQ_REMOVE(&tx_frag_desc[cpu], node, list);
tx_desc = node->ptr;
TAILQ_INSERT_HEAD(&free_tx_frag_desc[cpu], node, list);
} else {
/* Increment p2d desc fail count */
get_p2d_desc_failed++;
}
mtx_unlock_spin(&tx_desc_lock[cpu]);
return tx_desc;
}
static void
free_p2d_desc(struct p2d_tx_desc *tx_desc)
{
struct tx_desc_node *node;
int cpu = xlr_core_id();
mtx_lock_spin(&tx_desc_lock[cpu]);
node = TAILQ_FIRST(&free_tx_frag_desc[cpu]);
KASSERT((node != NULL), ("Free TX frag node list is empty\n"));
TAILQ_REMOVE(&free_tx_frag_desc[cpu], node, list);
node->ptr = tx_desc;
TAILQ_INSERT_HEAD(&tx_frag_desc[cpu], node, list);
xlr_tot_avail_p2d[cpu]++;
mtx_unlock_spin(&tx_desc_lock[cpu]);
}
static int
build_frag_list(struct mbuf *m_head, struct msgrng_msg *p2p_msg, struct p2d_tx_desc *tx_desc)
{
struct mbuf *m;
vm_paddr_t paddr;
uint64_t p2d_len;
int nfrag;
vm_paddr_t p1, p2;
uint32_t len1, len2;
vm_offset_t taddr;
uint64_t fr_stid;
fr_stid = (xlr_core_id() << 3) + xlr_thr_id() + 4;
if (tx_desc == NULL)
return 1;
nfrag = 0;
for (m = m_head; m != NULL; m = m->m_next) {
if ((nfrag + 1) >= XLR_MAX_TX_FRAGS) {
free_p2d_desc(tx_desc);
return 1;
}
if (m->m_len != 0) {
paddr = vtophys(mtod(m, vm_offset_t));
p1 = paddr + m->m_len;
p2 = vtophys(((vm_offset_t)m->m_data + m->m_len));
if (p1 != p2) {
len1 = (uint32_t)
(PAGE_SIZE - (paddr & PAGE_MASK));
tx_desc->frag[nfrag] = (127ULL << 54) |
((uint64_t) len1 << 40) | paddr;
nfrag++;
taddr = (vm_offset_t)m->m_data + len1;
p2 = vtophys(taddr);
len2 = m->m_len - len1;
if (len2 == 0)
continue;
if (nfrag >= XLR_MAX_TX_FRAGS)
panic("TX frags exceeded");
tx_desc->frag[nfrag] = (127ULL << 54) |
((uint64_t) len2 << 40) | p2;
taddr += len2;
p1 = vtophys(taddr);
if ((p2 + len2) != p1) {
printf("p1 = %p p2 = %p\n", (void *)p1, (void *)p2);
printf("len1 = %x len2 = %x\n", len1,
len2);
printf("m_data %p\n", m->m_data);
DELAY(1000000);
panic("Multiple Mbuf segment discontiguous\n");
}
} else {
tx_desc->frag[nfrag] = (127ULL << 54) |
((uint64_t) m->m_len << 40) | paddr;
}
nfrag++;
}
}
/* set eop in the last tx p2d desc */
tx_desc->frag[nfrag - 1] |= (1ULL << 63);
paddr = vtophys((vm_offset_t)tx_desc);
tx_desc->frag[nfrag] = (1ULL << 63) | (fr_stid << 54) | paddr;
nfrag++;
tx_desc->frag[XLR_MAX_TX_FRAGS] = (uint64_t)(intptr_t)tx_desc;
tx_desc->frag[XLR_MAX_TX_FRAGS + 1] = (uint64_t)(intptr_t)m_head;
p2d_len = (nfrag * 8);
p2p_msg->msg0 = (1ULL << 63) | (1ULL << 62) | (127ULL << 54) |
(p2d_len << 40) | paddr;
return 0;
}
static void
release_tx_desc(struct msgrng_msg *msg, int rel_buf)
{
struct p2d_tx_desc *tx_desc, *chk_addr;
struct mbuf *m;
tx_desc = (struct p2d_tx_desc *)MIPS_PHYS_TO_KSEG0(msg->msg0);
chk_addr = (struct p2d_tx_desc *)(intptr_t)tx_desc->frag[XLR_MAX_TX_FRAGS];
if (tx_desc != chk_addr) {
printf("Address %p does not match with stored addr %p - we leaked a descriptor\n",
tx_desc, chk_addr);
return;
}
if (rel_buf) {
m = (struct mbuf *)(intptr_t)tx_desc->frag[XLR_MAX_TX_FRAGS + 1];
m_freem(m);
}
free_p2d_desc(tx_desc);
}
static struct mbuf *
get_mbuf(void)
{
struct mbuf *m_new = NULL;
if ((m_new = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR)) == NULL)
return NULL;
m_new->m_len = MCLBYTES;
m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
return m_new;
}
static void
free_buf(vm_paddr_t paddr)
{
struct mbuf *m;
uint64_t mag;
uint32_t sr;
sr = xlr_enable_kx();
m = (struct mbuf *)(intptr_t)xlr_paddr_ld(paddr - XLR_CACHELINE_SIZE);
mag = xlr_paddr_ld(paddr - XLR_CACHELINE_SIZE + sizeof(uint64_t));
xlr_restore_kx(sr);
if (mag != 0xf00bad) {
printf("Something is wrong kseg:%lx found mag:%lx not 0xf00bad\n",
(u_long)paddr, (u_long)mag);
return;
}
if (m != NULL)
m_freem(m);
}
static void *
get_buf(void)
{
struct mbuf *m_new = NULL;
uint64_t *md;
#ifdef INVARIANTS
vm_paddr_t temp1, temp2;
#endif
m_new = get_mbuf();
if (m_new == NULL)
return NULL;
m_adj(m_new, XLR_CACHELINE_SIZE - ((uintptr_t)m_new->m_data & 0x1f));
md = (uint64_t *)m_new->m_data;
md[0] = (uintptr_t)m_new; /* Back Ptr */
md[1] = 0xf00bad;
m_adj(m_new, XLR_CACHELINE_SIZE);
#ifdef INVARIANTS
temp1 = vtophys((vm_offset_t)m_new->m_data);
temp2 = vtophys((vm_offset_t)m_new->m_data + 1536);
if ((temp1 + 1536) != temp2)
panic("ALLOCED BUFFER IS NOT CONTIGUOUS\n");
#endif
return (void *)m_new->m_data;
}
/**********************************************************************
**********************************************************************/
static void
rmi_xlr_mac_set_enable(struct driver_data *priv, int flag)
{
uint32_t regval;
int tx_threshold = 1518;
if (flag) {
regval = xlr_read_reg(priv->mmio, R_TX_CONTROL);
regval |= (1 << O_TX_CONTROL__TxEnable) |
(tx_threshold << O_TX_CONTROL__TxThreshold);
xlr_write_reg(priv->mmio, R_TX_CONTROL, regval);
regval = xlr_read_reg(priv->mmio, R_RX_CONTROL);
regval |= 1 << O_RX_CONTROL__RxEnable;
if (priv->mode == XLR_PORT0_RGMII)
regval |= 1 << O_RX_CONTROL__RGMII;
xlr_write_reg(priv->mmio, R_RX_CONTROL, regval);
regval = xlr_read_reg(priv->mmio, R_MAC_CONFIG_1);
regval |= (O_MAC_CONFIG_1__txen | O_MAC_CONFIG_1__rxen);
xlr_write_reg(priv->mmio, R_MAC_CONFIG_1, regval);
} else {
regval = xlr_read_reg(priv->mmio, R_TX_CONTROL);
regval &= ~((1 << O_TX_CONTROL__TxEnable) |
(tx_threshold << O_TX_CONTROL__TxThreshold));
xlr_write_reg(priv->mmio, R_TX_CONTROL, regval);
regval = xlr_read_reg(priv->mmio, R_RX_CONTROL);
regval &= ~(1 << O_RX_CONTROL__RxEnable);
xlr_write_reg(priv->mmio, R_RX_CONTROL, regval);
regval = xlr_read_reg(priv->mmio, R_MAC_CONFIG_1);
regval &= ~(O_MAC_CONFIG_1__txen | O_MAC_CONFIG_1__rxen);
xlr_write_reg(priv->mmio, R_MAC_CONFIG_1, regval);
}
}
/**********************************************************************
**********************************************************************/
static __inline__ int
xlr_mac_send_fr(struct driver_data *priv,
vm_paddr_t addr, int len)
{
struct msgrng_msg msg;
int stid = priv->rfrbucket;
int code, ret;
uint32_t msgrng_flags;
#ifdef INVARIANTS
int i = 0;
#endif
mac_make_desc_rfr(&msg, addr);
/* Send the packet to MAC */
dbg_msg("mac_%d: Sending free packet %lx to stid %d\n",
priv->instance, (u_long)addr, stid);
if (priv->type == XLR_XGMAC)
code = MSGRNG_CODE_XGMAC; /* WHY? */
else
code = MSGRNG_CODE_MAC;
do {
msgrng_flags = msgrng_access_enable();
ret = message_send(1, code, stid, &msg);
msgrng_restore(msgrng_flags);
KASSERT(i++ < 100000, ("Too many credit fails\n"));
} while (ret != 0);
return 0;
}
/**************************************************************/
static void
xgmac_mdio_setup(volatile unsigned int *_mmio)
{
int i;
uint32_t rd_data;
for (i = 0; i < 4; i++) {
rd_data = xmdio_read(_mmio, 1, 0x8000 + i);
rd_data = rd_data & 0xffffdfff; /* clear isolate bit */
xmdio_write(_mmio, 1, 0x8000 + i, rd_data);
}
}
/**********************************************************************
* Init MII interface
*
* Input parameters:
* s - priv structure
********************************************************************* */
#define PHY_STATUS_RETRIES 25000
static void
rmi_xlr_mac_mii_init(struct driver_data *priv)
{
xlr_reg_t *mii_mmio = priv->mii_mmio;
/* use the lowest clock divisor - divisor 28 */
xlr_write_reg(mii_mmio, R_MII_MGMT_CONFIG, 0x07);
}
/**********************************************************************
* Read a PHY register.
*
* Input parameters:
* s - priv structure
* phyaddr - PHY's address
* regidx = index of register to read
*
* Return value:
* value read, or 0 if an error occurred.
********************************************************************* */
static int
rge_mii_read_internal(xlr_reg_t * mii_mmio, int phyaddr, int regidx)
{
int i = 0;
/* setup the phy reg to be used */
xlr_write_reg(mii_mmio, R_MII_MGMT_ADDRESS,
(phyaddr << 8) | (regidx << 0));
/* Issue the read command */
xlr_write_reg(mii_mmio, R_MII_MGMT_COMMAND,
(1 << O_MII_MGMT_COMMAND__rstat));
/* poll for the read cycle to complete */
for (i = 0; i < PHY_STATUS_RETRIES; i++) {
if (xlr_read_reg(mii_mmio, R_MII_MGMT_INDICATORS) == 0)
break;
}
/* clear the read cycle */
xlr_write_reg(mii_mmio, R_MII_MGMT_COMMAND, 0);
if (i == PHY_STATUS_RETRIES) {
return 0xffffffff;
}
/* Read the data back */
return xlr_read_reg(mii_mmio, R_MII_MGMT_STATUS);
}
static int
rge_mii_read(device_t dev, int phyaddr, int regidx)
{
struct rge_softc *sc = device_get_softc(dev);
return rge_mii_read_internal(sc->priv.mii_mmio, phyaddr, regidx);
}
/**********************************************************************
* Set MII hooks to newly selected media
*
* Input parameters:
* ifp - Interface Pointer
*
* Return value:
* nothing
********************************************************************* */
static int
rmi_xlr_mac_mediachange(struct ifnet *ifp)
{
struct rge_softc *sc = ifp->if_softc;
if (ifp->if_flags & IFF_UP)
mii_mediachg(&sc->rge_mii);
return 0;
}
/**********************************************************************
* Get the current interface media status
*
* Input parameters:
* ifp - Interface Pointer
* ifmr - Interface media request ptr
*
* Return value:
* nothing
********************************************************************* */
static void
rmi_xlr_mac_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
{
struct rge_softc *sc = ifp->if_softc;
/* Check whether this is interface is active or not. */
ifmr->ifm_status = IFM_AVALID;
if (sc->link_up) {
ifmr->ifm_status |= IFM_ACTIVE;
} else {
ifmr->ifm_active = IFM_ETHER;
}
}
/**********************************************************************
* Write a value to a PHY register.
*
* Input parameters:
* s - priv structure
* phyaddr - PHY to use
* regidx - register within the PHY
* regval - data to write to register
*
* Return value:
* nothing
********************************************************************* */
static void
rge_mii_write_internal(xlr_reg_t * mii_mmio, int phyaddr, int regidx, int regval)
{
int i = 0;
xlr_write_reg(mii_mmio, R_MII_MGMT_ADDRESS,
(phyaddr << 8) | (regidx << 0));
/* Write the data which starts the write cycle */
xlr_write_reg(mii_mmio, R_MII_MGMT_WRITE_DATA, regval);
/* poll for the write cycle to complete */
for (i = 0; i < PHY_STATUS_RETRIES; i++) {
if (xlr_read_reg(mii_mmio, R_MII_MGMT_INDICATORS) == 0)
break;
}
return;
}
static int
rge_mii_write(device_t dev, int phyaddr, int regidx, int regval)
{
struct rge_softc *sc = device_get_softc(dev);
rge_mii_write_internal(sc->priv.mii_mmio, phyaddr, regidx, regval);
return (0);
}
static void
rmi_xlr_mac_mii_statchg(struct device *dev)
{
}
static void
serdes_regs_init(struct driver_data *priv)
{
xlr_reg_t *mmio_gpio = (xlr_reg_t *) (xlr_io_base + XLR_IO_GPIO_OFFSET);
/* Initialize SERDES CONTROL Registers */
rge_mii_write_internal(priv->serdes_mmio, 26, 0, 0x6DB0);
rge_mii_write_internal(priv->serdes_mmio, 26, 1, 0xFFFF);
rge_mii_write_internal(priv->serdes_mmio, 26, 2, 0xB6D0);
rge_mii_write_internal(priv->serdes_mmio, 26, 3, 0x00FF);
rge_mii_write_internal(priv->serdes_mmio, 26, 4, 0x0000);
rge_mii_write_internal(priv->serdes_mmio, 26, 5, 0x0000);
rge_mii_write_internal(priv->serdes_mmio, 26, 6, 0x0005);
rge_mii_write_internal(priv->serdes_mmio, 26, 7, 0x0001);
rge_mii_write_internal(priv->serdes_mmio, 26, 8, 0x0000);
rge_mii_write_internal(priv->serdes_mmio, 26, 9, 0x0000);
rge_mii_write_internal(priv->serdes_mmio, 26, 10, 0x0000);
/*
* GPIO setting which affect the serdes - needs figuring out
*/
DELAY(100);
xlr_write_reg(mmio_gpio, 0x20, 0x7e6802);
xlr_write_reg(mmio_gpio, 0x10, 0x7104);
DELAY(100);
/*
* This kludge is needed to setup serdes (?) clock correctly on some
* XLS boards
*/
if ((xlr_boot1_info.board_major_version == RMI_XLR_BOARD_ARIZONA_XI ||
xlr_boot1_info.board_major_version == RMI_XLR_BOARD_ARIZONA_XII) &&
xlr_boot1_info.board_minor_version == 4) {
/* use 125 Mhz instead of 156.25Mhz ref clock */
DELAY(100);
xlr_write_reg(mmio_gpio, 0x10, 0x7103);
xlr_write_reg(mmio_gpio, 0x21, 0x7103);
DELAY(100);
}
return;
}
static void
serdes_autoconfig(struct driver_data *priv)
{
int delay = 100000;
/* Enable Auto negotiation in the PCS Layer */
rge_mii_write_internal(priv->pcs_mmio, 27, 0, 0x1000);
DELAY(delay);
rge_mii_write_internal(priv->pcs_mmio, 27, 0, 0x0200);
DELAY(delay);
rge_mii_write_internal(priv->pcs_mmio, 28, 0, 0x1000);
DELAY(delay);
rge_mii_write_internal(priv->pcs_mmio, 28, 0, 0x0200);
DELAY(delay);
rge_mii_write_internal(priv->pcs_mmio, 29, 0, 0x1000);
DELAY(delay);
rge_mii_write_internal(priv->pcs_mmio, 29, 0, 0x0200);
DELAY(delay);
rge_mii_write_internal(priv->pcs_mmio, 30, 0, 0x1000);
DELAY(delay);
rge_mii_write_internal(priv->pcs_mmio, 30, 0, 0x0200);
DELAY(delay);
}
/*****************************************************************
* Initialize GMAC
*****************************************************************/
static void
rmi_xlr_config_pde(struct driver_data *priv)
{
int i = 0, cpu = 0, bucket = 0;
uint64_t bucket_map = 0;
/* uint32_t desc_pack_ctrl = 0; */
uint32_t cpumask;
cpumask = 0x1;
#ifdef SMP
/*
* rge may be called before SMP start in a BOOTP/NFSROOT
* setup. we will distribute packets to other cpus only when
* the SMP is started.
*/
if (smp_started)
cpumask = xlr_hw_thread_mask;
#endif
for (i = 0; i < MAXCPU; i++) {
if (cpumask & (1 << i)) {
cpu = i;
bucket = ((cpu >> 2) << 3);
bucket_map |= (3ULL << bucket);
}
}
printf("rmi_xlr_config_pde: bucket_map=%jx\n", (uintmax_t)bucket_map);
/* bucket_map = 0x1; */
xlr_write_reg(priv->mmio, R_PDE_CLASS_0, (bucket_map & 0xffffffff));
xlr_write_reg(priv->mmio, R_PDE_CLASS_0 + 1,
((bucket_map >> 32) & 0xffffffff));
xlr_write_reg(priv->mmio, R_PDE_CLASS_1, (bucket_map & 0xffffffff));
xlr_write_reg(priv->mmio, R_PDE_CLASS_1 + 1,
((bucket_map >> 32) & 0xffffffff));
xlr_write_reg(priv->mmio, R_PDE_CLASS_2, (bucket_map & 0xffffffff));
xlr_write_reg(priv->mmio, R_PDE_CLASS_2 + 1,
((bucket_map >> 32) & 0xffffffff));
xlr_write_reg(priv->mmio, R_PDE_CLASS_3, (bucket_map & 0xffffffff));
xlr_write_reg(priv->mmio, R_PDE_CLASS_3 + 1,
((bucket_map >> 32) & 0xffffffff));
}
static void
rge_smp_update_pde(void *dummy __unused)
{
int i;
struct driver_data *priv;
struct rge_softc *sc;
printf("Updating packet distribution for SMP\n");
for (i = 0; i < XLR_MAX_MACS; i++) {
sc = dev_mac[i];
if (!sc)
continue;
priv = &(sc->priv);
rmi_xlr_mac_set_enable(priv, 0);
rmi_xlr_config_pde(priv);
rmi_xlr_mac_set_enable(priv, 1);
}
}
SYSINIT(rge_smp_update_pde, SI_SUB_SMP, SI_ORDER_ANY, rge_smp_update_pde, NULL);
static void
rmi_xlr_config_parser(struct driver_data *priv)
{
/*
* Mark it as no classification The parser extract is gauranteed to
* be zero with no classfication
*/
xlr_write_reg(priv->mmio, R_L2TYPE_0, 0x00);
xlr_write_reg(priv->mmio, R_L2TYPE_0, 0x01);
/* configure the parser : L2 Type is configured in the bootloader */
/* extract IP: src, dest protocol */
xlr_write_reg(priv->mmio, R_L3CTABLE,
(9 << 20) | (1 << 19) | (1 << 18) | (0x01 << 16) |
(0x0800 << 0));
xlr_write_reg(priv->mmio, R_L3CTABLE + 1,
(12 << 25) | (4 << 21) | (16 << 14) | (4 << 10));
}
static void
rmi_xlr_config_classifier(struct driver_data *priv)
{
int i = 0;
if (priv->type == XLR_XGMAC) {
/* xgmac translation table doesn't have sane values on reset */
for (i = 0; i < 64; i++)
xlr_write_reg(priv->mmio, R_TRANSLATETABLE + i, 0x0);
/*
* use upper 7 bits of the parser extract to index the
* translate table
*/
xlr_write_reg(priv->mmio, R_PARSERCONFIGREG, 0x0);
}
}
enum {
SGMII_SPEED_10 = 0x00000000,
SGMII_SPEED_100 = 0x02000000,
SGMII_SPEED_1000 = 0x04000000,
};
static void
rmi_xlr_gmac_config_speed(struct driver_data *priv)
{
int phy_addr = priv->phy_addr;
xlr_reg_t *mmio = priv->mmio;
struct rge_softc *sc = priv->sc;
priv->speed = rge_mii_read_internal(priv->mii_mmio, phy_addr, 28);
priv->link = rge_mii_read_internal(priv->mii_mmio, phy_addr, 1) & 0x4;
priv->speed = (priv->speed >> 3) & 0x03;
if (priv->speed == xlr_mac_speed_10) {
if (priv->mode != XLR_RGMII)
xlr_write_reg(mmio, R_INTERFACE_CONTROL, SGMII_SPEED_10);
xlr_write_reg(mmio, R_MAC_CONFIG_2, 0x7117);
xlr_write_reg(mmio, R_CORECONTROL, 0x02);
printf("%s: [10Mbps]\n", device_get_nameunit(sc->rge_dev));
sc->rge_mii.mii_media.ifm_media = IFM_ETHER | IFM_AUTO | IFM_10_T | IFM_FDX;
sc->rge_mii.mii_media.ifm_cur->ifm_media = IFM_ETHER | IFM_AUTO | IFM_10_T | IFM_FDX;
sc->rge_mii.mii_media_active = IFM_ETHER | IFM_AUTO | IFM_10_T | IFM_FDX;
} else if (priv->speed == xlr_mac_speed_100) {
if (priv->mode != XLR_RGMII)
xlr_write_reg(mmio, R_INTERFACE_CONTROL, SGMII_SPEED_100);
xlr_write_reg(mmio, R_MAC_CONFIG_2, 0x7117);
xlr_write_reg(mmio, R_CORECONTROL, 0x01);
printf("%s: [100Mbps]\n", device_get_nameunit(sc->rge_dev));
sc->rge_mii.mii_media.ifm_media = IFM_ETHER | IFM_AUTO | IFM_100_TX | IFM_FDX;
sc->rge_mii.mii_media.ifm_cur->ifm_media = IFM_ETHER | IFM_AUTO | IFM_100_TX | IFM_FDX;
sc->rge_mii.mii_media_active = IFM_ETHER | IFM_AUTO | IFM_100_TX | IFM_FDX;
} else {
if (priv->speed != xlr_mac_speed_1000) {
if (priv->mode != XLR_RGMII)
xlr_write_reg(mmio, R_INTERFACE_CONTROL, SGMII_SPEED_100);
printf("PHY reported unknown MAC speed, defaulting to 100Mbps\n");
xlr_write_reg(mmio, R_MAC_CONFIG_2, 0x7117);
xlr_write_reg(mmio, R_CORECONTROL, 0x01);
sc->rge_mii.mii_media.ifm_media = IFM_ETHER | IFM_AUTO | IFM_100_TX | IFM_FDX;
sc->rge_mii.mii_media.ifm_cur->ifm_media = IFM_ETHER | IFM_AUTO | IFM_100_TX | IFM_FDX;
sc->rge_mii.mii_media_active = IFM_ETHER | IFM_AUTO | IFM_100_TX | IFM_FDX;
} else {
if (priv->mode != XLR_RGMII)
xlr_write_reg(mmio, R_INTERFACE_CONTROL, SGMII_SPEED_1000);
xlr_write_reg(mmio, R_MAC_CONFIG_2, 0x7217);
xlr_write_reg(mmio, R_CORECONTROL, 0x00);
printf("%s: [1000Mbps]\n", device_get_nameunit(sc->rge_dev));
sc->rge_mii.mii_media.ifm_media = IFM_ETHER | IFM_AUTO | IFM_1000_T | IFM_FDX;
sc->rge_mii.mii_media.ifm_cur->ifm_media = IFM_ETHER | IFM_AUTO | IFM_1000_T | IFM_FDX;
sc->rge_mii.mii_media_active = IFM_ETHER | IFM_AUTO | IFM_1000_T | IFM_FDX;
}
}
if (!priv->link) {
sc->rge_mii.mii_media.ifm_cur->ifm_media = IFM_ETHER;
sc->link_up = 0;
} else {
sc->link_up = 1;
}
}
/*****************************************************************
* Initialize XGMAC
*****************************************************************/
static void
rmi_xlr_xgmac_init(struct driver_data *priv)
{
int i = 0;
xlr_reg_t *mmio = priv->mmio;
int id = priv->instance;
struct rge_softc *sc = priv->sc;
volatile unsigned short *cpld;
cpld = (volatile unsigned short *)0xBD840000;
xlr_write_reg(priv->mmio, R_DESC_PACK_CTRL,
(MAX_FRAME_SIZE << O_DESC_PACK_CTRL__RegularSize) | (4 << 20));
xlr_write_reg(priv->mmio, R_BYTEOFFSET0, BYTE_OFFSET);
rmi_xlr_config_pde(priv);
rmi_xlr_config_parser(priv);
rmi_xlr_config_classifier(priv);
xlr_write_reg(priv->mmio, R_MSG_TX_THRESHOLD, 1);
/* configure the XGMAC Registers */
xlr_write_reg(mmio, R_XGMAC_CONFIG_1, 0x50000026);
/* configure the XGMAC_GLUE Registers */
xlr_write_reg(mmio, R_DMACR0, 0xffffffff);
xlr_write_reg(mmio, R_DMACR1, 0xffffffff);
xlr_write_reg(mmio, R_DMACR2, 0xffffffff);
xlr_write_reg(mmio, R_DMACR3, 0xffffffff);
xlr_write_reg(mmio, R_STATCTRL, 0x04);
xlr_write_reg(mmio, R_L2ALLOCCTRL, 0xffffffff);
xlr_write_reg(mmio, R_XGMACPADCALIBRATION, 0x030);
xlr_write_reg(mmio, R_EGRESSFIFOCARVINGSLOTS, 0x0f);
xlr_write_reg(mmio, R_L2ALLOCCTRL, 0xffffffff);
xlr_write_reg(mmio, R_XGMAC_MIIM_CONFIG, 0x3e);
/*
* take XGMII phy out of reset
*/
/*
* we are pulling everything out of reset because writing a 0 would
* reset other devices on the chip
*/
cpld[ATX_CPLD_RESET_1] = 0xffff;
cpld[ATX_CPLD_MISC_CTRL] = 0xffff;
cpld[ATX_CPLD_RESET_2] = 0xffff;
xgmac_mdio_setup(mmio);
rmi_xlr_config_spill_area(priv);
if (id == 0) {
for (i = 0; i < 16; i++) {
xlr_write_reg(mmio, R_XGS_TX0_BUCKET_SIZE + i,
bucket_sizes.
bucket[MSGRNG_STNID_XGS0_TX + i]);
}
xlr_write_reg(mmio, R_XGS_JFR_BUCKET_SIZE,
bucket_sizes.bucket[MSGRNG_STNID_XMAC0JFR]);
xlr_write_reg(mmio, R_XGS_RFR_BUCKET_SIZE,
bucket_sizes.bucket[MSGRNG_STNID_XMAC0RFR]);
for (i = 0; i < MAX_NUM_MSGRNG_STN_CC; i++) {
xlr_write_reg(mmio, R_CC_CPU0_0 + i,
cc_table_xgs_0.
counters[i >> 3][i & 0x07]);
}
} else if (id == 1) {
for (i = 0; i < 16; i++) {
xlr_write_reg(mmio, R_XGS_TX0_BUCKET_SIZE + i,
bucket_sizes.
bucket[MSGRNG_STNID_XGS1_TX + i]);
}
xlr_write_reg(mmio, R_XGS_JFR_BUCKET_SIZE,
bucket_sizes.bucket[MSGRNG_STNID_XMAC1JFR]);
xlr_write_reg(mmio, R_XGS_RFR_BUCKET_SIZE,
bucket_sizes.bucket[MSGRNG_STNID_XMAC1RFR]);
for (i = 0; i < MAX_NUM_MSGRNG_STN_CC; i++) {
xlr_write_reg(mmio, R_CC_CPU0_0 + i,
cc_table_xgs_1.
counters[i >> 3][i & 0x07]);
}
}
sc->rge_mii.mii_media.ifm_media = IFM_ETHER | IFM_AUTO | IFM_10G_SR | IFM_FDX;
sc->rge_mii.mii_media.ifm_media |= (IFM_AVALID | IFM_ACTIVE);
sc->rge_mii.mii_media.ifm_cur->ifm_media = IFM_ETHER | IFM_AUTO | IFM_10G_SR | IFM_FDX;
sc->rge_mii.mii_media_active = IFM_ETHER | IFM_AUTO | IFM_10G_SR | IFM_FDX;
sc->rge_mii.mii_media.ifm_cur->ifm_media |= (IFM_AVALID | IFM_ACTIVE);
priv->init_frin_desc = 1;
}
/*******************************************************
* Initialization gmac
*******************************************************/
static int
rmi_xlr_gmac_reset(struct driver_data *priv)
{
volatile uint32_t val;
xlr_reg_t *mmio = priv->mmio;
int i, maxloops = 100;
/* Disable MAC RX */
val = xlr_read_reg(mmio, R_MAC_CONFIG_1);
val &= ~0x4;
xlr_write_reg(mmio, R_MAC_CONFIG_1, val);
/* Disable Core RX */
val = xlr_read_reg(mmio, R_RX_CONTROL);
val &= ~0x1;
xlr_write_reg(mmio, R_RX_CONTROL, val);
/* wait for rx to halt */
for (i = 0; i < maxloops; i++) {
val = xlr_read_reg(mmio, R_RX_CONTROL);
if (val & 0x2)
break;
DELAY(1000);
}
if (i == maxloops)
return -1;
/* Issue a soft reset */
val = xlr_read_reg(mmio, R_RX_CONTROL);
val |= 0x4;
xlr_write_reg(mmio, R_RX_CONTROL, val);
/* wait for reset to complete */
for (i = 0; i < maxloops; i++) {
val = xlr_read_reg(mmio, R_RX_CONTROL);
if (val & 0x8)
break;
DELAY(1000);
}
if (i == maxloops)
return -1;
/* Clear the soft reset bit */
val = xlr_read_reg(mmio, R_RX_CONTROL);
val &= ~0x4;
xlr_write_reg(mmio, R_RX_CONTROL, val);
return 0;
}
static void
rmi_xlr_gmac_init(struct driver_data *priv)
{
int i = 0;
xlr_reg_t *mmio = priv->mmio;
int id = priv->instance;
struct stn_cc *gmac_cc_config;
uint32_t value = 0;
int blk = id / 4, port = id % 4;
rmi_xlr_mac_set_enable(priv, 0);
rmi_xlr_config_spill_area(priv);
xlr_write_reg(mmio, R_DESC_PACK_CTRL,
(BYTE_OFFSET << O_DESC_PACK_CTRL__ByteOffset) |
(1 << O_DESC_PACK_CTRL__MaxEntry) |
(MAX_FRAME_SIZE << O_DESC_PACK_CTRL__RegularSize));
rmi_xlr_config_pde(priv);
rmi_xlr_config_parser(priv);
rmi_xlr_config_classifier(priv);
xlr_write_reg(mmio, R_MSG_TX_THRESHOLD, 3);
xlr_write_reg(mmio, R_MAC_CONFIG_1, 0x35);
xlr_write_reg(mmio, R_RX_CONTROL, (0x7 << 6));
if (priv->mode == XLR_PORT0_RGMII) {
printf("Port 0 set in RGMII mode\n");
value = xlr_read_reg(mmio, R_RX_CONTROL);
value |= 1 << O_RX_CONTROL__RGMII;
xlr_write_reg(mmio, R_RX_CONTROL, value);
}
rmi_xlr_mac_mii_init(priv);
#if 0
priv->advertising = ADVERTISED_10baseT_Full | ADVERTISED_10baseT_Half |
ADVERTISED_100baseT_Full | ADVERTISED_100baseT_Half |
ADVERTISED_1000baseT_Full | ADVERTISED_Autoneg |
ADVERTISED_MII;
#endif
/*
* Enable all MDIO interrupts in the phy RX_ER bit seems to be get
* set about every 1 sec in GigE mode, ignore it for now...
*/
rge_mii_write_internal(priv->mii_mmio, priv->phy_addr, 25, 0xfffffffe);
if (priv->mode != XLR_RGMII) {
serdes_regs_init(priv);
serdes_autoconfig(priv);
}
rmi_xlr_gmac_config_speed(priv);
value = xlr_read_reg(mmio, R_IPG_IFG);
xlr_write_reg(mmio, R_IPG_IFG, ((value & ~0x7f) | MAC_B2B_IPG));
xlr_write_reg(mmio, R_DMACR0, 0xffffffff);
xlr_write_reg(mmio, R_DMACR1, 0xffffffff);
xlr_write_reg(mmio, R_DMACR2, 0xffffffff);
xlr_write_reg(mmio, R_DMACR3, 0xffffffff);
xlr_write_reg(mmio, R_STATCTRL, 0x04);
xlr_write_reg(mmio, R_L2ALLOCCTRL, 0xffffffff);
xlr_write_reg(mmio, R_INTMASK, 0);
xlr_write_reg(mmio, R_FREEQCARVE, 0);
xlr_write_reg(mmio, R_GMAC_TX0_BUCKET_SIZE + port,
xlr_board_info.bucket_sizes->bucket[priv->txbucket]);
xlr_write_reg(mmio, R_GMAC_JFR0_BUCKET_SIZE,
xlr_board_info.bucket_sizes->bucket[MSGRNG_STNID_GMACJFR_0]);
xlr_write_reg(mmio, R_GMAC_RFR0_BUCKET_SIZE,
xlr_board_info.bucket_sizes->bucket[MSGRNG_STNID_GMACRFR_0]);
xlr_write_reg(mmio, R_GMAC_JFR1_BUCKET_SIZE,
xlr_board_info.bucket_sizes->bucket[MSGRNG_STNID_GMACJFR_1]);
xlr_write_reg(mmio, R_GMAC_RFR1_BUCKET_SIZE,
xlr_board_info.bucket_sizes->bucket[MSGRNG_STNID_GMACRFR_1]);
dbg_msg("Programming credit counter %d : %d -> %d\n", blk, R_GMAC_TX0_BUCKET_SIZE + port,
xlr_board_info.bucket_sizes->bucket[priv->txbucket]);
gmac_cc_config = xlr_board_info.gmac_block[blk].credit_config;
for (i = 0; i < MAX_NUM_MSGRNG_STN_CC; i++) {
xlr_write_reg(mmio, R_CC_CPU0_0 + i,
gmac_cc_config->counters[i >> 3][i & 0x07]);
dbg_msg("%d: %d -> %d\n", priv->instance,
R_CC_CPU0_0 + i, gmac_cc_config->counters[i >> 3][i & 0x07]);
}
priv->init_frin_desc = 1;
}
/**********************************************************************
* Set promiscuous mode
**********************************************************************/
static void
xlr_mac_set_rx_mode(struct rge_softc *sc)
{
struct driver_data *priv = &(sc->priv);
uint32_t regval;
regval = xlr_read_reg(priv->mmio, R_MAC_FILTER_CONFIG);
if (sc->flags & IFF_PROMISC) {
regval |= (1 << O_MAC_FILTER_CONFIG__BROADCAST_EN) |
(1 << O_MAC_FILTER_CONFIG__PAUSE_FRAME_EN) |
(1 << O_MAC_FILTER_CONFIG__ALL_MCAST_EN) |
(1 << O_MAC_FILTER_CONFIG__ALL_UCAST_EN);
} else {
regval &= ~((1 << O_MAC_FILTER_CONFIG__PAUSE_FRAME_EN) |
(1 << O_MAC_FILTER_CONFIG__ALL_UCAST_EN));
}
xlr_write_reg(priv->mmio, R_MAC_FILTER_CONFIG, regval);
}
/**********************************************************************
* Configure LAN speed for the specified MAC.
********************************************************************* */
static int
rmi_xlr_mac_set_speed(struct driver_data *s, xlr_mac_speed_t speed)
{
return 0;
}
/**********************************************************************
* Set Ethernet duplex and flow control options for this MAC
********************************************************************* */
static int
rmi_xlr_mac_set_duplex(struct driver_data *s,
xlr_mac_duplex_t duplex, xlr_mac_fc_t fc)
{
return 0;
}
/*****************************************************************
* Kernel Net Stack <-> MAC Driver Interface
*****************************************************************/
/**********************************************************************
**********************************************************************/
#define MAC_TX_FAIL 2
#define MAC_TX_PASS 0
#define MAC_TX_RETRY 1
int xlr_dev_queue_xmit_hack = 0;
static int
mac_xmit(struct mbuf *m, struct rge_softc *sc,
struct driver_data *priv, int len, struct p2d_tx_desc *tx_desc)
{
struct msgrng_msg msg = {0,0,0,0};
int stid = priv->txbucket;
uint32_t tx_cycles = 0;
uint32_t mflags;
int vcpu = xlr_cpu_id();
int rv;
tx_cycles = mips_rd_count();
if (build_frag_list(m, &msg, tx_desc) != 0)
return MAC_TX_FAIL;
else {
mflags = msgrng_access_enable();
if ((rv = message_send(1, MSGRNG_CODE_MAC, stid, &msg)) != 0) {
msg_snd_failed++;
msgrng_restore(mflags);
release_tx_desc(&msg, 0);
xlr_rge_msg_snd_failed[vcpu]++;
dbg_msg("Failed packet to cpu %d, rv = %d, stid %d, msg0=%jx\n",
vcpu, rv, stid, (uintmax_t)msg.msg0);
return MAC_TX_FAIL;
}
msgrng_restore(mflags);
port_inc_counter(priv->instance, PORT_TX);
}
/* Send the packet to MAC */
dbg_msg("Sent tx packet to stid %d, msg0=%jx, msg1=%jx \n", stid,
(uintmax_t)msg.msg0, (uintmax_t)msg.msg1);
#ifdef DUMP_PACKETS
{
int i = 0;
unsigned char *buf = (char *)m->m_data;
printf("Tx Packet: length=%d\n", len);
for (i = 0; i < 64; i++) {
if (i && (i % 16) == 0)
printf("\n");
printf("%02x ", buf[i]);
}
printf("\n");
}
#endif
xlr_inc_counter(NETIF_TX);
return MAC_TX_PASS;
}
static int
rmi_xlr_mac_xmit(struct mbuf *m, struct rge_softc *sc, int len, struct p2d_tx_desc *tx_desc)
{
struct driver_data *priv = &(sc->priv);
int ret = -ENOSPC;
dbg_msg("IN\n");
xlr_inc_counter(NETIF_STACK_TX);
retry:
ret = mac_xmit(m, sc, priv, len, tx_desc);
if (ret == MAC_TX_RETRY)
goto retry;
dbg_msg("OUT, ret = %d\n", ret);
if (ret == MAC_TX_FAIL) {
/* FULL */
dbg_msg("Msg Ring Full. Stopping upper layer Q\n");
port_inc_counter(priv->instance, PORT_STOPQ);
}
return ret;
}
static void
mac_frin_replenish(void *args /* ignored */ )
{
int cpu = xlr_core_id();
int done = 0;
int i = 0;
xlr_inc_counter(REPLENISH_ENTER);
/*
* xlr_set_counter(REPLENISH_ENTER_COUNT,
* atomic_read(frin_to_be_sent));
*/
xlr_set_counter(REPLENISH_CPU, PCPU_GET(cpuid));
for (;;) {
done = 0;
for (i = 0; i < XLR_MAX_MACS; i++) {
/* int offset = 0; */
void *m;
uint32_t cycles;
struct rge_softc *sc;
struct driver_data *priv;
int frin_to_be_sent;
sc = dev_mac[i];
if (!sc)
goto skip;
priv = &(sc->priv);
frin_to_be_sent = priv->frin_to_be_sent[cpu];
/* if (atomic_read(frin_to_be_sent) < 0) */
if (frin_to_be_sent < 0) {
panic("BUG?: [%s]: gmac_%d illegal value for frin_to_be_sent=%d\n",
__FUNCTION__, i,
frin_to_be_sent);
}
/* if (!atomic_read(frin_to_be_sent)) */
if (!frin_to_be_sent)
goto skip;
cycles = mips_rd_count();
{
m = get_buf();
if (!m) {
device_printf(sc->rge_dev, "No buffer\n");
goto skip;
}
}
xlr_inc_counter(REPLENISH_FRIN);
if (xlr_mac_send_fr(priv, vtophys(m), MAX_FRAME_SIZE)) {
free_buf(vtophys(m));
printf("[%s]: rx free message_send failed!\n", __FUNCTION__);
break;
}
xlr_set_counter(REPLENISH_CYCLES,
(read_c0_count() - cycles));
atomic_subtract_int((&priv->frin_to_be_sent[cpu]), 1);
continue;
skip:
done++;
}
if (done == XLR_MAX_MACS)
break;
}
}
static volatile uint32_t g_tx_frm_tx_ok=0;
static void
rge_tx_bkp_func(void *arg, int npending)
{
int i = 0;
for (i = 0; i < xlr_board_info.gmacports; i++) {
if (!dev_mac[i] || !dev_mac[i]->active)
continue;
rge_start_locked(dev_mac[i]->rge_ifp, RGE_TX_THRESHOLD);
}
atomic_subtract_int(&g_tx_frm_tx_ok, 1);
}
/* This function is called from an interrupt handler */
void
rmi_xlr_mac_msgring_handler(int bucket, int size, int code,
int stid, struct msgrng_msg *msg,
void *data /* ignored */ )
{
uint64_t phys_addr = 0;
unsigned long addr = 0;
uint32_t length = 0;
int ctrl = 0, port = 0;
struct rge_softc *sc = NULL;
struct driver_data *priv = 0;
struct ifnet *ifp;
int vcpu = xlr_cpu_id();
int cpu = xlr_core_id();
dbg_msg("mac: bucket=%d, size=%d, code=%d, stid=%d, msg0=%jx msg1=%jx\n",
bucket, size, code, stid, (uintmax_t)msg->msg0, (uintmax_t)msg->msg1);
phys_addr = (uint64_t) (msg->msg0 & 0xffffffffe0ULL);
length = (msg->msg0 >> 40) & 0x3fff;
if (length == 0) {
ctrl = CTRL_REG_FREE;
port = (msg->msg0 >> 54) & 0x0f;
addr = 0;
} else {
ctrl = CTRL_SNGL;
length = length - BYTE_OFFSET - MAC_CRC_LEN;
port = msg->msg0 & 0x0f;
addr = 0;
}
if (xlr_board_info.is_xls) {
if (stid == MSGRNG_STNID_GMAC1)
port += 4;
sc = dev_mac[dev_mac_gmac0 + port];
} else {
if (stid == MSGRNG_STNID_XGS0FR)
sc = dev_mac[dev_mac_xgs0];
else if (stid == MSGRNG_STNID_XGS1FR)
sc = dev_mac[dev_mac_xgs0 + 1];
else
sc = dev_mac[dev_mac_gmac0 + port];
}
if (sc == NULL)
return;
priv = &(sc->priv);
dbg_msg("msg0 = %jx, stid = %d, port = %d, addr=%lx, length=%d, ctrl=%d\n",
(uintmax_t)msg->msg0, stid, port, addr, length, ctrl);
if (ctrl == CTRL_REG_FREE || ctrl == CTRL_JUMBO_FREE) {
xlr_rge_tx_ok_done[vcpu]++;
release_tx_desc(msg, 1);
ifp = sc->rge_ifp;
if (ifp->if_drv_flags & IFF_DRV_OACTIVE) {
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
}
if (atomic_cmpset_int(&g_tx_frm_tx_ok, 0, 1))
rge_tx_bkp_func(NULL, 0);
xlr_set_counter(NETIF_TX_COMPLETE_CYCLES,
(read_c0_count() - msgrng_msg_cycles));
} else if (ctrl == CTRL_SNGL || ctrl == CTRL_START) {
/* Rx Packet */
/* struct mbuf *m = 0; */
/* int logical_cpu = 0; */
dbg_msg("Received packet, port = %d\n", port);
/*
* if num frins to be sent exceeds threshold, wake up the
* helper thread
*/
atomic_add_int(&(priv->frin_to_be_sent[cpu]), 1);
if ((priv->frin_to_be_sent[cpu]) > MAC_FRIN_TO_BE_SENT_THRESHOLD) {
mac_frin_replenish(NULL);
}
dbg_msg("gmac_%d: rx packet: phys_addr = %jx, length = %x\n",
priv->instance, (uintmax_t)phys_addr, length);
mac_stats_add(priv->stats.rx_packets, 1);
mac_stats_add(priv->stats.rx_bytes, length);
xlr_inc_counter(NETIF_RX);
xlr_set_counter(NETIF_RX_CYCLES,
(read_c0_count() - msgrng_msg_cycles));
rge_rx(sc, phys_addr, length);
xlr_rge_rx_done[vcpu]++;
} else {
printf("[%s]: unrecognized ctrl=%d!\n", __FUNCTION__, ctrl);
}
}
/**********************************************************************
**********************************************************************/
static int
rge_probe(dev)
device_t dev;
{
device_set_desc(dev, "RMI Gigabit Ethernet");
/* Always return 0 */
return 0;
}
volatile unsigned long xlr_debug_enabled;
struct callout rge_dbg_count;
static void
xlr_debug_count(void *addr)
{
struct driver_data *priv = &dev_mac[0]->priv;
/* uint32_t crdt; */
if (xlr_debug_enabled) {
printf("\nAvailRxIn %#x\n", xlr_read_reg(priv->mmio, 0x23e));
}
callout_reset(&rge_dbg_count, hz, xlr_debug_count, NULL);
}
static void
xlr_tx_q_wakeup(void *addr)
{
int i = 0;
int j = 0;
for (i = 0; i < xlr_board_info.gmacports; i++) {
if (!dev_mac[i] || !dev_mac[i]->active)
continue;
if ((dev_mac[i]->rge_ifp->if_drv_flags) & IFF_DRV_OACTIVE) {
for (j = 0; j < XLR_MAX_CORE; j++) {
if (xlr_tot_avail_p2d[j]) {
dev_mac[i]->rge_ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
break;
}
}
}
}
if (atomic_cmpset_int(&g_tx_frm_tx_ok, 0, 1))
rge_tx_bkp_func(NULL, 0);
callout_reset(&xlr_tx_stop_bkp, 5 * hz, xlr_tx_q_wakeup, NULL);
}
static int
rge_attach(device_t dev)
{
struct ifnet *ifp;
struct rge_softc *sc;
struct driver_data *priv = 0;
int ret = 0;
struct xlr_gmac_block_t *gmac_conf = device_get_ivars(dev);
sc = device_get_softc(dev);
sc->rge_dev = dev;
/* Initialize mac's */
sc->unit = device_get_unit(dev);
if (sc->unit > XLR_MAX_MACS) {
ret = ENXIO;
goto out;
}
RGE_LOCK_INIT(sc, device_get_nameunit(dev));
priv = &(sc->priv);
priv->sc = sc;
sc->flags = 0; /* TODO : fix me up later */
priv->id = sc->unit;
if (gmac_conf->type == XLR_GMAC) {
priv->instance = priv->id;
priv->mmio = (xlr_reg_t *) (xlr_io_base + gmac_conf->baseaddr +
0x1000 * (sc->unit % 4));
if ((ret = rmi_xlr_gmac_reset(priv)) == -1)
goto out;
} else if (gmac_conf->type == XLR_XGMAC) {
priv->instance = priv->id - xlr_board_info.gmacports;
priv->mmio = (xlr_reg_t *) (xlr_io_base + gmac_conf->baseaddr);
}
if (xlr_boot1_info.board_major_version == RMI_XLR_BOARD_ARIZONA_VI ||
(xlr_boot1_info.board_major_version == RMI_XLR_BOARD_ARIZONA_XI &&
priv->instance >=4)) {
dbg_msg("Arizona board - offset 4 \n");
priv->mii_mmio = (xlr_reg_t *) (xlr_io_base + XLR_IO_GMAC_4_OFFSET);
} else
priv->mii_mmio = (xlr_reg_t *) (xlr_io_base + XLR_IO_GMAC_0_OFFSET);
priv->pcs_mmio = (xlr_reg_t *) (xlr_io_base + gmac_conf->baseaddr);
priv->serdes_mmio = (xlr_reg_t *) (xlr_io_base + XLR_IO_GMAC_0_OFFSET);
sc->base_addr = (unsigned long)priv->mmio;
sc->mem_end = (unsigned long)priv->mmio + XLR_IO_SIZE - 1;
sc->xmit = rge_start;
sc->stop = rge_stop;
sc->get_stats = rmi_xlr_mac_get_stats;
sc->ioctl = rge_ioctl;
/* Initialize the device specific driver data */
mtx_init(&priv->lock, "rge", NULL, MTX_SPIN);
priv->type = gmac_conf->type;
priv->mode = gmac_conf->mode;
if (xlr_board_info.is_xls == 0) {
/* TODO - check II and IIB boards */
if (xlr_boot1_info.board_major_version == RMI_XLR_BOARD_ARIZONA_II &&
xlr_boot1_info.board_minor_version != 1)
priv->phy_addr = priv->instance - 2;
else
priv->phy_addr = priv->instance;
priv->mode = XLR_RGMII;
} else {
if (gmac_conf->mode == XLR_PORT0_RGMII &&
priv->instance == 0) {
priv->mode = XLR_PORT0_RGMII;
priv->phy_addr = 0;
} else {
priv->mode = XLR_SGMII;
/* Board 11 has SGMII daughter cards with the XLS chips, in this case
the phy number is 0-3 for both GMAC blocks */
if (xlr_boot1_info.board_major_version == RMI_XLR_BOARD_ARIZONA_XI)
priv->phy_addr = priv->instance % 4 + 16;
else
priv->phy_addr = priv->instance + 16;
}
}
priv->txbucket = gmac_conf->station_txbase + priv->instance % 4;
priv->rfrbucket = gmac_conf->station_rfr;
priv->spill_configured = 0;
dbg_msg("priv->mmio=%p\n", priv->mmio);
/* Set up ifnet structure */
ifp = sc->rge_ifp = if_alloc(IFT_ETHER);
if (ifp == NULL) {
device_printf(sc->rge_dev, "failed to if_alloc()\n");
rge_release_resources(sc);
ret = ENXIO;
RGE_LOCK_DESTROY(sc);
goto out;
}
ifp->if_softc = sc;
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = rge_ioctl;
ifp->if_start = rge_start;
ifp->if_init = rge_init;
ifp->if_mtu = ETHERMTU;
ifp->if_snd.ifq_drv_maxlen = RGE_TX_Q_SIZE;
IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
IFQ_SET_READY(&ifp->if_snd);
sc->active = 1;
ifp->if_hwassist = 0;
ifp->if_capabilities = IFCAP_TXCSUM | IFCAP_VLAN_HWTAGGING;
ifp->if_capenable = ifp->if_capabilities;
/* Initialize the rge_softc */
sc->irq = gmac_conf->baseirq + priv->instance % 4;
/* Set the IRQ into the rid field */
/*
* note this is a hack to pass the irq to the iodi interrupt setup
* routines
*/
sc->rge_irq.__r_i = (struct resource_i *)(intptr_t)sc->irq;
ret = bus_setup_intr(dev, &sc->rge_irq, INTR_TYPE_NET | INTR_MPSAFE,
NULL, rge_intr, sc, &sc->rge_intrhand);
if (ret) {
rge_detach(dev);
device_printf(sc->rge_dev, "couldn't set up irq\n");
RGE_LOCK_DESTROY(sc);
goto out;
}
xlr_mac_get_hwaddr(sc);
xlr_mac_setup_hwaddr(priv);
dbg_msg("MMIO %08lx, MII %08lx, PCS %08lx, base %08lx PHY %d IRQ %d\n",
(u_long)priv->mmio, (u_long)priv->mii_mmio, (u_long)priv->pcs_mmio,
(u_long)sc->base_addr, priv->phy_addr, sc->irq);
dbg_msg("HWADDR %02x:%02x tx %d rfr %d\n", (u_int)sc->dev_addr[4],
(u_int)sc->dev_addr[5], priv->txbucket, priv->rfrbucket);
/*
* Set up ifmedia support.
*/
/*
* Initialize MII/media info.
*/
sc->rge_mii.mii_ifp = ifp;
sc->rge_mii.mii_readreg = rge_mii_read;
sc->rge_mii.mii_writereg = (mii_writereg_t) rge_mii_write;
sc->rge_mii.mii_statchg = rmi_xlr_mac_mii_statchg;
ifmedia_init(&sc->rge_mii.mii_media, 0, rmi_xlr_mac_mediachange,
rmi_xlr_mac_mediastatus);
ifmedia_add(&sc->rge_mii.mii_media, IFM_ETHER | IFM_AUTO, 0, NULL);
ifmedia_set(&sc->rge_mii.mii_media, IFM_ETHER | IFM_AUTO);
sc->rge_mii.mii_media.ifm_media = sc->rge_mii.mii_media.ifm_cur->ifm_media;
/*
* Call MI attach routine.
*/
ether_ifattach(ifp, sc->dev_addr);
if (priv->type == XLR_GMAC) {
rmi_xlr_gmac_init(priv);
} else if (priv->type == XLR_XGMAC) {
rmi_xlr_xgmac_init(priv);
}
dbg_msg("rge_%d: Phoenix Mac at 0x%p (mtu=%d)\n",
sc->unit, priv->mmio, sc->mtu);
dev_mac[sc->unit] = sc;
if (priv->type == XLR_XGMAC && priv->instance == 0)
dev_mac_xgs0 = sc->unit;
if (priv->type == XLR_GMAC && priv->instance == 0)
dev_mac_gmac0 = sc->unit;
if (!gmac_common_init_done) {
mac_common_init();
gmac_common_init_done = 1;
- callout_init(&xlr_tx_stop_bkp, CALLOUT_MPSAFE);
+ callout_init(&xlr_tx_stop_bkp, 1);
callout_reset(&xlr_tx_stop_bkp, hz, xlr_tx_q_wakeup, NULL);
- callout_init(&rge_dbg_count, CALLOUT_MPSAFE);
+ callout_init(&rge_dbg_count, 1);
//callout_reset(&rge_dbg_count, hz, xlr_debug_count, NULL);
}
if ((ret = rmi_xlr_mac_open(sc)) == -1) {
RGE_LOCK_DESTROY(sc);
goto out;
}
out:
if (ret < 0) {
device_printf(dev, "error - skipping\n");
}
return ret;
}
static void
rge_reset(struct rge_softc *sc)
{
}
static int
rge_detach(dev)
device_t dev;
{
#ifdef FREEBSD_MAC_NOT_YET
struct rge_softc *sc;
struct ifnet *ifp;
sc = device_get_softc(dev);
ifp = sc->rge_ifp;
RGE_LOCK(sc);
rge_stop(sc);
rge_reset(sc);
RGE_UNLOCK(sc);
ether_ifdetach(ifp);
if (sc->rge_tbi) {
ifmedia_removeall(&sc->rge_ifmedia);
} else {
bus_generic_detach(dev);
device_delete_child(dev, sc->rge_miibus);
}
rge_release_resources(sc);
#endif /* FREEBSD_MAC_NOT_YET */
return (0);
}
static int
rge_suspend(device_t dev)
{
struct rge_softc *sc;
sc = device_get_softc(dev);
RGE_LOCK(sc);
rge_stop(sc);
RGE_UNLOCK(sc);
return 0;
}
static int
rge_resume(device_t dev)
{
panic("rge_resume(): unimplemented\n");
return 0;
}
static void
rge_release_resources(struct rge_softc *sc)
{
if (sc->rge_ifp != NULL)
if_free(sc->rge_ifp);
if (mtx_initialized(&sc->rge_mtx)) /* XXX */
RGE_LOCK_DESTROY(sc);
}
uint32_t gmac_rx_fail[32];
uint32_t gmac_rx_pass[32];
static void
rge_rx(struct rge_softc *sc, vm_paddr_t paddr, int len)
{
struct mbuf *m;
struct ifnet *ifp = sc->rge_ifp;
uint64_t mag;
uint32_t sr;
/*
* On 32 bit machines we use XKPHYS to get the values stores with
* the mbuf, need to explicitly enable KX. Disable interrupts while
* KX is enabled to prevent this setting leaking to other code.
*/
sr = xlr_enable_kx();
m = (struct mbuf *)(intptr_t)xlr_paddr_ld(paddr - XLR_CACHELINE_SIZE);
mag = xlr_paddr_ld(paddr - XLR_CACHELINE_SIZE + sizeof(uint64_t));
xlr_restore_kx(sr);
if (mag != 0xf00bad) {
/* somebody else packet Error - FIXME in intialization */
printf("cpu %d: *ERROR* Not my packet paddr %p\n",
xlr_cpu_id(), (void *)paddr);
return;
}
/* align the data */
m->m_data += BYTE_OFFSET;
m->m_pkthdr.len = m->m_len = len;
m->m_pkthdr.rcvif = ifp;
#ifdef DUMP_PACKETS
{
int i = 0;
unsigned char *buf = (char *)m->m_data;
printf("Rx Packet: length=%d\n", len);
for (i = 0; i < 64; i++) {
if (i && (i % 16) == 0)
printf("\n");
printf("%02x ", buf[i]);
}
printf("\n");
}
#endif
if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
(*ifp->if_input) (ifp, m);
}
static void
rge_intr(void *arg)
{
struct rge_softc *sc = (struct rge_softc *)arg;
struct driver_data *priv = &(sc->priv);
xlr_reg_t *mmio = priv->mmio;
uint32_t intreg = xlr_read_reg(mmio, R_INTREG);
if (intreg & (1 << O_INTREG__MDInt)) {
uint32_t phy_int_status = 0;
int i = 0;
for (i = 0; i < XLR_MAX_MACS; i++) {
struct rge_softc *phy_dev = 0;
struct driver_data *phy_priv = 0;
phy_dev = dev_mac[i];
if (phy_dev == NULL)
continue;
phy_priv = &phy_dev->priv;
if (phy_priv->type == XLR_XGMAC)
continue;
phy_int_status = rge_mii_read_internal(phy_priv->mii_mmio,
phy_priv->phy_addr, 26);
printf("rge%d: Phy addr %d, MII MMIO %lx status %x\n", phy_priv->instance,
(int)phy_priv->phy_addr, (u_long)phy_priv->mii_mmio, phy_int_status);
rmi_xlr_gmac_config_speed(phy_priv);
}
} else {
printf("[%s]: mac type = %d, instance %d error "
"interrupt: INTREG = 0x%08x\n",
__FUNCTION__, priv->type, priv->instance, intreg);
}
/* clear all interrupts and hope to make progress */
xlr_write_reg(mmio, R_INTREG, 0xffffffff);
/* (not yet) on A0 and B0, xgmac interrupts are routed only to xgs_1 irq */
if ((xlr_revision() < 2) && (priv->type == XLR_XGMAC)) {
struct rge_softc *xgs0_dev = dev_mac[dev_mac_xgs0];
struct driver_data *xgs0_priv = &xgs0_dev->priv;
xlr_reg_t *xgs0_mmio = xgs0_priv->mmio;
uint32_t xgs0_intreg = xlr_read_reg(xgs0_mmio, R_INTREG);
if (xgs0_intreg) {
printf("[%s]: mac type = %d, instance %d error "
"interrupt: INTREG = 0x%08x\n",
__FUNCTION__, xgs0_priv->type, xgs0_priv->instance, xgs0_intreg);
xlr_write_reg(xgs0_mmio, R_INTREG, 0xffffffff);
}
}
}
static void
rge_start_locked(struct ifnet *ifp, int threshold)
{
struct rge_softc *sc = ifp->if_softc;
struct mbuf *m = NULL;
int prepend_pkt = 0;
int i = 0;
struct p2d_tx_desc *tx_desc = NULL;
int cpu = xlr_core_id();
uint32_t vcpu = xlr_cpu_id();
if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
return;
for (i = 0; i < xlr_tot_avail_p2d[cpu]; i++) {
if (IFQ_DRV_IS_EMPTY(&ifp->if_snd))
return;
tx_desc = get_p2d_desc();
if (!tx_desc) {
xlr_rge_get_p2d_failed[vcpu]++;
return;
}
/* Grab a packet off the queue. */
IFQ_DEQUEUE(&ifp->if_snd, m);
if (m == NULL) {
free_p2d_desc(tx_desc);
return;
}
prepend_pkt = rmi_xlr_mac_xmit(m, sc, 0, tx_desc);
if (prepend_pkt) {
xlr_rge_tx_prepend[vcpu]++;
IF_PREPEND(&ifp->if_snd, m);
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
return;
} else {
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
xlr_rge_tx_done[vcpu]++;
}
}
}
static void
rge_start(struct ifnet *ifp)
{
rge_start_locked(ifp, RGE_TX_Q_SIZE);
}
static int
rge_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
{
struct rge_softc *sc = ifp->if_softc;
struct ifreq *ifr = (struct ifreq *)data;
int mask, error = 0;
/* struct mii_data *mii; */
switch (command) {
case SIOCSIFMTU:
ifp->if_mtu = ifr->ifr_mtu;
error = rmi_xlr_mac_change_mtu(sc, ifr->ifr_mtu);
break;
case SIOCSIFFLAGS:
RGE_LOCK(sc);
if (ifp->if_flags & IFF_UP) {
/*
* If only the state of the PROMISC flag changed,
* then just use the 'set promisc mode' command
* instead of reinitializing the entire NIC. Doing a
* full re-init means reloading the firmware and
* waiting for it to start up, which may take a
* second or two. Similarly for ALLMULTI.
*/
if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
ifp->if_flags & IFF_PROMISC &&
!(sc->flags & IFF_PROMISC)) {
sc->flags |= IFF_PROMISC;
xlr_mac_set_rx_mode(sc);
} else if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
!(ifp->if_flags & IFF_PROMISC) &&
sc->flags & IFF_PROMISC) {
sc->flags &= IFF_PROMISC;
xlr_mac_set_rx_mode(sc);
} else if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
(ifp->if_flags ^ sc->flags) & IFF_ALLMULTI) {
rmi_xlr_mac_set_multicast_list(sc);
} else
xlr_mac_set_rx_mode(sc);
} else {
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
xlr_mac_set_rx_mode(sc);
}
}
sc->flags = ifp->if_flags;
RGE_UNLOCK(sc);
error = 0;
break;
case SIOCADDMULTI:
case SIOCDELMULTI:
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
RGE_LOCK(sc);
rmi_xlr_mac_set_multicast_list(sc);
RGE_UNLOCK(sc);
error = 0;
}
break;
case SIOCSIFMEDIA:
case SIOCGIFMEDIA:
error = ifmedia_ioctl(ifp, ifr,
&sc->rge_mii.mii_media, command);
break;
case SIOCSIFCAP:
mask = ifr->ifr_reqcap ^ ifp->if_capenable;
ifp->if_hwassist = 0;
break;
default:
error = ether_ioctl(ifp, command, data);
break;
}
return (error);
}
static void
rge_init(void *addr)
{
struct rge_softc *sc = (struct rge_softc *)addr;
struct ifnet *ifp;
struct driver_data *priv = &(sc->priv);
ifp = sc->rge_ifp;
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
return;
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
rmi_xlr_mac_set_enable(priv, 1);
}
static void
rge_stop(struct rge_softc *sc)
{
rmi_xlr_mac_close(sc);
}
static int
rge_shutdown(device_t dev)
{
struct rge_softc *sc;
sc = device_get_softc(dev);
RGE_LOCK(sc);
rge_stop(sc);
rge_reset(sc);
RGE_UNLOCK(sc);
return (0);
}
static int
rmi_xlr_mac_open(struct rge_softc *sc)
{
struct driver_data *priv = &(sc->priv);
int i;
dbg_msg("IN\n");
if (rmi_xlr_mac_fill_rxfr(sc)) {
return -1;
}
mtx_lock_spin(&priv->lock);
xlr_mac_set_rx_mode(sc);
if (sc->unit == xlr_board_info.gmacports - 1) {
printf("Enabling MDIO interrupts\n");
struct rge_softc *tmp = NULL;
for (i = 0; i < xlr_board_info.gmacports; i++) {
tmp = dev_mac[i];
if (tmp)
xlr_write_reg(tmp->priv.mmio, R_INTMASK,
((tmp->priv.instance == 0) << O_INTMASK__MDInt));
}
}
/*
* Configure the speed, duplex, and flow control
*/
rmi_xlr_mac_set_speed(priv, priv->speed);
rmi_xlr_mac_set_duplex(priv, priv->duplex, priv->flow_ctrl);
rmi_xlr_mac_set_enable(priv, 0);
mtx_unlock_spin(&priv->lock);
for (i = 0; i < 8; i++) {
priv->frin_to_be_sent[i] = 0;
}
return 0;
}
/**********************************************************************
**********************************************************************/
static int
rmi_xlr_mac_close(struct rge_softc *sc)
{
struct driver_data *priv = &(sc->priv);
mtx_lock_spin(&priv->lock);
/*
* There may have left over mbufs in the ring as well as in free in
* they will be reused next time open is called
*/
rmi_xlr_mac_set_enable(priv, 0);
xlr_inc_counter(NETIF_STOP_Q);
port_inc_counter(priv->instance, PORT_STOPQ);
mtx_unlock_spin(&priv->lock);
return 0;
}
/**********************************************************************
**********************************************************************/
static struct rge_softc_stats *
rmi_xlr_mac_get_stats(struct rge_softc *sc)
{
struct driver_data *priv = &(sc->priv);
/* unsigned long flags; */
mtx_lock_spin(&priv->lock);
/* XXX update other stats here */
mtx_unlock_spin(&priv->lock);
return &priv->stats;
}
/**********************************************************************
**********************************************************************/
static void
rmi_xlr_mac_set_multicast_list(struct rge_softc *sc)
{
}
/**********************************************************************
**********************************************************************/
static int
rmi_xlr_mac_change_mtu(struct rge_softc *sc, int new_mtu)
{
struct driver_data *priv = &(sc->priv);
if ((new_mtu > 9500) || (new_mtu < 64)) {
return -EINVAL;
}
mtx_lock_spin(&priv->lock);
sc->mtu = new_mtu;
/* Disable MAC TX/RX */
rmi_xlr_mac_set_enable(priv, 0);
/* Flush RX FR IN */
/* Flush TX IN */
rmi_xlr_mac_set_enable(priv, 1);
mtx_unlock_spin(&priv->lock);
return 0;
}
/**********************************************************************
**********************************************************************/
static int
rmi_xlr_mac_fill_rxfr(struct rge_softc *sc)
{
struct driver_data *priv = &(sc->priv);
int i;
int ret = 0;
void *ptr;
dbg_msg("\n");
if (!priv->init_frin_desc)
return ret;
priv->init_frin_desc = 0;
dbg_msg("\n");
for (i = 0; i < MAX_NUM_DESC; i++) {
ptr = get_buf();
if (!ptr) {
ret = -ENOMEM;
break;
}
/* Send the free Rx desc to the MAC */
xlr_mac_send_fr(priv, vtophys(ptr), MAX_FRAME_SIZE);
}
return ret;
}
/**********************************************************************
**********************************************************************/
static __inline__ void *
rmi_xlr_config_spill(xlr_reg_t * mmio,
int reg_start_0, int reg_start_1,
int reg_size, int size)
{
uint32_t spill_size = size;
void *spill = NULL;
uint64_t phys_addr = 0;
spill = contigmalloc((spill_size + XLR_CACHELINE_SIZE), M_DEVBUF,
M_NOWAIT | M_ZERO, 0, 0xffffffff, XLR_CACHELINE_SIZE, 0);
if (!spill || ((vm_offset_t)spill & (XLR_CACHELINE_SIZE - 1))) {
panic("Unable to allocate memory for spill area!\n");
}
phys_addr = vtophys(spill);
dbg_msg("Allocate spill %d bytes at %jx\n", size, (uintmax_t)phys_addr);
xlr_write_reg(mmio, reg_start_0, (phys_addr >> 5) & 0xffffffff);
xlr_write_reg(mmio, reg_start_1, (phys_addr >> 37) & 0x07);
xlr_write_reg(mmio, reg_size, spill_size);
return spill;
}
static void
rmi_xlr_config_spill_area(struct driver_data *priv)
{
/*
* if driver initialization is done parallely on multiple cpus
* spill_configured needs synchronization
*/
if (priv->spill_configured)
return;
if (priv->type == XLR_GMAC && priv->instance % 4 != 0) {
priv->spill_configured = 1;
return;
}
priv->spill_configured = 1;
priv->frin_spill =
rmi_xlr_config_spill(priv->mmio,
R_REG_FRIN_SPILL_MEM_START_0,
R_REG_FRIN_SPILL_MEM_START_1,
R_REG_FRIN_SPILL_MEM_SIZE,
MAX_FRIN_SPILL *
sizeof(struct fr_desc));
priv->class_0_spill =
rmi_xlr_config_spill(priv->mmio,
R_CLASS0_SPILL_MEM_START_0,
R_CLASS0_SPILL_MEM_START_1,
R_CLASS0_SPILL_MEM_SIZE,
MAX_CLASS_0_SPILL *
sizeof(union rx_tx_desc));
priv->class_1_spill =
rmi_xlr_config_spill(priv->mmio,
R_CLASS1_SPILL_MEM_START_0,
R_CLASS1_SPILL_MEM_START_1,
R_CLASS1_SPILL_MEM_SIZE,
MAX_CLASS_1_SPILL *
sizeof(union rx_tx_desc));
priv->frout_spill =
rmi_xlr_config_spill(priv->mmio, R_FROUT_SPILL_MEM_START_0,
R_FROUT_SPILL_MEM_START_1,
R_FROUT_SPILL_MEM_SIZE,
MAX_FROUT_SPILL *
sizeof(struct fr_desc));
priv->class_2_spill =
rmi_xlr_config_spill(priv->mmio,
R_CLASS2_SPILL_MEM_START_0,
R_CLASS2_SPILL_MEM_START_1,
R_CLASS2_SPILL_MEM_SIZE,
MAX_CLASS_2_SPILL *
sizeof(union rx_tx_desc));
priv->class_3_spill =
rmi_xlr_config_spill(priv->mmio,
R_CLASS3_SPILL_MEM_START_0,
R_CLASS3_SPILL_MEM_START_1,
R_CLASS3_SPILL_MEM_SIZE,
MAX_CLASS_3_SPILL *
sizeof(union rx_tx_desc));
priv->spill_configured = 1;
}
/*****************************************************************
* Write the MAC address to the XLR registers
* All 4 addresses are the same for now
*****************************************************************/
static void
xlr_mac_setup_hwaddr(struct driver_data *priv)
{
struct rge_softc *sc = priv->sc;
xlr_write_reg(priv->mmio, R_MAC_ADDR0,
((sc->dev_addr[5] << 24) | (sc->dev_addr[4] << 16)
| (sc->dev_addr[3] << 8) | (sc->dev_addr[2]))
);
xlr_write_reg(priv->mmio, R_MAC_ADDR0 + 1,
((sc->dev_addr[1] << 24) | (sc->
dev_addr[0] << 16)));
xlr_write_reg(priv->mmio, R_MAC_ADDR_MASK2, 0xffffffff);
xlr_write_reg(priv->mmio, R_MAC_ADDR_MASK2 + 1, 0xffffffff);
xlr_write_reg(priv->mmio, R_MAC_ADDR_MASK3, 0xffffffff);
xlr_write_reg(priv->mmio, R_MAC_ADDR_MASK3 + 1, 0xffffffff);
xlr_write_reg(priv->mmio, R_MAC_FILTER_CONFIG,
(1 << O_MAC_FILTER_CONFIG__BROADCAST_EN) |
(1 << O_MAC_FILTER_CONFIG__ALL_MCAST_EN) |
(1 << O_MAC_FILTER_CONFIG__MAC_ADDR0_VALID)
);
}
/*****************************************************************
* Read the MAC address from the XLR registers
* All 4 addresses are the same for now
*****************************************************************/
static void
xlr_mac_get_hwaddr(struct rge_softc *sc)
{
struct driver_data *priv = &(sc->priv);
sc->dev_addr[0] = (xlr_boot1_info.mac_addr >> 40) & 0xff;
sc->dev_addr[1] = (xlr_boot1_info.mac_addr >> 32) & 0xff;
sc->dev_addr[2] = (xlr_boot1_info.mac_addr >> 24) & 0xff;
sc->dev_addr[3] = (xlr_boot1_info.mac_addr >> 16) & 0xff;
sc->dev_addr[4] = (xlr_boot1_info.mac_addr >> 8) & 0xff;
sc->dev_addr[5] = ((xlr_boot1_info.mac_addr >> 0) & 0xff) + priv->instance;
}
/*****************************************************************
* Mac Module Initialization
*****************************************************************/
static void
mac_common_init(void)
{
init_p2d_allocation();
init_tx_ring();
if (xlr_board_info.is_xls) {
if (register_msgring_handler(MSGRNG_STNID_GMAC,
MSGRNG_STNID_GMAC + 1, rmi_xlr_mac_msgring_handler,
NULL)) {
panic("Couldn't register msgring handler\n");
}
if (register_msgring_handler(MSGRNG_STNID_GMAC1,
MSGRNG_STNID_GMAC1 + 1, rmi_xlr_mac_msgring_handler,
NULL)) {
panic("Couldn't register msgring handler\n");
}
} else {
if (register_msgring_handler(MSGRNG_STNID_GMAC,
MSGRNG_STNID_GMAC + 1, rmi_xlr_mac_msgring_handler,
NULL)) {
panic("Couldn't register msgring handler\n");
}
}
/*
* Not yet if (xlr_board_atx_ii()) { if (register_msgring_handler
* (TX_STN_XGS_0, rmi_xlr_mac_msgring_handler, NULL)) {
* panic("Couldn't register msgring handler for TX_STN_XGS_0\n"); }
* if (register_msgring_handler (TX_STN_XGS_1,
* rmi_xlr_mac_msgring_handler, NULL)) { panic("Couldn't register
* msgring handler for TX_STN_XGS_1\n"); } }
*/
}
Index: head/sys/net/if_spppsubr.c
===================================================================
--- head/sys/net/if_spppsubr.c (revision 283290)
+++ head/sys/net/if_spppsubr.c (revision 283291)
@@ -1,5421 +1,5421 @@
/*
* Synchronous PPP/Cisco/Frame Relay link level subroutines.
* Keepalive protocol implemented in both Cisco and PPP modes.
*/
/*-
* Copyright (C) 1994-2000 Cronyx Engineering.
* Author: Serge Vakulenko, <vak@cronyx.ru>
*
* Heavily revamped to conform to RFC 1661.
* Copyright (C) 1997, 2001 Joerg Wunsch.
*
* This software is distributed with NO WARRANTIES, not even the implied
* warranties for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* Authors grant any other persons or organisations permission to use
* or modify this software as long as this message is kept with the software,
* all derivative works or modified versions.
*
* From: Version 2.4, Thu Apr 30 17:17:21 MSD 1997
*
* $FreeBSD$
*/
#include <sys/param.h>
#include "opt_inet.h"
#include "opt_inet6.h"
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/sockio.h>
#include <sys/socket.h>
#include <sys/syslog.h>
#include <sys/random.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/md5.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/netisr.h>
#include <net/if_types.h>
#include <net/route.h>
#include <net/vnet.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <net/slcompress.h>
#include <machine/stdarg.h>
#include <netinet/in_var.h>
#ifdef INET
#include <netinet/ip.h>
#include <netinet/tcp.h>
#endif
#ifdef INET6
#include <netinet6/scope6_var.h>
#endif
#include <netinet/if_ether.h>
#include <net/if_sppp.h>
#define IOCTL_CMD_T u_long
#define MAXALIVECNT 3 /* max. alive packets */
/*
* Interface flags that can be set in an ifconfig command.
*
* Setting link0 will make the link passive, i.e. it will be marked
* as being administrative openable, but won't be opened to begin
* with. Incoming calls will be answered, or subsequent calls with
* -link1 will cause the administrative open of the LCP layer.
*
* Setting link1 will cause the link to auto-dial only as packets
* arrive to be sent.
*
* Setting IFF_DEBUG will syslog the option negotiation and state
* transitions at level kern.debug. Note: all logs consistently look
* like
*
* <if-name><unit>: <proto-name> <additional info...>
*
* with <if-name><unit> being something like "bppp0", and <proto-name>
* being one of "lcp", "ipcp", "cisco", "chap", "pap", etc.
*/
#define IFF_PASSIVE IFF_LINK0 /* wait passively for connection */
#define IFF_AUTO IFF_LINK1 /* auto-dial on output */
#define IFF_CISCO IFF_LINK2 /* auto-dial on output */
#define PPP_ALLSTATIONS 0xff /* All-Stations broadcast address */
#define PPP_UI 0x03 /* Unnumbered Information */
#define PPP_IP 0x0021 /* Internet Protocol */
#define PPP_ISO 0x0023 /* ISO OSI Protocol */
#define PPP_XNS 0x0025 /* Xerox NS Protocol */
#define PPP_IPX 0x002b /* Novell IPX Protocol */
#define PPP_VJ_COMP 0x002d /* VJ compressed TCP/IP */
#define PPP_VJ_UCOMP 0x002f /* VJ uncompressed TCP/IP */
#define PPP_IPV6 0x0057 /* Internet Protocol Version 6 */
#define PPP_LCP 0xc021 /* Link Control Protocol */
#define PPP_PAP 0xc023 /* Password Authentication Protocol */
#define PPP_CHAP 0xc223 /* Challenge-Handshake Auth Protocol */
#define PPP_IPCP 0x8021 /* Internet Protocol Control Protocol */
#define PPP_IPV6CP 0x8057 /* IPv6 Control Protocol */
#define CONF_REQ 1 /* PPP configure request */
#define CONF_ACK 2 /* PPP configure acknowledge */
#define CONF_NAK 3 /* PPP configure negative ack */
#define CONF_REJ 4 /* PPP configure reject */
#define TERM_REQ 5 /* PPP terminate request */
#define TERM_ACK 6 /* PPP terminate acknowledge */
#define CODE_REJ 7 /* PPP code reject */
#define PROTO_REJ 8 /* PPP protocol reject */
#define ECHO_REQ 9 /* PPP echo request */
#define ECHO_REPLY 10 /* PPP echo reply */
#define DISC_REQ 11 /* PPP discard request */
#define LCP_OPT_MRU 1 /* maximum receive unit */
#define LCP_OPT_ASYNC_MAP 2 /* async control character map */
#define LCP_OPT_AUTH_PROTO 3 /* authentication protocol */
#define LCP_OPT_QUAL_PROTO 4 /* quality protocol */
#define LCP_OPT_MAGIC 5 /* magic number */
#define LCP_OPT_RESERVED 6 /* reserved */
#define LCP_OPT_PROTO_COMP 7 /* protocol field compression */
#define LCP_OPT_ADDR_COMP 8 /* address/control field compression */
#define IPCP_OPT_ADDRESSES 1 /* both IP addresses; deprecated */
#define IPCP_OPT_COMPRESSION 2 /* IP compression protocol (VJ) */
#define IPCP_OPT_ADDRESS 3 /* local IP address */
#define IPV6CP_OPT_IFID 1 /* interface identifier */
#define IPV6CP_OPT_COMPRESSION 2 /* IPv6 compression protocol */
#define IPCP_COMP_VJ 0x2d /* Code for VJ compression */
#define PAP_REQ 1 /* PAP name/password request */
#define PAP_ACK 2 /* PAP acknowledge */
#define PAP_NAK 3 /* PAP fail */
#define CHAP_CHALLENGE 1 /* CHAP challenge request */
#define CHAP_RESPONSE 2 /* CHAP challenge response */
#define CHAP_SUCCESS 3 /* CHAP response ok */
#define CHAP_FAILURE 4 /* CHAP response failed */
#define CHAP_MD5 5 /* hash algorithm - MD5 */
#define CISCO_MULTICAST 0x8f /* Cisco multicast address */
#define CISCO_UNICAST 0x0f /* Cisco unicast address */
#define CISCO_KEEPALIVE 0x8035 /* Cisco keepalive protocol */
#define CISCO_ADDR_REQ 0 /* Cisco address request */
#define CISCO_ADDR_REPLY 1 /* Cisco address reply */
#define CISCO_KEEPALIVE_REQ 2 /* Cisco keepalive request */
/* states are named and numbered according to RFC 1661 */
#define STATE_INITIAL 0
#define STATE_STARTING 1
#define STATE_CLOSED 2
#define STATE_STOPPED 3
#define STATE_CLOSING 4
#define STATE_STOPPING 5
#define STATE_REQ_SENT 6
#define STATE_ACK_RCVD 7
#define STATE_ACK_SENT 8
#define STATE_OPENED 9
static MALLOC_DEFINE(M_SPPP, "sppp", "synchronous PPP interface internals");
struct ppp_header {
u_char address;
u_char control;
u_short protocol;
} __packed;
#define PPP_HEADER_LEN sizeof (struct ppp_header)
struct lcp_header {
u_char type;
u_char ident;
u_short len;
} __packed;
#define LCP_HEADER_LEN sizeof (struct lcp_header)
struct cisco_packet {
u_long type;
u_long par1;
u_long par2;
u_short rel;
u_short time0;
u_short time1;
} __packed;
#define CISCO_PACKET_LEN sizeof (struct cisco_packet)
/*
* We follow the spelling and capitalization of RFC 1661 here, to make
* it easier comparing with the standard. Please refer to this RFC in
* case you can't make sense out of these abbreviation; it will also
* explain the semantics related to the various events and actions.
*/
struct cp {
u_short proto; /* PPP control protocol number */
u_char protoidx; /* index into state table in struct sppp */
u_char flags;
#define CP_LCP 0x01 /* this is the LCP */
#define CP_AUTH 0x02 /* this is an authentication protocol */
#define CP_NCP 0x04 /* this is a NCP */
#define CP_QUAL 0x08 /* this is a quality reporting protocol */
const char *name; /* name of this control protocol */
/* event handlers */
void (*Up)(struct sppp *sp);
void (*Down)(struct sppp *sp);
void (*Open)(struct sppp *sp);
void (*Close)(struct sppp *sp);
void (*TO)(void *sp);
int (*RCR)(struct sppp *sp, struct lcp_header *h, int len);
void (*RCN_rej)(struct sppp *sp, struct lcp_header *h, int len);
void (*RCN_nak)(struct sppp *sp, struct lcp_header *h, int len);
/* actions */
void (*tlu)(struct sppp *sp);
void (*tld)(struct sppp *sp);
void (*tls)(struct sppp *sp);
void (*tlf)(struct sppp *sp);
void (*scr)(struct sppp *sp);
};
#define SPP_FMT "%s: "
#define SPP_ARGS(ifp) (ifp)->if_xname
#define SPPP_LOCK(sp) mtx_lock (&(sp)->mtx)
#define SPPP_UNLOCK(sp) mtx_unlock (&(sp)->mtx)
#define SPPP_LOCK_ASSERT(sp) mtx_assert (&(sp)->mtx, MA_OWNED)
#define SPPP_LOCK_OWNED(sp) mtx_owned (&(sp)->mtx)
#ifdef INET
/*
* The following disgusting hack gets around the problem that IP TOS
* can't be set yet. We want to put "interactive" traffic on a high
* priority queue. To decide if traffic is interactive, we check that
* a) it is TCP and b) one of its ports is telnet, rlogin or ftp control.
*
* XXX is this really still necessary? - joerg -
*/
static const u_short interactive_ports[8] = {
0, 513, 0, 0,
0, 21, 0, 23,
};
#define INTERACTIVE(p) (interactive_ports[(p) & 7] == (p))
#endif
/* almost every function needs these */
#define STDDCL \
struct ifnet *ifp = SP2IFP(sp); \
int debug = ifp->if_flags & IFF_DEBUG
static int sppp_output(struct ifnet *ifp, struct mbuf *m,
const struct sockaddr *dst, struct route *ro);
static void sppp_cisco_send(struct sppp *sp, int type, long par1, long par2);
static void sppp_cisco_input(struct sppp *sp, struct mbuf *m);
static void sppp_cp_input(const struct cp *cp, struct sppp *sp,
struct mbuf *m);
static void sppp_cp_send(struct sppp *sp, u_short proto, u_char type,
u_char ident, u_short len, void *data);
/* static void sppp_cp_timeout(void *arg); */
static void sppp_cp_change_state(const struct cp *cp, struct sppp *sp,
int newstate);
static void sppp_auth_send(const struct cp *cp,
struct sppp *sp, unsigned int type, unsigned int id,
...);
static void sppp_up_event(const struct cp *cp, struct sppp *sp);
static void sppp_down_event(const struct cp *cp, struct sppp *sp);
static void sppp_open_event(const struct cp *cp, struct sppp *sp);
static void sppp_close_event(const struct cp *cp, struct sppp *sp);
static void sppp_to_event(const struct cp *cp, struct sppp *sp);
static void sppp_null(struct sppp *sp);
static void sppp_pp_up(struct sppp *sp);
static void sppp_pp_down(struct sppp *sp);
static void sppp_lcp_init(struct sppp *sp);
static void sppp_lcp_up(struct sppp *sp);
static void sppp_lcp_down(struct sppp *sp);
static void sppp_lcp_open(struct sppp *sp);
static void sppp_lcp_close(struct sppp *sp);
static void sppp_lcp_TO(void *sp);
static int sppp_lcp_RCR(struct sppp *sp, struct lcp_header *h, int len);
static void sppp_lcp_RCN_rej(struct sppp *sp, struct lcp_header *h, int len);
static void sppp_lcp_RCN_nak(struct sppp *sp, struct lcp_header *h, int len);
static void sppp_lcp_tlu(struct sppp *sp);
static void sppp_lcp_tld(struct sppp *sp);
static void sppp_lcp_tls(struct sppp *sp);
static void sppp_lcp_tlf(struct sppp *sp);
static void sppp_lcp_scr(struct sppp *sp);
static void sppp_lcp_check_and_close(struct sppp *sp);
static int sppp_ncp_check(struct sppp *sp);
static void sppp_ipcp_init(struct sppp *sp);
static void sppp_ipcp_up(struct sppp *sp);
static void sppp_ipcp_down(struct sppp *sp);
static void sppp_ipcp_open(struct sppp *sp);
static void sppp_ipcp_close(struct sppp *sp);
static void sppp_ipcp_TO(void *sp);
static int sppp_ipcp_RCR(struct sppp *sp, struct lcp_header *h, int len);
static void sppp_ipcp_RCN_rej(struct sppp *sp, struct lcp_header *h, int len);
static void sppp_ipcp_RCN_nak(struct sppp *sp, struct lcp_header *h, int len);
static void sppp_ipcp_tlu(struct sppp *sp);
static void sppp_ipcp_tld(struct sppp *sp);
static void sppp_ipcp_tls(struct sppp *sp);
static void sppp_ipcp_tlf(struct sppp *sp);
static void sppp_ipcp_scr(struct sppp *sp);
static void sppp_ipv6cp_init(struct sppp *sp);
static void sppp_ipv6cp_up(struct sppp *sp);
static void sppp_ipv6cp_down(struct sppp *sp);
static void sppp_ipv6cp_open(struct sppp *sp);
static void sppp_ipv6cp_close(struct sppp *sp);
static void sppp_ipv6cp_TO(void *sp);
static int sppp_ipv6cp_RCR(struct sppp *sp, struct lcp_header *h, int len);
static void sppp_ipv6cp_RCN_rej(struct sppp *sp, struct lcp_header *h, int len);
static void sppp_ipv6cp_RCN_nak(struct sppp *sp, struct lcp_header *h, int len);
static void sppp_ipv6cp_tlu(struct sppp *sp);
static void sppp_ipv6cp_tld(struct sppp *sp);
static void sppp_ipv6cp_tls(struct sppp *sp);
static void sppp_ipv6cp_tlf(struct sppp *sp);
static void sppp_ipv6cp_scr(struct sppp *sp);
static void sppp_pap_input(struct sppp *sp, struct mbuf *m);
static void sppp_pap_init(struct sppp *sp);
static void sppp_pap_open(struct sppp *sp);
static void sppp_pap_close(struct sppp *sp);
static void sppp_pap_TO(void *sp);
static void sppp_pap_my_TO(void *sp);
static void sppp_pap_tlu(struct sppp *sp);
static void sppp_pap_tld(struct sppp *sp);
static void sppp_pap_scr(struct sppp *sp);
static void sppp_chap_input(struct sppp *sp, struct mbuf *m);
static void sppp_chap_init(struct sppp *sp);
static void sppp_chap_open(struct sppp *sp);
static void sppp_chap_close(struct sppp *sp);
static void sppp_chap_TO(void *sp);
static void sppp_chap_tlu(struct sppp *sp);
static void sppp_chap_tld(struct sppp *sp);
static void sppp_chap_scr(struct sppp *sp);
static const char *sppp_auth_type_name(u_short proto, u_char type);
static const char *sppp_cp_type_name(u_char type);
#ifdef INET
static const char *sppp_dotted_quad(u_long addr);
static const char *sppp_ipcp_opt_name(u_char opt);
#endif
#ifdef INET6
static const char *sppp_ipv6cp_opt_name(u_char opt);
#endif
static const char *sppp_lcp_opt_name(u_char opt);
static const char *sppp_phase_name(enum ppp_phase phase);
static const char *sppp_proto_name(u_short proto);
static const char *sppp_state_name(int state);
static int sppp_params(struct sppp *sp, u_long cmd, void *data);
static int sppp_strnlen(u_char *p, int max);
static void sppp_keepalive(void *dummy);
static void sppp_phase_network(struct sppp *sp);
static void sppp_print_bytes(const u_char *p, u_short len);
static void sppp_print_string(const char *p, u_short len);
static void sppp_qflush(struct ifqueue *ifq);
#ifdef INET
static void sppp_set_ip_addr(struct sppp *sp, u_long src);
#endif
#ifdef INET6
static void sppp_get_ip6_addrs(struct sppp *sp, struct in6_addr *src,
struct in6_addr *dst, struct in6_addr *srcmask);
#ifdef IPV6CP_MYIFID_DYN
static void sppp_set_ip6_addr(struct sppp *sp, const struct in6_addr *src);
static void sppp_gen_ip6_addr(struct sppp *sp, const struct in6_addr *src);
#endif
static void sppp_suggest_ip6_addr(struct sppp *sp, struct in6_addr *src);
#endif
/* if_start () wrapper */
static void sppp_ifstart (struct ifnet *ifp);
/* our control protocol descriptors */
static const struct cp lcp = {
PPP_LCP, IDX_LCP, CP_LCP, "lcp",
sppp_lcp_up, sppp_lcp_down, sppp_lcp_open, sppp_lcp_close,
sppp_lcp_TO, sppp_lcp_RCR, sppp_lcp_RCN_rej, sppp_lcp_RCN_nak,
sppp_lcp_tlu, sppp_lcp_tld, sppp_lcp_tls, sppp_lcp_tlf,
sppp_lcp_scr
};
static const struct cp ipcp = {
PPP_IPCP, IDX_IPCP,
#ifdef INET /* don't run IPCP if there's no IPv4 support */
CP_NCP,
#else
0,
#endif
"ipcp",
sppp_ipcp_up, sppp_ipcp_down, sppp_ipcp_open, sppp_ipcp_close,
sppp_ipcp_TO, sppp_ipcp_RCR, sppp_ipcp_RCN_rej, sppp_ipcp_RCN_nak,
sppp_ipcp_tlu, sppp_ipcp_tld, sppp_ipcp_tls, sppp_ipcp_tlf,
sppp_ipcp_scr
};
static const struct cp ipv6cp = {
PPP_IPV6CP, IDX_IPV6CP,
#ifdef INET6 /*don't run IPv6CP if there's no IPv6 support*/
CP_NCP,
#else
0,
#endif
"ipv6cp",
sppp_ipv6cp_up, sppp_ipv6cp_down, sppp_ipv6cp_open, sppp_ipv6cp_close,
sppp_ipv6cp_TO, sppp_ipv6cp_RCR, sppp_ipv6cp_RCN_rej, sppp_ipv6cp_RCN_nak,
sppp_ipv6cp_tlu, sppp_ipv6cp_tld, sppp_ipv6cp_tls, sppp_ipv6cp_tlf,
sppp_ipv6cp_scr
};
static const struct cp pap = {
PPP_PAP, IDX_PAP, CP_AUTH, "pap",
sppp_null, sppp_null, sppp_pap_open, sppp_pap_close,
sppp_pap_TO, 0, 0, 0,
sppp_pap_tlu, sppp_pap_tld, sppp_null, sppp_null,
sppp_pap_scr
};
static const struct cp chap = {
PPP_CHAP, IDX_CHAP, CP_AUTH, "chap",
sppp_null, sppp_null, sppp_chap_open, sppp_chap_close,
sppp_chap_TO, 0, 0, 0,
sppp_chap_tlu, sppp_chap_tld, sppp_null, sppp_null,
sppp_chap_scr
};
static const struct cp *cps[IDX_COUNT] = {
&lcp, /* IDX_LCP */
&ipcp, /* IDX_IPCP */
&ipv6cp, /* IDX_IPV6CP */
&pap, /* IDX_PAP */
&chap, /* IDX_CHAP */
};
static void*
sppp_alloc(u_char type, struct ifnet *ifp)
{
struct sppp *sp;
sp = malloc(sizeof(struct sppp), M_SPPP, M_WAITOK | M_ZERO);
sp->pp_ifp = ifp;
return (sp);
}
static void
sppp_free(void *com, u_char type)
{
free(com, M_SPPP);
}
static int
sppp_modevent(module_t mod, int type, void *unused)
{
switch (type) {
case MOD_LOAD:
/*
* XXX: should probably be IFT_SPPP, but it's fairly
* harmless to allocate struct sppp's for non-sppp
* interfaces.
*/
if_register_com_alloc(IFT_PPP, sppp_alloc, sppp_free);
break;
case MOD_UNLOAD:
/* if_deregister_com_alloc(IFT_PPP); */
return EACCES;
default:
return EOPNOTSUPP;
}
return 0;
}
static moduledata_t spppmod = {
"sppp",
sppp_modevent,
0
};
MODULE_VERSION(sppp, 1);
DECLARE_MODULE(sppp, spppmod, SI_SUB_DRIVERS, SI_ORDER_ANY);
/*
* Exported functions, comprising our interface to the lower layer.
*/
/*
* Process the received packet.
*/
void
sppp_input(struct ifnet *ifp, struct mbuf *m)
{
struct ppp_header *h;
int isr = -1;
struct sppp *sp = IFP2SP(ifp);
int debug, do_account = 0;
#ifdef INET
int hlen, vjlen;
u_char *iphdr;
#endif
SPPP_LOCK(sp);
debug = ifp->if_flags & IFF_DEBUG;
if (ifp->if_flags & IFF_UP)
/* Count received bytes, add FCS and one flag */
if_inc_counter(ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len + 3);
if (m->m_pkthdr.len <= PPP_HEADER_LEN) {
/* Too small packet, drop it. */
if (debug)
log(LOG_DEBUG,
SPP_FMT "input packet is too small, %d bytes\n",
SPP_ARGS(ifp), m->m_pkthdr.len);
drop:
m_freem (m);
SPPP_UNLOCK(sp);
drop2:
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
return;
}
if (sp->pp_mode == PP_FR) {
sppp_fr_input (sp, m);
SPPP_UNLOCK(sp);
return;
}
/* Get PPP header. */
h = mtod (m, struct ppp_header*);
m_adj (m, PPP_HEADER_LEN);
switch (h->address) {
case PPP_ALLSTATIONS:
if (h->control != PPP_UI)
goto invalid;
if (sp->pp_mode == IFF_CISCO) {
if (debug)
log(LOG_DEBUG,
SPP_FMT "PPP packet in Cisco mode "
"<addr=0x%x ctrl=0x%x proto=0x%x>\n",
SPP_ARGS(ifp),
h->address, h->control, ntohs(h->protocol));
goto drop;
}
switch (ntohs (h->protocol)) {
default:
if (debug)
log(LOG_DEBUG,
SPP_FMT "rejecting protocol "
"<addr=0x%x ctrl=0x%x proto=0x%x>\n",
SPP_ARGS(ifp),
h->address, h->control, ntohs(h->protocol));
if (sp->state[IDX_LCP] == STATE_OPENED)
sppp_cp_send (sp, PPP_LCP, PROTO_REJ,
++sp->pp_seq[IDX_LCP], m->m_pkthdr.len + 2,
&h->protocol);
if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1);
goto drop;
case PPP_LCP:
sppp_cp_input(&lcp, sp, m);
m_freem (m);
SPPP_UNLOCK(sp);
return;
case PPP_PAP:
if (sp->pp_phase >= PHASE_AUTHENTICATE)
sppp_pap_input(sp, m);
m_freem (m);
SPPP_UNLOCK(sp);
return;
case PPP_CHAP:
if (sp->pp_phase >= PHASE_AUTHENTICATE)
sppp_chap_input(sp, m);
m_freem (m);
SPPP_UNLOCK(sp);
return;
#ifdef INET
case PPP_IPCP:
if (sp->pp_phase == PHASE_NETWORK)
sppp_cp_input(&ipcp, sp, m);
m_freem (m);
SPPP_UNLOCK(sp);
return;
case PPP_IP:
if (sp->state[IDX_IPCP] == STATE_OPENED) {
isr = NETISR_IP;
}
do_account++;
break;
case PPP_VJ_COMP:
if (sp->state[IDX_IPCP] == STATE_OPENED) {
if ((vjlen =
sl_uncompress_tcp_core(mtod(m, u_char *),
m->m_len, m->m_len,
TYPE_COMPRESSED_TCP,
sp->pp_comp,
&iphdr, &hlen)) <= 0) {
if (debug)
log(LOG_INFO,
SPP_FMT "VJ uncompress failed on compressed packet\n",
SPP_ARGS(ifp));
goto drop;
}
/*
* Trim the VJ header off the packet, and prepend
* the uncompressed IP header (which will usually
* end up in two chained mbufs since there's not
* enough leading space in the existing mbuf).
*/
m_adj(m, vjlen);
M_PREPEND(m, hlen, M_NOWAIT);
if (m == NULL) {
SPPP_UNLOCK(sp);
goto drop2;
}
bcopy(iphdr, mtod(m, u_char *), hlen);
isr = NETISR_IP;
}
do_account++;
break;
case PPP_VJ_UCOMP:
if (sp->state[IDX_IPCP] == STATE_OPENED) {
if (sl_uncompress_tcp_core(mtod(m, u_char *),
m->m_len, m->m_len,
TYPE_UNCOMPRESSED_TCP,
sp->pp_comp,
&iphdr, &hlen) != 0) {
if (debug)
log(LOG_INFO,
SPP_FMT "VJ uncompress failed on uncompressed packet\n",
SPP_ARGS(ifp));
goto drop;
}
isr = NETISR_IP;
}
do_account++;
break;
#endif
#ifdef INET6
case PPP_IPV6CP:
if (sp->pp_phase == PHASE_NETWORK)
sppp_cp_input(&ipv6cp, sp, m);
m_freem (m);
SPPP_UNLOCK(sp);
return;
case PPP_IPV6:
if (sp->state[IDX_IPV6CP] == STATE_OPENED)
isr = NETISR_IPV6;
do_account++;
break;
#endif
}
break;
case CISCO_MULTICAST:
case CISCO_UNICAST:
/* Don't check the control field here (RFC 1547). */
if (sp->pp_mode != IFF_CISCO) {
if (debug)
log(LOG_DEBUG,
SPP_FMT "Cisco packet in PPP mode "
"<addr=0x%x ctrl=0x%x proto=0x%x>\n",
SPP_ARGS(ifp),
h->address, h->control, ntohs(h->protocol));
goto drop;
}
switch (ntohs (h->protocol)) {
default:
if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1);
goto invalid;
case CISCO_KEEPALIVE:
sppp_cisco_input (sp, m);
m_freem (m);
SPPP_UNLOCK(sp);
return;
#ifdef INET
case ETHERTYPE_IP:
isr = NETISR_IP;
do_account++;
break;
#endif
#ifdef INET6
case ETHERTYPE_IPV6:
isr = NETISR_IPV6;
do_account++;
break;
#endif
}
break;
default: /* Invalid PPP packet. */
invalid:
if (debug)
log(LOG_DEBUG,
SPP_FMT "invalid input packet "
"<addr=0x%x ctrl=0x%x proto=0x%x>\n",
SPP_ARGS(ifp),
h->address, h->control, ntohs(h->protocol));
goto drop;
}
if (! (ifp->if_flags & IFF_UP) || isr == -1)
goto drop;
SPPP_UNLOCK(sp);
M_SETFIB(m, ifp->if_fib);
/* Check queue. */
if (netisr_queue(isr, m)) { /* (0) on success. */
if (debug)
log(LOG_DEBUG, SPP_FMT "protocol queue overflow\n",
SPP_ARGS(ifp));
goto drop2;
}
if (do_account)
/*
* Do only account for network packets, not for control
* packets. This is used by some subsystems to detect
* idle lines.
*/
sp->pp_last_recv = time_uptime;
}
static void
sppp_ifstart_sched(void *dummy)
{
struct sppp *sp = dummy;
sp->if_start(SP2IFP(sp));
}
/* if_start () wrapper function. We use it to schedule real if_start () for
* execution. We can't call it directly
*/
static void
sppp_ifstart(struct ifnet *ifp)
{
struct sppp *sp = IFP2SP(ifp);
if (SPPP_LOCK_OWNED(sp)) {
if (callout_pending(&sp->ifstart_callout))
return;
callout_reset(&sp->ifstart_callout, 1, sppp_ifstart_sched,
(void *)sp);
} else {
sp->if_start(ifp);
}
}
/*
* Enqueue transmit packet.
*/
static int
sppp_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
struct route *ro)
{
struct sppp *sp = IFP2SP(ifp);
struct ppp_header *h;
struct ifqueue *ifq = NULL;
int error, rv = 0;
#ifdef INET
int ipproto = PPP_IP;
#endif
int debug = ifp->if_flags & IFF_DEBUG;
SPPP_LOCK(sp);
if (!(ifp->if_flags & IFF_UP) ||
(!(ifp->if_flags & IFF_AUTO) &&
!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
#ifdef INET6
drop:
#endif
m_freem (m);
SPPP_UNLOCK(sp);
return (ENETDOWN);
}
if ((ifp->if_flags & IFF_AUTO) &&
!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
#ifdef INET6
/*
* XXX
*
* Hack to prevent the initialization-time generated
* IPv6 multicast packet to erroneously cause a
* dialout event in case IPv6 has been
* administratively disabled on that interface.
*/
if (dst->sa_family == AF_INET6 &&
!(sp->confflags & CONF_ENABLE_IPV6))
goto drop;
#endif
/*
* Interface is not yet running, but auto-dial. Need
* to start LCP for it.
*/
ifp->if_drv_flags |= IFF_DRV_RUNNING;
lcp.Open(sp);
}
#ifdef INET
if (dst->sa_family == AF_INET) {
/* XXX Check mbuf length here? */
struct ip *ip = mtod (m, struct ip*);
struct tcphdr *tcp = (struct tcphdr*) ((long*)ip + ip->ip_hl);
/*
* When using dynamic local IP address assignment by using
* 0.0.0.0 as a local address, the first TCP session will
* not connect because the local TCP checksum is computed
* using 0.0.0.0 which will later become our real IP address
* so the TCP checksum computed at the remote end will
* become invalid. So we
* - don't let packets with src ip addr 0 thru
* - we flag TCP packets with src ip 0 as an error
*/
if(ip->ip_src.s_addr == INADDR_ANY) /* -hm */
{
m_freem(m);
SPPP_UNLOCK(sp);
if(ip->ip_p == IPPROTO_TCP)
return(EADDRNOTAVAIL);
else
return(0);
}
/*
* Put low delay, telnet, rlogin and ftp control packets
* in front of the queue or let ALTQ take care.
*/
if (ALTQ_IS_ENABLED(&ifp->if_snd))
;
else if (_IF_QFULL(&sp->pp_fastq))
;
else if (ip->ip_tos & IPTOS_LOWDELAY)
ifq = &sp->pp_fastq;
else if (m->m_len < sizeof *ip + sizeof *tcp)
;
else if (ip->ip_p != IPPROTO_TCP)
;
else if (INTERACTIVE (ntohs (tcp->th_sport)))
ifq = &sp->pp_fastq;
else if (INTERACTIVE (ntohs (tcp->th_dport)))
ifq = &sp->pp_fastq;
/*
* Do IP Header compression
*/
if (sp->pp_mode != IFF_CISCO && sp->pp_mode != PP_FR &&
(sp->ipcp.flags & IPCP_VJ) && ip->ip_p == IPPROTO_TCP)
switch (sl_compress_tcp(m, ip, sp->pp_comp,
sp->ipcp.compress_cid)) {
case TYPE_COMPRESSED_TCP:
ipproto = PPP_VJ_COMP;
break;
case TYPE_UNCOMPRESSED_TCP:
ipproto = PPP_VJ_UCOMP;
break;
case TYPE_IP:
ipproto = PPP_IP;
break;
default:
m_freem(m);
SPPP_UNLOCK(sp);
return (EINVAL);
}
}
#endif
#ifdef INET6
if (dst->sa_family == AF_INET6) {
/* XXX do something tricky here? */
}
#endif
if (sp->pp_mode == PP_FR) {
/* Add frame relay header. */
m = sppp_fr_header (sp, m, dst->sa_family);
if (! m)
goto nobufs;
goto out;
}
/*
* Prepend general data packet PPP header. For now, IP only.
*/
M_PREPEND (m, PPP_HEADER_LEN, M_NOWAIT);
if (! m) {
nobufs: if (debug)
log(LOG_DEBUG, SPP_FMT "no memory for transmit header\n",
SPP_ARGS(ifp));
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
SPPP_UNLOCK(sp);
return (ENOBUFS);
}
/*
* May want to check size of packet
* (albeit due to the implementation it's always enough)
*/
h = mtod (m, struct ppp_header*);
if (sp->pp_mode == IFF_CISCO) {
h->address = CISCO_UNICAST; /* unicast address */
h->control = 0;
} else {
h->address = PPP_ALLSTATIONS; /* broadcast address */
h->control = PPP_UI; /* Unnumbered Info */
}
switch (dst->sa_family) {
#ifdef INET
case AF_INET: /* Internet Protocol */
if (sp->pp_mode == IFF_CISCO)
h->protocol = htons (ETHERTYPE_IP);
else {
/*
* Don't choke with an ENETDOWN early. It's
* possible that we just started dialing out,
* so don't drop the packet immediately. If
* we notice that we run out of buffer space
* below, we will however remember that we are
* not ready to carry IP packets, and return
* ENETDOWN, as opposed to ENOBUFS.
*/
h->protocol = htons(ipproto);
if (sp->state[IDX_IPCP] != STATE_OPENED)
rv = ENETDOWN;
}
break;
#endif
#ifdef INET6
case AF_INET6: /* Internet Protocol */
if (sp->pp_mode == IFF_CISCO)
h->protocol = htons (ETHERTYPE_IPV6);
else {
/*
* Don't choke with an ENETDOWN early. It's
* possible that we just started dialing out,
* so don't drop the packet immediately. If
* we notice that we run out of buffer space
* below, we will however remember that we are
* not ready to carry IP packets, and return
* ENETDOWN, as opposed to ENOBUFS.
*/
h->protocol = htons(PPP_IPV6);
if (sp->state[IDX_IPV6CP] != STATE_OPENED)
rv = ENETDOWN;
}
break;
#endif
default:
m_freem (m);
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
SPPP_UNLOCK(sp);
return (EAFNOSUPPORT);
}
/*
* Queue message on interface, and start output if interface
* not yet active.
*/
out:
if (ifq != NULL)
error = !(IF_HANDOFF_ADJ(ifq, m, ifp, 3));
else
IFQ_HANDOFF_ADJ(ifp, m, 3, error);
if (error) {
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
SPPP_UNLOCK(sp);
return (rv? rv: ENOBUFS);
}
SPPP_UNLOCK(sp);
/*
* Unlike in sppp_input(), we can always bump the timestamp
* here since sppp_output() is only called on behalf of
* network-layer traffic; control-layer traffic is handled
* by sppp_cp_send().
*/
sp->pp_last_sent = time_uptime;
return (0);
}
void
sppp_attach(struct ifnet *ifp)
{
struct sppp *sp = IFP2SP(ifp);
/* Initialize mtx lock */
mtx_init(&sp->mtx, "sppp", MTX_NETWORK_LOCK, MTX_DEF | MTX_RECURSE);
/* Initialize keepalive handler. */
- callout_init(&sp->keepalive_callout, CALLOUT_MPSAFE);
+ callout_init(&sp->keepalive_callout, 1);
callout_reset(&sp->keepalive_callout, hz * 10, sppp_keepalive,
(void *)sp);
ifp->if_mtu = PP_MTU;
ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
ifp->if_output = sppp_output;
#if 0
sp->pp_flags = PP_KEEPALIVE;
#endif
ifp->if_snd.ifq_maxlen = 32;
sp->pp_fastq.ifq_maxlen = 32;
sp->pp_cpq.ifq_maxlen = 20;
sp->pp_loopcnt = 0;
sp->pp_alivecnt = 0;
bzero(&sp->pp_seq[0], sizeof(sp->pp_seq));
bzero(&sp->pp_rseq[0], sizeof(sp->pp_rseq));
sp->pp_phase = PHASE_DEAD;
sp->pp_up = sppp_pp_up;
sp->pp_down = sppp_pp_down;
if(!mtx_initialized(&sp->pp_cpq.ifq_mtx))
mtx_init(&sp->pp_cpq.ifq_mtx, "sppp_cpq", NULL, MTX_DEF);
if(!mtx_initialized(&sp->pp_fastq.ifq_mtx))
mtx_init(&sp->pp_fastq.ifq_mtx, "sppp_fastq", NULL, MTX_DEF);
sp->pp_last_recv = sp->pp_last_sent = time_uptime;
sp->confflags = 0;
#ifdef INET
sp->confflags |= CONF_ENABLE_VJ;
#endif
#ifdef INET6
sp->confflags |= CONF_ENABLE_IPV6;
#endif
- callout_init(&sp->ifstart_callout, CALLOUT_MPSAFE);
+ callout_init(&sp->ifstart_callout, 1);
sp->if_start = ifp->if_start;
ifp->if_start = sppp_ifstart;
sp->pp_comp = malloc(sizeof(struct slcompress), M_TEMP, M_WAITOK);
sl_compress_init(sp->pp_comp, -1);
sppp_lcp_init(sp);
sppp_ipcp_init(sp);
sppp_ipv6cp_init(sp);
sppp_pap_init(sp);
sppp_chap_init(sp);
}
void
sppp_detach(struct ifnet *ifp)
{
struct sppp *sp = IFP2SP(ifp);
int i;
KASSERT(mtx_initialized(&sp->mtx), ("sppp mutex is not initialized"));
/* Stop keepalive handler. */
if (!callout_drain(&sp->keepalive_callout))
callout_stop(&sp->keepalive_callout);
for (i = 0; i < IDX_COUNT; i++) {
if (!callout_drain(&sp->ch[i]))
callout_stop(&sp->ch[i]);
}
if (!callout_drain(&sp->pap_my_to_ch))
callout_stop(&sp->pap_my_to_ch);
mtx_destroy(&sp->pp_cpq.ifq_mtx);
mtx_destroy(&sp->pp_fastq.ifq_mtx);
mtx_destroy(&sp->mtx);
}
/*
* Flush the interface output queue.
*/
static void
sppp_flush_unlocked(struct ifnet *ifp)
{
struct sppp *sp = IFP2SP(ifp);
sppp_qflush ((struct ifqueue *)&SP2IFP(sp)->if_snd);
sppp_qflush (&sp->pp_fastq);
sppp_qflush (&sp->pp_cpq);
}
void
sppp_flush(struct ifnet *ifp)
{
struct sppp *sp = IFP2SP(ifp);
SPPP_LOCK(sp);
sppp_flush_unlocked (ifp);
SPPP_UNLOCK(sp);
}
/*
* Check if the output queue is empty.
*/
int
sppp_isempty(struct ifnet *ifp)
{
struct sppp *sp = IFP2SP(ifp);
int empty;
SPPP_LOCK(sp);
empty = !sp->pp_fastq.ifq_head && !sp->pp_cpq.ifq_head &&
!SP2IFP(sp)->if_snd.ifq_head;
SPPP_UNLOCK(sp);
return (empty);
}
/*
* Get next packet to send.
*/
struct mbuf *
sppp_dequeue(struct ifnet *ifp)
{
struct sppp *sp = IFP2SP(ifp);
struct mbuf *m;
SPPP_LOCK(sp);
/*
* Process only the control protocol queue until we have at
* least one NCP open.
*
* Do always serve all three queues in Cisco mode.
*/
IF_DEQUEUE(&sp->pp_cpq, m);
if (m == NULL &&
(sppp_ncp_check(sp) || sp->pp_mode == IFF_CISCO ||
sp->pp_mode == PP_FR)) {
IF_DEQUEUE(&sp->pp_fastq, m);
if (m == NULL)
IF_DEQUEUE (&SP2IFP(sp)->if_snd, m);
}
SPPP_UNLOCK(sp);
return m;
}
/*
* Pick the next packet, do not remove it from the queue.
*/
struct mbuf *
sppp_pick(struct ifnet *ifp)
{
struct sppp *sp = IFP2SP(ifp);
struct mbuf *m;
SPPP_LOCK(sp);
m = sp->pp_cpq.ifq_head;
if (m == NULL &&
(sp->pp_phase == PHASE_NETWORK ||
sp->pp_mode == IFF_CISCO ||
sp->pp_mode == PP_FR))
if ((m = sp->pp_fastq.ifq_head) == NULL)
m = SP2IFP(sp)->if_snd.ifq_head;
SPPP_UNLOCK(sp);
return (m);
}
/*
* Process an ioctl request. Called on low priority level.
*/
int
sppp_ioctl(struct ifnet *ifp, IOCTL_CMD_T cmd, void *data)
{
struct ifreq *ifr = (struct ifreq*) data;
struct sppp *sp = IFP2SP(ifp);
int rv, going_up, going_down, newmode;
SPPP_LOCK(sp);
rv = 0;
switch (cmd) {
case SIOCAIFADDR:
break;
case SIOCSIFADDR:
/* set the interface "up" when assigning an IP address */
ifp->if_flags |= IFF_UP;
/* FALLTHROUGH */
case SIOCSIFFLAGS:
going_up = ifp->if_flags & IFF_UP &&
(ifp->if_drv_flags & IFF_DRV_RUNNING) == 0;
going_down = (ifp->if_flags & IFF_UP) == 0 &&
ifp->if_drv_flags & IFF_DRV_RUNNING;
newmode = ifp->if_flags & IFF_PASSIVE;
if (!newmode)
newmode = ifp->if_flags & IFF_AUTO;
if (!newmode)
newmode = ifp->if_flags & IFF_CISCO;
ifp->if_flags &= ~(IFF_PASSIVE | IFF_AUTO | IFF_CISCO);
ifp->if_flags |= newmode;
if (!newmode)
newmode = sp->pp_flags & PP_FR;
if (newmode != sp->pp_mode) {
going_down = 1;
if (!going_up)
going_up = ifp->if_drv_flags & IFF_DRV_RUNNING;
}
if (going_down) {
if (sp->pp_mode != IFF_CISCO &&
sp->pp_mode != PP_FR)
lcp.Close(sp);
else if (sp->pp_tlf)
(sp->pp_tlf)(sp);
sppp_flush_unlocked(ifp);
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
sp->pp_mode = newmode;
}
if (going_up) {
if (sp->pp_mode != IFF_CISCO &&
sp->pp_mode != PP_FR)
lcp.Close(sp);
sp->pp_mode = newmode;
if (sp->pp_mode == 0) {
ifp->if_drv_flags |= IFF_DRV_RUNNING;
lcp.Open(sp);
}
if ((sp->pp_mode == IFF_CISCO) ||
(sp->pp_mode == PP_FR)) {
if (sp->pp_tls)
(sp->pp_tls)(sp);
ifp->if_drv_flags |= IFF_DRV_RUNNING;
}
}
break;
#ifdef SIOCSIFMTU
#ifndef ifr_mtu
#define ifr_mtu ifr_metric
#endif
case SIOCSIFMTU:
if (ifr->ifr_mtu < 128 || ifr->ifr_mtu > sp->lcp.their_mru)
return (EINVAL);
ifp->if_mtu = ifr->ifr_mtu;
break;
#endif
#ifdef SLIOCSETMTU
case SLIOCSETMTU:
if (*(short*)data < 128 || *(short*)data > sp->lcp.their_mru)
return (EINVAL);
ifp->if_mtu = *(short*)data;
break;
#endif
#ifdef SIOCGIFMTU
case SIOCGIFMTU:
ifr->ifr_mtu = ifp->if_mtu;
break;
#endif
#ifdef SLIOCGETMTU
case SLIOCGETMTU:
*(short*)data = ifp->if_mtu;
break;
#endif
case SIOCADDMULTI:
case SIOCDELMULTI:
break;
case SIOCGIFGENERIC:
case SIOCSIFGENERIC:
rv = sppp_params(sp, cmd, data);
break;
default:
rv = ENOTTY;
}
SPPP_UNLOCK(sp);
return rv;
}
/*
* Cisco framing implementation.
*/
/*
* Handle incoming Cisco keepalive protocol packets.
*/
static void
sppp_cisco_input(struct sppp *sp, struct mbuf *m)
{
STDDCL;
struct cisco_packet *h;
u_long me, mymask;
if (m->m_pkthdr.len < CISCO_PACKET_LEN) {
if (debug)
log(LOG_DEBUG,
SPP_FMT "cisco invalid packet length: %d bytes\n",
SPP_ARGS(ifp), m->m_pkthdr.len);
return;
}
h = mtod (m, struct cisco_packet*);
if (debug)
log(LOG_DEBUG,
SPP_FMT "cisco input: %d bytes "
"<0x%lx 0x%lx 0x%lx 0x%x 0x%x-0x%x>\n",
SPP_ARGS(ifp), m->m_pkthdr.len,
(u_long)ntohl (h->type), (u_long)h->par1, (u_long)h->par2, (u_int)h->rel,
(u_int)h->time0, (u_int)h->time1);
switch (ntohl (h->type)) {
default:
if (debug)
log(-1, SPP_FMT "cisco unknown packet type: 0x%lx\n",
SPP_ARGS(ifp), (u_long)ntohl (h->type));
break;
case CISCO_ADDR_REPLY:
/* Reply on address request, ignore */
break;
case CISCO_KEEPALIVE_REQ:
sp->pp_alivecnt = 0;
sp->pp_rseq[IDX_LCP] = ntohl (h->par1);
if (sp->pp_seq[IDX_LCP] == sp->pp_rseq[IDX_LCP]) {
/* Local and remote sequence numbers are equal.
* Probably, the line is in loopback mode. */
if (sp->pp_loopcnt >= MAXALIVECNT) {
printf (SPP_FMT "loopback\n",
SPP_ARGS(ifp));
sp->pp_loopcnt = 0;
if (ifp->if_flags & IFF_UP) {
if_down (ifp);
sppp_qflush (&sp->pp_cpq);
}
}
++sp->pp_loopcnt;
/* Generate new local sequence number */
sp->pp_seq[IDX_LCP] = random();
break;
}
sp->pp_loopcnt = 0;
if (! (ifp->if_flags & IFF_UP) &&
(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
if_up(ifp);
printf (SPP_FMT "up\n", SPP_ARGS(ifp));
}
break;
case CISCO_ADDR_REQ:
sppp_get_ip_addrs(sp, &me, 0, &mymask);
if (me != 0L)
sppp_cisco_send(sp, CISCO_ADDR_REPLY, me, mymask);
break;
}
}
/*
* Send Cisco keepalive packet.
*/
static void
sppp_cisco_send(struct sppp *sp, int type, long par1, long par2)
{
STDDCL;
struct ppp_header *h;
struct cisco_packet *ch;
struct mbuf *m;
struct timeval tv;
getmicrouptime(&tv);
MGETHDR (m, M_NOWAIT, MT_DATA);
if (! m)
return;
m->m_pkthdr.len = m->m_len = PPP_HEADER_LEN + CISCO_PACKET_LEN;
m->m_pkthdr.rcvif = 0;
h = mtod (m, struct ppp_header*);
h->address = CISCO_MULTICAST;
h->control = 0;
h->protocol = htons (CISCO_KEEPALIVE);
ch = (struct cisco_packet*) (h + 1);
ch->type = htonl (type);
ch->par1 = htonl (par1);
ch->par2 = htonl (par2);
ch->rel = -1;
ch->time0 = htons ((u_short) (tv.tv_sec >> 16));
ch->time1 = htons ((u_short) tv.tv_sec);
if (debug)
log(LOG_DEBUG,
SPP_FMT "cisco output: <0x%lx 0x%lx 0x%lx 0x%x 0x%x-0x%x>\n",
SPP_ARGS(ifp), (u_long)ntohl (ch->type), (u_long)ch->par1,
(u_long)ch->par2, (u_int)ch->rel, (u_int)ch->time0, (u_int)ch->time1);
if (! IF_HANDOFF_ADJ(&sp->pp_cpq, m, ifp, 3))
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
}
/*
* PPP protocol implementation.
*/
/*
* Send PPP control protocol packet.
*/
static void
sppp_cp_send(struct sppp *sp, u_short proto, u_char type,
u_char ident, u_short len, void *data)
{
STDDCL;
struct ppp_header *h;
struct lcp_header *lh;
struct mbuf *m;
if (len > MHLEN - PPP_HEADER_LEN - LCP_HEADER_LEN)
len = MHLEN - PPP_HEADER_LEN - LCP_HEADER_LEN;
MGETHDR (m, M_NOWAIT, MT_DATA);
if (! m)
return;
m->m_pkthdr.len = m->m_len = PPP_HEADER_LEN + LCP_HEADER_LEN + len;
m->m_pkthdr.rcvif = 0;
h = mtod (m, struct ppp_header*);
h->address = PPP_ALLSTATIONS; /* broadcast address */
h->control = PPP_UI; /* Unnumbered Info */
h->protocol = htons (proto); /* Link Control Protocol */
lh = (struct lcp_header*) (h + 1);
lh->type = type;
lh->ident = ident;
lh->len = htons (LCP_HEADER_LEN + len);
if (len)
bcopy (data, lh+1, len);
if (debug) {
log(LOG_DEBUG, SPP_FMT "%s output <%s id=0x%x len=%d",
SPP_ARGS(ifp),
sppp_proto_name(proto),
sppp_cp_type_name (lh->type), lh->ident,
ntohs (lh->len));
sppp_print_bytes ((u_char*) (lh+1), len);
log(-1, ">\n");
}
if (! IF_HANDOFF_ADJ(&sp->pp_cpq, m, ifp, 3))
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
}
/*
* Handle incoming PPP control protocol packets.
*/
static void
sppp_cp_input(const struct cp *cp, struct sppp *sp, struct mbuf *m)
{
STDDCL;
struct lcp_header *h;
int len = m->m_pkthdr.len;
int rv;
u_char *p;
if (len < 4) {
if (debug)
log(LOG_DEBUG,
SPP_FMT "%s invalid packet length: %d bytes\n",
SPP_ARGS(ifp), cp->name, len);
return;
}
h = mtod (m, struct lcp_header*);
if (debug) {
log(LOG_DEBUG,
SPP_FMT "%s input(%s): <%s id=0x%x len=%d",
SPP_ARGS(ifp), cp->name,
sppp_state_name(sp->state[cp->protoidx]),
sppp_cp_type_name (h->type), h->ident, ntohs (h->len));
sppp_print_bytes ((u_char*) (h+1), len-4);
log(-1, ">\n");
}
if (len > ntohs (h->len))
len = ntohs (h->len);
p = (u_char *)(h + 1);
switch (h->type) {
case CONF_REQ:
if (len < 4) {
if (debug)
log(-1, SPP_FMT "%s invalid conf-req length %d\n",
SPP_ARGS(ifp), cp->name,
len);
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
break;
}
/* handle states where RCR doesn't get a SCA/SCN */
switch (sp->state[cp->protoidx]) {
case STATE_CLOSING:
case STATE_STOPPING:
return;
case STATE_CLOSED:
sppp_cp_send(sp, cp->proto, TERM_ACK, h->ident,
0, 0);
return;
}
rv = (cp->RCR)(sp, h, len);
switch (sp->state[cp->protoidx]) {
case STATE_OPENED:
(cp->tld)(sp);
(cp->scr)(sp);
/* FALLTHROUGH */
case STATE_ACK_SENT:
case STATE_REQ_SENT:
/*
* sppp_cp_change_state() have the side effect of
* restarting the timeouts. We want to avoid that
* if the state don't change, otherwise we won't
* ever timeout and resend a configuration request
* that got lost.
*/
if (sp->state[cp->protoidx] == (rv ? STATE_ACK_SENT:
STATE_REQ_SENT))
break;
sppp_cp_change_state(cp, sp, rv?
STATE_ACK_SENT: STATE_REQ_SENT);
break;
case STATE_STOPPED:
sp->rst_counter[cp->protoidx] = sp->lcp.max_configure;
(cp->scr)(sp);
sppp_cp_change_state(cp, sp, rv?
STATE_ACK_SENT: STATE_REQ_SENT);
break;
case STATE_ACK_RCVD:
if (rv) {
sppp_cp_change_state(cp, sp, STATE_OPENED);
if (debug)
log(LOG_DEBUG, SPP_FMT "%s tlu\n",
SPP_ARGS(ifp),
cp->name);
(cp->tlu)(sp);
} else
sppp_cp_change_state(cp, sp, STATE_ACK_RCVD);
break;
default:
printf(SPP_FMT "%s illegal %s in state %s\n",
SPP_ARGS(ifp), cp->name,
sppp_cp_type_name(h->type),
sppp_state_name(sp->state[cp->protoidx]));
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
}
break;
case CONF_ACK:
if (h->ident != sp->confid[cp->protoidx]) {
if (debug)
log(-1, SPP_FMT "%s id mismatch 0x%x != 0x%x\n",
SPP_ARGS(ifp), cp->name,
h->ident, sp->confid[cp->protoidx]);
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
break;
}
switch (sp->state[cp->protoidx]) {
case STATE_CLOSED:
case STATE_STOPPED:
sppp_cp_send(sp, cp->proto, TERM_ACK, h->ident, 0, 0);
break;
case STATE_CLOSING:
case STATE_STOPPING:
break;
case STATE_REQ_SENT:
sp->rst_counter[cp->protoidx] = sp->lcp.max_configure;
sppp_cp_change_state(cp, sp, STATE_ACK_RCVD);
break;
case STATE_OPENED:
(cp->tld)(sp);
/* FALLTHROUGH */
case STATE_ACK_RCVD:
(cp->scr)(sp);
sppp_cp_change_state(cp, sp, STATE_REQ_SENT);
break;
case STATE_ACK_SENT:
sp->rst_counter[cp->protoidx] = sp->lcp.max_configure;
sppp_cp_change_state(cp, sp, STATE_OPENED);
if (debug)
log(LOG_DEBUG, SPP_FMT "%s tlu\n",
SPP_ARGS(ifp), cp->name);
(cp->tlu)(sp);
break;
default:
printf(SPP_FMT "%s illegal %s in state %s\n",
SPP_ARGS(ifp), cp->name,
sppp_cp_type_name(h->type),
sppp_state_name(sp->state[cp->protoidx]));
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
}
break;
case CONF_NAK:
case CONF_REJ:
if (h->ident != sp->confid[cp->protoidx]) {
if (debug)
log(-1, SPP_FMT "%s id mismatch 0x%x != 0x%x\n",
SPP_ARGS(ifp), cp->name,
h->ident, sp->confid[cp->protoidx]);
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
break;
}
if (h->type == CONF_NAK)
(cp->RCN_nak)(sp, h, len);
else /* CONF_REJ */
(cp->RCN_rej)(sp, h, len);
switch (sp->state[cp->protoidx]) {
case STATE_CLOSED:
case STATE_STOPPED:
sppp_cp_send(sp, cp->proto, TERM_ACK, h->ident, 0, 0);
break;
case STATE_REQ_SENT:
case STATE_ACK_SENT:
sp->rst_counter[cp->protoidx] = sp->lcp.max_configure;
/*
* Slow things down a bit if we think we might be
* in loopback. Depend on the timeout to send the
* next configuration request.
*/
if (sp->pp_loopcnt)
break;
(cp->scr)(sp);
break;
case STATE_OPENED:
(cp->tld)(sp);
/* FALLTHROUGH */
case STATE_ACK_RCVD:
sppp_cp_change_state(cp, sp, STATE_REQ_SENT);
(cp->scr)(sp);
break;
case STATE_CLOSING:
case STATE_STOPPING:
break;
default:
printf(SPP_FMT "%s illegal %s in state %s\n",
SPP_ARGS(ifp), cp->name,
sppp_cp_type_name(h->type),
sppp_state_name(sp->state[cp->protoidx]));
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
}
break;
case TERM_REQ:
switch (sp->state[cp->protoidx]) {
case STATE_ACK_RCVD:
case STATE_ACK_SENT:
sppp_cp_change_state(cp, sp, STATE_REQ_SENT);
/* FALLTHROUGH */
case STATE_CLOSED:
case STATE_STOPPED:
case STATE_CLOSING:
case STATE_STOPPING:
case STATE_REQ_SENT:
sta:
/* Send Terminate-Ack packet. */
if (debug)
log(LOG_DEBUG, SPP_FMT "%s send terminate-ack\n",
SPP_ARGS(ifp), cp->name);
sppp_cp_send(sp, cp->proto, TERM_ACK, h->ident, 0, 0);
break;
case STATE_OPENED:
(cp->tld)(sp);
sp->rst_counter[cp->protoidx] = 0;
sppp_cp_change_state(cp, sp, STATE_STOPPING);
goto sta;
break;
default:
printf(SPP_FMT "%s illegal %s in state %s\n",
SPP_ARGS(ifp), cp->name,
sppp_cp_type_name(h->type),
sppp_state_name(sp->state[cp->protoidx]));
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
}
break;
case TERM_ACK:
switch (sp->state[cp->protoidx]) {
case STATE_CLOSED:
case STATE_STOPPED:
case STATE_REQ_SENT:
case STATE_ACK_SENT:
break;
case STATE_CLOSING:
sppp_cp_change_state(cp, sp, STATE_CLOSED);
(cp->tlf)(sp);
break;
case STATE_STOPPING:
sppp_cp_change_state(cp, sp, STATE_STOPPED);
(cp->tlf)(sp);
break;
case STATE_ACK_RCVD:
sppp_cp_change_state(cp, sp, STATE_REQ_SENT);
break;
case STATE_OPENED:
(cp->tld)(sp);
(cp->scr)(sp);
sppp_cp_change_state(cp, sp, STATE_ACK_RCVD);
break;
default:
printf(SPP_FMT "%s illegal %s in state %s\n",
SPP_ARGS(ifp), cp->name,
sppp_cp_type_name(h->type),
sppp_state_name(sp->state[cp->protoidx]));
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
}
break;
case CODE_REJ:
/* XXX catastrophic rejects (RXJ-) aren't handled yet. */
log(LOG_INFO,
SPP_FMT "%s: ignoring RXJ (%s) for proto 0x%x, "
"danger will robinson\n",
SPP_ARGS(ifp), cp->name,
sppp_cp_type_name(h->type), ntohs(*((u_short *)p)));
switch (sp->state[cp->protoidx]) {
case STATE_CLOSED:
case STATE_STOPPED:
case STATE_REQ_SENT:
case STATE_ACK_SENT:
case STATE_CLOSING:
case STATE_STOPPING:
case STATE_OPENED:
break;
case STATE_ACK_RCVD:
sppp_cp_change_state(cp, sp, STATE_REQ_SENT);
break;
default:
printf(SPP_FMT "%s illegal %s in state %s\n",
SPP_ARGS(ifp), cp->name,
sppp_cp_type_name(h->type),
sppp_state_name(sp->state[cp->protoidx]));
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
}
break;
case PROTO_REJ:
{
int catastrophic;
const struct cp *upper;
int i;
u_int16_t proto;
catastrophic = 0;
upper = NULL;
proto = ntohs(*((u_int16_t *)p));
for (i = 0; i < IDX_COUNT; i++) {
if (cps[i]->proto == proto) {
upper = cps[i];
break;
}
}
if (upper == NULL)
catastrophic++;
if (catastrophic || debug)
log(catastrophic? LOG_INFO: LOG_DEBUG,
SPP_FMT "%s: RXJ%c (%s) for proto 0x%x (%s/%s)\n",
SPP_ARGS(ifp), cp->name, catastrophic ? '-' : '+',
sppp_cp_type_name(h->type), proto,
upper ? upper->name : "unknown",
upper ? sppp_state_name(sp->state[upper->protoidx]) : "?");
/*
* if we got RXJ+ against conf-req, the peer does not implement
* this particular protocol type. terminate the protocol.
*/
if (upper && !catastrophic) {
if (sp->state[upper->protoidx] == STATE_REQ_SENT) {
upper->Close(sp);
break;
}
}
/* XXX catastrophic rejects (RXJ-) aren't handled yet. */
switch (sp->state[cp->protoidx]) {
case STATE_CLOSED:
case STATE_STOPPED:
case STATE_REQ_SENT:
case STATE_ACK_SENT:
case STATE_CLOSING:
case STATE_STOPPING:
case STATE_OPENED:
break;
case STATE_ACK_RCVD:
sppp_cp_change_state(cp, sp, STATE_REQ_SENT);
break;
default:
printf(SPP_FMT "%s illegal %s in state %s\n",
SPP_ARGS(ifp), cp->name,
sppp_cp_type_name(h->type),
sppp_state_name(sp->state[cp->protoidx]));
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
}
break;
}
case DISC_REQ:
if (cp->proto != PPP_LCP)
goto illegal;
/* Discard the packet. */
break;
case ECHO_REQ:
if (cp->proto != PPP_LCP)
goto illegal;
if (sp->state[cp->protoidx] != STATE_OPENED) {
if (debug)
log(-1, SPP_FMT "lcp echo req but lcp closed\n",
SPP_ARGS(ifp));
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
break;
}
if (len < 8) {
if (debug)
log(-1, SPP_FMT "invalid lcp echo request "
"packet length: %d bytes\n",
SPP_ARGS(ifp), len);
break;
}
if ((sp->lcp.opts & (1 << LCP_OPT_MAGIC)) &&
ntohl (*(long*)(h+1)) == sp->lcp.magic) {
/* Line loopback mode detected. */
printf(SPP_FMT "loopback\n", SPP_ARGS(ifp));
sp->pp_loopcnt = MAXALIVECNT * 5;
if_down (ifp);
sppp_qflush (&sp->pp_cpq);
/* Shut down the PPP link. */
/* XXX */
lcp.Down(sp);
lcp.Up(sp);
break;
}
*(long*)(h+1) = htonl (sp->lcp.magic);
if (debug)
log(-1, SPP_FMT "got lcp echo req, sending echo rep\n",
SPP_ARGS(ifp));
sppp_cp_send (sp, PPP_LCP, ECHO_REPLY, h->ident, len-4, h+1);
break;
case ECHO_REPLY:
if (cp->proto != PPP_LCP)
goto illegal;
if (h->ident != sp->lcp.echoid) {
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
break;
}
if (len < 8) {
if (debug)
log(-1, SPP_FMT "lcp invalid echo reply "
"packet length: %d bytes\n",
SPP_ARGS(ifp), len);
break;
}
if (debug)
log(-1, SPP_FMT "lcp got echo rep\n",
SPP_ARGS(ifp));
if (!(sp->lcp.opts & (1 << LCP_OPT_MAGIC)) ||
ntohl (*(long*)(h+1)) != sp->lcp.magic)
sp->pp_alivecnt = 0;
break;
default:
/* Unknown packet type -- send Code-Reject packet. */
illegal:
if (debug)
log(-1, SPP_FMT "%s send code-rej for 0x%x\n",
SPP_ARGS(ifp), cp->name, h->type);
sppp_cp_send(sp, cp->proto, CODE_REJ,
++sp->pp_seq[cp->protoidx], m->m_pkthdr.len, h);
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
}
}
/*
* The generic part of all Up/Down/Open/Close/TO event handlers.
* Basically, the state transition handling in the automaton.
*/
static void
sppp_up_event(const struct cp *cp, struct sppp *sp)
{
STDDCL;
if (debug)
log(LOG_DEBUG, SPP_FMT "%s up(%s)\n",
SPP_ARGS(ifp), cp->name,
sppp_state_name(sp->state[cp->protoidx]));
switch (sp->state[cp->protoidx]) {
case STATE_INITIAL:
sppp_cp_change_state(cp, sp, STATE_CLOSED);
break;
case STATE_STARTING:
sp->rst_counter[cp->protoidx] = sp->lcp.max_configure;
(cp->scr)(sp);
sppp_cp_change_state(cp, sp, STATE_REQ_SENT);
break;
default:
printf(SPP_FMT "%s illegal up in state %s\n",
SPP_ARGS(ifp), cp->name,
sppp_state_name(sp->state[cp->protoidx]));
}
}
static void
sppp_down_event(const struct cp *cp, struct sppp *sp)
{
STDDCL;
if (debug)
log(LOG_DEBUG, SPP_FMT "%s down(%s)\n",
SPP_ARGS(ifp), cp->name,
sppp_state_name(sp->state[cp->protoidx]));
switch (sp->state[cp->protoidx]) {
case STATE_CLOSED:
case STATE_CLOSING:
sppp_cp_change_state(cp, sp, STATE_INITIAL);
break;
case STATE_STOPPED:
sppp_cp_change_state(cp, sp, STATE_STARTING);
(cp->tls)(sp);
break;
case STATE_STOPPING:
case STATE_REQ_SENT:
case STATE_ACK_RCVD:
case STATE_ACK_SENT:
sppp_cp_change_state(cp, sp, STATE_STARTING);
break;
case STATE_OPENED:
(cp->tld)(sp);
sppp_cp_change_state(cp, sp, STATE_STARTING);
break;
default:
printf(SPP_FMT "%s illegal down in state %s\n",
SPP_ARGS(ifp), cp->name,
sppp_state_name(sp->state[cp->protoidx]));
}
}
static void
sppp_open_event(const struct cp *cp, struct sppp *sp)
{
STDDCL;
if (debug)
log(LOG_DEBUG, SPP_FMT "%s open(%s)\n",
SPP_ARGS(ifp), cp->name,
sppp_state_name(sp->state[cp->protoidx]));
switch (sp->state[cp->protoidx]) {
case STATE_INITIAL:
sppp_cp_change_state(cp, sp, STATE_STARTING);
(cp->tls)(sp);
break;
case STATE_STARTING:
break;
case STATE_CLOSED:
sp->rst_counter[cp->protoidx] = sp->lcp.max_configure;
(cp->scr)(sp);
sppp_cp_change_state(cp, sp, STATE_REQ_SENT);
break;
case STATE_STOPPED:
/*
* Try escaping stopped state. This seems to bite
* people occasionally, in particular for IPCP,
* presumably following previous IPCP negotiation
* aborts. Somehow, we must have missed a Down event
* which would have caused a transition into starting
* state, so as a bandaid we force the Down event now.
* This effectively implements (something like the)
* `restart' option mentioned in the state transition
* table of RFC 1661.
*/
sppp_cp_change_state(cp, sp, STATE_STARTING);
(cp->tls)(sp);
break;
case STATE_STOPPING:
case STATE_REQ_SENT:
case STATE_ACK_RCVD:
case STATE_ACK_SENT:
case STATE_OPENED:
break;
case STATE_CLOSING:
sppp_cp_change_state(cp, sp, STATE_STOPPING);
break;
}
}
static void
sppp_close_event(const struct cp *cp, struct sppp *sp)
{
STDDCL;
if (debug)
log(LOG_DEBUG, SPP_FMT "%s close(%s)\n",
SPP_ARGS(ifp), cp->name,
sppp_state_name(sp->state[cp->protoidx]));
switch (sp->state[cp->protoidx]) {
case STATE_INITIAL:
case STATE_CLOSED:
case STATE_CLOSING:
break;
case STATE_STARTING:
sppp_cp_change_state(cp, sp, STATE_INITIAL);
(cp->tlf)(sp);
break;
case STATE_STOPPED:
sppp_cp_change_state(cp, sp, STATE_CLOSED);
break;
case STATE_STOPPING:
sppp_cp_change_state(cp, sp, STATE_CLOSING);
break;
case STATE_OPENED:
(cp->tld)(sp);
/* FALLTHROUGH */
case STATE_REQ_SENT:
case STATE_ACK_RCVD:
case STATE_ACK_SENT:
sp->rst_counter[cp->protoidx] = sp->lcp.max_terminate;
sppp_cp_send(sp, cp->proto, TERM_REQ,
++sp->pp_seq[cp->protoidx], 0, 0);
sppp_cp_change_state(cp, sp, STATE_CLOSING);
break;
}
}
static void
sppp_to_event(const struct cp *cp, struct sppp *sp)
{
STDDCL;
SPPP_LOCK(sp);
if (debug)
log(LOG_DEBUG, SPP_FMT "%s TO(%s) rst_counter = %d\n",
SPP_ARGS(ifp), cp->name,
sppp_state_name(sp->state[cp->protoidx]),
sp->rst_counter[cp->protoidx]);
if (--sp->rst_counter[cp->protoidx] < 0)
/* TO- event */
switch (sp->state[cp->protoidx]) {
case STATE_CLOSING:
sppp_cp_change_state(cp, sp, STATE_CLOSED);
(cp->tlf)(sp);
break;
case STATE_STOPPING:
sppp_cp_change_state(cp, sp, STATE_STOPPED);
(cp->tlf)(sp);
break;
case STATE_REQ_SENT:
case STATE_ACK_RCVD:
case STATE_ACK_SENT:
sppp_cp_change_state(cp, sp, STATE_STOPPED);
(cp->tlf)(sp);
break;
}
else
/* TO+ event */
switch (sp->state[cp->protoidx]) {
case STATE_CLOSING:
case STATE_STOPPING:
sppp_cp_send(sp, cp->proto, TERM_REQ,
++sp->pp_seq[cp->protoidx], 0, 0);
callout_reset(&sp->ch[cp->protoidx], sp->lcp.timeout,
cp->TO, (void *)sp);
break;
case STATE_REQ_SENT:
case STATE_ACK_RCVD:
(cp->scr)(sp);
/* sppp_cp_change_state() will restart the timer */
sppp_cp_change_state(cp, sp, STATE_REQ_SENT);
break;
case STATE_ACK_SENT:
(cp->scr)(sp);
callout_reset(&sp->ch[cp->protoidx], sp->lcp.timeout,
cp->TO, (void *)sp);
break;
}
SPPP_UNLOCK(sp);
}
/*
* Change the state of a control protocol in the state automaton.
* Takes care of starting/stopping the restart timer.
*/
static void
sppp_cp_change_state(const struct cp *cp, struct sppp *sp, int newstate)
{
sp->state[cp->protoidx] = newstate;
callout_stop (&sp->ch[cp->protoidx]);
switch (newstate) {
case STATE_INITIAL:
case STATE_STARTING:
case STATE_CLOSED:
case STATE_STOPPED:
case STATE_OPENED:
break;
case STATE_CLOSING:
case STATE_STOPPING:
case STATE_REQ_SENT:
case STATE_ACK_RCVD:
case STATE_ACK_SENT:
callout_reset(&sp->ch[cp->protoidx], sp->lcp.timeout,
cp->TO, (void *)sp);
break;
}
}
/*
*--------------------------------------------------------------------------*
* *
* The LCP implementation. *
* *
*--------------------------------------------------------------------------*
*/
static void
sppp_pp_up(struct sppp *sp)
{
SPPP_LOCK(sp);
lcp.Up(sp);
SPPP_UNLOCK(sp);
}
static void
sppp_pp_down(struct sppp *sp)
{
SPPP_LOCK(sp);
lcp.Down(sp);
SPPP_UNLOCK(sp);
}
static void
sppp_lcp_init(struct sppp *sp)
{
sp->lcp.opts = (1 << LCP_OPT_MAGIC);
sp->lcp.magic = 0;
sp->state[IDX_LCP] = STATE_INITIAL;
sp->fail_counter[IDX_LCP] = 0;
sp->pp_seq[IDX_LCP] = 0;
sp->pp_rseq[IDX_LCP] = 0;
sp->lcp.protos = 0;
sp->lcp.mru = sp->lcp.their_mru = PP_MTU;
/* Note that these values are relevant for all control protocols */
sp->lcp.timeout = 3 * hz;
sp->lcp.max_terminate = 2;
sp->lcp.max_configure = 10;
sp->lcp.max_failure = 10;
- callout_init(&sp->ch[IDX_LCP], CALLOUT_MPSAFE);
+ callout_init(&sp->ch[IDX_LCP], 1);
}
static void
sppp_lcp_up(struct sppp *sp)
{
STDDCL;
sp->pp_alivecnt = 0;
sp->lcp.opts = (1 << LCP_OPT_MAGIC);
sp->lcp.magic = 0;
sp->lcp.protos = 0;
sp->lcp.mru = sp->lcp.their_mru = PP_MTU;
/*
* If we are authenticator, negotiate LCP_AUTH
*/
if (sp->hisauth.proto != 0)
sp->lcp.opts |= (1 << LCP_OPT_AUTH_PROTO);
else
sp->lcp.opts &= ~(1 << LCP_OPT_AUTH_PROTO);
sp->pp_flags &= ~PP_NEEDAUTH;
/*
* If this interface is passive or dial-on-demand, and we are
* still in Initial state, it means we've got an incoming
* call. Activate the interface.
*/
if ((ifp->if_flags & (IFF_AUTO | IFF_PASSIVE)) != 0) {
if (debug)
log(LOG_DEBUG,
SPP_FMT "Up event", SPP_ARGS(ifp));
ifp->if_drv_flags |= IFF_DRV_RUNNING;
if (sp->state[IDX_LCP] == STATE_INITIAL) {
if (debug)
log(-1, "(incoming call)\n");
sp->pp_flags |= PP_CALLIN;
lcp.Open(sp);
} else if (debug)
log(-1, "\n");
} else if ((ifp->if_flags & (IFF_AUTO | IFF_PASSIVE)) == 0 &&
(sp->state[IDX_LCP] == STATE_INITIAL)) {
ifp->if_drv_flags |= IFF_DRV_RUNNING;
lcp.Open(sp);
}
sppp_up_event(&lcp, sp);
}
static void
sppp_lcp_down(struct sppp *sp)
{
STDDCL;
sppp_down_event(&lcp, sp);
/*
* If this is neither a dial-on-demand nor a passive
* interface, simulate an ``ifconfig down'' action, so the
* administrator can force a redial by another ``ifconfig
* up''. XXX For leased line operation, should we immediately
* try to reopen the connection here?
*/
if ((ifp->if_flags & (IFF_AUTO | IFF_PASSIVE)) == 0) {
log(LOG_INFO,
SPP_FMT "Down event, taking interface down.\n",
SPP_ARGS(ifp));
if_down(ifp);
} else {
if (debug)
log(LOG_DEBUG,
SPP_FMT "Down event (carrier loss)\n",
SPP_ARGS(ifp));
sp->pp_flags &= ~PP_CALLIN;
if (sp->state[IDX_LCP] != STATE_INITIAL)
lcp.Close(sp);
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
}
}
static void
sppp_lcp_open(struct sppp *sp)
{
sppp_open_event(&lcp, sp);
}
static void
sppp_lcp_close(struct sppp *sp)
{
sppp_close_event(&lcp, sp);
}
static void
sppp_lcp_TO(void *cookie)
{
sppp_to_event(&lcp, (struct sppp *)cookie);
}
/*
* Analyze a configure request. Return true if it was agreeable, and
* caused action sca, false if it has been rejected or nak'ed, and
* caused action scn. (The return value is used to make the state
* transition decision in the state automaton.)
*/
static int
sppp_lcp_RCR(struct sppp *sp, struct lcp_header *h, int len)
{
STDDCL;
u_char *buf, *r, *p;
int origlen, rlen;
u_long nmagic;
u_short authproto;
len -= 4;
origlen = len;
buf = r = malloc (len, M_TEMP, M_NOWAIT);
if (! buf)
return (0);
if (debug)
log(LOG_DEBUG, SPP_FMT "lcp parse opts: ",
SPP_ARGS(ifp));
/* pass 1: check for things that need to be rejected */
p = (void*) (h+1);
for (rlen=0; len >= 2 && p[1] >= 2 && len >= p[1];
len-=p[1], p+=p[1]) {
if (debug)
log(-1, " %s ", sppp_lcp_opt_name(*p));
switch (*p) {
case LCP_OPT_MAGIC:
/* Magic number. */
if (len >= 6 && p[1] == 6)
continue;
if (debug)
log(-1, "[invalid] ");
break;
case LCP_OPT_ASYNC_MAP:
/* Async control character map. */
if (len >= 6 && p[1] == 6)
continue;
if (debug)
log(-1, "[invalid] ");
break;
case LCP_OPT_MRU:
/* Maximum receive unit. */
if (len >= 4 && p[1] == 4)
continue;
if (debug)
log(-1, "[invalid] ");
break;
case LCP_OPT_AUTH_PROTO:
if (len < 4) {
if (debug)
log(-1, "[invalid] ");
break;
}
authproto = (p[2] << 8) + p[3];
if (authproto == PPP_CHAP && p[1] != 5) {
if (debug)
log(-1, "[invalid chap len] ");
break;
}
if (sp->myauth.proto == 0) {
/* we are not configured to do auth */
if (debug)
log(-1, "[not configured] ");
break;
}
/*
* Remote want us to authenticate, remember this,
* so we stay in PHASE_AUTHENTICATE after LCP got
* up.
*/
sp->pp_flags |= PP_NEEDAUTH;
continue;
default:
/* Others not supported. */
if (debug)
log(-1, "[rej] ");
break;
}
/* Add the option to rejected list. */
bcopy (p, r, p[1]);
r += p[1];
rlen += p[1];
}
if (rlen) {
if (debug)
log(-1, " send conf-rej\n");
sppp_cp_send (sp, PPP_LCP, CONF_REJ, h->ident, rlen, buf);
return 0;
} else if (debug)
log(-1, "\n");
/*
* pass 2: check for option values that are unacceptable and
* thus require to be nak'ed.
*/
if (debug)
log(LOG_DEBUG, SPP_FMT "lcp parse opt values: ",
SPP_ARGS(ifp));
p = (void*) (h+1);
len = origlen;
for (rlen=0; len >= 2 && p[1] >= 2 && len >= p[1];
len-=p[1], p+=p[1]) {
if (debug)
log(-1, " %s ", sppp_lcp_opt_name(*p));
switch (*p) {
case LCP_OPT_MAGIC:
/* Magic number -- extract. */
nmagic = (u_long)p[2] << 24 |
(u_long)p[3] << 16 | p[4] << 8 | p[5];
if (nmagic != sp->lcp.magic) {
sp->pp_loopcnt = 0;
if (debug)
log(-1, "0x%lx ", nmagic);
continue;
}
if (debug && sp->pp_loopcnt < MAXALIVECNT*5)
log(-1, "[glitch] ");
++sp->pp_loopcnt;
/*
* We negate our magic here, and NAK it. If
* we see it later in an NAK packet, we
* suggest a new one.
*/
nmagic = ~sp->lcp.magic;
/* Gonna NAK it. */
p[2] = nmagic >> 24;
p[3] = nmagic >> 16;
p[4] = nmagic >> 8;
p[5] = nmagic;
break;
case LCP_OPT_ASYNC_MAP:
/*
* Async control character map -- just ignore it.
*
* Quote from RFC 1662, chapter 6:
* To enable this functionality, synchronous PPP
* implementations MUST always respond to the
* Async-Control-Character-Map Configuration
* Option with the LCP Configure-Ack. However,
* acceptance of the Configuration Option does
* not imply that the synchronous implementation
* will do any ACCM mapping. Instead, all such
* octet mapping will be performed by the
* asynchronous-to-synchronous converter.
*/
continue;
case LCP_OPT_MRU:
/*
* Maximum receive unit. Always agreeable,
* but ignored by now.
*/
sp->lcp.their_mru = p[2] * 256 + p[3];
if (debug)
log(-1, "%lu ", sp->lcp.their_mru);
continue;
case LCP_OPT_AUTH_PROTO:
authproto = (p[2] << 8) + p[3];
if (sp->myauth.proto != authproto) {
/* not agreed, nak */
if (debug)
log(-1, "[mine %s != his %s] ",
sppp_proto_name(sp->hisauth.proto),
sppp_proto_name(authproto));
p[2] = sp->myauth.proto >> 8;
p[3] = sp->myauth.proto;
break;
}
if (authproto == PPP_CHAP && p[4] != CHAP_MD5) {
if (debug)
log(-1, "[chap not MD5] ");
p[4] = CHAP_MD5;
break;
}
continue;
}
/* Add the option to nak'ed list. */
bcopy (p, r, p[1]);
r += p[1];
rlen += p[1];
}
if (rlen) {
/*
* Local and remote magics equal -- loopback?
*/
if (sp->pp_loopcnt >= MAXALIVECNT*5) {
if (sp->pp_loopcnt == MAXALIVECNT*5)
printf (SPP_FMT "loopback\n",
SPP_ARGS(ifp));
if (ifp->if_flags & IFF_UP) {
if_down(ifp);
sppp_qflush(&sp->pp_cpq);
/* XXX ? */
lcp.Down(sp);
lcp.Up(sp);
}
} else if (!sp->pp_loopcnt &&
++sp->fail_counter[IDX_LCP] >= sp->lcp.max_failure) {
if (debug)
log(-1, " max_failure (%d) exceeded, "
"send conf-rej\n",
sp->lcp.max_failure);
sppp_cp_send(sp, PPP_LCP, CONF_REJ, h->ident, rlen, buf);
} else {
if (debug)
log(-1, " send conf-nak\n");
sppp_cp_send (sp, PPP_LCP, CONF_NAK, h->ident, rlen, buf);
}
} else {
if (debug)
log(-1, " send conf-ack\n");
sp->fail_counter[IDX_LCP] = 0;
sp->pp_loopcnt = 0;
sppp_cp_send (sp, PPP_LCP, CONF_ACK,
h->ident, origlen, h+1);
}
free (buf, M_TEMP);
return (rlen == 0);
}
/*
* Analyze the LCP Configure-Reject option list, and adjust our
* negotiation.
*/
static void
sppp_lcp_RCN_rej(struct sppp *sp, struct lcp_header *h, int len)
{
STDDCL;
u_char *buf, *p;
len -= 4;
buf = malloc (len, M_TEMP, M_NOWAIT);
if (!buf)
return;
if (debug)
log(LOG_DEBUG, SPP_FMT "lcp rej opts: ",
SPP_ARGS(ifp));
p = (void*) (h+1);
for (; len >= 2 && p[1] >= 2 && len >= p[1];
len -= p[1], p += p[1]) {
if (debug)
log(-1, " %s ", sppp_lcp_opt_name(*p));
switch (*p) {
case LCP_OPT_MAGIC:
/* Magic number -- can't use it, use 0 */
sp->lcp.opts &= ~(1 << LCP_OPT_MAGIC);
sp->lcp.magic = 0;
break;
case LCP_OPT_MRU:
/*
* Should not be rejected anyway, since we only
* negotiate a MRU if explicitly requested by
* peer.
*/
sp->lcp.opts &= ~(1 << LCP_OPT_MRU);
break;
case LCP_OPT_AUTH_PROTO:
/*
* Peer doesn't want to authenticate himself,
* deny unless this is a dialout call, and
* AUTHFLAG_NOCALLOUT is set.
*/
if ((sp->pp_flags & PP_CALLIN) == 0 &&
(sp->hisauth.flags & AUTHFLAG_NOCALLOUT) != 0) {
if (debug)
log(-1, "[don't insist on auth "
"for callout]");
sp->lcp.opts &= ~(1 << LCP_OPT_AUTH_PROTO);
break;
}
if (debug)
log(-1, "[access denied]\n");
lcp.Close(sp);
break;
}
}
if (debug)
log(-1, "\n");
free (buf, M_TEMP);
return;
}
/*
* Analyze the LCP Configure-NAK option list, and adjust our
* negotiation.
*/
static void
sppp_lcp_RCN_nak(struct sppp *sp, struct lcp_header *h, int len)
{
STDDCL;
u_char *buf, *p;
u_long magic;
len -= 4;
buf = malloc (len, M_TEMP, M_NOWAIT);
if (!buf)
return;
if (debug)
log(LOG_DEBUG, SPP_FMT "lcp nak opts: ",
SPP_ARGS(ifp));
p = (void*) (h+1);
for (; len >= 2 && p[1] >= 2 && len >= p[1];
len -= p[1], p += p[1]) {
if (debug)
log(-1, " %s ", sppp_lcp_opt_name(*p));
switch (*p) {
case LCP_OPT_MAGIC:
/* Magic number -- renegotiate */
if ((sp->lcp.opts & (1 << LCP_OPT_MAGIC)) &&
len >= 6 && p[1] == 6) {
magic = (u_long)p[2] << 24 |
(u_long)p[3] << 16 | p[4] << 8 | p[5];
/*
* If the remote magic is our negated one,
* this looks like a loopback problem.
* Suggest a new magic to make sure.
*/
if (magic == ~sp->lcp.magic) {
if (debug)
log(-1, "magic glitch ");
sp->lcp.magic = random();
} else {
sp->lcp.magic = magic;
if (debug)
log(-1, "%lu ", magic);
}
}
break;
case LCP_OPT_MRU:
/*
* Peer wants to advise us to negotiate an MRU.
* Agree on it if it's reasonable, or use
* default otherwise.
*/
if (len >= 4 && p[1] == 4) {
u_int mru = p[2] * 256 + p[3];
if (debug)
log(-1, "%d ", mru);
if (mru < PP_MTU || mru > PP_MAX_MRU)
mru = PP_MTU;
sp->lcp.mru = mru;
sp->lcp.opts |= (1 << LCP_OPT_MRU);
}
break;
case LCP_OPT_AUTH_PROTO:
/*
* Peer doesn't like our authentication method,
* deny.
*/
if (debug)
log(-1, "[access denied]\n");
lcp.Close(sp);
break;
}
}
if (debug)
log(-1, "\n");
free (buf, M_TEMP);
return;
}
static void
sppp_lcp_tlu(struct sppp *sp)
{
STDDCL;
int i;
u_long mask;
/* XXX ? */
if (! (ifp->if_flags & IFF_UP) &&
(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
/* Coming out of loopback mode. */
if_up(ifp);
printf (SPP_FMT "up\n", SPP_ARGS(ifp));
}
for (i = 0; i < IDX_COUNT; i++)
if ((cps[i])->flags & CP_QUAL)
(cps[i])->Open(sp);
if ((sp->lcp.opts & (1 << LCP_OPT_AUTH_PROTO)) != 0 ||
(sp->pp_flags & PP_NEEDAUTH) != 0)
sp->pp_phase = PHASE_AUTHENTICATE;
else
sp->pp_phase = PHASE_NETWORK;
if (debug)
log(LOG_DEBUG, SPP_FMT "phase %s\n", SPP_ARGS(ifp),
sppp_phase_name(sp->pp_phase));
/*
* Open all authentication protocols. This is even required
* if we already proceeded to network phase, since it might be
* that remote wants us to authenticate, so we might have to
* send a PAP request. Undesired authentication protocols
* don't do anything when they get an Open event.
*/
for (i = 0; i < IDX_COUNT; i++)
if ((cps[i])->flags & CP_AUTH)
(cps[i])->Open(sp);
if (sp->pp_phase == PHASE_NETWORK) {
/* Notify all NCPs. */
for (i = 0; i < IDX_COUNT; i++)
if (((cps[i])->flags & CP_NCP) &&
/*
* XXX
* Hack to administratively disable IPv6 if
* not desired. Perhaps we should have another
* flag for this, but right now, we can make
* all struct cp's read/only.
*/
(cps[i] != &ipv6cp ||
(sp->confflags & CONF_ENABLE_IPV6)))
(cps[i])->Open(sp);
}
/* Send Up events to all started protos. */
for (i = 0, mask = 1; i < IDX_COUNT; i++, mask <<= 1)
if ((sp->lcp.protos & mask) && ((cps[i])->flags & CP_LCP) == 0)
(cps[i])->Up(sp);
/* notify low-level driver of state change */
if (sp->pp_chg)
sp->pp_chg(sp, (int)sp->pp_phase);
if (sp->pp_phase == PHASE_NETWORK)
/* if no NCP is starting, close down */
sppp_lcp_check_and_close(sp);
}
static void
sppp_lcp_tld(struct sppp *sp)
{
STDDCL;
int i;
u_long mask;
sp->pp_phase = PHASE_TERMINATE;
if (debug)
log(LOG_DEBUG, SPP_FMT "phase %s\n", SPP_ARGS(ifp),
sppp_phase_name(sp->pp_phase));
/*
* Take upper layers down. We send the Down event first and
* the Close second to prevent the upper layers from sending
* ``a flurry of terminate-request packets'', as the RFC
* describes it.
*/
for (i = 0, mask = 1; i < IDX_COUNT; i++, mask <<= 1)
if ((sp->lcp.protos & mask) && ((cps[i])->flags & CP_LCP) == 0) {
(cps[i])->Down(sp);
(cps[i])->Close(sp);
}
}
static void
sppp_lcp_tls(struct sppp *sp)
{
STDDCL;
sp->pp_phase = PHASE_ESTABLISH;
if (debug)
log(LOG_DEBUG, SPP_FMT "phase %s\n", SPP_ARGS(ifp),
sppp_phase_name(sp->pp_phase));
/* Notify lower layer if desired. */
if (sp->pp_tls)
(sp->pp_tls)(sp);
else
(sp->pp_up)(sp);
}
static void
sppp_lcp_tlf(struct sppp *sp)
{
STDDCL;
sp->pp_phase = PHASE_DEAD;
if (debug)
log(LOG_DEBUG, SPP_FMT "phase %s\n", SPP_ARGS(ifp),
sppp_phase_name(sp->pp_phase));
/* Notify lower layer if desired. */
if (sp->pp_tlf)
(sp->pp_tlf)(sp);
else
(sp->pp_down)(sp);
}
static void
sppp_lcp_scr(struct sppp *sp)
{
char opt[6 /* magicnum */ + 4 /* mru */ + 5 /* chap */];
int i = 0;
u_short authproto;
if (sp->lcp.opts & (1 << LCP_OPT_MAGIC)) {
if (! sp->lcp.magic)
sp->lcp.magic = random();
opt[i++] = LCP_OPT_MAGIC;
opt[i++] = 6;
opt[i++] = sp->lcp.magic >> 24;
opt[i++] = sp->lcp.magic >> 16;
opt[i++] = sp->lcp.magic >> 8;
opt[i++] = sp->lcp.magic;
}
if (sp->lcp.opts & (1 << LCP_OPT_MRU)) {
opt[i++] = LCP_OPT_MRU;
opt[i++] = 4;
opt[i++] = sp->lcp.mru >> 8;
opt[i++] = sp->lcp.mru;
}
if (sp->lcp.opts & (1 << LCP_OPT_AUTH_PROTO)) {
authproto = sp->hisauth.proto;
opt[i++] = LCP_OPT_AUTH_PROTO;
opt[i++] = authproto == PPP_CHAP? 5: 4;
opt[i++] = authproto >> 8;
opt[i++] = authproto;
if (authproto == PPP_CHAP)
opt[i++] = CHAP_MD5;
}
sp->confid[IDX_LCP] = ++sp->pp_seq[IDX_LCP];
sppp_cp_send (sp, PPP_LCP, CONF_REQ, sp->confid[IDX_LCP], i, &opt);
}
/*
* Check the open NCPs, return true if at least one NCP is open.
*/
static int
sppp_ncp_check(struct sppp *sp)
{
int i, mask;
for (i = 0, mask = 1; i < IDX_COUNT; i++, mask <<= 1)
if ((sp->lcp.protos & mask) && (cps[i])->flags & CP_NCP)
return 1;
return 0;
}
/*
* Re-check the open NCPs and see if we should terminate the link.
* Called by the NCPs during their tlf action handling.
*/
static void
sppp_lcp_check_and_close(struct sppp *sp)
{
if (sp->pp_phase < PHASE_NETWORK)
/* don't bother, we are already going down */
return;
if (sppp_ncp_check(sp))
return;
lcp.Close(sp);
}
/*
*--------------------------------------------------------------------------*
* *
* The IPCP implementation. *
* *
*--------------------------------------------------------------------------*
*/
#ifdef INET
static void
sppp_ipcp_init(struct sppp *sp)
{
sp->ipcp.opts = 0;
sp->ipcp.flags = 0;
sp->state[IDX_IPCP] = STATE_INITIAL;
sp->fail_counter[IDX_IPCP] = 0;
sp->pp_seq[IDX_IPCP] = 0;
sp->pp_rseq[IDX_IPCP] = 0;
- callout_init(&sp->ch[IDX_IPCP], CALLOUT_MPSAFE);
+ callout_init(&sp->ch[IDX_IPCP], 1);
}
static void
sppp_ipcp_up(struct sppp *sp)
{
sppp_up_event(&ipcp, sp);
}
static void
sppp_ipcp_down(struct sppp *sp)
{
sppp_down_event(&ipcp, sp);
}
static void
sppp_ipcp_open(struct sppp *sp)
{
STDDCL;
u_long myaddr, hisaddr;
sp->ipcp.flags &= ~(IPCP_HISADDR_SEEN | IPCP_MYADDR_SEEN |
IPCP_MYADDR_DYN | IPCP_VJ);
sp->ipcp.opts = 0;
sppp_get_ip_addrs(sp, &myaddr, &hisaddr, 0);
/*
* If we don't have his address, this probably means our
* interface doesn't want to talk IP at all. (This could
* be the case if somebody wants to speak only IPX, for
* example.) Don't open IPCP in this case.
*/
if (hisaddr == 0L) {
/* XXX this message should go away */
if (debug)
log(LOG_DEBUG, SPP_FMT "ipcp_open(): no IP interface\n",
SPP_ARGS(ifp));
return;
}
if (myaddr == 0L) {
/*
* I don't have an assigned address, so i need to
* negotiate my address.
*/
sp->ipcp.flags |= IPCP_MYADDR_DYN;
sp->ipcp.opts |= (1 << IPCP_OPT_ADDRESS);
} else
sp->ipcp.flags |= IPCP_MYADDR_SEEN;
if (sp->confflags & CONF_ENABLE_VJ) {
sp->ipcp.opts |= (1 << IPCP_OPT_COMPRESSION);
sp->ipcp.max_state = MAX_STATES - 1;
sp->ipcp.compress_cid = 1;
}
sppp_open_event(&ipcp, sp);
}
static void
sppp_ipcp_close(struct sppp *sp)
{
sppp_close_event(&ipcp, sp);
if (sp->ipcp.flags & IPCP_MYADDR_DYN)
/*
* My address was dynamic, clear it again.
*/
sppp_set_ip_addr(sp, 0L);
}
static void
sppp_ipcp_TO(void *cookie)
{
sppp_to_event(&ipcp, (struct sppp *)cookie);
}
/*
* Analyze a configure request. Return true if it was agreeable, and
* caused action sca, false if it has been rejected or nak'ed, and
* caused action scn. (The return value is used to make the state
* transition decision in the state automaton.)
*/
static int
sppp_ipcp_RCR(struct sppp *sp, struct lcp_header *h, int len)
{
u_char *buf, *r, *p;
struct ifnet *ifp = SP2IFP(sp);
int rlen, origlen, debug = ifp->if_flags & IFF_DEBUG;
u_long hisaddr, desiredaddr;
int gotmyaddr = 0;
int desiredcomp;
len -= 4;
origlen = len;
/*
* Make sure to allocate a buf that can at least hold a
* conf-nak with an `address' option. We might need it below.
*/
buf = r = malloc ((len < 6? 6: len), M_TEMP, M_NOWAIT);
if (! buf)
return (0);
/* pass 1: see if we can recognize them */
if (debug)
log(LOG_DEBUG, SPP_FMT "ipcp parse opts: ",
SPP_ARGS(ifp));
p = (void*) (h+1);
for (rlen=0; len >= 2 && p[1] >= 2 && len >= p[1];
len-=p[1], p+=p[1]) {
if (debug)
log(-1, " %s ", sppp_ipcp_opt_name(*p));
switch (*p) {
case IPCP_OPT_COMPRESSION:
if (!(sp->confflags & CONF_ENABLE_VJ)) {
/* VJ compression administratively disabled */
if (debug)
log(-1, "[locally disabled] ");
break;
}
/*
* In theory, we should only conf-rej an
* option that is shorter than RFC 1618
* requires (i.e. < 4), and should conf-nak
* anything else that is not VJ. However,
* since our algorithm always uses the
* original option to NAK it with new values,
* things would become more complicated. In
* pratice, the only commonly implemented IP
* compression option is VJ anyway, so the
* difference is negligible.
*/
if (len >= 6 && p[1] == 6) {
/*
* correctly formed compression option
* that could be VJ compression
*/
continue;
}
if (debug)
log(-1,
"optlen %d [invalid/unsupported] ",
p[1]);
break;
case IPCP_OPT_ADDRESS:
if (len >= 6 && p[1] == 6) {
/* correctly formed address option */
continue;
}
if (debug)
log(-1, "[invalid] ");
break;
default:
/* Others not supported. */
if (debug)
log(-1, "[rej] ");
break;
}
/* Add the option to rejected list. */
bcopy (p, r, p[1]);
r += p[1];
rlen += p[1];
}
if (rlen) {
if (debug)
log(-1, " send conf-rej\n");
sppp_cp_send (sp, PPP_IPCP, CONF_REJ, h->ident, rlen, buf);
return 0;
} else if (debug)
log(-1, "\n");
/* pass 2: parse option values */
sppp_get_ip_addrs(sp, 0, &hisaddr, 0);
if (debug)
log(LOG_DEBUG, SPP_FMT "ipcp parse opt values: ",
SPP_ARGS(ifp));
p = (void*) (h+1);
len = origlen;
for (rlen=0; len >= 2 && p[1] >= 2 && len >= p[1];
len-=p[1], p+=p[1]) {
if (debug)
log(-1, " %s ", sppp_ipcp_opt_name(*p));
switch (*p) {
case IPCP_OPT_COMPRESSION:
desiredcomp = p[2] << 8 | p[3];
/* We only support VJ */
if (desiredcomp == IPCP_COMP_VJ) {
if (debug)
log(-1, "VJ [ack] ");
sp->ipcp.flags |= IPCP_VJ;
sl_compress_init(sp->pp_comp, p[4]);
sp->ipcp.max_state = p[4];
sp->ipcp.compress_cid = p[5];
continue;
}
if (debug)
log(-1,
"compproto %#04x [not supported] ",
desiredcomp);
p[2] = IPCP_COMP_VJ >> 8;
p[3] = IPCP_COMP_VJ;
p[4] = sp->ipcp.max_state;
p[5] = sp->ipcp.compress_cid;
break;
case IPCP_OPT_ADDRESS:
/* This is the address he wants in his end */
desiredaddr = p[2] << 24 | p[3] << 16 |
p[4] << 8 | p[5];
if (desiredaddr == hisaddr ||
(hisaddr >= 1 && hisaddr <= 254 && desiredaddr != 0)) {
/*
* Peer's address is same as our value,
* or we have set it to 0.0.0.* to
* indicate that we do not really care,
* this is agreeable. Gonna conf-ack
* it.
*/
if (debug)
log(-1, "%s [ack] ",
sppp_dotted_quad(hisaddr));
/* record that we've seen it already */
sp->ipcp.flags |= IPCP_HISADDR_SEEN;
continue;
}
/*
* The address wasn't agreeable. This is either
* he sent us 0.0.0.0, asking to assign him an
* address, or he send us another address not
* matching our value. Either case, we gonna
* conf-nak it with our value.
* XXX: we should "rej" if hisaddr == 0
*/
if (debug) {
if (desiredaddr == 0)
log(-1, "[addr requested] ");
else
log(-1, "%s [not agreed] ",
sppp_dotted_quad(desiredaddr));
}
p[2] = hisaddr >> 24;
p[3] = hisaddr >> 16;
p[4] = hisaddr >> 8;
p[5] = hisaddr;
break;
}
/* Add the option to nak'ed list. */
bcopy (p, r, p[1]);
r += p[1];
rlen += p[1];
}
/*
* If we are about to conf-ack the request, but haven't seen
* his address so far, gonna conf-nak it instead, with the
* `address' option present and our idea of his address being
* filled in there, to request negotiation of both addresses.
*
* XXX This can result in an endless req - nak loop if peer
* doesn't want to send us his address. Q: What should we do
* about it? XXX A: implement the max-failure counter.
*/
if (rlen == 0 && !(sp->ipcp.flags & IPCP_HISADDR_SEEN) && !gotmyaddr) {
buf[0] = IPCP_OPT_ADDRESS;
buf[1] = 6;
buf[2] = hisaddr >> 24;
buf[3] = hisaddr >> 16;
buf[4] = hisaddr >> 8;
buf[5] = hisaddr;
rlen = 6;
if (debug)
log(-1, "still need hisaddr ");
}
if (rlen) {
if (debug)
log(-1, " send conf-nak\n");
sppp_cp_send (sp, PPP_IPCP, CONF_NAK, h->ident, rlen, buf);
} else {
if (debug)
log(-1, " send conf-ack\n");
sppp_cp_send (sp, PPP_IPCP, CONF_ACK,
h->ident, origlen, h+1);
}
free (buf, M_TEMP);
return (rlen == 0);
}
/*
* Analyze the IPCP Configure-Reject option list, and adjust our
* negotiation.
*/
static void
sppp_ipcp_RCN_rej(struct sppp *sp, struct lcp_header *h, int len)
{
u_char *buf, *p;
struct ifnet *ifp = SP2IFP(sp);
int debug = ifp->if_flags & IFF_DEBUG;
len -= 4;
buf = malloc (len, M_TEMP, M_NOWAIT);
if (!buf)
return;
if (debug)
log(LOG_DEBUG, SPP_FMT "ipcp rej opts: ",
SPP_ARGS(ifp));
p = (void*) (h+1);
for (; len >= 2 && p[1] >= 2 && len >= p[1];
len -= p[1], p += p[1]) {
if (debug)
log(-1, " %s ", sppp_ipcp_opt_name(*p));
switch (*p) {
case IPCP_OPT_COMPRESSION:
sp->ipcp.opts &= ~(1 << IPCP_OPT_COMPRESSION);
break;
case IPCP_OPT_ADDRESS:
/*
* Peer doesn't grok address option. This is
* bad. XXX Should we better give up here?
* XXX We could try old "addresses" option...
*/
sp->ipcp.opts &= ~(1 << IPCP_OPT_ADDRESS);
break;
}
}
if (debug)
log(-1, "\n");
free (buf, M_TEMP);
return;
}
/*
* Analyze the IPCP Configure-NAK option list, and adjust our
* negotiation.
*/
static void
sppp_ipcp_RCN_nak(struct sppp *sp, struct lcp_header *h, int len)
{
u_char *buf, *p;
struct ifnet *ifp = SP2IFP(sp);
int debug = ifp->if_flags & IFF_DEBUG;
int desiredcomp;
u_long wantaddr;
len -= 4;
buf = malloc (len, M_TEMP, M_NOWAIT);
if (!buf)
return;
if (debug)
log(LOG_DEBUG, SPP_FMT "ipcp nak opts: ",
SPP_ARGS(ifp));
p = (void*) (h+1);
for (; len >= 2 && p[1] >= 2 && len >= p[1];
len -= p[1], p += p[1]) {
if (debug)
log(-1, " %s ", sppp_ipcp_opt_name(*p));
switch (*p) {
case IPCP_OPT_COMPRESSION:
if (len >= 6 && p[1] == 6) {
desiredcomp = p[2] << 8 | p[3];
if (debug)
log(-1, "[wantcomp %#04x] ",
desiredcomp);
if (desiredcomp == IPCP_COMP_VJ) {
sl_compress_init(sp->pp_comp, p[4]);
sp->ipcp.max_state = p[4];
sp->ipcp.compress_cid = p[5];
if (debug)
log(-1, "[agree] ");
} else
sp->ipcp.opts &=
~(1 << IPCP_OPT_COMPRESSION);
}
break;
case IPCP_OPT_ADDRESS:
/*
* Peer doesn't like our local IP address. See
* if we can do something for him. We'll drop
* him our address then.
*/
if (len >= 6 && p[1] == 6) {
wantaddr = p[2] << 24 | p[3] << 16 |
p[4] << 8 | p[5];
sp->ipcp.opts |= (1 << IPCP_OPT_ADDRESS);
if (debug)
log(-1, "[wantaddr %s] ",
sppp_dotted_quad(wantaddr));
/*
* When doing dynamic address assignment,
* we accept his offer. Otherwise, we
* ignore it and thus continue to negotiate
* our already existing value.
* XXX: Bogus, if he said no once, he'll
* just say no again, might as well die.
*/
if (sp->ipcp.flags & IPCP_MYADDR_DYN) {
sppp_set_ip_addr(sp, wantaddr);
if (debug)
log(-1, "[agree] ");
sp->ipcp.flags |= IPCP_MYADDR_SEEN;
}
}
break;
}
}
if (debug)
log(-1, "\n");
free (buf, M_TEMP);
return;
}
static void
sppp_ipcp_tlu(struct sppp *sp)
{
/* we are up - notify isdn daemon */
if (sp->pp_con)
sp->pp_con(sp);
}
static void
sppp_ipcp_tld(struct sppp *sp)
{
}
static void
sppp_ipcp_tls(struct sppp *sp)
{
/* indicate to LCP that it must stay alive */
sp->lcp.protos |= (1 << IDX_IPCP);
}
static void
sppp_ipcp_tlf(struct sppp *sp)
{
/* we no longer need LCP */
sp->lcp.protos &= ~(1 << IDX_IPCP);
sppp_lcp_check_and_close(sp);
}
static void
sppp_ipcp_scr(struct sppp *sp)
{
char opt[6 /* compression */ + 6 /* address */];
u_long ouraddr;
int i = 0;
if (sp->ipcp.opts & (1 << IPCP_OPT_COMPRESSION)) {
opt[i++] = IPCP_OPT_COMPRESSION;
opt[i++] = 6;
opt[i++] = IPCP_COMP_VJ >> 8;
opt[i++] = IPCP_COMP_VJ;
opt[i++] = sp->ipcp.max_state;
opt[i++] = sp->ipcp.compress_cid;
}
if (sp->ipcp.opts & (1 << IPCP_OPT_ADDRESS)) {
sppp_get_ip_addrs(sp, &ouraddr, 0, 0);
opt[i++] = IPCP_OPT_ADDRESS;
opt[i++] = 6;
opt[i++] = ouraddr >> 24;
opt[i++] = ouraddr >> 16;
opt[i++] = ouraddr >> 8;
opt[i++] = ouraddr;
}
sp->confid[IDX_IPCP] = ++sp->pp_seq[IDX_IPCP];
sppp_cp_send(sp, PPP_IPCP, CONF_REQ, sp->confid[IDX_IPCP], i, &opt);
}
#else /* !INET */
static void
sppp_ipcp_init(struct sppp *sp)
{
}
static void
sppp_ipcp_up(struct sppp *sp)
{
}
static void
sppp_ipcp_down(struct sppp *sp)
{
}
static void
sppp_ipcp_open(struct sppp *sp)
{
}
static void
sppp_ipcp_close(struct sppp *sp)
{
}
static void
sppp_ipcp_TO(void *cookie)
{
}
static int
sppp_ipcp_RCR(struct sppp *sp, struct lcp_header *h, int len)
{
return (0);
}
static void
sppp_ipcp_RCN_rej(struct sppp *sp, struct lcp_header *h, int len)
{
}
static void
sppp_ipcp_RCN_nak(struct sppp *sp, struct lcp_header *h, int len)
{
}
static void
sppp_ipcp_tlu(struct sppp *sp)
{
}
static void
sppp_ipcp_tld(struct sppp *sp)
{
}
static void
sppp_ipcp_tls(struct sppp *sp)
{
}
static void
sppp_ipcp_tlf(struct sppp *sp)
{
}
static void
sppp_ipcp_scr(struct sppp *sp)
{
}
#endif
/*
*--------------------------------------------------------------------------*
* *
* The IPv6CP implementation. *
* *
*--------------------------------------------------------------------------*
*/
#ifdef INET6
static void
sppp_ipv6cp_init(struct sppp *sp)
{
sp->ipv6cp.opts = 0;
sp->ipv6cp.flags = 0;
sp->state[IDX_IPV6CP] = STATE_INITIAL;
sp->fail_counter[IDX_IPV6CP] = 0;
sp->pp_seq[IDX_IPV6CP] = 0;
sp->pp_rseq[IDX_IPV6CP] = 0;
- callout_init(&sp->ch[IDX_IPV6CP], CALLOUT_MPSAFE);
+ callout_init(&sp->ch[IDX_IPV6CP], 1);
}
static void
sppp_ipv6cp_up(struct sppp *sp)
{
sppp_up_event(&ipv6cp, sp);
}
static void
sppp_ipv6cp_down(struct sppp *sp)
{
sppp_down_event(&ipv6cp, sp);
}
static void
sppp_ipv6cp_open(struct sppp *sp)
{
STDDCL;
struct in6_addr myaddr, hisaddr;
#ifdef IPV6CP_MYIFID_DYN
sp->ipv6cp.flags &= ~(IPV6CP_MYIFID_SEEN|IPV6CP_MYIFID_DYN);
#else
sp->ipv6cp.flags &= ~IPV6CP_MYIFID_SEEN;
#endif
sppp_get_ip6_addrs(sp, &myaddr, &hisaddr, 0);
/*
* If we don't have our address, this probably means our
* interface doesn't want to talk IPv6 at all. (This could
* be the case if somebody wants to speak only IPX, for
* example.) Don't open IPv6CP in this case.
*/
if (IN6_IS_ADDR_UNSPECIFIED(&myaddr)) {
/* XXX this message should go away */
if (debug)
log(LOG_DEBUG, SPP_FMT "ipv6cp_open(): no IPv6 interface\n",
SPP_ARGS(ifp));
return;
}
sp->ipv6cp.flags |= IPV6CP_MYIFID_SEEN;
sp->ipv6cp.opts |= (1 << IPV6CP_OPT_IFID);
sppp_open_event(&ipv6cp, sp);
}
static void
sppp_ipv6cp_close(struct sppp *sp)
{
sppp_close_event(&ipv6cp, sp);
}
static void
sppp_ipv6cp_TO(void *cookie)
{
sppp_to_event(&ipv6cp, (struct sppp *)cookie);
}
/*
* Analyze a configure request. Return true if it was agreeable, and
* caused action sca, false if it has been rejected or nak'ed, and
* caused action scn. (The return value is used to make the state
* transition decision in the state automaton.)
*/
static int
sppp_ipv6cp_RCR(struct sppp *sp, struct lcp_header *h, int len)
{
u_char *buf, *r, *p;
struct ifnet *ifp = SP2IFP(sp);
int rlen, origlen, debug = ifp->if_flags & IFF_DEBUG;
struct in6_addr myaddr, desiredaddr, suggestaddr;
int ifidcount;
int type;
int collision, nohisaddr;
char ip6buf[INET6_ADDRSTRLEN];
len -= 4;
origlen = len;
/*
* Make sure to allocate a buf that can at least hold a
* conf-nak with an `address' option. We might need it below.
*/
buf = r = malloc ((len < 6? 6: len), M_TEMP, M_NOWAIT);
if (! buf)
return (0);
/* pass 1: see if we can recognize them */
if (debug)
log(LOG_DEBUG, SPP_FMT "ipv6cp parse opts:",
SPP_ARGS(ifp));
p = (void*) (h+1);
ifidcount = 0;
for (rlen=0; len >= 2 && p[1] >= 2 && len >= p[1];
len-=p[1], p+=p[1]) {
if (debug)
log(-1, " %s", sppp_ipv6cp_opt_name(*p));
switch (*p) {
case IPV6CP_OPT_IFID:
if (len >= 10 && p[1] == 10 && ifidcount == 0) {
/* correctly formed address option */
ifidcount++;
continue;
}
if (debug)
log(-1, " [invalid]");
break;
#ifdef notyet
case IPV6CP_OPT_COMPRESSION:
if (len >= 4 && p[1] >= 4) {
/* correctly formed compress option */
continue;
}
if (debug)
log(-1, " [invalid]");
break;
#endif
default:
/* Others not supported. */
if (debug)
log(-1, " [rej]");
break;
}
/* Add the option to rejected list. */
bcopy (p, r, p[1]);
r += p[1];
rlen += p[1];
}
if (rlen) {
if (debug)
log(-1, " send conf-rej\n");
sppp_cp_send (sp, PPP_IPV6CP, CONF_REJ, h->ident, rlen, buf);
goto end;
} else if (debug)
log(-1, "\n");
/* pass 2: parse option values */
sppp_get_ip6_addrs(sp, &myaddr, 0, 0);
if (debug)
log(LOG_DEBUG, SPP_FMT "ipv6cp parse opt values: ",
SPP_ARGS(ifp));
p = (void*) (h+1);
len = origlen;
type = CONF_ACK;
for (rlen=0; len >= 2 && p[1] >= 2 && len >= p[1];
len-=p[1], p+=p[1]) {
if (debug)
log(-1, " %s", sppp_ipv6cp_opt_name(*p));
switch (*p) {
#ifdef notyet
case IPV6CP_OPT_COMPRESSION:
continue;
#endif
case IPV6CP_OPT_IFID:
bzero(&desiredaddr, sizeof(desiredaddr));
bcopy(&p[2], &desiredaddr.s6_addr[8], 8);
collision = (bcmp(&desiredaddr.s6_addr[8],
&myaddr.s6_addr[8], 8) == 0);
nohisaddr = IN6_IS_ADDR_UNSPECIFIED(&desiredaddr);
desiredaddr.s6_addr16[0] = htons(0xfe80);
(void)in6_setscope(&desiredaddr, SP2IFP(sp), NULL);
if (!collision && !nohisaddr) {
/* no collision, hisaddr known - Conf-Ack */
type = CONF_ACK;
if (debug) {
log(-1, " %s [%s]",
ip6_sprintf(ip6buf, &desiredaddr),
sppp_cp_type_name(type));
}
continue;
}
bzero(&suggestaddr, sizeof(suggestaddr));
if (collision && nohisaddr) {
/* collision, hisaddr unknown - Conf-Rej */
type = CONF_REJ;
bzero(&p[2], 8);
} else {
/*
* - no collision, hisaddr unknown, or
* - collision, hisaddr known
* Conf-Nak, suggest hisaddr
*/
type = CONF_NAK;
sppp_suggest_ip6_addr(sp, &suggestaddr);
bcopy(&suggestaddr.s6_addr[8], &p[2], 8);
}
if (debug)
log(-1, " %s [%s]",
ip6_sprintf(ip6buf, &desiredaddr),
sppp_cp_type_name(type));
break;
}
/* Add the option to nak'ed list. */
bcopy (p, r, p[1]);
r += p[1];
rlen += p[1];
}
if (rlen == 0 && type == CONF_ACK) {
if (debug)
log(-1, " send %s\n", sppp_cp_type_name(type));
sppp_cp_send (sp, PPP_IPV6CP, type, h->ident, origlen, h+1);
} else {
#ifdef DIAGNOSTIC
if (type == CONF_ACK)
panic("IPv6CP RCR: CONF_ACK with non-zero rlen");
#endif
if (debug) {
log(-1, " send %s suggest %s\n",
sppp_cp_type_name(type),
ip6_sprintf(ip6buf, &suggestaddr));
}
sppp_cp_send (sp, PPP_IPV6CP, type, h->ident, rlen, buf);
}
end:
free (buf, M_TEMP);
return (rlen == 0);
}
/*
* Analyze the IPv6CP Configure-Reject option list, and adjust our
* negotiation.
*/
static void
sppp_ipv6cp_RCN_rej(struct sppp *sp, struct lcp_header *h, int len)
{
u_char *buf, *p;
struct ifnet *ifp = SP2IFP(sp);
int debug = ifp->if_flags & IFF_DEBUG;
len -= 4;
buf = malloc (len, M_TEMP, M_NOWAIT);
if (!buf)
return;
if (debug)
log(LOG_DEBUG, SPP_FMT "ipv6cp rej opts:",
SPP_ARGS(ifp));
p = (void*) (h+1);
for (; len >= 2 && p[1] >= 2 && len >= p[1];
len -= p[1], p += p[1]) {
if (debug)
log(-1, " %s", sppp_ipv6cp_opt_name(*p));
switch (*p) {
case IPV6CP_OPT_IFID:
/*
* Peer doesn't grok address option. This is
* bad. XXX Should we better give up here?
*/
sp->ipv6cp.opts &= ~(1 << IPV6CP_OPT_IFID);
break;
#ifdef notyet
case IPV6CP_OPT_COMPRESS:
sp->ipv6cp.opts &= ~(1 << IPV6CP_OPT_COMPRESS);
break;
#endif
}
}
if (debug)
log(-1, "\n");
free (buf, M_TEMP);
return;
}
/*
* Analyze the IPv6CP Configure-NAK option list, and adjust our
* negotiation.
*/
static void
sppp_ipv6cp_RCN_nak(struct sppp *sp, struct lcp_header *h, int len)
{
u_char *buf, *p;
struct ifnet *ifp = SP2IFP(sp);
int debug = ifp->if_flags & IFF_DEBUG;
struct in6_addr suggestaddr;
char ip6buf[INET6_ADDRSTRLEN];
len -= 4;
buf = malloc (len, M_TEMP, M_NOWAIT);
if (!buf)
return;
if (debug)
log(LOG_DEBUG, SPP_FMT "ipv6cp nak opts:",
SPP_ARGS(ifp));
p = (void*) (h+1);
for (; len >= 2 && p[1] >= 2 && len >= p[1];
len -= p[1], p += p[1]) {
if (debug)
log(-1, " %s", sppp_ipv6cp_opt_name(*p));
switch (*p) {
case IPV6CP_OPT_IFID:
/*
* Peer doesn't like our local ifid. See
* if we can do something for him. We'll drop
* him our address then.
*/
if (len < 10 || p[1] != 10)
break;
bzero(&suggestaddr, sizeof(suggestaddr));
suggestaddr.s6_addr16[0] = htons(0xfe80);
(void)in6_setscope(&suggestaddr, SP2IFP(sp), NULL);
bcopy(&p[2], &suggestaddr.s6_addr[8], 8);
sp->ipv6cp.opts |= (1 << IPV6CP_OPT_IFID);
if (debug)
log(-1, " [suggestaddr %s]",
ip6_sprintf(ip6buf, &suggestaddr));
#ifdef IPV6CP_MYIFID_DYN
/*
* When doing dynamic address assignment,
* we accept his offer.
*/
if (sp->ipv6cp.flags & IPV6CP_MYIFID_DYN) {
struct in6_addr lastsuggest;
/*
* If <suggested myaddr from peer> equals to
* <hisaddr we have suggested last time>,
* we have a collision. generate new random
* ifid.
*/
sppp_suggest_ip6_addr(&lastsuggest);
if (IN6_ARE_ADDR_EQUAL(&suggestaddr,
lastsuggest)) {
if (debug)
log(-1, " [random]");
sppp_gen_ip6_addr(sp, &suggestaddr);
}
sppp_set_ip6_addr(sp, &suggestaddr, 0);
if (debug)
log(-1, " [agree]");
sp->ipv6cp.flags |= IPV6CP_MYIFID_SEEN;
}
#else
/*
* Since we do not do dynamic address assignment,
* we ignore it and thus continue to negotiate
* our already existing value. This can possibly
* go into infinite request-reject loop.
*
* This is not likely because we normally use
* ifid based on MAC-address.
* If you have no ethernet card on the node, too bad.
* XXX should we use fail_counter?
*/
#endif
break;
#ifdef notyet
case IPV6CP_OPT_COMPRESS:
/*
* Peer wants different compression parameters.
*/
break;
#endif
}
}
if (debug)
log(-1, "\n");
free (buf, M_TEMP);
return;
}
static void
sppp_ipv6cp_tlu(struct sppp *sp)
{
/* we are up - notify isdn daemon */
if (sp->pp_con)
sp->pp_con(sp);
}
static void
sppp_ipv6cp_tld(struct sppp *sp)
{
}
static void
sppp_ipv6cp_tls(struct sppp *sp)
{
/* indicate to LCP that it must stay alive */
sp->lcp.protos |= (1 << IDX_IPV6CP);
}
static void
sppp_ipv6cp_tlf(struct sppp *sp)
{
#if 0 /* need #if 0 to close IPv6CP properly */
/* we no longer need LCP */
sp->lcp.protos &= ~(1 << IDX_IPV6CP);
sppp_lcp_check_and_close(sp);
#endif
}
static void
sppp_ipv6cp_scr(struct sppp *sp)
{
char opt[10 /* ifid */ + 4 /* compression, minimum */];
struct in6_addr ouraddr;
int i = 0;
if (sp->ipv6cp.opts & (1 << IPV6CP_OPT_IFID)) {
sppp_get_ip6_addrs(sp, &ouraddr, 0, 0);
opt[i++] = IPV6CP_OPT_IFID;
opt[i++] = 10;
bcopy(&ouraddr.s6_addr[8], &opt[i], 8);
i += 8;
}
#ifdef notyet
if (sp->ipv6cp.opts & (1 << IPV6CP_OPT_COMPRESSION)) {
opt[i++] = IPV6CP_OPT_COMPRESSION;
opt[i++] = 4;
opt[i++] = 0; /* TBD */
opt[i++] = 0; /* TBD */
/* variable length data may follow */
}
#endif
sp->confid[IDX_IPV6CP] = ++sp->pp_seq[IDX_IPV6CP];
sppp_cp_send(sp, PPP_IPV6CP, CONF_REQ, sp->confid[IDX_IPV6CP], i, &opt);
}
#else /*INET6*/
static void sppp_ipv6cp_init(struct sppp *sp)
{
}
static void sppp_ipv6cp_up(struct sppp *sp)
{
}
static void sppp_ipv6cp_down(struct sppp *sp)
{
}
static void sppp_ipv6cp_open(struct sppp *sp)
{
}
static void sppp_ipv6cp_close(struct sppp *sp)
{
}
static void sppp_ipv6cp_TO(void *sp)
{
}
static int sppp_ipv6cp_RCR(struct sppp *sp, struct lcp_header *h, int len)
{
return 0;
}
static void sppp_ipv6cp_RCN_rej(struct sppp *sp, struct lcp_header *h, int len)
{
}
static void sppp_ipv6cp_RCN_nak(struct sppp *sp, struct lcp_header *h, int len)
{
}
static void sppp_ipv6cp_tlu(struct sppp *sp)
{
}
static void sppp_ipv6cp_tld(struct sppp *sp)
{
}
static void sppp_ipv6cp_tls(struct sppp *sp)
{
}
static void sppp_ipv6cp_tlf(struct sppp *sp)
{
}
static void sppp_ipv6cp_scr(struct sppp *sp)
{
}
#endif /*INET6*/
/*
*--------------------------------------------------------------------------*
* *
* The CHAP implementation. *
* *
*--------------------------------------------------------------------------*
*/
/*
* The authentication protocols don't employ a full-fledged state machine as
* the control protocols do, since they do have Open and Close events, but
* not Up and Down, nor are they explicitly terminated. Also, use of the
* authentication protocols may be different in both directions (this makes
* sense, think of a machine that never accepts incoming calls but only
* calls out, it doesn't require the called party to authenticate itself).
*
* Our state machine for the local authentication protocol (we are requesting
* the peer to authenticate) looks like:
*
* RCA-
* +--------------------------------------------+
* V scn,tld|
* +--------+ Close +---------+ RCA+
* | |<----------------------------------| |------+
* +--->| Closed | TO* | Opened | sca |
* | | |-----+ +-------| |<-----+
* | +--------+ irc | | +---------+
* | ^ | | ^
* | | | | |
* | | | | |
* | TO-| | | |
* | |tld TO+ V | |
* | | +------->+ | |
* | | | | | |
* | +--------+ V | |
* | | |<----+<--------------------+ |
* | | Req- | scr |
* | | Sent | |
* | | | |
* | +--------+ |
* | RCA- | | RCA+ |
* +------+ +------------------------------------------+
* scn,tld sca,irc,ict,tlu
*
*
* with:
*
* Open: LCP reached authentication phase
* Close: LCP reached terminate phase
*
* RCA+: received reply (pap-req, chap-response), acceptable
* RCN: received reply (pap-req, chap-response), not acceptable
* TO+: timeout with restart counter >= 0
* TO-: timeout with restart counter < 0
* TO*: reschedule timeout for CHAP
*
* scr: send request packet (none for PAP, chap-challenge)
* sca: send ack packet (pap-ack, chap-success)
* scn: send nak packet (pap-nak, chap-failure)
* ict: initialize re-challenge timer (CHAP only)
*
* tlu: this-layer-up, LCP reaches network phase
* tld: this-layer-down, LCP enters terminate phase
*
* Note that in CHAP mode, after sending a new challenge, while the state
* automaton falls back into Req-Sent state, it doesn't signal a tld
* event to LCP, so LCP remains in network phase. Only after not getting
* any response (or after getting an unacceptable response), CHAP closes,
* causing LCP to enter terminate phase.
*
* With PAP, there is no initial request that can be sent. The peer is
* expected to send one based on the successful negotiation of PAP as
* the authentication protocol during the LCP option negotiation.
*
* Incoming authentication protocol requests (remote requests
* authentication, we are peer) don't employ a state machine at all,
* they are simply answered. Some peers [Ascend P50 firmware rev
* 4.50] react allergically when sending IPCP requests while they are
* still in authentication phase (thereby violating the standard that
* demands that these NCP packets are to be discarded), so we keep
* track of the peer demanding us to authenticate, and only proceed to
* phase network once we've seen a positive acknowledge for the
* authentication.
*/
/*
* Handle incoming CHAP packets.
*/
static void
sppp_chap_input(struct sppp *sp, struct mbuf *m)
{
STDDCL;
struct lcp_header *h;
int len;
u_char *value, *name, digest[AUTHKEYLEN], dsize;
int value_len, name_len;
MD5_CTX ctx;
len = m->m_pkthdr.len;
if (len < 4) {
if (debug)
log(LOG_DEBUG,
SPP_FMT "chap invalid packet length: %d bytes\n",
SPP_ARGS(ifp), len);
return;
}
h = mtod (m, struct lcp_header*);
if (len > ntohs (h->len))
len = ntohs (h->len);
switch (h->type) {
/* challenge, failure and success are his authproto */
case CHAP_CHALLENGE:
value = 1 + (u_char*)(h+1);
value_len = value[-1];
name = value + value_len;
name_len = len - value_len - 5;
if (name_len < 0) {
if (debug) {
log(LOG_DEBUG,
SPP_FMT "chap corrupted challenge "
"<%s id=0x%x len=%d",
SPP_ARGS(ifp),
sppp_auth_type_name(PPP_CHAP, h->type),
h->ident, ntohs(h->len));
sppp_print_bytes((u_char*) (h+1), len-4);
log(-1, ">\n");
}
break;
}
if (debug) {
log(LOG_DEBUG,
SPP_FMT "chap input <%s id=0x%x len=%d name=",
SPP_ARGS(ifp),
sppp_auth_type_name(PPP_CHAP, h->type), h->ident,
ntohs(h->len));
sppp_print_string((char*) name, name_len);
log(-1, " value-size=%d value=", value_len);
sppp_print_bytes(value, value_len);
log(-1, ">\n");
}
/* Compute reply value. */
MD5Init(&ctx);
MD5Update(&ctx, &h->ident, 1);
MD5Update(&ctx, sp->myauth.secret,
sppp_strnlen(sp->myauth.secret, AUTHKEYLEN));
MD5Update(&ctx, value, value_len);
MD5Final(digest, &ctx);
dsize = sizeof digest;
sppp_auth_send(&chap, sp, CHAP_RESPONSE, h->ident,
sizeof dsize, (const char *)&dsize,
sizeof digest, digest,
(size_t)sppp_strnlen(sp->myauth.name, AUTHNAMELEN),
sp->myauth.name,
0);
break;
case CHAP_SUCCESS:
if (debug) {
log(LOG_DEBUG, SPP_FMT "chap success",
SPP_ARGS(ifp));
if (len > 4) {
log(-1, ": ");
sppp_print_string((char*)(h + 1), len - 4);
}
log(-1, "\n");
}
SPPP_LOCK(sp);
sp->pp_flags &= ~PP_NEEDAUTH;
if (sp->myauth.proto == PPP_CHAP &&
(sp->lcp.opts & (1 << LCP_OPT_AUTH_PROTO)) &&
(sp->lcp.protos & (1 << IDX_CHAP)) == 0) {
/*
* We are authenticator for CHAP but didn't
* complete yet. Leave it to tlu to proceed
* to network phase.
*/
SPPP_UNLOCK(sp);
break;
}
SPPP_UNLOCK(sp);
sppp_phase_network(sp);
break;
case CHAP_FAILURE:
if (debug) {
log(LOG_INFO, SPP_FMT "chap failure",
SPP_ARGS(ifp));
if (len > 4) {
log(-1, ": ");
sppp_print_string((char*)(h + 1), len - 4);
}
log(-1, "\n");
} else
log(LOG_INFO, SPP_FMT "chap failure\n",
SPP_ARGS(ifp));
/* await LCP shutdown by authenticator */
break;
/* response is my authproto */
case CHAP_RESPONSE:
value = 1 + (u_char*)(h+1);
value_len = value[-1];
name = value + value_len;
name_len = len - value_len - 5;
if (name_len < 0) {
if (debug) {
log(LOG_DEBUG,
SPP_FMT "chap corrupted response "
"<%s id=0x%x len=%d",
SPP_ARGS(ifp),
sppp_auth_type_name(PPP_CHAP, h->type),
h->ident, ntohs(h->len));
sppp_print_bytes((u_char*)(h+1), len-4);
log(-1, ">\n");
}
break;
}
if (h->ident != sp->confid[IDX_CHAP]) {
if (debug)
log(LOG_DEBUG,
SPP_FMT "chap dropping response for old ID "
"(got %d, expected %d)\n",
SPP_ARGS(ifp),
h->ident, sp->confid[IDX_CHAP]);
break;
}
if (name_len != sppp_strnlen(sp->hisauth.name, AUTHNAMELEN)
|| bcmp(name, sp->hisauth.name, name_len) != 0) {
log(LOG_INFO, SPP_FMT "chap response, his name ",
SPP_ARGS(ifp));
sppp_print_string(name, name_len);
log(-1, " != expected ");
sppp_print_string(sp->hisauth.name,
sppp_strnlen(sp->hisauth.name, AUTHNAMELEN));
log(-1, "\n");
}
if (debug) {
log(LOG_DEBUG, SPP_FMT "chap input(%s) "
"<%s id=0x%x len=%d name=",
SPP_ARGS(ifp),
sppp_state_name(sp->state[IDX_CHAP]),
sppp_auth_type_name(PPP_CHAP, h->type),
h->ident, ntohs (h->len));
sppp_print_string((char*)name, name_len);
log(-1, " value-size=%d value=", value_len);
sppp_print_bytes(value, value_len);
log(-1, ">\n");
}
if (value_len != AUTHKEYLEN) {
if (debug)
log(LOG_DEBUG,
SPP_FMT "chap bad hash value length: "
"%d bytes, should be %d\n",
SPP_ARGS(ifp), value_len,
AUTHKEYLEN);
break;
}
MD5Init(&ctx);
MD5Update(&ctx, &h->ident, 1);
MD5Update(&ctx, sp->hisauth.secret,
sppp_strnlen(sp->hisauth.secret, AUTHKEYLEN));
MD5Update(&ctx, sp->myauth.challenge, AUTHKEYLEN);
MD5Final(digest, &ctx);
#define FAILMSG "Failed..."
#define SUCCMSG "Welcome!"
if (value_len != sizeof digest ||
bcmp(digest, value, value_len) != 0) {
/* action scn, tld */
sppp_auth_send(&chap, sp, CHAP_FAILURE, h->ident,
sizeof(FAILMSG) - 1, (u_char *)FAILMSG,
0);
chap.tld(sp);
break;
}
/* action sca, perhaps tlu */
if (sp->state[IDX_CHAP] == STATE_REQ_SENT ||
sp->state[IDX_CHAP] == STATE_OPENED)
sppp_auth_send(&chap, sp, CHAP_SUCCESS, h->ident,
sizeof(SUCCMSG) - 1, (u_char *)SUCCMSG,
0);
if (sp->state[IDX_CHAP] == STATE_REQ_SENT) {
sppp_cp_change_state(&chap, sp, STATE_OPENED);
chap.tlu(sp);
}
break;
default:
/* Unknown CHAP packet type -- ignore. */
if (debug) {
log(LOG_DEBUG, SPP_FMT "chap unknown input(%s) "
"<0x%x id=0x%xh len=%d",
SPP_ARGS(ifp),
sppp_state_name(sp->state[IDX_CHAP]),
h->type, h->ident, ntohs(h->len));
sppp_print_bytes((u_char*)(h+1), len-4);
log(-1, ">\n");
}
break;
}
}
static void
sppp_chap_init(struct sppp *sp)
{
/* Chap doesn't have STATE_INITIAL at all. */
sp->state[IDX_CHAP] = STATE_CLOSED;
sp->fail_counter[IDX_CHAP] = 0;
sp->pp_seq[IDX_CHAP] = 0;
sp->pp_rseq[IDX_CHAP] = 0;
- callout_init(&sp->ch[IDX_CHAP], CALLOUT_MPSAFE);
+ callout_init(&sp->ch[IDX_CHAP], 1);
}
static void
sppp_chap_open(struct sppp *sp)
{
if (sp->myauth.proto == PPP_CHAP &&
(sp->lcp.opts & (1 << LCP_OPT_AUTH_PROTO)) != 0) {
/* we are authenticator for CHAP, start it */
chap.scr(sp);
sp->rst_counter[IDX_CHAP] = sp->lcp.max_configure;
sppp_cp_change_state(&chap, sp, STATE_REQ_SENT);
}
/* nothing to be done if we are peer, await a challenge */
}
static void
sppp_chap_close(struct sppp *sp)
{
if (sp->state[IDX_CHAP] != STATE_CLOSED)
sppp_cp_change_state(&chap, sp, STATE_CLOSED);
}
static void
sppp_chap_TO(void *cookie)
{
struct sppp *sp = (struct sppp *)cookie;
STDDCL;
SPPP_LOCK(sp);
if (debug)
log(LOG_DEBUG, SPP_FMT "chap TO(%s) rst_counter = %d\n",
SPP_ARGS(ifp),
sppp_state_name(sp->state[IDX_CHAP]),
sp->rst_counter[IDX_CHAP]);
if (--sp->rst_counter[IDX_CHAP] < 0)
/* TO- event */
switch (sp->state[IDX_CHAP]) {
case STATE_REQ_SENT:
chap.tld(sp);
sppp_cp_change_state(&chap, sp, STATE_CLOSED);
break;
}
else
/* TO+ (or TO*) event */
switch (sp->state[IDX_CHAP]) {
case STATE_OPENED:
/* TO* event */
sp->rst_counter[IDX_CHAP] = sp->lcp.max_configure;
/* FALLTHROUGH */
case STATE_REQ_SENT:
chap.scr(sp);
/* sppp_cp_change_state() will restart the timer */
sppp_cp_change_state(&chap, sp, STATE_REQ_SENT);
break;
}
SPPP_UNLOCK(sp);
}
static void
sppp_chap_tlu(struct sppp *sp)
{
STDDCL;
int i;
i = 0;
sp->rst_counter[IDX_CHAP] = sp->lcp.max_configure;
/*
* Some broken CHAP implementations (Conware CoNet, firmware
* 4.0.?) don't want to re-authenticate their CHAP once the
* initial challenge-response exchange has taken place.
* Provide for an option to avoid rechallenges.
*/
if ((sp->hisauth.flags & AUTHFLAG_NORECHALLENGE) == 0) {
/*
* Compute the re-challenge timeout. This will yield
* a number between 300 and 810 seconds.
*/
i = 300 + ((unsigned)(random() & 0xff00) >> 7);
callout_reset(&sp->ch[IDX_CHAP], i * hz, chap.TO, (void *)sp);
}
if (debug) {
log(LOG_DEBUG,
SPP_FMT "chap %s, ",
SPP_ARGS(ifp),
sp->pp_phase == PHASE_NETWORK? "reconfirmed": "tlu");
if ((sp->hisauth.flags & AUTHFLAG_NORECHALLENGE) == 0)
log(-1, "next re-challenge in %d seconds\n", i);
else
log(-1, "re-challenging supressed\n");
}
SPPP_LOCK(sp);
/* indicate to LCP that we need to be closed down */
sp->lcp.protos |= (1 << IDX_CHAP);
if (sp->pp_flags & PP_NEEDAUTH) {
/*
* Remote is authenticator, but his auth proto didn't
* complete yet. Defer the transition to network
* phase.
*/
SPPP_UNLOCK(sp);
return;
}
SPPP_UNLOCK(sp);
/*
* If we are already in phase network, we are done here. This
* is the case if this is a dummy tlu event after a re-challenge.
*/
if (sp->pp_phase != PHASE_NETWORK)
sppp_phase_network(sp);
}
static void
sppp_chap_tld(struct sppp *sp)
{
STDDCL;
if (debug)
log(LOG_DEBUG, SPP_FMT "chap tld\n", SPP_ARGS(ifp));
callout_stop(&sp->ch[IDX_CHAP]);
sp->lcp.protos &= ~(1 << IDX_CHAP);
lcp.Close(sp);
}
static void
sppp_chap_scr(struct sppp *sp)
{
u_long *ch, seed;
u_char clen;
/* Compute random challenge. */
ch = (u_long *)sp->myauth.challenge;
read_random(&seed, sizeof seed);
ch[0] = seed ^ random();
ch[1] = seed ^ random();
ch[2] = seed ^ random();
ch[3] = seed ^ random();
clen = AUTHKEYLEN;
sp->confid[IDX_CHAP] = ++sp->pp_seq[IDX_CHAP];
sppp_auth_send(&chap, sp, CHAP_CHALLENGE, sp->confid[IDX_CHAP],
sizeof clen, (const char *)&clen,
(size_t)AUTHKEYLEN, sp->myauth.challenge,
(size_t)sppp_strnlen(sp->myauth.name, AUTHNAMELEN),
sp->myauth.name,
0);
}
/*
*--------------------------------------------------------------------------*
* *
* The PAP implementation. *
* *
*--------------------------------------------------------------------------*
*/
/*
* For PAP, we need to keep a little state also if we are the peer, not the
* authenticator. This is since we don't get a request to authenticate, but
* have to repeatedly authenticate ourself until we got a response (or the
* retry counter is expired).
*/
/*
* Handle incoming PAP packets. */
static void
sppp_pap_input(struct sppp *sp, struct mbuf *m)
{
STDDCL;
struct lcp_header *h;
int len;
u_char *name, *passwd, mlen;
int name_len, passwd_len;
len = m->m_pkthdr.len;
if (len < 5) {
if (debug)
log(LOG_DEBUG,
SPP_FMT "pap invalid packet length: %d bytes\n",
SPP_ARGS(ifp), len);
return;
}
h = mtod (m, struct lcp_header*);
if (len > ntohs (h->len))
len = ntohs (h->len);
switch (h->type) {
/* PAP request is my authproto */
case PAP_REQ:
name = 1 + (u_char*)(h+1);
name_len = name[-1];
passwd = name + name_len + 1;
if (name_len > len - 6 ||
(passwd_len = passwd[-1]) > len - 6 - name_len) {
if (debug) {
log(LOG_DEBUG, SPP_FMT "pap corrupted input "
"<%s id=0x%x len=%d",
SPP_ARGS(ifp),
sppp_auth_type_name(PPP_PAP, h->type),
h->ident, ntohs(h->len));
sppp_print_bytes((u_char*)(h+1), len-4);
log(-1, ">\n");
}
break;
}
if (debug) {
log(LOG_DEBUG, SPP_FMT "pap input(%s) "
"<%s id=0x%x len=%d name=",
SPP_ARGS(ifp),
sppp_state_name(sp->state[IDX_PAP]),
sppp_auth_type_name(PPP_PAP, h->type),
h->ident, ntohs(h->len));
sppp_print_string((char*)name, name_len);
log(-1, " passwd=");
sppp_print_string((char*)passwd, passwd_len);
log(-1, ">\n");
}
if (name_len != sppp_strnlen(sp->hisauth.name, AUTHNAMELEN) ||
passwd_len != sppp_strnlen(sp->hisauth.secret, AUTHKEYLEN) ||
bcmp(name, sp->hisauth.name, name_len) != 0 ||
bcmp(passwd, sp->hisauth.secret, passwd_len) != 0) {
/* action scn, tld */
mlen = sizeof(FAILMSG) - 1;
sppp_auth_send(&pap, sp, PAP_NAK, h->ident,
sizeof mlen, (const char *)&mlen,
sizeof(FAILMSG) - 1, (u_char *)FAILMSG,
0);
pap.tld(sp);
break;
}
/* action sca, perhaps tlu */
if (sp->state[IDX_PAP] == STATE_REQ_SENT ||
sp->state[IDX_PAP] == STATE_OPENED) {
mlen = sizeof(SUCCMSG) - 1;
sppp_auth_send(&pap, sp, PAP_ACK, h->ident,
sizeof mlen, (const char *)&mlen,
sizeof(SUCCMSG) - 1, (u_char *)SUCCMSG,
0);
}
if (sp->state[IDX_PAP] == STATE_REQ_SENT) {
sppp_cp_change_state(&pap, sp, STATE_OPENED);
pap.tlu(sp);
}
break;
/* ack and nak are his authproto */
case PAP_ACK:
callout_stop(&sp->pap_my_to_ch);
if (debug) {
log(LOG_DEBUG, SPP_FMT "pap success",
SPP_ARGS(ifp));
name_len = *((char *)h);
if (len > 5 && name_len) {
log(-1, ": ");
sppp_print_string((char*)(h+1), name_len);
}
log(-1, "\n");
}
SPPP_LOCK(sp);
sp->pp_flags &= ~PP_NEEDAUTH;
if (sp->myauth.proto == PPP_PAP &&
(sp->lcp.opts & (1 << LCP_OPT_AUTH_PROTO)) &&
(sp->lcp.protos & (1 << IDX_PAP)) == 0) {
/*
* We are authenticator for PAP but didn't
* complete yet. Leave it to tlu to proceed
* to network phase.
*/
SPPP_UNLOCK(sp);
break;
}
SPPP_UNLOCK(sp);
sppp_phase_network(sp);
break;
case PAP_NAK:
callout_stop (&sp->pap_my_to_ch);
if (debug) {
log(LOG_INFO, SPP_FMT "pap failure",
SPP_ARGS(ifp));
name_len = *((char *)h);
if (len > 5 && name_len) {
log(-1, ": ");
sppp_print_string((char*)(h+1), name_len);
}
log(-1, "\n");
} else
log(LOG_INFO, SPP_FMT "pap failure\n",
SPP_ARGS(ifp));
/* await LCP shutdown by authenticator */
break;
default:
/* Unknown PAP packet type -- ignore. */
if (debug) {
log(LOG_DEBUG, SPP_FMT "pap corrupted input "
"<0x%x id=0x%x len=%d",
SPP_ARGS(ifp),
h->type, h->ident, ntohs(h->len));
sppp_print_bytes((u_char*)(h+1), len-4);
log(-1, ">\n");
}
break;
}
}
static void
sppp_pap_init(struct sppp *sp)
{
/* PAP doesn't have STATE_INITIAL at all. */
sp->state[IDX_PAP] = STATE_CLOSED;
sp->fail_counter[IDX_PAP] = 0;
sp->pp_seq[IDX_PAP] = 0;
sp->pp_rseq[IDX_PAP] = 0;
- callout_init(&sp->ch[IDX_PAP], CALLOUT_MPSAFE);
- callout_init(&sp->pap_my_to_ch, CALLOUT_MPSAFE);
+ callout_init(&sp->ch[IDX_PAP], 1);
+ callout_init(&sp->pap_my_to_ch, 1);
}
static void
sppp_pap_open(struct sppp *sp)
{
if (sp->hisauth.proto == PPP_PAP &&
(sp->lcp.opts & (1 << LCP_OPT_AUTH_PROTO)) != 0) {
/* we are authenticator for PAP, start our timer */
sp->rst_counter[IDX_PAP] = sp->lcp.max_configure;
sppp_cp_change_state(&pap, sp, STATE_REQ_SENT);
}
if (sp->myauth.proto == PPP_PAP) {
/* we are peer, send a request, and start a timer */
pap.scr(sp);
callout_reset(&sp->pap_my_to_ch, sp->lcp.timeout,
sppp_pap_my_TO, (void *)sp);
}
}
static void
sppp_pap_close(struct sppp *sp)
{
if (sp->state[IDX_PAP] != STATE_CLOSED)
sppp_cp_change_state(&pap, sp, STATE_CLOSED);
}
/*
* That's the timeout routine if we are authenticator. Since the
* authenticator is basically passive in PAP, we can't do much here.
*/
static void
sppp_pap_TO(void *cookie)
{
struct sppp *sp = (struct sppp *)cookie;
STDDCL;
SPPP_LOCK(sp);
if (debug)
log(LOG_DEBUG, SPP_FMT "pap TO(%s) rst_counter = %d\n",
SPP_ARGS(ifp),
sppp_state_name(sp->state[IDX_PAP]),
sp->rst_counter[IDX_PAP]);
if (--sp->rst_counter[IDX_PAP] < 0)
/* TO- event */
switch (sp->state[IDX_PAP]) {
case STATE_REQ_SENT:
pap.tld(sp);
sppp_cp_change_state(&pap, sp, STATE_CLOSED);
break;
}
else
/* TO+ event, not very much we could do */
switch (sp->state[IDX_PAP]) {
case STATE_REQ_SENT:
/* sppp_cp_change_state() will restart the timer */
sppp_cp_change_state(&pap, sp, STATE_REQ_SENT);
break;
}
SPPP_UNLOCK(sp);
}
/*
* That's the timeout handler if we are peer. Since the peer is active,
* we need to retransmit our PAP request since it is apparently lost.
* XXX We should impose a max counter.
*/
static void
sppp_pap_my_TO(void *cookie)
{
struct sppp *sp = (struct sppp *)cookie;
STDDCL;
if (debug)
log(LOG_DEBUG, SPP_FMT "pap peer TO\n",
SPP_ARGS(ifp));
SPPP_LOCK(sp);
pap.scr(sp);
SPPP_UNLOCK(sp);
}
static void
sppp_pap_tlu(struct sppp *sp)
{
STDDCL;
sp->rst_counter[IDX_PAP] = sp->lcp.max_configure;
if (debug)
log(LOG_DEBUG, SPP_FMT "%s tlu\n",
SPP_ARGS(ifp), pap.name);
SPPP_LOCK(sp);
/* indicate to LCP that we need to be closed down */
sp->lcp.protos |= (1 << IDX_PAP);
if (sp->pp_flags & PP_NEEDAUTH) {
/*
* Remote is authenticator, but his auth proto didn't
* complete yet. Defer the transition to network
* phase.
*/
SPPP_UNLOCK(sp);
return;
}
SPPP_UNLOCK(sp);
sppp_phase_network(sp);
}
static void
sppp_pap_tld(struct sppp *sp)
{
STDDCL;
if (debug)
log(LOG_DEBUG, SPP_FMT "pap tld\n", SPP_ARGS(ifp));
callout_stop (&sp->ch[IDX_PAP]);
callout_stop (&sp->pap_my_to_ch);
sp->lcp.protos &= ~(1 << IDX_PAP);
lcp.Close(sp);
}
static void
sppp_pap_scr(struct sppp *sp)
{
u_char idlen, pwdlen;
sp->confid[IDX_PAP] = ++sp->pp_seq[IDX_PAP];
pwdlen = sppp_strnlen(sp->myauth.secret, AUTHKEYLEN);
idlen = sppp_strnlen(sp->myauth.name, AUTHNAMELEN);
sppp_auth_send(&pap, sp, PAP_REQ, sp->confid[IDX_PAP],
sizeof idlen, (const char *)&idlen,
(size_t)idlen, sp->myauth.name,
sizeof pwdlen, (const char *)&pwdlen,
(size_t)pwdlen, sp->myauth.secret,
0);
}
/*
* Random miscellaneous functions.
*/
/*
* Send a PAP or CHAP proto packet.
*
* Varadic function, each of the elements for the ellipsis is of type
* ``size_t mlen, const u_char *msg''. Processing will stop iff
* mlen == 0.
* NOTE: never declare variadic functions with types subject to type
* promotion (i.e. u_char). This is asking for big trouble depending
* on the architecture you are on...
*/
static void
sppp_auth_send(const struct cp *cp, struct sppp *sp,
unsigned int type, unsigned int id,
...)
{
STDDCL;
struct ppp_header *h;
struct lcp_header *lh;
struct mbuf *m;
u_char *p;
int len;
unsigned int mlen;
const char *msg;
va_list ap;
MGETHDR (m, M_NOWAIT, MT_DATA);
if (! m)
return;
m->m_pkthdr.rcvif = 0;
h = mtod (m, struct ppp_header*);
h->address = PPP_ALLSTATIONS; /* broadcast address */
h->control = PPP_UI; /* Unnumbered Info */
h->protocol = htons(cp->proto);
lh = (struct lcp_header*)(h + 1);
lh->type = type;
lh->ident = id;
p = (u_char*) (lh+1);
va_start(ap, id);
len = 0;
while ((mlen = (unsigned int)va_arg(ap, size_t)) != 0) {
msg = va_arg(ap, const char *);
len += mlen;
if (len > MHLEN - PPP_HEADER_LEN - LCP_HEADER_LEN) {
va_end(ap);
m_freem(m);
return;
}
bcopy(msg, p, mlen);
p += mlen;
}
va_end(ap);
m->m_pkthdr.len = m->m_len = PPP_HEADER_LEN + LCP_HEADER_LEN + len;
lh->len = htons (LCP_HEADER_LEN + len);
if (debug) {
log(LOG_DEBUG, SPP_FMT "%s output <%s id=0x%x len=%d",
SPP_ARGS(ifp), cp->name,
sppp_auth_type_name(cp->proto, lh->type),
lh->ident, ntohs(lh->len));
sppp_print_bytes((u_char*) (lh+1), len);
log(-1, ">\n");
}
if (! IF_HANDOFF_ADJ(&sp->pp_cpq, m, ifp, 3))
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
}
/*
* Flush interface queue.
*/
static void
sppp_qflush(struct ifqueue *ifq)
{
struct mbuf *m, *n;
n = ifq->ifq_head;
while ((m = n)) {
n = m->m_nextpkt;
m_freem (m);
}
ifq->ifq_head = 0;
ifq->ifq_tail = 0;
ifq->ifq_len = 0;
}
/*
* Send keepalive packets, every 10 seconds.
*/
static void
sppp_keepalive(void *dummy)
{
struct sppp *sp = (struct sppp*)dummy;
struct ifnet *ifp = SP2IFP(sp);
SPPP_LOCK(sp);
/* Keepalive mode disabled or channel down? */
if (! (sp->pp_flags & PP_KEEPALIVE) ||
! (ifp->if_drv_flags & IFF_DRV_RUNNING))
goto out;
if (sp->pp_mode == PP_FR) {
sppp_fr_keepalive (sp);
goto out;
}
/* No keepalive in PPP mode if LCP not opened yet. */
if (sp->pp_mode != IFF_CISCO &&
sp->pp_phase < PHASE_AUTHENTICATE)
goto out;
if (sp->pp_alivecnt == MAXALIVECNT) {
/* No keepalive packets got. Stop the interface. */
printf (SPP_FMT "down\n", SPP_ARGS(ifp));
if_down (ifp);
sppp_qflush (&sp->pp_cpq);
if (sp->pp_mode != IFF_CISCO) {
/* XXX */
/* Shut down the PPP link. */
lcp.Down(sp);
/* Initiate negotiation. XXX */
lcp.Up(sp);
}
}
if (sp->pp_alivecnt <= MAXALIVECNT)
++sp->pp_alivecnt;
if (sp->pp_mode == IFF_CISCO)
sppp_cisco_send (sp, CISCO_KEEPALIVE_REQ,
++sp->pp_seq[IDX_LCP], sp->pp_rseq[IDX_LCP]);
else if (sp->pp_phase >= PHASE_AUTHENTICATE) {
long nmagic = htonl (sp->lcp.magic);
sp->lcp.echoid = ++sp->pp_seq[IDX_LCP];
sppp_cp_send (sp, PPP_LCP, ECHO_REQ,
sp->lcp.echoid, 4, &nmagic);
}
out:
SPPP_UNLOCK(sp);
callout_reset(&sp->keepalive_callout, hz * 10, sppp_keepalive,
(void *)sp);
}
/*
* Get both IP addresses.
*/
void
sppp_get_ip_addrs(struct sppp *sp, u_long *src, u_long *dst, u_long *srcmask)
{
struct ifnet *ifp = SP2IFP(sp);
struct ifaddr *ifa;
struct sockaddr_in *si, *sm;
u_long ssrc, ddst;
sm = NULL;
ssrc = ddst = 0L;
/*
* Pick the first AF_INET address from the list,
* aliases don't make any sense on a p2p link anyway.
*/
si = 0;
if_addr_rlock(ifp);
TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link)
if (ifa->ifa_addr->sa_family == AF_INET) {
si = (struct sockaddr_in *)ifa->ifa_addr;
sm = (struct sockaddr_in *)ifa->ifa_netmask;
if (si)
break;
}
if (ifa) {
if (si && si->sin_addr.s_addr) {
ssrc = si->sin_addr.s_addr;
if (srcmask)
*srcmask = ntohl(sm->sin_addr.s_addr);
}
si = (struct sockaddr_in *)ifa->ifa_dstaddr;
if (si && si->sin_addr.s_addr)
ddst = si->sin_addr.s_addr;
}
if_addr_runlock(ifp);
if (dst) *dst = ntohl(ddst);
if (src) *src = ntohl(ssrc);
}
#ifdef INET
/*
* Set my IP address.
*/
static void
sppp_set_ip_addr(struct sppp *sp, u_long src)
{
STDDCL;
struct ifaddr *ifa;
struct sockaddr_in *si;
struct in_ifaddr *ia;
/*
* Pick the first AF_INET address from the list,
* aliases don't make any sense on a p2p link anyway.
*/
si = 0;
if_addr_rlock(ifp);
TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family == AF_INET) {
si = (struct sockaddr_in *)ifa->ifa_addr;
if (si != NULL) {
ifa_ref(ifa);
break;
}
}
}
if_addr_runlock(ifp);
if (ifa != NULL) {
int error;
/* delete old route */
error = rtinit(ifa, (int)RTM_DELETE, RTF_HOST);
if (debug && error) {
log(LOG_DEBUG, SPP_FMT "sppp_set_ip_addr: rtinit DEL failed, error=%d\n",
SPP_ARGS(ifp), error);
}
/* set new address */
si->sin_addr.s_addr = htonl(src);
ia = ifatoia(ifa);
IN_IFADDR_WLOCK();
LIST_REMOVE(ia, ia_hash);
LIST_INSERT_HEAD(INADDR_HASH(si->sin_addr.s_addr), ia, ia_hash);
IN_IFADDR_WUNLOCK();
/* add new route */
error = rtinit(ifa, (int)RTM_ADD, RTF_HOST);
if (debug && error) {
log(LOG_DEBUG, SPP_FMT "sppp_set_ip_addr: rtinit ADD failed, error=%d",
SPP_ARGS(ifp), error);
}
ifa_free(ifa);
}
}
#endif
#ifdef INET6
/*
* Get both IPv6 addresses.
*/
static void
sppp_get_ip6_addrs(struct sppp *sp, struct in6_addr *src, struct in6_addr *dst,
struct in6_addr *srcmask)
{
struct ifnet *ifp = SP2IFP(sp);
struct ifaddr *ifa;
struct sockaddr_in6 *si, *sm;
struct in6_addr ssrc, ddst;
sm = NULL;
bzero(&ssrc, sizeof(ssrc));
bzero(&ddst, sizeof(ddst));
/*
* Pick the first link-local AF_INET6 address from the list,
* aliases don't make any sense on a p2p link anyway.
*/
si = NULL;
if_addr_rlock(ifp);
TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link)
if (ifa->ifa_addr->sa_family == AF_INET6) {
si = (struct sockaddr_in6 *)ifa->ifa_addr;
sm = (struct sockaddr_in6 *)ifa->ifa_netmask;
if (si && IN6_IS_ADDR_LINKLOCAL(&si->sin6_addr))
break;
}
if (ifa) {
if (si && !IN6_IS_ADDR_UNSPECIFIED(&si->sin6_addr)) {
bcopy(&si->sin6_addr, &ssrc, sizeof(ssrc));
if (srcmask) {
bcopy(&sm->sin6_addr, srcmask,
sizeof(*srcmask));
}
}
si = (struct sockaddr_in6 *)ifa->ifa_dstaddr;
if (si && !IN6_IS_ADDR_UNSPECIFIED(&si->sin6_addr))
bcopy(&si->sin6_addr, &ddst, sizeof(ddst));
}
if (dst)
bcopy(&ddst, dst, sizeof(*dst));
if (src)
bcopy(&ssrc, src, sizeof(*src));
if_addr_runlock(ifp);
}
#ifdef IPV6CP_MYIFID_DYN
/*
* Generate random ifid.
*/
static void
sppp_gen_ip6_addr(struct sppp *sp, struct in6_addr *addr)
{
/* TBD */
}
/*
* Set my IPv6 address.
*/
static void
sppp_set_ip6_addr(struct sppp *sp, const struct in6_addr *src)
{
STDDCL;
struct ifaddr *ifa;
struct sockaddr_in6 *sin6;
/*
* Pick the first link-local AF_INET6 address from the list,
* aliases don't make any sense on a p2p link anyway.
*/
sin6 = NULL;
if_addr_rlock(ifp);
TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family == AF_INET6) {
sin6 = (struct sockaddr_in6 *)ifa->ifa_addr;
if (sin6 && IN6_IS_ADDR_LINKLOCAL(&sin6->sin6_addr)) {
ifa_ref(ifa);
break;
}
}
}
if_addr_runlock(ifp);
if (ifa != NULL) {
int error;
struct sockaddr_in6 new_sin6 = *sin6;
bcopy(src, &new_sin6.sin6_addr, sizeof(new_sin6.sin6_addr));
error = in6_ifinit(ifp, ifatoia6(ifa), &new_sin6, 1);
if (debug && error) {
log(LOG_DEBUG, SPP_FMT "sppp_set_ip6_addr: in6_ifinit "
" failed, error=%d\n", SPP_ARGS(ifp), error);
}
ifa_free(ifa);
}
}
#endif
/*
* Suggest a candidate address to be used by peer.
*/
static void
sppp_suggest_ip6_addr(struct sppp *sp, struct in6_addr *suggest)
{
struct in6_addr myaddr;
struct timeval tv;
sppp_get_ip6_addrs(sp, &myaddr, 0, 0);
myaddr.s6_addr[8] &= ~0x02; /* u bit to "local" */
microtime(&tv);
if ((tv.tv_usec & 0xff) == 0 && (tv.tv_sec & 0xff) == 0) {
myaddr.s6_addr[14] ^= 0xff;
myaddr.s6_addr[15] ^= 0xff;
} else {
myaddr.s6_addr[14] ^= (tv.tv_usec & 0xff);
myaddr.s6_addr[15] ^= (tv.tv_sec & 0xff);
}
if (suggest)
bcopy(&myaddr, suggest, sizeof(myaddr));
}
#endif /*INET6*/
static int
sppp_params(struct sppp *sp, u_long cmd, void *data)
{
u_long subcmd;
struct ifreq *ifr = (struct ifreq *)data;
struct spppreq *spr;
int rv = 0;
if ((spr = malloc(sizeof(struct spppreq), M_TEMP, M_NOWAIT)) == 0)
return (EAGAIN);
/*
* ifr->ifr_data is supposed to point to a struct spppreq.
* Check the cmd word first before attempting to fetch all the
* data.
*/
rv = fueword(ifr->ifr_data, &subcmd);
if (rv == -1) {
rv = EFAULT;
goto quit;
}
if (copyin((caddr_t)ifr->ifr_data, spr, sizeof(struct spppreq)) != 0) {
rv = EFAULT;
goto quit;
}
switch (subcmd) {
case (u_long)SPPPIOGDEFS:
if (cmd != SIOCGIFGENERIC) {
rv = EINVAL;
break;
}
/*
* We copy over the entire current state, but clean
* out some of the stuff we don't wanna pass up.
* Remember, SIOCGIFGENERIC is unprotected, and can be
* called by any user. No need to ever get PAP or
* CHAP secrets back to userland anyway.
*/
spr->defs.pp_phase = sp->pp_phase;
spr->defs.enable_vj = (sp->confflags & CONF_ENABLE_VJ) != 0;
spr->defs.enable_ipv6 = (sp->confflags & CONF_ENABLE_IPV6) != 0;
spr->defs.lcp = sp->lcp;
spr->defs.ipcp = sp->ipcp;
spr->defs.ipv6cp = sp->ipv6cp;
spr->defs.myauth = sp->myauth;
spr->defs.hisauth = sp->hisauth;
bzero(spr->defs.myauth.secret, AUTHKEYLEN);
bzero(spr->defs.myauth.challenge, AUTHKEYLEN);
bzero(spr->defs.hisauth.secret, AUTHKEYLEN);
bzero(spr->defs.hisauth.challenge, AUTHKEYLEN);
/*
* Fixup the LCP timeout value to milliseconds so
* spppcontrol doesn't need to bother about the value
* of "hz". We do the reverse calculation below when
* setting it.
*/
spr->defs.lcp.timeout = sp->lcp.timeout * 1000 / hz;
rv = copyout(spr, (caddr_t)ifr->ifr_data,
sizeof(struct spppreq));
break;
case (u_long)SPPPIOSDEFS:
if (cmd != SIOCSIFGENERIC) {
rv = EINVAL;
break;
}
/*
* We have a very specific idea of which fields we
* allow being passed back from userland, so to not
* clobber our current state. For one, we only allow
* setting anything if LCP is in dead or establish
* phase. Once the authentication negotiations
* started, the authentication settings must not be
* changed again. (The administrator can force an
* ifconfig down in order to get LCP back into dead
* phase.)
*
* Also, we only allow for authentication parameters to be
* specified.
*
* XXX Should allow to set or clear pp_flags.
*
* Finally, if the respective authentication protocol to
* be used is set differently than 0, but the secret is
* passed as all zeros, we don't trash the existing secret.
* This allows an administrator to change the system name
* only without clobbering the secret (which he didn't get
* back in a previous SPPPIOGDEFS call). However, the
* secrets are cleared if the authentication protocol is
* reset to 0. */
if (sp->pp_phase != PHASE_DEAD &&
sp->pp_phase != PHASE_ESTABLISH) {
rv = EBUSY;
break;
}
if ((spr->defs.myauth.proto != 0 && spr->defs.myauth.proto != PPP_PAP &&
spr->defs.myauth.proto != PPP_CHAP) ||
(spr->defs.hisauth.proto != 0 && spr->defs.hisauth.proto != PPP_PAP &&
spr->defs.hisauth.proto != PPP_CHAP)) {
rv = EINVAL;
break;
}
if (spr->defs.myauth.proto == 0)
/* resetting myauth */
bzero(&sp->myauth, sizeof sp->myauth);
else {
/* setting/changing myauth */
sp->myauth.proto = spr->defs.myauth.proto;
bcopy(spr->defs.myauth.name, sp->myauth.name, AUTHNAMELEN);
if (spr->defs.myauth.secret[0] != '\0')
bcopy(spr->defs.myauth.secret, sp->myauth.secret,
AUTHKEYLEN);
}
if (spr->defs.hisauth.proto == 0)
/* resetting hisauth */
bzero(&sp->hisauth, sizeof sp->hisauth);
else {
/* setting/changing hisauth */
sp->hisauth.proto = spr->defs.hisauth.proto;
sp->hisauth.flags = spr->defs.hisauth.flags;
bcopy(spr->defs.hisauth.name, sp->hisauth.name, AUTHNAMELEN);
if (spr->defs.hisauth.secret[0] != '\0')
bcopy(spr->defs.hisauth.secret, sp->hisauth.secret,
AUTHKEYLEN);
}
/* set LCP restart timer timeout */
if (spr->defs.lcp.timeout != 0)
sp->lcp.timeout = spr->defs.lcp.timeout * hz / 1000;
/* set VJ enable and IPv6 disable flags */
#ifdef INET
if (spr->defs.enable_vj)
sp->confflags |= CONF_ENABLE_VJ;
else
sp->confflags &= ~CONF_ENABLE_VJ;
#endif
#ifdef INET6
if (spr->defs.enable_ipv6)
sp->confflags |= CONF_ENABLE_IPV6;
else
sp->confflags &= ~CONF_ENABLE_IPV6;
#endif
break;
default:
rv = EINVAL;
}
quit:
free(spr, M_TEMP);
return (rv);
}
static void
sppp_phase_network(struct sppp *sp)
{
STDDCL;
int i;
u_long mask;
sp->pp_phase = PHASE_NETWORK;
if (debug)
log(LOG_DEBUG, SPP_FMT "phase %s\n", SPP_ARGS(ifp),
sppp_phase_name(sp->pp_phase));
/* Notify NCPs now. */
for (i = 0; i < IDX_COUNT; i++)
if ((cps[i])->flags & CP_NCP)
(cps[i])->Open(sp);
/* Send Up events to all NCPs. */
for (i = 0, mask = 1; i < IDX_COUNT; i++, mask <<= 1)
if ((sp->lcp.protos & mask) && ((cps[i])->flags & CP_NCP))
(cps[i])->Up(sp);
/* if no NCP is starting, all this was in vain, close down */
sppp_lcp_check_and_close(sp);
}
static const char *
sppp_cp_type_name(u_char type)
{
static char buf[12];
switch (type) {
case CONF_REQ: return "conf-req";
case CONF_ACK: return "conf-ack";
case CONF_NAK: return "conf-nak";
case CONF_REJ: return "conf-rej";
case TERM_REQ: return "term-req";
case TERM_ACK: return "term-ack";
case CODE_REJ: return "code-rej";
case PROTO_REJ: return "proto-rej";
case ECHO_REQ: return "echo-req";
case ECHO_REPLY: return "echo-reply";
case DISC_REQ: return "discard-req";
}
snprintf (buf, sizeof(buf), "cp/0x%x", type);
return buf;
}
static const char *
sppp_auth_type_name(u_short proto, u_char type)
{
static char buf[12];
switch (proto) {
case PPP_CHAP:
switch (type) {
case CHAP_CHALLENGE: return "challenge";
case CHAP_RESPONSE: return "response";
case CHAP_SUCCESS: return "success";
case CHAP_FAILURE: return "failure";
}
case PPP_PAP:
switch (type) {
case PAP_REQ: return "req";
case PAP_ACK: return "ack";
case PAP_NAK: return "nak";
}
}
snprintf (buf, sizeof(buf), "auth/0x%x", type);
return buf;
}
static const char *
sppp_lcp_opt_name(u_char opt)
{
static char buf[12];
switch (opt) {
case LCP_OPT_MRU: return "mru";
case LCP_OPT_ASYNC_MAP: return "async-map";
case LCP_OPT_AUTH_PROTO: return "auth-proto";
case LCP_OPT_QUAL_PROTO: return "qual-proto";
case LCP_OPT_MAGIC: return "magic";
case LCP_OPT_PROTO_COMP: return "proto-comp";
case LCP_OPT_ADDR_COMP: return "addr-comp";
}
snprintf (buf, sizeof(buf), "lcp/0x%x", opt);
return buf;
}
#ifdef INET
static const char *
sppp_ipcp_opt_name(u_char opt)
{
static char buf[12];
switch (opt) {
case IPCP_OPT_ADDRESSES: return "addresses";
case IPCP_OPT_COMPRESSION: return "compression";
case IPCP_OPT_ADDRESS: return "address";
}
snprintf (buf, sizeof(buf), "ipcp/0x%x", opt);
return buf;
}
#endif
#ifdef INET6
static const char *
sppp_ipv6cp_opt_name(u_char opt)
{
static char buf[12];
switch (opt) {
case IPV6CP_OPT_IFID: return "ifid";
case IPV6CP_OPT_COMPRESSION: return "compression";
}
sprintf (buf, "0x%x", opt);
return buf;
}
#endif
static const char *
sppp_state_name(int state)
{
switch (state) {
case STATE_INITIAL: return "initial";
case STATE_STARTING: return "starting";
case STATE_CLOSED: return "closed";
case STATE_STOPPED: return "stopped";
case STATE_CLOSING: return "closing";
case STATE_STOPPING: return "stopping";
case STATE_REQ_SENT: return "req-sent";
case STATE_ACK_RCVD: return "ack-rcvd";
case STATE_ACK_SENT: return "ack-sent";
case STATE_OPENED: return "opened";
}
return "illegal";
}
static const char *
sppp_phase_name(enum ppp_phase phase)
{
switch (phase) {
case PHASE_DEAD: return "dead";
case PHASE_ESTABLISH: return "establish";
case PHASE_TERMINATE: return "terminate";
case PHASE_AUTHENTICATE: return "authenticate";
case PHASE_NETWORK: return "network";
}
return "illegal";
}
static const char *
sppp_proto_name(u_short proto)
{
static char buf[12];
switch (proto) {
case PPP_LCP: return "lcp";
case PPP_IPCP: return "ipcp";
case PPP_PAP: return "pap";
case PPP_CHAP: return "chap";
case PPP_IPV6CP: return "ipv6cp";
}
snprintf(buf, sizeof(buf), "proto/0x%x", (unsigned)proto);
return buf;
}
static void
sppp_print_bytes(const u_char *p, u_short len)
{
if (len)
log(-1, " %*D", len, p, "-");
}
static void
sppp_print_string(const char *p, u_short len)
{
u_char c;
while (len-- > 0) {
c = *p++;
/*
* Print only ASCII chars directly. RFC 1994 recommends
* using only them, but we don't rely on it. */
if (c < ' ' || c > '~')
log(-1, "\\x%x", c);
else
log(-1, "%c", c);
}
}
#ifdef INET
static const char *
sppp_dotted_quad(u_long addr)
{
static char s[16];
sprintf(s, "%d.%d.%d.%d",
(int)((addr >> 24) & 0xff),
(int)((addr >> 16) & 0xff),
(int)((addr >> 8) & 0xff),
(int)(addr & 0xff));
return s;
}
#endif
static int
sppp_strnlen(u_char *p, int max)
{
int len;
for (len = 0; len < max && *p; ++p)
++len;
return len;
}
/* a dummy, used to drop uninteresting events */
static void
sppp_null(struct sppp *unused)
{
/* do just nothing */
}
Index: head/sys/net80211/ieee80211_ht.c
===================================================================
--- head/sys/net80211/ieee80211_ht.c (revision 283290)
+++ head/sys/net80211/ieee80211_ht.c (revision 283291)
@@ -1,2902 +1,2902 @@
/*-
* Copyright (c) 2007-2008 Sam Leffler, Errno Consulting
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#ifdef __FreeBSD__
__FBSDID("$FreeBSD$");
#endif
/*
* IEEE 802.11n protocol support.
*/
#include "opt_inet.h"
#include "opt_wlan.h"
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/endian.h>
#include <sys/socket.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_media.h>
#include <net/ethernet.h>
#include <net80211/ieee80211_var.h>
#include <net80211/ieee80211_action.h>
#include <net80211/ieee80211_input.h>
/* define here, used throughout file */
#define MS(_v, _f) (((_v) & _f) >> _f##_S)
#define SM(_v, _f) (((_v) << _f##_S) & _f)
const struct ieee80211_mcs_rates ieee80211_htrates[IEEE80211_HTRATE_MAXSIZE] = {
{ 13, 14, 27, 30 }, /* MCS 0 */
{ 26, 29, 54, 60 }, /* MCS 1 */
{ 39, 43, 81, 90 }, /* MCS 2 */
{ 52, 58, 108, 120 }, /* MCS 3 */
{ 78, 87, 162, 180 }, /* MCS 4 */
{ 104, 116, 216, 240 }, /* MCS 5 */
{ 117, 130, 243, 270 }, /* MCS 6 */
{ 130, 144, 270, 300 }, /* MCS 7 */
{ 26, 29, 54, 60 }, /* MCS 8 */
{ 52, 58, 108, 120 }, /* MCS 9 */
{ 78, 87, 162, 180 }, /* MCS 10 */
{ 104, 116, 216, 240 }, /* MCS 11 */
{ 156, 173, 324, 360 }, /* MCS 12 */
{ 208, 231, 432, 480 }, /* MCS 13 */
{ 234, 260, 486, 540 }, /* MCS 14 */
{ 260, 289, 540, 600 }, /* MCS 15 */
{ 39, 43, 81, 90 }, /* MCS 16 */
{ 78, 87, 162, 180 }, /* MCS 17 */
{ 117, 130, 243, 270 }, /* MCS 18 */
{ 156, 173, 324, 360 }, /* MCS 19 */
{ 234, 260, 486, 540 }, /* MCS 20 */
{ 312, 347, 648, 720 }, /* MCS 21 */
{ 351, 390, 729, 810 }, /* MCS 22 */
{ 390, 433, 810, 900 }, /* MCS 23 */
{ 52, 58, 108, 120 }, /* MCS 24 */
{ 104, 116, 216, 240 }, /* MCS 25 */
{ 156, 173, 324, 360 }, /* MCS 26 */
{ 208, 231, 432, 480 }, /* MCS 27 */
{ 312, 347, 648, 720 }, /* MCS 28 */
{ 416, 462, 864, 960 }, /* MCS 29 */
{ 468, 520, 972, 1080 }, /* MCS 30 */
{ 520, 578, 1080, 1200 }, /* MCS 31 */
{ 0, 0, 12, 13 }, /* MCS 32 */
{ 78, 87, 162, 180 }, /* MCS 33 */
{ 104, 116, 216, 240 }, /* MCS 34 */
{ 130, 144, 270, 300 }, /* MCS 35 */
{ 117, 130, 243, 270 }, /* MCS 36 */
{ 156, 173, 324, 360 }, /* MCS 37 */
{ 195, 217, 405, 450 }, /* MCS 38 */
{ 104, 116, 216, 240 }, /* MCS 39 */
{ 130, 144, 270, 300 }, /* MCS 40 */
{ 130, 144, 270, 300 }, /* MCS 41 */
{ 156, 173, 324, 360 }, /* MCS 42 */
{ 182, 202, 378, 420 }, /* MCS 43 */
{ 182, 202, 378, 420 }, /* MCS 44 */
{ 208, 231, 432, 480 }, /* MCS 45 */
{ 156, 173, 324, 360 }, /* MCS 46 */
{ 195, 217, 405, 450 }, /* MCS 47 */
{ 195, 217, 405, 450 }, /* MCS 48 */
{ 234, 260, 486, 540 }, /* MCS 49 */
{ 273, 303, 567, 630 }, /* MCS 50 */
{ 273, 303, 567, 630 }, /* MCS 51 */
{ 312, 347, 648, 720 }, /* MCS 52 */
{ 130, 144, 270, 300 }, /* MCS 53 */
{ 156, 173, 324, 360 }, /* MCS 54 */
{ 182, 202, 378, 420 }, /* MCS 55 */
{ 156, 173, 324, 360 }, /* MCS 56 */
{ 182, 202, 378, 420 }, /* MCS 57 */
{ 208, 231, 432, 480 }, /* MCS 58 */
{ 234, 260, 486, 540 }, /* MCS 59 */
{ 208, 231, 432, 480 }, /* MCS 60 */
{ 234, 260, 486, 540 }, /* MCS 61 */
{ 260, 289, 540, 600 }, /* MCS 62 */
{ 260, 289, 540, 600 }, /* MCS 63 */
{ 286, 318, 594, 660 }, /* MCS 64 */
{ 195, 217, 405, 450 }, /* MCS 65 */
{ 234, 260, 486, 540 }, /* MCS 66 */
{ 273, 303, 567, 630 }, /* MCS 67 */
{ 234, 260, 486, 540 }, /* MCS 68 */
{ 273, 303, 567, 630 }, /* MCS 69 */
{ 312, 347, 648, 720 }, /* MCS 70 */
{ 351, 390, 729, 810 }, /* MCS 71 */
{ 312, 347, 648, 720 }, /* MCS 72 */
{ 351, 390, 729, 810 }, /* MCS 73 */
{ 390, 433, 810, 900 }, /* MCS 74 */
{ 390, 433, 810, 900 }, /* MCS 75 */
{ 429, 477, 891, 990 }, /* MCS 76 */
};
#ifdef IEEE80211_AMPDU_AGE
static int ieee80211_ampdu_age = -1; /* threshold for ampdu reorder q (ms) */
SYSCTL_PROC(_net_wlan, OID_AUTO, ampdu_age, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_ampdu_age, 0, ieee80211_sysctl_msecs_ticks, "I",
"AMPDU max reorder age (ms)");
#endif
static int ieee80211_recv_bar_ena = 1;
SYSCTL_INT(_net_wlan, OID_AUTO, recv_bar, CTLFLAG_RW, &ieee80211_recv_bar_ena,
0, "BAR frame processing (ena/dis)");
static int ieee80211_addba_timeout = -1;/* timeout for ADDBA response */
SYSCTL_PROC(_net_wlan, OID_AUTO, addba_timeout, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_addba_timeout, 0, ieee80211_sysctl_msecs_ticks, "I",
"ADDBA request timeout (ms)");
static int ieee80211_addba_backoff = -1;/* backoff after max ADDBA requests */
SYSCTL_PROC(_net_wlan, OID_AUTO, addba_backoff, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_addba_backoff, 0, ieee80211_sysctl_msecs_ticks, "I",
"ADDBA request backoff (ms)");
static int ieee80211_addba_maxtries = 3;/* max ADDBA requests before backoff */
SYSCTL_INT(_net_wlan, OID_AUTO, addba_maxtries, CTLFLAG_RW,
&ieee80211_addba_maxtries, 0, "max ADDBA requests sent before backoff");
static int ieee80211_bar_timeout = -1; /* timeout waiting for BAR response */
static int ieee80211_bar_maxtries = 50;/* max BAR requests before DELBA */
static ieee80211_recv_action_func ht_recv_action_ba_addba_request;
static ieee80211_recv_action_func ht_recv_action_ba_addba_response;
static ieee80211_recv_action_func ht_recv_action_ba_delba;
static ieee80211_recv_action_func ht_recv_action_ht_mimopwrsave;
static ieee80211_recv_action_func ht_recv_action_ht_txchwidth;
static ieee80211_send_action_func ht_send_action_ba_addba;
static ieee80211_send_action_func ht_send_action_ba_delba;
static ieee80211_send_action_func ht_send_action_ht_txchwidth;
static void
ieee80211_ht_init(void)
{
/*
* Setup HT parameters that depends on the clock frequency.
*/
#ifdef IEEE80211_AMPDU_AGE
ieee80211_ampdu_age = msecs_to_ticks(500);
#endif
ieee80211_addba_timeout = msecs_to_ticks(250);
ieee80211_addba_backoff = msecs_to_ticks(10*1000);
ieee80211_bar_timeout = msecs_to_ticks(250);
/*
* Register action frame handlers.
*/
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_BA,
IEEE80211_ACTION_BA_ADDBA_REQUEST, ht_recv_action_ba_addba_request);
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_BA,
IEEE80211_ACTION_BA_ADDBA_RESPONSE, ht_recv_action_ba_addba_response);
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_BA,
IEEE80211_ACTION_BA_DELBA, ht_recv_action_ba_delba);
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_HT,
IEEE80211_ACTION_HT_MIMOPWRSAVE, ht_recv_action_ht_mimopwrsave);
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_HT,
IEEE80211_ACTION_HT_TXCHWIDTH, ht_recv_action_ht_txchwidth);
ieee80211_send_action_register(IEEE80211_ACTION_CAT_BA,
IEEE80211_ACTION_BA_ADDBA_REQUEST, ht_send_action_ba_addba);
ieee80211_send_action_register(IEEE80211_ACTION_CAT_BA,
IEEE80211_ACTION_BA_ADDBA_RESPONSE, ht_send_action_ba_addba);
ieee80211_send_action_register(IEEE80211_ACTION_CAT_BA,
IEEE80211_ACTION_BA_DELBA, ht_send_action_ba_delba);
ieee80211_send_action_register(IEEE80211_ACTION_CAT_HT,
IEEE80211_ACTION_HT_TXCHWIDTH, ht_send_action_ht_txchwidth);
}
SYSINIT(wlan_ht, SI_SUB_DRIVERS, SI_ORDER_FIRST, ieee80211_ht_init, NULL);
static int ieee80211_ampdu_enable(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap);
static int ieee80211_addba_request(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap,
int dialogtoken, int baparamset, int batimeout);
static int ieee80211_addba_response(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap,
int code, int baparamset, int batimeout);
static void ieee80211_addba_stop(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap);
static void null_addba_response_timeout(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap);
static void ieee80211_bar_response(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap, int status);
static void ampdu_tx_stop(struct ieee80211_tx_ampdu *tap);
static void bar_stop_timer(struct ieee80211_tx_ampdu *tap);
static int ampdu_rx_start(struct ieee80211_node *, struct ieee80211_rx_ampdu *,
int baparamset, int batimeout, int baseqctl);
static void ampdu_rx_stop(struct ieee80211_node *, struct ieee80211_rx_ampdu *);
void
ieee80211_ht_attach(struct ieee80211com *ic)
{
/* setup default aggregation policy */
ic->ic_recv_action = ieee80211_recv_action;
ic->ic_send_action = ieee80211_send_action;
ic->ic_ampdu_enable = ieee80211_ampdu_enable;
ic->ic_addba_request = ieee80211_addba_request;
ic->ic_addba_response = ieee80211_addba_response;
ic->ic_addba_response_timeout = null_addba_response_timeout;
ic->ic_addba_stop = ieee80211_addba_stop;
ic->ic_bar_response = ieee80211_bar_response;
ic->ic_ampdu_rx_start = ampdu_rx_start;
ic->ic_ampdu_rx_stop = ampdu_rx_stop;
ic->ic_htprotmode = IEEE80211_PROT_RTSCTS;
ic->ic_curhtprotmode = IEEE80211_HTINFO_OPMODE_PURE;
}
void
ieee80211_ht_detach(struct ieee80211com *ic)
{
}
void
ieee80211_ht_vattach(struct ieee80211vap *vap)
{
/* driver can override defaults */
vap->iv_ampdu_rxmax = IEEE80211_HTCAP_MAXRXAMPDU_8K;
vap->iv_ampdu_density = IEEE80211_HTCAP_MPDUDENSITY_NA;
vap->iv_ampdu_limit = vap->iv_ampdu_rxmax;
vap->iv_amsdu_limit = vap->iv_htcaps & IEEE80211_HTCAP_MAXAMSDU;
/* tx aggregation traffic thresholds */
vap->iv_ampdu_mintraffic[WME_AC_BK] = 128;
vap->iv_ampdu_mintraffic[WME_AC_BE] = 64;
vap->iv_ampdu_mintraffic[WME_AC_VO] = 32;
vap->iv_ampdu_mintraffic[WME_AC_VI] = 32;
if (vap->iv_htcaps & IEEE80211_HTC_HT) {
/*
* Device is HT capable; enable all HT-related
* facilities by default.
* XXX these choices may be too aggressive.
*/
vap->iv_flags_ht |= IEEE80211_FHT_HT
| IEEE80211_FHT_HTCOMPAT
;
if (vap->iv_htcaps & IEEE80211_HTCAP_SHORTGI20)
vap->iv_flags_ht |= IEEE80211_FHT_SHORTGI20;
/* XXX infer from channel list? */
if (vap->iv_htcaps & IEEE80211_HTCAP_CHWIDTH40) {
vap->iv_flags_ht |= IEEE80211_FHT_USEHT40;
if (vap->iv_htcaps & IEEE80211_HTCAP_SHORTGI40)
vap->iv_flags_ht |= IEEE80211_FHT_SHORTGI40;
}
/* enable RIFS if capable */
if (vap->iv_htcaps & IEEE80211_HTC_RIFS)
vap->iv_flags_ht |= IEEE80211_FHT_RIFS;
/* NB: A-MPDU and A-MSDU rx are mandated, these are tx only */
vap->iv_flags_ht |= IEEE80211_FHT_AMPDU_RX;
if (vap->iv_htcaps & IEEE80211_HTC_AMPDU)
vap->iv_flags_ht |= IEEE80211_FHT_AMPDU_TX;
vap->iv_flags_ht |= IEEE80211_FHT_AMSDU_RX;
if (vap->iv_htcaps & IEEE80211_HTC_AMSDU)
vap->iv_flags_ht |= IEEE80211_FHT_AMSDU_TX;
}
/* NB: disable default legacy WDS, too many issues right now */
if (vap->iv_flags_ext & IEEE80211_FEXT_WDSLEGACY)
vap->iv_flags_ht &= ~IEEE80211_FHT_HT;
}
void
ieee80211_ht_vdetach(struct ieee80211vap *vap)
{
}
static int
ht_getrate(struct ieee80211com *ic, int index, enum ieee80211_phymode mode,
int ratetype)
{
int mword, rate;
mword = ieee80211_rate2media(ic, index | IEEE80211_RATE_MCS, mode);
if (IFM_SUBTYPE(mword) != IFM_IEEE80211_MCS)
return (0);
switch (ratetype) {
case 0:
rate = ieee80211_htrates[index].ht20_rate_800ns;
break;
case 1:
rate = ieee80211_htrates[index].ht20_rate_400ns;
break;
case 2:
rate = ieee80211_htrates[index].ht40_rate_800ns;
break;
default:
rate = ieee80211_htrates[index].ht40_rate_400ns;
break;
}
return (rate);
}
static struct printranges {
int minmcs;
int maxmcs;
int txstream;
int ratetype;
int htcapflags;
} ranges[] = {
{ 0, 7, 1, 0, 0 },
{ 8, 15, 2, 0, 0 },
{ 16, 23, 3, 0, 0 },
{ 24, 31, 4, 0, 0 },
{ 32, 0, 1, 2, IEEE80211_HTC_TXMCS32 },
{ 33, 38, 2, 0, IEEE80211_HTC_TXUNEQUAL },
{ 39, 52, 3, 0, IEEE80211_HTC_TXUNEQUAL },
{ 53, 76, 4, 0, IEEE80211_HTC_TXUNEQUAL },
{ 0, 0, 0, 0, 0 },
};
static void
ht_rateprint(struct ieee80211com *ic, enum ieee80211_phymode mode, int ratetype)
{
struct ifnet *ifp = ic->ic_ifp;
int minrate, maxrate;
struct printranges *range;
for (range = ranges; range->txstream != 0; range++) {
if (ic->ic_txstream < range->txstream)
continue;
if (range->htcapflags &&
(ic->ic_htcaps & range->htcapflags) == 0)
continue;
if (ratetype < range->ratetype)
continue;
minrate = ht_getrate(ic, range->minmcs, mode, ratetype);
maxrate = ht_getrate(ic, range->maxmcs, mode, ratetype);
if (range->maxmcs) {
if_printf(ifp, "MCS %d-%d: %d%sMbps - %d%sMbps\n",
range->minmcs, range->maxmcs,
minrate/2, ((minrate & 0x1) != 0 ? ".5" : ""),
maxrate/2, ((maxrate & 0x1) != 0 ? ".5" : ""));
} else {
if_printf(ifp, "MCS %d: %d%sMbps\n", range->minmcs,
minrate/2, ((minrate & 0x1) != 0 ? ".5" : ""));
}
}
}
static void
ht_announce(struct ieee80211com *ic, enum ieee80211_phymode mode)
{
struct ifnet *ifp = ic->ic_ifp;
const char *modestr = ieee80211_phymode_name[mode];
if_printf(ifp, "%s MCS 20MHz\n", modestr);
ht_rateprint(ic, mode, 0);
if (ic->ic_htcaps & IEEE80211_HTCAP_SHORTGI20) {
if_printf(ifp, "%s MCS 20MHz SGI\n", modestr);
ht_rateprint(ic, mode, 1);
}
if (ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40) {
if_printf(ifp, "%s MCS 40MHz:\n", modestr);
ht_rateprint(ic, mode, 2);
}
if ((ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40) &&
(ic->ic_htcaps & IEEE80211_HTCAP_SHORTGI40)) {
if_printf(ifp, "%s MCS 40MHz SGI:\n", modestr);
ht_rateprint(ic, mode, 3);
}
}
void
ieee80211_ht_announce(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
if (isset(ic->ic_modecaps, IEEE80211_MODE_11NA) ||
isset(ic->ic_modecaps, IEEE80211_MODE_11NG))
if_printf(ifp, "%dT%dR\n", ic->ic_txstream, ic->ic_rxstream);
if (isset(ic->ic_modecaps, IEEE80211_MODE_11NA))
ht_announce(ic, IEEE80211_MODE_11NA);
if (isset(ic->ic_modecaps, IEEE80211_MODE_11NG))
ht_announce(ic, IEEE80211_MODE_11NG);
}
static struct ieee80211_htrateset htrateset;
const struct ieee80211_htrateset *
ieee80211_get_suphtrates(struct ieee80211com *ic,
const struct ieee80211_channel *c)
{
#define ADDRATE(x) do { \
htrateset.rs_rates[htrateset.rs_nrates] = x; \
htrateset.rs_nrates++; \
} while (0)
int i;
memset(&htrateset, 0, sizeof(struct ieee80211_htrateset));
for (i = 0; i < ic->ic_txstream * 8; i++)
ADDRATE(i);
if ((ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40) &&
(ic->ic_htcaps & IEEE80211_HTC_TXMCS32))
ADDRATE(32);
if (ic->ic_htcaps & IEEE80211_HTC_TXUNEQUAL) {
if (ic->ic_txstream >= 2) {
for (i = 33; i <= 38; i++)
ADDRATE(i);
}
if (ic->ic_txstream >= 3) {
for (i = 39; i <= 52; i++)
ADDRATE(i);
}
if (ic->ic_txstream == 4) {
for (i = 53; i <= 76; i++)
ADDRATE(i);
}
}
return &htrateset;
#undef ADDRATE
}
/*
* Receive processing.
*/
/*
* Decap the encapsulated A-MSDU frames and dispatch all but
* the last for delivery. The last frame is returned for
* delivery via the normal path.
*/
struct mbuf *
ieee80211_decap_amsdu(struct ieee80211_node *ni, struct mbuf *m)
{
struct ieee80211vap *vap = ni->ni_vap;
int framelen;
struct mbuf *n;
/* discard 802.3 header inserted by ieee80211_decap */
m_adj(m, sizeof(struct ether_header));
vap->iv_stats.is_amsdu_decap++;
for (;;) {
/*
* Decap the first frame, bust it apart from the
* remainder and deliver. We leave the last frame
* delivery to the caller (for consistency with other
* code paths, could also do it here).
*/
m = ieee80211_decap1(m, &framelen);
if (m == NULL) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
ni->ni_macaddr, "a-msdu", "%s", "decap failed");
vap->iv_stats.is_amsdu_tooshort++;
return NULL;
}
if (m->m_pkthdr.len == framelen)
break;
n = m_split(m, framelen, M_NOWAIT);
if (n == NULL) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
ni->ni_macaddr, "a-msdu",
"%s", "unable to split encapsulated frames");
vap->iv_stats.is_amsdu_split++;
m_freem(m); /* NB: must reclaim */
return NULL;
}
vap->iv_deliver_data(vap, ni, m);
/*
* Remove frame contents; each intermediate frame
* is required to be aligned to a 4-byte boundary.
*/
m = n;
m_adj(m, roundup2(framelen, 4) - framelen); /* padding */
}
return m; /* last delivered by caller */
}
/*
* Purge all frames in the A-MPDU re-order queue.
*/
static void
ampdu_rx_purge(struct ieee80211_rx_ampdu *rap)
{
struct mbuf *m;
int i;
for (i = 0; i < rap->rxa_wnd; i++) {
m = rap->rxa_m[i];
if (m != NULL) {
rap->rxa_m[i] = NULL;
rap->rxa_qbytes -= m->m_pkthdr.len;
m_freem(m);
if (--rap->rxa_qframes == 0)
break;
}
}
KASSERT(rap->rxa_qbytes == 0 && rap->rxa_qframes == 0,
("lost %u data, %u frames on ampdu rx q",
rap->rxa_qbytes, rap->rxa_qframes));
}
/*
* Start A-MPDU rx/re-order processing for the specified TID.
*/
static int
ampdu_rx_start(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap,
int baparamset, int batimeout, int baseqctl)
{
int bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ);
if (rap->rxa_flags & IEEE80211_AGGR_RUNNING) {
/*
* AMPDU previously setup and not terminated with a DELBA,
* flush the reorder q's in case anything remains.
*/
ampdu_rx_purge(rap);
}
memset(rap, 0, sizeof(*rap));
rap->rxa_wnd = (bufsiz == 0) ?
IEEE80211_AGGR_BAWMAX : min(bufsiz, IEEE80211_AGGR_BAWMAX);
rap->rxa_start = MS(baseqctl, IEEE80211_BASEQ_START);
rap->rxa_flags |= IEEE80211_AGGR_RUNNING | IEEE80211_AGGR_XCHGPEND;
return 0;
}
/*
* Stop A-MPDU rx processing for the specified TID.
*/
static void
ampdu_rx_stop(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap)
{
ampdu_rx_purge(rap);
rap->rxa_flags &= ~(IEEE80211_AGGR_RUNNING | IEEE80211_AGGR_XCHGPEND);
}
/*
* Dispatch a frame from the A-MPDU reorder queue. The
* frame is fed back into ieee80211_input marked with an
* M_AMPDU_MPDU flag so it doesn't come back to us (it also
* permits ieee80211_input to optimize re-processing).
*/
static __inline void
ampdu_dispatch(struct ieee80211_node *ni, struct mbuf *m)
{
m->m_flags |= M_AMPDU_MPDU; /* bypass normal processing */
/* NB: rssi and noise are ignored w/ M_AMPDU_MPDU set */
(void) ieee80211_input(ni, m, 0, 0);
}
/*
* Dispatch as many frames as possible from the re-order queue.
* Frames will always be "at the front"; we process all frames
* up to the first empty slot in the window. On completion we
* cleanup state if there are still pending frames in the current
* BA window. We assume the frame at slot 0 is already handled
* by the caller; we always start at slot 1.
*/
static void
ampdu_rx_dispatch(struct ieee80211_rx_ampdu *rap, struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
struct mbuf *m;
int i;
/* flush run of frames */
for (i = 1; i < rap->rxa_wnd; i++) {
m = rap->rxa_m[i];
if (m == NULL)
break;
rap->rxa_m[i] = NULL;
rap->rxa_qbytes -= m->m_pkthdr.len;
rap->rxa_qframes--;
ampdu_dispatch(ni, m);
}
/*
* If frames remain, copy the mbuf pointers down so
* they correspond to the offsets in the new window.
*/
if (rap->rxa_qframes != 0) {
int n = rap->rxa_qframes, j;
for (j = i+1; j < rap->rxa_wnd; j++) {
if (rap->rxa_m[j] != NULL) {
rap->rxa_m[j-i] = rap->rxa_m[j];
rap->rxa_m[j] = NULL;
if (--n == 0)
break;
}
}
KASSERT(n == 0, ("lost %d frames", n));
vap->iv_stats.is_ampdu_rx_copy += rap->rxa_qframes;
}
/*
* Adjust the start of the BA window to
* reflect the frames just dispatched.
*/
rap->rxa_start = IEEE80211_SEQ_ADD(rap->rxa_start, i);
vap->iv_stats.is_ampdu_rx_oor += i;
}
#ifdef IEEE80211_AMPDU_AGE
/*
* Dispatch all frames in the A-MPDU re-order queue.
*/
static void
ampdu_rx_flush(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap)
{
struct ieee80211vap *vap = ni->ni_vap;
struct mbuf *m;
int i;
for (i = 0; i < rap->rxa_wnd; i++) {
m = rap->rxa_m[i];
if (m == NULL)
continue;
rap->rxa_m[i] = NULL;
rap->rxa_qbytes -= m->m_pkthdr.len;
rap->rxa_qframes--;
vap->iv_stats.is_ampdu_rx_oor++;
ampdu_dispatch(ni, m);
if (rap->rxa_qframes == 0)
break;
}
}
#endif /* IEEE80211_AMPDU_AGE */
/*
* Dispatch all frames in the A-MPDU re-order queue
* preceding the specified sequence number. This logic
* handles window moves due to a received MSDU or BAR.
*/
static void
ampdu_rx_flush_upto(struct ieee80211_node *ni,
struct ieee80211_rx_ampdu *rap, ieee80211_seq winstart)
{
struct ieee80211vap *vap = ni->ni_vap;
struct mbuf *m;
ieee80211_seq seqno;
int i;
/*
* Flush any complete MSDU's with a sequence number lower
* than winstart. Gaps may exist. Note that we may actually
* dispatch frames past winstart if a run continues; this is
* an optimization that avoids having to do a separate pass
* to dispatch frames after moving the BA window start.
*/
seqno = rap->rxa_start;
for (i = 0; i < rap->rxa_wnd; i++) {
m = rap->rxa_m[i];
if (m != NULL) {
rap->rxa_m[i] = NULL;
rap->rxa_qbytes -= m->m_pkthdr.len;
rap->rxa_qframes--;
vap->iv_stats.is_ampdu_rx_oor++;
ampdu_dispatch(ni, m);
} else {
if (!IEEE80211_SEQ_BA_BEFORE(seqno, winstart))
break;
}
seqno = IEEE80211_SEQ_INC(seqno);
}
/*
* If frames remain, copy the mbuf pointers down so
* they correspond to the offsets in the new window.
*/
if (rap->rxa_qframes != 0) {
int n = rap->rxa_qframes, j;
/* NB: this loop assumes i > 0 and/or rxa_m[0] is NULL */
KASSERT(rap->rxa_m[0] == NULL,
("%s: BA window slot 0 occupied", __func__));
for (j = i+1; j < rap->rxa_wnd; j++) {
if (rap->rxa_m[j] != NULL) {
rap->rxa_m[j-i] = rap->rxa_m[j];
rap->rxa_m[j] = NULL;
if (--n == 0)
break;
}
}
KASSERT(n == 0, ("%s: lost %d frames, qframes %d off %d "
"BA win <%d:%d> winstart %d",
__func__, n, rap->rxa_qframes, i, rap->rxa_start,
IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1),
winstart));
vap->iv_stats.is_ampdu_rx_copy += rap->rxa_qframes;
}
/*
* Move the start of the BA window; we use the
* sequence number of the last MSDU that was
* passed up the stack+1 or winstart if stopped on
* a gap in the reorder buffer.
*/
rap->rxa_start = seqno;
}
/*
* Process a received QoS data frame for an HT station. Handle
* A-MPDU reordering: if this frame is received out of order
* and falls within the BA window hold onto it. Otherwise if
* this frame completes a run, flush any pending frames. We
* return 1 if the frame is consumed. A 0 is returned if
* the frame should be processed normally by the caller.
*/
int
ieee80211_ampdu_reorder(struct ieee80211_node *ni, struct mbuf *m)
{
#define IEEE80211_FC0_QOSDATA \
(IEEE80211_FC0_TYPE_DATA|IEEE80211_FC0_SUBTYPE_QOS|IEEE80211_FC0_VERSION_0)
#define PROCESS 0 /* caller should process frame */
#define CONSUMED 1 /* frame consumed, caller does nothing */
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_qosframe *wh;
struct ieee80211_rx_ampdu *rap;
ieee80211_seq rxseq;
uint8_t tid;
int off;
KASSERT((m->m_flags & (M_AMPDU | M_AMPDU_MPDU)) == M_AMPDU,
("!a-mpdu or already re-ordered, flags 0x%x", m->m_flags));
KASSERT(ni->ni_flags & IEEE80211_NODE_HT, ("not an HT sta"));
/* NB: m_len known to be sufficient */
wh = mtod(m, struct ieee80211_qosframe *);
if (wh->i_fc[0] != IEEE80211_FC0_QOSDATA) {
/*
* Not QoS data, shouldn't get here but just
* return it to the caller for processing.
*/
return PROCESS;
}
if (IEEE80211_IS_DSTODS(wh))
tid = ((struct ieee80211_qosframe_addr4 *)wh)->i_qos[0];
else
tid = wh->i_qos[0];
tid &= IEEE80211_QOS_TID;
rap = &ni->ni_rx_ampdu[tid];
if ((rap->rxa_flags & IEEE80211_AGGR_XCHGPEND) == 0) {
/*
* No ADDBA request yet, don't touch.
*/
return PROCESS;
}
rxseq = le16toh(*(uint16_t *)wh->i_seq);
if ((rxseq & IEEE80211_SEQ_FRAG_MASK) != 0) {
/*
* Fragments are not allowed; toss.
*/
IEEE80211_DISCARD_MAC(vap,
IEEE80211_MSG_INPUT | IEEE80211_MSG_11N, ni->ni_macaddr,
"A-MPDU", "fragment, rxseq 0x%x tid %u%s", rxseq, tid,
wh->i_fc[1] & IEEE80211_FC1_RETRY ? " (retransmit)" : "");
vap->iv_stats.is_ampdu_rx_drop++;
IEEE80211_NODE_STAT(ni, rx_drop);
m_freem(m);
return CONSUMED;
}
rxseq >>= IEEE80211_SEQ_SEQ_SHIFT;
rap->rxa_nframes++;
again:
if (rxseq == rap->rxa_start) {
/*
* First frame in window.
*/
if (rap->rxa_qframes != 0) {
/*
* Dispatch as many packets as we can.
*/
KASSERT(rap->rxa_m[0] == NULL, ("unexpected dup"));
ampdu_dispatch(ni, m);
ampdu_rx_dispatch(rap, ni);
return CONSUMED;
} else {
/*
* In order; advance window and notify
* caller to dispatch directly.
*/
rap->rxa_start = IEEE80211_SEQ_INC(rxseq);
return PROCESS;
}
}
/*
* Frame is out of order; store if in the BA window.
*/
/* calculate offset in BA window */
off = IEEE80211_SEQ_SUB(rxseq, rap->rxa_start);
if (off < rap->rxa_wnd) {
/*
* Common case (hopefully): in the BA window.
* Sec 9.10.7.6.2 a) (p.137)
*/
#ifdef IEEE80211_AMPDU_AGE
/*
* Check for frames sitting too long in the reorder queue.
* This should only ever happen if frames are not delivered
* without the sender otherwise notifying us (e.g. with a
* BAR to move the window). Typically this happens because
* of vendor bugs that cause the sequence number to jump.
* When this happens we get a gap in the reorder queue that
* leaves frame sitting on the queue until they get pushed
* out due to window moves. When the vendor does not send
* BAR this move only happens due to explicit packet sends
*
* NB: we only track the time of the oldest frame in the
* reorder q; this means that if we flush we might push
* frames that still "new"; if this happens then subsequent
* frames will result in BA window moves which cost something
* but is still better than a big throughput dip.
*/
if (rap->rxa_qframes != 0) {
/* XXX honor batimeout? */
if (ticks - rap->rxa_age > ieee80211_ampdu_age) {
/*
* Too long since we received the first
* frame; flush the reorder buffer.
*/
if (rap->rxa_qframes != 0) {
vap->iv_stats.is_ampdu_rx_age +=
rap->rxa_qframes;
ampdu_rx_flush(ni, rap);
}
rap->rxa_start = IEEE80211_SEQ_INC(rxseq);
return PROCESS;
}
} else {
/*
* First frame, start aging timer.
*/
rap->rxa_age = ticks;
}
#endif /* IEEE80211_AMPDU_AGE */
/* save packet */
if (rap->rxa_m[off] == NULL) {
rap->rxa_m[off] = m;
rap->rxa_qframes++;
rap->rxa_qbytes += m->m_pkthdr.len;
vap->iv_stats.is_ampdu_rx_reorder++;
} else {
IEEE80211_DISCARD_MAC(vap,
IEEE80211_MSG_INPUT | IEEE80211_MSG_11N,
ni->ni_macaddr, "a-mpdu duplicate",
"seqno %u tid %u BA win <%u:%u>",
rxseq, tid, rap->rxa_start,
IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1));
vap->iv_stats.is_rx_dup++;
IEEE80211_NODE_STAT(ni, rx_dup);
m_freem(m);
}
return CONSUMED;
}
if (off < IEEE80211_SEQ_BA_RANGE) {
/*
* Outside the BA window, but within range;
* flush the reorder q and move the window.
* Sec 9.10.7.6.2 b) (p.138)
*/
IEEE80211_NOTE(vap, IEEE80211_MSG_11N, ni,
"move BA win <%u:%u> (%u frames) rxseq %u tid %u",
rap->rxa_start,
IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1),
rap->rxa_qframes, rxseq, tid);
vap->iv_stats.is_ampdu_rx_move++;
/*
* The spec says to flush frames up to but not including:
* WinStart_B = rxseq - rap->rxa_wnd + 1
* Then insert the frame or notify the caller to process
* it immediately. We can safely do this by just starting
* over again because we know the frame will now be within
* the BA window.
*/
/* NB: rxa_wnd known to be >0 */
ampdu_rx_flush_upto(ni, rap,
IEEE80211_SEQ_SUB(rxseq, rap->rxa_wnd-1));
goto again;
} else {
/*
* Outside the BA window and out of range; toss.
* Sec 9.10.7.6.2 c) (p.138)
*/
IEEE80211_DISCARD_MAC(vap,
IEEE80211_MSG_INPUT | IEEE80211_MSG_11N, ni->ni_macaddr,
"MPDU", "BA win <%u:%u> (%u frames) rxseq %u tid %u%s",
rap->rxa_start,
IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1),
rap->rxa_qframes, rxseq, tid,
wh->i_fc[1] & IEEE80211_FC1_RETRY ? " (retransmit)" : "");
vap->iv_stats.is_ampdu_rx_drop++;
IEEE80211_NODE_STAT(ni, rx_drop);
m_freem(m);
return CONSUMED;
}
#undef CONSUMED
#undef PROCESS
#undef IEEE80211_FC0_QOSDATA
}
/*
* Process a BAR ctl frame. Dispatch all frames up to
* the sequence number of the frame. If this frame is
* out of range it's discarded.
*/
void
ieee80211_recv_bar(struct ieee80211_node *ni, struct mbuf *m0)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_frame_bar *wh;
struct ieee80211_rx_ampdu *rap;
ieee80211_seq rxseq;
int tid, off;
if (!ieee80211_recv_bar_ena) {
#if 0
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_11N,
ni->ni_macaddr, "BAR", "%s", "processing disabled");
#endif
vap->iv_stats.is_ampdu_bar_bad++;
return;
}
wh = mtod(m0, struct ieee80211_frame_bar *);
/* XXX check basic BAR */
tid = MS(le16toh(wh->i_ctl), IEEE80211_BAR_TID);
rap = &ni->ni_rx_ampdu[tid];
if ((rap->rxa_flags & IEEE80211_AGGR_XCHGPEND) == 0) {
/*
* No ADDBA request yet, don't touch.
*/
IEEE80211_DISCARD_MAC(vap,
IEEE80211_MSG_INPUT | IEEE80211_MSG_11N,
ni->ni_macaddr, "BAR", "no BA stream, tid %u", tid);
vap->iv_stats.is_ampdu_bar_bad++;
return;
}
vap->iv_stats.is_ampdu_bar_rx++;
rxseq = le16toh(wh->i_seq) >> IEEE80211_SEQ_SEQ_SHIFT;
if (rxseq == rap->rxa_start)
return;
/* calculate offset in BA window */
off = IEEE80211_SEQ_SUB(rxseq, rap->rxa_start);
if (off < IEEE80211_SEQ_BA_RANGE) {
/*
* Flush the reorder q up to rxseq and move the window.
* Sec 9.10.7.6.3 a) (p.138)
*/
IEEE80211_NOTE(vap, IEEE80211_MSG_11N, ni,
"BAR moves BA win <%u:%u> (%u frames) rxseq %u tid %u",
rap->rxa_start,
IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1),
rap->rxa_qframes, rxseq, tid);
vap->iv_stats.is_ampdu_bar_move++;
ampdu_rx_flush_upto(ni, rap, rxseq);
if (off >= rap->rxa_wnd) {
/*
* BAR specifies a window start to the right of BA
* window; we must move it explicitly since
* ampdu_rx_flush_upto will not.
*/
rap->rxa_start = rxseq;
}
} else {
/*
* Out of range; toss.
* Sec 9.10.7.6.3 b) (p.138)
*/
IEEE80211_DISCARD_MAC(vap,
IEEE80211_MSG_INPUT | IEEE80211_MSG_11N, ni->ni_macaddr,
"BAR", "BA win <%u:%u> (%u frames) rxseq %u tid %u%s",
rap->rxa_start,
IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1),
rap->rxa_qframes, rxseq, tid,
wh->i_fc[1] & IEEE80211_FC1_RETRY ? " (retransmit)" : "");
vap->iv_stats.is_ampdu_bar_oow++;
IEEE80211_NODE_STAT(ni, rx_drop);
}
}
/*
* Setup HT-specific state in a node. Called only
* when HT use is negotiated so we don't do extra
* work for temporary and/or legacy sta's.
*/
void
ieee80211_ht_node_init(struct ieee80211_node *ni)
{
struct ieee80211_tx_ampdu *tap;
int tid;
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N,
ni,
"%s: called",
__func__);
if (ni->ni_flags & IEEE80211_NODE_HT) {
/*
* Clean AMPDU state on re-associate. This handles the case
* where a station leaves w/o notifying us and then returns
* before node is reaped for inactivity.
*/
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N,
ni,
"%s: calling cleanup",
__func__);
ieee80211_ht_node_cleanup(ni);
}
for (tid = 0; tid < WME_NUM_TID; tid++) {
tap = &ni->ni_tx_ampdu[tid];
tap->txa_tid = tid;
tap->txa_ni = ni;
tap->txa_lastsample = ticks;
/* NB: further initialization deferred */
}
ni->ni_flags |= IEEE80211_NODE_HT | IEEE80211_NODE_AMPDU;
}
/*
* Cleanup HT-specific state in a node. Called only
* when HT use has been marked.
*/
void
ieee80211_ht_node_cleanup(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
int i;
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N,
ni,
"%s: called",
__func__);
KASSERT(ni->ni_flags & IEEE80211_NODE_HT, ("not an HT node"));
/* XXX optimize this */
for (i = 0; i < WME_NUM_TID; i++) {
struct ieee80211_tx_ampdu *tap = &ni->ni_tx_ampdu[i];
if (tap->txa_flags & IEEE80211_AGGR_SETUP)
ampdu_tx_stop(tap);
}
for (i = 0; i < WME_NUM_TID; i++)
ic->ic_ampdu_rx_stop(ni, &ni->ni_rx_ampdu[i]);
ni->ni_htcap = 0;
ni->ni_flags &= ~IEEE80211_NODE_HT_ALL;
}
/*
* Age out HT resources for a station.
*/
void
ieee80211_ht_node_age(struct ieee80211_node *ni)
{
#ifdef IEEE80211_AMPDU_AGE
struct ieee80211vap *vap = ni->ni_vap;
uint8_t tid;
#endif
KASSERT(ni->ni_flags & IEEE80211_NODE_HT, ("not an HT sta"));
#ifdef IEEE80211_AMPDU_AGE
for (tid = 0; tid < WME_NUM_TID; tid++) {
struct ieee80211_rx_ampdu *rap;
rap = &ni->ni_rx_ampdu[tid];
if ((rap->rxa_flags & IEEE80211_AGGR_XCHGPEND) == 0)
continue;
if (rap->rxa_qframes == 0)
continue;
/*
* Check for frames sitting too long in the reorder queue.
* See above for more details on what's happening here.
*/
/* XXX honor batimeout? */
if (ticks - rap->rxa_age > ieee80211_ampdu_age) {
/*
* Too long since we received the first
* frame; flush the reorder buffer.
*/
vap->iv_stats.is_ampdu_rx_age += rap->rxa_qframes;
ampdu_rx_flush(ni, rap);
}
}
#endif /* IEEE80211_AMPDU_AGE */
}
static struct ieee80211_channel *
findhtchan(struct ieee80211com *ic, struct ieee80211_channel *c, int htflags)
{
return ieee80211_find_channel(ic, c->ic_freq,
(c->ic_flags &~ IEEE80211_CHAN_HT) | htflags);
}
/*
* Adjust a channel to be HT/non-HT according to the vap's configuration.
*/
struct ieee80211_channel *
ieee80211_ht_adjust_channel(struct ieee80211com *ic,
struct ieee80211_channel *chan, int flags)
{
struct ieee80211_channel *c;
if (flags & IEEE80211_FHT_HT) {
/* promote to HT if possible */
if (flags & IEEE80211_FHT_USEHT40) {
if (!IEEE80211_IS_CHAN_HT40(chan)) {
/* NB: arbitrarily pick ht40+ over ht40- */
c = findhtchan(ic, chan, IEEE80211_CHAN_HT40U);
if (c == NULL)
c = findhtchan(ic, chan,
IEEE80211_CHAN_HT40D);
if (c == NULL)
c = findhtchan(ic, chan,
IEEE80211_CHAN_HT20);
if (c != NULL)
chan = c;
}
} else if (!IEEE80211_IS_CHAN_HT20(chan)) {
c = findhtchan(ic, chan, IEEE80211_CHAN_HT20);
if (c != NULL)
chan = c;
}
} else if (IEEE80211_IS_CHAN_HT(chan)) {
/* demote to legacy, HT use is disabled */
c = ieee80211_find_channel(ic, chan->ic_freq,
chan->ic_flags &~ IEEE80211_CHAN_HT);
if (c != NULL)
chan = c;
}
return chan;
}
/*
* Setup HT-specific state for a legacy WDS peer.
*/
void
ieee80211_ht_wds_init(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_tx_ampdu *tap;
int tid;
KASSERT(vap->iv_flags_ht & IEEE80211_FHT_HT, ("no HT requested"));
/* XXX check scan cache in case peer has an ap and we have info */
/*
* If setup with a legacy channel; locate an HT channel.
* Otherwise if the inherited channel (from a companion
* AP) is suitable use it so we use the same location
* for the extension channel).
*/
ni->ni_chan = ieee80211_ht_adjust_channel(ni->ni_ic,
ni->ni_chan, ieee80211_htchanflags(ni->ni_chan));
ni->ni_htcap = 0;
if (vap->iv_flags_ht & IEEE80211_FHT_SHORTGI20)
ni->ni_htcap |= IEEE80211_HTCAP_SHORTGI20;
if (IEEE80211_IS_CHAN_HT40(ni->ni_chan)) {
ni->ni_htcap |= IEEE80211_HTCAP_CHWIDTH40;
ni->ni_chw = 40;
if (IEEE80211_IS_CHAN_HT40U(ni->ni_chan))
ni->ni_ht2ndchan = IEEE80211_HTINFO_2NDCHAN_ABOVE;
else if (IEEE80211_IS_CHAN_HT40D(ni->ni_chan))
ni->ni_ht2ndchan = IEEE80211_HTINFO_2NDCHAN_BELOW;
if (vap->iv_flags_ht & IEEE80211_FHT_SHORTGI40)
ni->ni_htcap |= IEEE80211_HTCAP_SHORTGI40;
} else {
ni->ni_chw = 20;
ni->ni_ht2ndchan = IEEE80211_HTINFO_2NDCHAN_NONE;
}
ni->ni_htctlchan = ni->ni_chan->ic_ieee;
if (vap->iv_flags_ht & IEEE80211_FHT_RIFS)
ni->ni_flags |= IEEE80211_NODE_RIFS;
/* XXX does it make sense to enable SMPS? */
ni->ni_htopmode = 0; /* XXX need protection state */
ni->ni_htstbc = 0; /* XXX need info */
for (tid = 0; tid < WME_NUM_TID; tid++) {
tap = &ni->ni_tx_ampdu[tid];
tap->txa_tid = tid;
tap->txa_lastsample = ticks;
}
/* NB: AMPDU tx/rx governed by IEEE80211_FHT_AMPDU_{TX,RX} */
ni->ni_flags |= IEEE80211_NODE_HT | IEEE80211_NODE_AMPDU;
}
/*
* Notify hostap vaps of a change in the HTINFO ie.
*/
static void
htinfo_notify(struct ieee80211com *ic)
{
struct ieee80211vap *vap;
int first = 1;
IEEE80211_LOCK_ASSERT(ic);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
if (vap->iv_opmode != IEEE80211_M_HOSTAP)
continue;
if (vap->iv_state != IEEE80211_S_RUN ||
!IEEE80211_IS_CHAN_HT(vap->iv_bss->ni_chan))
continue;
if (first) {
IEEE80211_NOTE(vap,
IEEE80211_MSG_ASSOC | IEEE80211_MSG_11N,
vap->iv_bss,
"HT bss occupancy change: %d sta, %d ht, "
"%d ht40%s, HT protmode now 0x%x"
, ic->ic_sta_assoc
, ic->ic_ht_sta_assoc
, ic->ic_ht40_sta_assoc
, (ic->ic_flags_ht & IEEE80211_FHT_NONHT_PR) ?
", non-HT sta present" : ""
, ic->ic_curhtprotmode);
first = 0;
}
ieee80211_beacon_notify(vap, IEEE80211_BEACON_HTINFO);
}
}
/*
* Calculate HT protection mode from current
* state and handle updates.
*/
static void
htinfo_update(struct ieee80211com *ic)
{
uint8_t protmode;
if (ic->ic_sta_assoc != ic->ic_ht_sta_assoc) {
protmode = IEEE80211_HTINFO_OPMODE_MIXED
| IEEE80211_HTINFO_NONHT_PRESENT;
} else if (ic->ic_flags_ht & IEEE80211_FHT_NONHT_PR) {
protmode = IEEE80211_HTINFO_OPMODE_PROTOPT
| IEEE80211_HTINFO_NONHT_PRESENT;
} else if (ic->ic_bsschan != IEEE80211_CHAN_ANYC &&
IEEE80211_IS_CHAN_HT40(ic->ic_bsschan) &&
ic->ic_sta_assoc != ic->ic_ht40_sta_assoc) {
protmode = IEEE80211_HTINFO_OPMODE_HT20PR;
} else {
protmode = IEEE80211_HTINFO_OPMODE_PURE;
}
if (protmode != ic->ic_curhtprotmode) {
ic->ic_curhtprotmode = protmode;
htinfo_notify(ic);
}
}
/*
* Handle an HT station joining a BSS.
*/
void
ieee80211_ht_node_join(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
IEEE80211_LOCK_ASSERT(ic);
if (ni->ni_flags & IEEE80211_NODE_HT) {
ic->ic_ht_sta_assoc++;
if (ni->ni_chw == 40)
ic->ic_ht40_sta_assoc++;
}
htinfo_update(ic);
}
/*
* Handle an HT station leaving a BSS.
*/
void
ieee80211_ht_node_leave(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
IEEE80211_LOCK_ASSERT(ic);
if (ni->ni_flags & IEEE80211_NODE_HT) {
ic->ic_ht_sta_assoc--;
if (ni->ni_chw == 40)
ic->ic_ht40_sta_assoc--;
}
htinfo_update(ic);
}
/*
* Public version of htinfo_update; used for processing
* beacon frames from overlapping bss.
*
* Caller can specify either IEEE80211_HTINFO_OPMODE_MIXED
* (on receipt of a beacon that advertises MIXED) or
* IEEE80211_HTINFO_OPMODE_PROTOPT (on receipt of a beacon
* from an overlapping legacy bss). We treat MIXED with
* a higher precedence than PROTOPT (i.e. we will not change
* change PROTOPT -> MIXED; only MIXED -> PROTOPT). This
* corresponds to how we handle things in htinfo_update.
*/
void
ieee80211_htprot_update(struct ieee80211com *ic, int protmode)
{
#define OPMODE(x) SM(x, IEEE80211_HTINFO_OPMODE)
IEEE80211_LOCK(ic);
/* track non-HT station presence */
KASSERT(protmode & IEEE80211_HTINFO_NONHT_PRESENT,
("protmode 0x%x", protmode));
ic->ic_flags_ht |= IEEE80211_FHT_NONHT_PR;
ic->ic_lastnonht = ticks;
if (protmode != ic->ic_curhtprotmode &&
(OPMODE(ic->ic_curhtprotmode) != IEEE80211_HTINFO_OPMODE_MIXED ||
OPMODE(protmode) == IEEE80211_HTINFO_OPMODE_PROTOPT)) {
/* push beacon update */
ic->ic_curhtprotmode = protmode;
htinfo_notify(ic);
}
IEEE80211_UNLOCK(ic);
#undef OPMODE
}
/*
* Time out presence of an overlapping bss with non-HT
* stations. When operating in hostap mode we listen for
* beacons from other stations and if we identify a non-HT
* station is present we update the opmode field of the
* HTINFO ie. To identify when all non-HT stations are
* gone we time out this condition.
*/
void
ieee80211_ht_timeout(struct ieee80211com *ic)
{
IEEE80211_LOCK_ASSERT(ic);
if ((ic->ic_flags_ht & IEEE80211_FHT_NONHT_PR) &&
time_after(ticks, ic->ic_lastnonht + IEEE80211_NONHT_PRESENT_AGE)) {
#if 0
IEEE80211_NOTE(vap, IEEE80211_MSG_11N, ni,
"%s", "time out non-HT STA present on channel");
#endif
ic->ic_flags_ht &= ~IEEE80211_FHT_NONHT_PR;
htinfo_update(ic);
}
}
/* unalligned little endian access */
#define LE_READ_2(p) \
((uint16_t) \
((((const uint8_t *)(p))[0] ) | \
(((const uint8_t *)(p))[1] << 8)))
/*
* Process an 802.11n HT capabilities ie.
*/
void
ieee80211_parse_htcap(struct ieee80211_node *ni, const uint8_t *ie)
{
if (ie[0] == IEEE80211_ELEMID_VENDOR) {
/*
* Station used Vendor OUI ie to associate;
* mark the node so when we respond we'll use
* the Vendor OUI's and not the standard ie's.
*/
ni->ni_flags |= IEEE80211_NODE_HTCOMPAT;
ie += 4;
} else
ni->ni_flags &= ~IEEE80211_NODE_HTCOMPAT;
ni->ni_htcap = LE_READ_2(ie +
__offsetof(struct ieee80211_ie_htcap, hc_cap));
ni->ni_htparam = ie[__offsetof(struct ieee80211_ie_htcap, hc_param)];
}
static void
htinfo_parse(struct ieee80211_node *ni,
const struct ieee80211_ie_htinfo *htinfo)
{
uint16_t w;
ni->ni_htctlchan = htinfo->hi_ctrlchannel;
ni->ni_ht2ndchan = SM(htinfo->hi_byte1, IEEE80211_HTINFO_2NDCHAN);
w = LE_READ_2(&htinfo->hi_byte2);
ni->ni_htopmode = SM(w, IEEE80211_HTINFO_OPMODE);
w = LE_READ_2(&htinfo->hi_byte45);
ni->ni_htstbc = SM(w, IEEE80211_HTINFO_BASIC_STBCMCS);
}
/*
* Parse an 802.11n HT info ie and save useful information
* to the node state. Note this does not effect any state
* changes such as for channel width change.
*/
void
ieee80211_parse_htinfo(struct ieee80211_node *ni, const uint8_t *ie)
{
if (ie[0] == IEEE80211_ELEMID_VENDOR)
ie += 4;
htinfo_parse(ni, (const struct ieee80211_ie_htinfo *) ie);
}
/*
* Handle 11n channel switch. Use the received HT ie's to
* identify the right channel to use. If we cannot locate it
* in the channel table then fallback to legacy operation.
* Note that we use this information to identify the node's
* channel only; the caller is responsible for insuring any
* required channel change is done (e.g. in sta mode when
* parsing the contents of a beacon frame).
*/
static int
htinfo_update_chw(struct ieee80211_node *ni, int htflags)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211_channel *c;
int chanflags;
int ret = 0;
chanflags = (ni->ni_chan->ic_flags &~ IEEE80211_CHAN_HT) | htflags;
if (chanflags != ni->ni_chan->ic_flags) {
/* XXX not right for ht40- */
c = ieee80211_find_channel(ic, ni->ni_chan->ic_freq, chanflags);
if (c == NULL && (htflags & IEEE80211_CHAN_HT40)) {
/*
* No HT40 channel entry in our table; fall back
* to HT20 operation. This should not happen.
*/
c = findhtchan(ic, ni->ni_chan, IEEE80211_CHAN_HT20);
#if 0
IEEE80211_NOTE(ni->ni_vap,
IEEE80211_MSG_ASSOC | IEEE80211_MSG_11N, ni,
"no HT40 channel (freq %u), falling back to HT20",
ni->ni_chan->ic_freq);
#endif
/* XXX stat */
}
if (c != NULL && c != ni->ni_chan) {
IEEE80211_NOTE(ni->ni_vap,
IEEE80211_MSG_ASSOC | IEEE80211_MSG_11N, ni,
"switch station to HT%d channel %u/0x%x",
IEEE80211_IS_CHAN_HT40(c) ? 40 : 20,
c->ic_freq, c->ic_flags);
ni->ni_chan = c;
ret = 1;
}
/* NB: caller responsible for forcing any channel change */
}
/* update node's tx channel width */
ni->ni_chw = IEEE80211_IS_CHAN_HT40(ni->ni_chan)? 40 : 20;
return (ret);
}
/*
* Update 11n MIMO PS state according to received htcap.
*/
static __inline int
htcap_update_mimo_ps(struct ieee80211_node *ni)
{
uint16_t oflags = ni->ni_flags;
switch (ni->ni_htcap & IEEE80211_HTCAP_SMPS) {
case IEEE80211_HTCAP_SMPS_DYNAMIC:
ni->ni_flags |= IEEE80211_NODE_MIMO_PS;
ni->ni_flags |= IEEE80211_NODE_MIMO_RTS;
break;
case IEEE80211_HTCAP_SMPS_ENA:
ni->ni_flags |= IEEE80211_NODE_MIMO_PS;
ni->ni_flags &= ~IEEE80211_NODE_MIMO_RTS;
break;
case IEEE80211_HTCAP_SMPS_OFF:
default: /* disable on rx of reserved value */
ni->ni_flags &= ~IEEE80211_NODE_MIMO_PS;
ni->ni_flags &= ~IEEE80211_NODE_MIMO_RTS;
break;
}
return (oflags ^ ni->ni_flags);
}
/*
* Update short GI state according to received htcap
* and local settings.
*/
static __inline void
htcap_update_shortgi(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
ni->ni_flags &= ~(IEEE80211_NODE_SGI20|IEEE80211_NODE_SGI40);
if ((ni->ni_htcap & IEEE80211_HTCAP_SHORTGI20) &&
(vap->iv_flags_ht & IEEE80211_FHT_SHORTGI20))
ni->ni_flags |= IEEE80211_NODE_SGI20;
if ((ni->ni_htcap & IEEE80211_HTCAP_SHORTGI40) &&
(vap->iv_flags_ht & IEEE80211_FHT_SHORTGI40))
ni->ni_flags |= IEEE80211_NODE_SGI40;
}
/*
* Parse and update HT-related state extracted from
* the HT cap and info ie's.
*/
int
ieee80211_ht_updateparams(struct ieee80211_node *ni,
const uint8_t *htcapie, const uint8_t *htinfoie)
{
struct ieee80211vap *vap = ni->ni_vap;
const struct ieee80211_ie_htinfo *htinfo;
int htflags;
int ret = 0;
ieee80211_parse_htcap(ni, htcapie);
if (vap->iv_htcaps & IEEE80211_HTCAP_SMPS)
htcap_update_mimo_ps(ni);
htcap_update_shortgi(ni);
if (htinfoie[0] == IEEE80211_ELEMID_VENDOR)
htinfoie += 4;
htinfo = (const struct ieee80211_ie_htinfo *) htinfoie;
htinfo_parse(ni, htinfo);
htflags = (vap->iv_flags_ht & IEEE80211_FHT_HT) ?
IEEE80211_CHAN_HT20 : 0;
/* NB: honor operating mode constraint */
if ((htinfo->hi_byte1 & IEEE80211_HTINFO_TXWIDTH_2040) &&
(vap->iv_flags_ht & IEEE80211_FHT_USEHT40)) {
if (ni->ni_ht2ndchan == IEEE80211_HTINFO_2NDCHAN_ABOVE)
htflags = IEEE80211_CHAN_HT40U;
else if (ni->ni_ht2ndchan == IEEE80211_HTINFO_2NDCHAN_BELOW)
htflags = IEEE80211_CHAN_HT40D;
}
if (htinfo_update_chw(ni, htflags))
ret = 1;
if ((htinfo->hi_byte1 & IEEE80211_HTINFO_RIFSMODE_PERM) &&
(vap->iv_flags_ht & IEEE80211_FHT_RIFS))
ni->ni_flags |= IEEE80211_NODE_RIFS;
else
ni->ni_flags &= ~IEEE80211_NODE_RIFS;
return (ret);
}
/*
* Parse and update HT-related state extracted from the HT cap ie
* for a station joining an HT BSS.
*/
void
ieee80211_ht_updatehtcap(struct ieee80211_node *ni, const uint8_t *htcapie)
{
struct ieee80211vap *vap = ni->ni_vap;
int htflags;
ieee80211_parse_htcap(ni, htcapie);
if (vap->iv_htcaps & IEEE80211_HTCAP_SMPS)
htcap_update_mimo_ps(ni);
htcap_update_shortgi(ni);
/* NB: honor operating mode constraint */
/* XXX 40 MHz intolerant */
htflags = (vap->iv_flags_ht & IEEE80211_FHT_HT) ?
IEEE80211_CHAN_HT20 : 0;
if ((ni->ni_htcap & IEEE80211_HTCAP_CHWIDTH40) &&
(vap->iv_flags_ht & IEEE80211_FHT_USEHT40)) {
if (IEEE80211_IS_CHAN_HT40U(vap->iv_bss->ni_chan))
htflags = IEEE80211_CHAN_HT40U;
else if (IEEE80211_IS_CHAN_HT40D(vap->iv_bss->ni_chan))
htflags = IEEE80211_CHAN_HT40D;
}
(void) htinfo_update_chw(ni, htflags);
}
/*
* Install received HT rate set by parsing the HT cap ie.
*/
int
ieee80211_setup_htrates(struct ieee80211_node *ni, const uint8_t *ie, int flags)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211vap *vap = ni->ni_vap;
const struct ieee80211_ie_htcap *htcap;
struct ieee80211_htrateset *rs;
int i, maxequalmcs, maxunequalmcs;
maxequalmcs = ic->ic_txstream * 8 - 1;
if (ic->ic_htcaps & IEEE80211_HTC_TXUNEQUAL) {
if (ic->ic_txstream >= 2)
maxunequalmcs = 38;
if (ic->ic_txstream >= 3)
maxunequalmcs = 52;
if (ic->ic_txstream >= 4)
maxunequalmcs = 76;
} else
maxunequalmcs = 0;
rs = &ni->ni_htrates;
memset(rs, 0, sizeof(*rs));
if (ie != NULL) {
if (ie[0] == IEEE80211_ELEMID_VENDOR)
ie += 4;
htcap = (const struct ieee80211_ie_htcap *) ie;
for (i = 0; i < IEEE80211_HTRATE_MAXSIZE; i++) {
if (isclr(htcap->hc_mcsset, i))
continue;
if (rs->rs_nrates == IEEE80211_HTRATE_MAXSIZE) {
IEEE80211_NOTE(vap,
IEEE80211_MSG_XRATE | IEEE80211_MSG_11N, ni,
"WARNING, HT rate set too large; only "
"using %u rates", IEEE80211_HTRATE_MAXSIZE);
vap->iv_stats.is_rx_rstoobig++;
break;
}
if (i <= 31 && i > maxequalmcs)
continue;
if (i == 32 &&
(ic->ic_htcaps & IEEE80211_HTC_TXMCS32) == 0)
continue;
if (i > 32 && i > maxunequalmcs)
continue;
rs->rs_rates[rs->rs_nrates++] = i;
}
}
return ieee80211_fix_rate(ni, (struct ieee80211_rateset *) rs, flags);
}
/*
* Mark rates in a node's HT rate set as basic according
* to the information in the supplied HT info ie.
*/
void
ieee80211_setup_basic_htrates(struct ieee80211_node *ni, const uint8_t *ie)
{
const struct ieee80211_ie_htinfo *htinfo;
struct ieee80211_htrateset *rs;
int i, j;
if (ie[0] == IEEE80211_ELEMID_VENDOR)
ie += 4;
htinfo = (const struct ieee80211_ie_htinfo *) ie;
rs = &ni->ni_htrates;
if (rs->rs_nrates == 0) {
IEEE80211_NOTE(ni->ni_vap,
IEEE80211_MSG_XRATE | IEEE80211_MSG_11N, ni,
"%s", "WARNING, empty HT rate set");
return;
}
for (i = 0; i < IEEE80211_HTRATE_MAXSIZE; i++) {
if (isclr(htinfo->hi_basicmcsset, i))
continue;
for (j = 0; j < rs->rs_nrates; j++)
if ((rs->rs_rates[j] & IEEE80211_RATE_VAL) == i)
rs->rs_rates[j] |= IEEE80211_RATE_BASIC;
}
}
static void
ampdu_tx_setup(struct ieee80211_tx_ampdu *tap)
{
- callout_init(&tap->txa_timer, CALLOUT_MPSAFE);
+ callout_init(&tap->txa_timer, 1);
tap->txa_flags |= IEEE80211_AGGR_SETUP;
tap->txa_lastsample = ticks;
}
static void
ampdu_tx_stop(struct ieee80211_tx_ampdu *tap)
{
struct ieee80211_node *ni = tap->txa_ni;
struct ieee80211com *ic = ni->ni_ic;
IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N,
tap->txa_ni,
"%s: called",
__func__);
KASSERT(tap->txa_flags & IEEE80211_AGGR_SETUP,
("txa_flags 0x%x tid %d ac %d", tap->txa_flags, tap->txa_tid,
TID_TO_WME_AC(tap->txa_tid)));
/*
* Stop BA stream if setup so driver has a chance
* to reclaim any resources it might have allocated.
*/
ic->ic_addba_stop(ni, tap);
/*
* Stop any pending BAR transmit.
*/
bar_stop_timer(tap);
/*
* Reset packet estimate.
*/
tap->txa_lastsample = ticks;
tap->txa_avgpps = 0;
/* NB: clearing NAK means we may re-send ADDBA */
tap->txa_flags &= ~(IEEE80211_AGGR_SETUP | IEEE80211_AGGR_NAK);
}
/*
* ADDBA response timeout.
*
* If software aggregation and per-TID queue management was done here,
* that queue would be unpaused after the ADDBA timeout occurs.
*/
static void
addba_timeout(void *arg)
{
struct ieee80211_tx_ampdu *tap = arg;
struct ieee80211_node *ni = tap->txa_ni;
struct ieee80211com *ic = ni->ni_ic;
/* XXX ? */
tap->txa_flags &= ~IEEE80211_AGGR_XCHGPEND;
tap->txa_attempts++;
ic->ic_addba_response_timeout(ni, tap);
}
static void
addba_start_timeout(struct ieee80211_tx_ampdu *tap)
{
/* XXX use CALLOUT_PENDING instead? */
callout_reset(&tap->txa_timer, ieee80211_addba_timeout,
addba_timeout, tap);
tap->txa_flags |= IEEE80211_AGGR_XCHGPEND;
tap->txa_nextrequest = ticks + ieee80211_addba_timeout;
}
static void
addba_stop_timeout(struct ieee80211_tx_ampdu *tap)
{
/* XXX use CALLOUT_PENDING instead? */
if (tap->txa_flags & IEEE80211_AGGR_XCHGPEND) {
callout_stop(&tap->txa_timer);
tap->txa_flags &= ~IEEE80211_AGGR_XCHGPEND;
}
}
static void
null_addba_response_timeout(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap)
{
}
/*
* Default method for requesting A-MPDU tx aggregation.
* We setup the specified state block and start a timer
* to wait for an ADDBA response frame.
*/
static int
ieee80211_addba_request(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap,
int dialogtoken, int baparamset, int batimeout)
{
int bufsiz;
/* XXX locking */
tap->txa_token = dialogtoken;
tap->txa_flags |= IEEE80211_AGGR_IMMEDIATE;
bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ);
tap->txa_wnd = (bufsiz == 0) ?
IEEE80211_AGGR_BAWMAX : min(bufsiz, IEEE80211_AGGR_BAWMAX);
addba_start_timeout(tap);
return 1;
}
/*
* Default method for processing an A-MPDU tx aggregation
* response. We shutdown any pending timer and update the
* state block according to the reply.
*/
static int
ieee80211_addba_response(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap,
int status, int baparamset, int batimeout)
{
int bufsiz, tid;
/* XXX locking */
addba_stop_timeout(tap);
if (status == IEEE80211_STATUS_SUCCESS) {
bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ);
/* XXX override our request? */
tap->txa_wnd = (bufsiz == 0) ?
IEEE80211_AGGR_BAWMAX : min(bufsiz, IEEE80211_AGGR_BAWMAX);
/* XXX AC/TID */
tid = MS(baparamset, IEEE80211_BAPS_TID);
tap->txa_flags |= IEEE80211_AGGR_RUNNING;
tap->txa_attempts = 0;
} else {
/* mark tid so we don't try again */
tap->txa_flags |= IEEE80211_AGGR_NAK;
}
return 1;
}
/*
* Default method for stopping A-MPDU tx aggregation.
* Any timer is cleared and we drain any pending frames.
*/
static void
ieee80211_addba_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap)
{
/* XXX locking */
addba_stop_timeout(tap);
if (tap->txa_flags & IEEE80211_AGGR_RUNNING) {
/* XXX clear aggregation queue */
tap->txa_flags &= ~IEEE80211_AGGR_RUNNING;
}
tap->txa_attempts = 0;
}
/*
* Process a received action frame using the default aggregation
* policy. We intercept ADDBA-related frames and use them to
* update our aggregation state. All other frames are passed up
* for processing by ieee80211_recv_action.
*/
static int
ht_recv_action_ba_addba_request(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_rx_ampdu *rap;
uint8_t dialogtoken;
uint16_t baparamset, batimeout, baseqctl;
uint16_t args[5];
int tid;
dialogtoken = frm[2];
baparamset = LE_READ_2(frm+3);
batimeout = LE_READ_2(frm+5);
baseqctl = LE_READ_2(frm+7);
tid = MS(baparamset, IEEE80211_BAPS_TID);
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni,
"recv ADDBA request: dialogtoken %u baparamset 0x%x "
"(tid %d bufsiz %d) batimeout %d baseqctl %d:%d",
dialogtoken, baparamset,
tid, MS(baparamset, IEEE80211_BAPS_BUFSIZ),
batimeout,
MS(baseqctl, IEEE80211_BASEQ_START),
MS(baseqctl, IEEE80211_BASEQ_FRAG));
rap = &ni->ni_rx_ampdu[tid];
/* Send ADDBA response */
args[0] = dialogtoken;
/*
* NB: We ack only if the sta associated with HT and
* the ap is configured to do AMPDU rx (the latter
* violates the 11n spec and is mostly for testing).
*/
if ((ni->ni_flags & IEEE80211_NODE_AMPDU_RX) &&
(vap->iv_flags_ht & IEEE80211_FHT_AMPDU_RX)) {
/* XXX handle ampdu_rx_start failure */
ic->ic_ampdu_rx_start(ni, rap,
baparamset, batimeout, baseqctl);
args[1] = IEEE80211_STATUS_SUCCESS;
} else {
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N,
ni, "reject ADDBA request: %s",
ni->ni_flags & IEEE80211_NODE_AMPDU_RX ?
"administratively disabled" :
"not negotiated for station");
vap->iv_stats.is_addba_reject++;
args[1] = IEEE80211_STATUS_UNSPECIFIED;
}
/* XXX honor rap flags? */
args[2] = IEEE80211_BAPS_POLICY_IMMEDIATE
| SM(tid, IEEE80211_BAPS_TID)
| SM(rap->rxa_wnd, IEEE80211_BAPS_BUFSIZ)
;
args[3] = 0;
args[4] = 0;
ic->ic_send_action(ni, IEEE80211_ACTION_CAT_BA,
IEEE80211_ACTION_BA_ADDBA_RESPONSE, args);
return 0;
}
static int
ht_recv_action_ba_addba_response(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_tx_ampdu *tap;
uint8_t dialogtoken, policy;
uint16_t baparamset, batimeout, code;
int tid, bufsiz;
dialogtoken = frm[2];
code = LE_READ_2(frm+3);
baparamset = LE_READ_2(frm+5);
tid = MS(baparamset, IEEE80211_BAPS_TID);
bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ);
policy = MS(baparamset, IEEE80211_BAPS_POLICY);
batimeout = LE_READ_2(frm+7);
tap = &ni->ni_tx_ampdu[tid];
if ((tap->txa_flags & IEEE80211_AGGR_XCHGPEND) == 0) {
IEEE80211_DISCARD_MAC(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_11N,
ni->ni_macaddr, "ADDBA response",
"no pending ADDBA, tid %d dialogtoken %u "
"code %d", tid, dialogtoken, code);
vap->iv_stats.is_addba_norequest++;
return 0;
}
if (dialogtoken != tap->txa_token) {
IEEE80211_DISCARD_MAC(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_11N,
ni->ni_macaddr, "ADDBA response",
"dialogtoken mismatch: waiting for %d, "
"received %d, tid %d code %d",
tap->txa_token, dialogtoken, tid, code);
vap->iv_stats.is_addba_badtoken++;
return 0;
}
/* NB: assumes IEEE80211_AGGR_IMMEDIATE is 1 */
if (policy != (tap->txa_flags & IEEE80211_AGGR_IMMEDIATE)) {
IEEE80211_DISCARD_MAC(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_11N,
ni->ni_macaddr, "ADDBA response",
"policy mismatch: expecting %s, "
"received %s, tid %d code %d",
tap->txa_flags & IEEE80211_AGGR_IMMEDIATE,
policy, tid, code);
vap->iv_stats.is_addba_badpolicy++;
return 0;
}
#if 0
/* XXX we take MIN in ieee80211_addba_response */
if (bufsiz > IEEE80211_AGGR_BAWMAX) {
IEEE80211_DISCARD_MAC(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_11N,
ni->ni_macaddr, "ADDBA response",
"BA window too large: max %d, "
"received %d, tid %d code %d",
bufsiz, IEEE80211_AGGR_BAWMAX, tid, code);
vap->iv_stats.is_addba_badbawinsize++;
return 0;
}
#endif
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni,
"recv ADDBA response: dialogtoken %u code %d "
"baparamset 0x%x (tid %d bufsiz %d) batimeout %d",
dialogtoken, code, baparamset, tid, bufsiz,
batimeout);
ic->ic_addba_response(ni, tap, code, baparamset, batimeout);
return 0;
}
static int
ht_recv_action_ba_delba(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211_rx_ampdu *rap;
struct ieee80211_tx_ampdu *tap;
uint16_t baparamset, code;
int tid;
baparamset = LE_READ_2(frm+2);
code = LE_READ_2(frm+4);
tid = MS(baparamset, IEEE80211_DELBAPS_TID);
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni,
"recv DELBA: baparamset 0x%x (tid %d initiator %d) "
"code %d", baparamset, tid,
MS(baparamset, IEEE80211_DELBAPS_INIT), code);
if ((baparamset & IEEE80211_DELBAPS_INIT) == 0) {
tap = &ni->ni_tx_ampdu[tid];
ic->ic_addba_stop(ni, tap);
} else {
rap = &ni->ni_rx_ampdu[tid];
ic->ic_ampdu_rx_stop(ni, rap);
}
return 0;
}
static int
ht_recv_action_ht_txchwidth(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
int chw;
chw = (frm[2] == IEEE80211_A_HT_TXCHWIDTH_2040) ? 40 : 20;
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni,
"%s: HT txchwidth, width %d%s",
__func__, chw, ni->ni_chw != chw ? "*" : "");
if (chw != ni->ni_chw) {
ni->ni_chw = chw;
/* XXX notify on change */
}
return 0;
}
static int
ht_recv_action_ht_mimopwrsave(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
const struct ieee80211_action_ht_mimopowersave *mps =
(const struct ieee80211_action_ht_mimopowersave *) frm;
/* XXX check iv_htcaps */
if (mps->am_control & IEEE80211_A_HT_MIMOPWRSAVE_ENA)
ni->ni_flags |= IEEE80211_NODE_MIMO_PS;
else
ni->ni_flags &= ~IEEE80211_NODE_MIMO_PS;
if (mps->am_control & IEEE80211_A_HT_MIMOPWRSAVE_MODE)
ni->ni_flags |= IEEE80211_NODE_MIMO_RTS;
else
ni->ni_flags &= ~IEEE80211_NODE_MIMO_RTS;
/* XXX notify on change */
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni,
"%s: HT MIMO PS (%s%s)", __func__,
(ni->ni_flags & IEEE80211_NODE_MIMO_PS) ? "on" : "off",
(ni->ni_flags & IEEE80211_NODE_MIMO_RTS) ? "+rts" : ""
);
return 0;
}
/*
* Transmit processing.
*/
/*
* Check if A-MPDU should be requested/enabled for a stream.
* We require a traffic rate above a per-AC threshold and we
* also handle backoff from previous failed attempts.
*
* Drivers may override this method to bring in information
* such as link state conditions in making the decision.
*/
static int
ieee80211_ampdu_enable(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap)
{
struct ieee80211vap *vap = ni->ni_vap;
if (tap->txa_avgpps <
vap->iv_ampdu_mintraffic[TID_TO_WME_AC(tap->txa_tid)])
return 0;
/* XXX check rssi? */
if (tap->txa_attempts >= ieee80211_addba_maxtries &&
ticks < tap->txa_nextrequest) {
/*
* Don't retry too often; txa_nextrequest is set
* to the minimum interval we'll retry after
* ieee80211_addba_maxtries failed attempts are made.
*/
return 0;
}
IEEE80211_NOTE(vap, IEEE80211_MSG_11N, ni,
"enable AMPDU on tid %d (%s), avgpps %d pkts %d",
tap->txa_tid, ieee80211_wme_acnames[TID_TO_WME_AC(tap->txa_tid)],
tap->txa_avgpps, tap->txa_pkts);
return 1;
}
/*
* Request A-MPDU tx aggregation. Setup local state and
* issue an ADDBA request. BA use will only happen after
* the other end replies with ADDBA response.
*/
int
ieee80211_ampdu_request(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap)
{
struct ieee80211com *ic = ni->ni_ic;
uint16_t args[5];
int tid, dialogtoken;
static int tokens = 0; /* XXX */
/* XXX locking */
if ((tap->txa_flags & IEEE80211_AGGR_SETUP) == 0) {
/* do deferred setup of state */
ampdu_tx_setup(tap);
}
/* XXX hack for not doing proper locking */
tap->txa_flags &= ~IEEE80211_AGGR_NAK;
dialogtoken = (tokens+1) % 63; /* XXX */
tid = tap->txa_tid;
tap->txa_start = ni->ni_txseqs[tid];
args[0] = dialogtoken;
args[1] = 0; /* NB: status code not used */
args[2] = IEEE80211_BAPS_POLICY_IMMEDIATE
| SM(tid, IEEE80211_BAPS_TID)
| SM(IEEE80211_AGGR_BAWMAX, IEEE80211_BAPS_BUFSIZ)
;
args[3] = 0; /* batimeout */
/* NB: do first so there's no race against reply */
if (!ic->ic_addba_request(ni, tap, dialogtoken, args[2], args[3])) {
/* unable to setup state, don't make request */
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N,
ni, "%s: could not setup BA stream for TID %d AC %d",
__func__, tap->txa_tid, TID_TO_WME_AC(tap->txa_tid));
/* defer next try so we don't slam the driver with requests */
tap->txa_attempts = ieee80211_addba_maxtries;
/* NB: check in case driver wants to override */
if (tap->txa_nextrequest <= ticks)
tap->txa_nextrequest = ticks + ieee80211_addba_backoff;
return 0;
}
tokens = dialogtoken; /* allocate token */
/* NB: after calling ic_addba_request so driver can set txa_start */
args[4] = SM(tap->txa_start, IEEE80211_BASEQ_START)
| SM(0, IEEE80211_BASEQ_FRAG)
;
return ic->ic_send_action(ni, IEEE80211_ACTION_CAT_BA,
IEEE80211_ACTION_BA_ADDBA_REQUEST, args);
}
/*
* Terminate an AMPDU tx stream. State is reclaimed
* and the peer notified with a DelBA Action frame.
*/
void
ieee80211_ampdu_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap,
int reason)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211vap *vap = ni->ni_vap;
uint16_t args[4];
/* XXX locking */
tap->txa_flags &= ~IEEE80211_AGGR_BARPEND;
if (IEEE80211_AMPDU_RUNNING(tap)) {
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N,
ni, "%s: stop BA stream for TID %d (reason %d)",
__func__, tap->txa_tid, reason);
vap->iv_stats.is_ampdu_stop++;
ic->ic_addba_stop(ni, tap);
args[0] = tap->txa_tid;
args[1] = IEEE80211_DELBAPS_INIT;
args[2] = reason; /* XXX reason code */
ic->ic_send_action(ni, IEEE80211_ACTION_CAT_BA,
IEEE80211_ACTION_BA_DELBA, args);
} else {
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N,
ni, "%s: BA stream for TID %d not running (reason %d)",
__func__, tap->txa_tid, reason);
vap->iv_stats.is_ampdu_stop_failed++;
}
}
/* XXX */
static void bar_start_timer(struct ieee80211_tx_ampdu *tap);
static void
bar_timeout(void *arg)
{
struct ieee80211_tx_ampdu *tap = arg;
struct ieee80211_node *ni = tap->txa_ni;
KASSERT((tap->txa_flags & IEEE80211_AGGR_XCHGPEND) == 0,
("bar/addba collision, flags 0x%x", tap->txa_flags));
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N,
ni, "%s: tid %u flags 0x%x attempts %d", __func__,
tap->txa_tid, tap->txa_flags, tap->txa_attempts);
/* guard against race with bar_tx_complete */
if ((tap->txa_flags & IEEE80211_AGGR_BARPEND) == 0)
return;
/* XXX ? */
if (tap->txa_attempts >= ieee80211_bar_maxtries) {
struct ieee80211com *ic = ni->ni_ic;
ni->ni_vap->iv_stats.is_ampdu_bar_tx_fail++;
/*
* If (at least) the last BAR TX timeout was due to
* an ieee80211_send_bar() failures, then we need
* to make sure we notify the driver that a BAR
* TX did occur and fail. This gives the driver
* a chance to undo any queue pause that may
* have occured.
*/
ic->ic_bar_response(ni, tap, 1);
ieee80211_ampdu_stop(ni, tap, IEEE80211_REASON_TIMEOUT);
} else {
ni->ni_vap->iv_stats.is_ampdu_bar_tx_retry++;
if (ieee80211_send_bar(ni, tap, tap->txa_seqpending) != 0) {
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N,
ni, "%s: failed to TX, starting timer\n",
__func__);
/*
* If ieee80211_send_bar() fails here, the
* timer may have stopped and/or the pending
* flag may be clear. Because of this,
* fake the BARPEND and reset the timer.
* A retransmission attempt will then occur
* during the next timeout.
*/
/* XXX locking */
tap->txa_flags |= IEEE80211_AGGR_BARPEND;
bar_start_timer(tap);
}
}
}
static void
bar_start_timer(struct ieee80211_tx_ampdu *tap)
{
IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N,
tap->txa_ni,
"%s: called",
__func__);
callout_reset(&tap->txa_timer, ieee80211_bar_timeout, bar_timeout, tap);
}
static void
bar_stop_timer(struct ieee80211_tx_ampdu *tap)
{
IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N,
tap->txa_ni,
"%s: called",
__func__);
callout_stop(&tap->txa_timer);
}
static void
bar_tx_complete(struct ieee80211_node *ni, void *arg, int status)
{
struct ieee80211_tx_ampdu *tap = arg;
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N,
ni, "%s: tid %u flags 0x%x pending %d status %d",
__func__, tap->txa_tid, tap->txa_flags,
callout_pending(&tap->txa_timer), status);
ni->ni_vap->iv_stats.is_ampdu_bar_tx++;
/* XXX locking */
if ((tap->txa_flags & IEEE80211_AGGR_BARPEND) &&
callout_pending(&tap->txa_timer)) {
struct ieee80211com *ic = ni->ni_ic;
if (status == 0) /* ACK'd */
bar_stop_timer(tap);
ic->ic_bar_response(ni, tap, status);
/* NB: just let timer expire so we pace requests */
}
}
static void
ieee80211_bar_response(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap, int status)
{
IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N,
tap->txa_ni,
"%s: called",
__func__);
if (status == 0) { /* got ACK */
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N,
ni, "BAR moves BA win <%u:%u> (%u frames) txseq %u tid %u",
tap->txa_start,
IEEE80211_SEQ_ADD(tap->txa_start, tap->txa_wnd-1),
tap->txa_qframes, tap->txa_seqpending,
tap->txa_tid);
/* NB: timer already stopped in bar_tx_complete */
tap->txa_start = tap->txa_seqpending;
tap->txa_flags &= ~IEEE80211_AGGR_BARPEND;
}
}
/*
* Transmit a BAR frame to the specified node. The
* BAR contents are drawn from the supplied aggregation
* state associated with the node.
*
* NB: we only handle immediate ACK w/ compressed bitmap.
*/
int
ieee80211_send_bar(struct ieee80211_node *ni,
struct ieee80211_tx_ampdu *tap, ieee80211_seq seq)
{
#define senderr(_x, _v) do { vap->iv_stats._v++; ret = _x; goto bad; } while (0)
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211_frame_bar *bar;
struct mbuf *m;
uint16_t barctl, barseqctl;
uint8_t *frm;
int tid, ret;
IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N,
tap->txa_ni,
"%s: called",
__func__);
if ((tap->txa_flags & IEEE80211_AGGR_RUNNING) == 0) {
/* no ADDBA response, should not happen */
/* XXX stat+msg */
return EINVAL;
}
/* XXX locking */
bar_stop_timer(tap);
ieee80211_ref_node(ni);
m = ieee80211_getmgtframe(&frm, ic->ic_headroom, sizeof(*bar));
if (m == NULL)
senderr(ENOMEM, is_tx_nobuf);
if (!ieee80211_add_callback(m, bar_tx_complete, tap)) {
m_freem(m);
senderr(ENOMEM, is_tx_nobuf); /* XXX */
/* NOTREACHED */
}
bar = mtod(m, struct ieee80211_frame_bar *);
bar->i_fc[0] = IEEE80211_FC0_VERSION_0 |
IEEE80211_FC0_TYPE_CTL | IEEE80211_FC0_SUBTYPE_BAR;
bar->i_fc[1] = 0;
IEEE80211_ADDR_COPY(bar->i_ra, ni->ni_macaddr);
IEEE80211_ADDR_COPY(bar->i_ta, vap->iv_myaddr);
tid = tap->txa_tid;
barctl = (tap->txa_flags & IEEE80211_AGGR_IMMEDIATE ?
0 : IEEE80211_BAR_NOACK)
| IEEE80211_BAR_COMP
| SM(tid, IEEE80211_BAR_TID)
;
barseqctl = SM(seq, IEEE80211_BAR_SEQ_START);
/* NB: known to have proper alignment */
bar->i_ctl = htole16(barctl);
bar->i_seq = htole16(barseqctl);
m->m_pkthdr.len = m->m_len = sizeof(struct ieee80211_frame_bar);
M_WME_SETAC(m, WME_AC_VO);
IEEE80211_NODE_STAT(ni, tx_mgmt); /* XXX tx_ctl? */
/* XXX locking */
/* init/bump attempts counter */
if ((tap->txa_flags & IEEE80211_AGGR_BARPEND) == 0)
tap->txa_attempts = 1;
else
tap->txa_attempts++;
tap->txa_seqpending = seq;
tap->txa_flags |= IEEE80211_AGGR_BARPEND;
IEEE80211_NOTE(vap, IEEE80211_MSG_DEBUG | IEEE80211_MSG_11N,
ni, "send BAR: tid %u ctl 0x%x start %u (attempt %d)",
tid, barctl, seq, tap->txa_attempts);
/*
* ic_raw_xmit will free the node reference
* regardless of queue/TX success or failure.
*/
IEEE80211_TX_LOCK(ic);
ret = ieee80211_raw_output(vap, ni, m, NULL);
IEEE80211_TX_UNLOCK(ic);
if (ret != 0) {
IEEE80211_NOTE(vap, IEEE80211_MSG_DEBUG | IEEE80211_MSG_11N,
ni, "send BAR: failed: (ret = %d)\n",
ret);
/* xmit failed, clear state flag */
tap->txa_flags &= ~IEEE80211_AGGR_BARPEND;
vap->iv_stats.is_ampdu_bar_tx_fail++;
return ret;
}
/* XXX hack against tx complete happening before timer is started */
if (tap->txa_flags & IEEE80211_AGGR_BARPEND)
bar_start_timer(tap);
return 0;
bad:
IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N,
tap->txa_ni,
"%s: bad! ret=%d",
__func__, ret);
vap->iv_stats.is_ampdu_bar_tx_fail++;
ieee80211_free_node(ni);
return ret;
#undef senderr
}
static int
ht_action_output(struct ieee80211_node *ni, struct mbuf *m)
{
struct ieee80211_bpf_params params;
memset(&params, 0, sizeof(params));
params.ibp_pri = WME_AC_VO;
params.ibp_rate0 = ni->ni_txparms->mgmtrate;
/* NB: we know all frames are unicast */
params.ibp_try0 = ni->ni_txparms->maxretry;
params.ibp_power = ni->ni_txpower;
return ieee80211_mgmt_output(ni, m, IEEE80211_FC0_SUBTYPE_ACTION,
&params);
}
#define ADDSHORT(frm, v) do { \
frm[0] = (v) & 0xff; \
frm[1] = (v) >> 8; \
frm += 2; \
} while (0)
/*
* Send an action management frame. The arguments are stuff
* into a frame without inspection; the caller is assumed to
* prepare them carefully (e.g. based on the aggregation state).
*/
static int
ht_send_action_ba_addba(struct ieee80211_node *ni,
int category, int action, void *arg0)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
uint16_t *args = arg0;
struct mbuf *m;
uint8_t *frm;
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni,
"send ADDBA %s: dialogtoken %d status %d "
"baparamset 0x%x (tid %d) batimeout 0x%x baseqctl 0x%x",
(action == IEEE80211_ACTION_BA_ADDBA_REQUEST) ?
"request" : "response",
args[0], args[1], args[2], MS(args[2], IEEE80211_BAPS_TID),
args[3], args[4]);
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__,
ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1);
ieee80211_ref_node(ni);
m = ieee80211_getmgtframe(&frm,
ic->ic_headroom + sizeof(struct ieee80211_frame),
sizeof(uint16_t) /* action+category */
/* XXX may action payload */
+ sizeof(struct ieee80211_action_ba_addbaresponse)
);
if (m != NULL) {
*frm++ = category;
*frm++ = action;
*frm++ = args[0]; /* dialog token */
if (action == IEEE80211_ACTION_BA_ADDBA_RESPONSE)
ADDSHORT(frm, args[1]); /* status code */
ADDSHORT(frm, args[2]); /* baparamset */
ADDSHORT(frm, args[3]); /* batimeout */
if (action == IEEE80211_ACTION_BA_ADDBA_REQUEST)
ADDSHORT(frm, args[4]); /* baseqctl */
m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
return ht_action_output(ni, m);
} else {
vap->iv_stats.is_tx_nobuf++;
ieee80211_free_node(ni);
return ENOMEM;
}
}
static int
ht_send_action_ba_delba(struct ieee80211_node *ni,
int category, int action, void *arg0)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
uint16_t *args = arg0;
struct mbuf *m;
uint16_t baparamset;
uint8_t *frm;
baparamset = SM(args[0], IEEE80211_DELBAPS_TID)
| args[1]
;
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni,
"send DELBA action: tid %d, initiator %d reason %d",
args[0], args[1], args[2]);
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__,
ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1);
ieee80211_ref_node(ni);
m = ieee80211_getmgtframe(&frm,
ic->ic_headroom + sizeof(struct ieee80211_frame),
sizeof(uint16_t) /* action+category */
/* XXX may action payload */
+ sizeof(struct ieee80211_action_ba_addbaresponse)
);
if (m != NULL) {
*frm++ = category;
*frm++ = action;
ADDSHORT(frm, baparamset);
ADDSHORT(frm, args[2]); /* reason code */
m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
return ht_action_output(ni, m);
} else {
vap->iv_stats.is_tx_nobuf++;
ieee80211_free_node(ni);
return ENOMEM;
}
}
static int
ht_send_action_ht_txchwidth(struct ieee80211_node *ni,
int category, int action, void *arg0)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
struct mbuf *m;
uint8_t *frm;
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni,
"send HT txchwidth: width %d",
IEEE80211_IS_CHAN_HT40(ni->ni_chan) ? 40 : 20);
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__,
ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1);
ieee80211_ref_node(ni);
m = ieee80211_getmgtframe(&frm,
ic->ic_headroom + sizeof(struct ieee80211_frame),
sizeof(uint16_t) /* action+category */
/* XXX may action payload */
+ sizeof(struct ieee80211_action_ba_addbaresponse)
);
if (m != NULL) {
*frm++ = category;
*frm++ = action;
*frm++ = IEEE80211_IS_CHAN_HT40(ni->ni_chan) ?
IEEE80211_A_HT_TXCHWIDTH_2040 :
IEEE80211_A_HT_TXCHWIDTH_20;
m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
return ht_action_output(ni, m);
} else {
vap->iv_stats.is_tx_nobuf++;
ieee80211_free_node(ni);
return ENOMEM;
}
}
#undef ADDSHORT
/*
* Construct the MCS bit mask for inclusion in an HT capabilities
* information element.
*/
static void
ieee80211_set_mcsset(struct ieee80211com *ic, uint8_t *frm)
{
int i;
uint8_t txparams;
KASSERT((ic->ic_rxstream > 0 && ic->ic_rxstream <= 4),
("ic_rxstream %d out of range", ic->ic_rxstream));
KASSERT((ic->ic_txstream > 0 && ic->ic_txstream <= 4),
("ic_txstream %d out of range", ic->ic_txstream));
for (i = 0; i < ic->ic_rxstream * 8; i++)
setbit(frm, i);
if ((ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40) &&
(ic->ic_htcaps & IEEE80211_HTC_RXMCS32))
setbit(frm, 32);
if (ic->ic_htcaps & IEEE80211_HTC_RXUNEQUAL) {
if (ic->ic_rxstream >= 2) {
for (i = 33; i <= 38; i++)
setbit(frm, i);
}
if (ic->ic_rxstream >= 3) {
for (i = 39; i <= 52; i++)
setbit(frm, i);
}
if (ic->ic_txstream >= 4) {
for (i = 53; i <= 76; i++)
setbit(frm, i);
}
}
if (ic->ic_rxstream != ic->ic_txstream) {
txparams = 0x1; /* TX MCS set defined */
txparams |= 0x2; /* TX RX MCS not equal */
txparams |= (ic->ic_txstream - 1) << 2; /* num TX streams */
if (ic->ic_htcaps & IEEE80211_HTC_TXUNEQUAL)
txparams |= 0x16; /* TX unequal modulation sup */
} else
txparams = 0;
frm[12] = txparams;
}
/*
* Add body of an HTCAP information element.
*/
static uint8_t *
ieee80211_add_htcap_body(uint8_t *frm, struct ieee80211_node *ni)
{
#define ADDSHORT(frm, v) do { \
frm[0] = (v) & 0xff; \
frm[1] = (v) >> 8; \
frm += 2; \
} while (0)
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211vap *vap = ni->ni_vap;
uint16_t caps, extcaps;
int rxmax, density;
/* HT capabilities */
caps = vap->iv_htcaps & 0xffff;
/*
* Note channel width depends on whether we are operating as
* a sta or not. When operating as a sta we are generating
* a request based on our desired configuration. Otherwise
* we are operational and the channel attributes identify
* how we've been setup (which might be different if a fixed
* channel is specified).
*/
if (vap->iv_opmode == IEEE80211_M_STA) {
/* override 20/40 use based on config */
if (vap->iv_flags_ht & IEEE80211_FHT_USEHT40)
caps |= IEEE80211_HTCAP_CHWIDTH40;
else
caps &= ~IEEE80211_HTCAP_CHWIDTH40;
/* Start by using the advertised settings */
rxmax = MS(ni->ni_htparam, IEEE80211_HTCAP_MAXRXAMPDU);
density = MS(ni->ni_htparam, IEEE80211_HTCAP_MPDUDENSITY);
/* Cap at VAP rxmax */
if (rxmax > vap->iv_ampdu_rxmax)
rxmax = vap->iv_ampdu_rxmax;
/*
* If the VAP ampdu density value greater, use that.
*
* (Larger density value == larger minimum gap between A-MPDU
* subframes.)
*/
if (vap->iv_ampdu_density > density)
density = vap->iv_ampdu_density;
/*
* NB: Hardware might support HT40 on some but not all
* channels. We can't determine this earlier because only
* after association the channel is upgraded to HT based
* on the negotiated capabilities.
*/
if (ni->ni_chan != IEEE80211_CHAN_ANYC &&
findhtchan(ic, ni->ni_chan, IEEE80211_CHAN_HT40U) == NULL &&
findhtchan(ic, ni->ni_chan, IEEE80211_CHAN_HT40D) == NULL)
caps &= ~IEEE80211_HTCAP_CHWIDTH40;
} else {
/* override 20/40 use based on current channel */
if (IEEE80211_IS_CHAN_HT40(ni->ni_chan))
caps |= IEEE80211_HTCAP_CHWIDTH40;
else
caps &= ~IEEE80211_HTCAP_CHWIDTH40;
/* XXX TODO should it start by using advertised settings? */
rxmax = vap->iv_ampdu_rxmax;
density = vap->iv_ampdu_density;
}
/* adjust short GI based on channel and config */
if ((vap->iv_flags_ht & IEEE80211_FHT_SHORTGI20) == 0)
caps &= ~IEEE80211_HTCAP_SHORTGI20;
if ((vap->iv_flags_ht & IEEE80211_FHT_SHORTGI40) == 0 ||
(caps & IEEE80211_HTCAP_CHWIDTH40) == 0)
caps &= ~IEEE80211_HTCAP_SHORTGI40;
ADDSHORT(frm, caps);
/* HT parameters */
*frm = SM(rxmax, IEEE80211_HTCAP_MAXRXAMPDU)
| SM(density, IEEE80211_HTCAP_MPDUDENSITY)
;
frm++;
/* pre-zero remainder of ie */
memset(frm, 0, sizeof(struct ieee80211_ie_htcap) -
__offsetof(struct ieee80211_ie_htcap, hc_mcsset));
/* supported MCS set */
/*
* XXX: For sta mode the rate set should be restricted based
* on the AP's capabilities, but ni_htrates isn't setup when
* we're called to form an AssocReq frame so for now we're
* restricted to the device capabilities.
*/
ieee80211_set_mcsset(ni->ni_ic, frm);
frm += __offsetof(struct ieee80211_ie_htcap, hc_extcap) -
__offsetof(struct ieee80211_ie_htcap, hc_mcsset);
/* HT extended capabilities */
extcaps = vap->iv_htextcaps & 0xffff;
ADDSHORT(frm, extcaps);
frm += sizeof(struct ieee80211_ie_htcap) -
__offsetof(struct ieee80211_ie_htcap, hc_txbf);
return frm;
#undef ADDSHORT
}
/*
* Add 802.11n HT capabilities information element
*/
uint8_t *
ieee80211_add_htcap(uint8_t *frm, struct ieee80211_node *ni)
{
frm[0] = IEEE80211_ELEMID_HTCAP;
frm[1] = sizeof(struct ieee80211_ie_htcap) - 2;
return ieee80211_add_htcap_body(frm + 2, ni);
}
/*
* Add Broadcom OUI wrapped standard HTCAP ie; this is
* used for compatibility w/ pre-draft implementations.
*/
uint8_t *
ieee80211_add_htcap_vendor(uint8_t *frm, struct ieee80211_node *ni)
{
frm[0] = IEEE80211_ELEMID_VENDOR;
frm[1] = 4 + sizeof(struct ieee80211_ie_htcap) - 2;
frm[2] = (BCM_OUI >> 0) & 0xff;
frm[3] = (BCM_OUI >> 8) & 0xff;
frm[4] = (BCM_OUI >> 16) & 0xff;
frm[5] = BCM_OUI_HTCAP;
return ieee80211_add_htcap_body(frm + 6, ni);
}
/*
* Construct the MCS bit mask of basic rates
* for inclusion in an HT information element.
*/
static void
ieee80211_set_basic_htrates(uint8_t *frm, const struct ieee80211_htrateset *rs)
{
int i;
for (i = 0; i < rs->rs_nrates; i++) {
int r = rs->rs_rates[i] & IEEE80211_RATE_VAL;
if ((rs->rs_rates[i] & IEEE80211_RATE_BASIC) &&
r < IEEE80211_HTRATE_MAXSIZE) {
/* NB: this assumes a particular implementation */
setbit(frm, r);
}
}
}
/*
* Update the HTINFO ie for a beacon frame.
*/
void
ieee80211_ht_update_beacon(struct ieee80211vap *vap,
struct ieee80211_beacon_offsets *bo)
{
#define PROTMODE (IEEE80211_HTINFO_OPMODE|IEEE80211_HTINFO_NONHT_PRESENT)
struct ieee80211_node *ni;
const struct ieee80211_channel *bsschan;
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_ie_htinfo *ht =
(struct ieee80211_ie_htinfo *) bo->bo_htinfo;
ni = ieee80211_ref_node(vap->iv_bss);
bsschan = ni->ni_chan;
/* XXX only update on channel change */
ht->hi_ctrlchannel = ieee80211_chan2ieee(ic, bsschan);
if (vap->iv_flags_ht & IEEE80211_FHT_RIFS)
ht->hi_byte1 = IEEE80211_HTINFO_RIFSMODE_PERM;
else
ht->hi_byte1 = IEEE80211_HTINFO_RIFSMODE_PROH;
if (IEEE80211_IS_CHAN_HT40U(bsschan))
ht->hi_byte1 |= IEEE80211_HTINFO_2NDCHAN_ABOVE;
else if (IEEE80211_IS_CHAN_HT40D(bsschan))
ht->hi_byte1 |= IEEE80211_HTINFO_2NDCHAN_BELOW;
else
ht->hi_byte1 |= IEEE80211_HTINFO_2NDCHAN_NONE;
if (IEEE80211_IS_CHAN_HT40(bsschan))
ht->hi_byte1 |= IEEE80211_HTINFO_TXWIDTH_2040;
/* protection mode */
ht->hi_byte2 = (ht->hi_byte2 &~ PROTMODE) | ic->ic_curhtprotmode;
ieee80211_free_node(ni);
/* XXX propagate to vendor ie's */
#undef PROTMODE
}
/*
* Add body of an HTINFO information element.
*
* NB: We don't use struct ieee80211_ie_htinfo because we can
* be called to fillin both a standard ie and a compat ie that
* has a vendor OUI at the front.
*/
static uint8_t *
ieee80211_add_htinfo_body(uint8_t *frm, struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
/* pre-zero remainder of ie */
memset(frm, 0, sizeof(struct ieee80211_ie_htinfo) - 2);
/* primary/control channel center */
*frm++ = ieee80211_chan2ieee(ic, ni->ni_chan);
if (vap->iv_flags_ht & IEEE80211_FHT_RIFS)
frm[0] = IEEE80211_HTINFO_RIFSMODE_PERM;
else
frm[0] = IEEE80211_HTINFO_RIFSMODE_PROH;
if (IEEE80211_IS_CHAN_HT40U(ni->ni_chan))
frm[0] |= IEEE80211_HTINFO_2NDCHAN_ABOVE;
else if (IEEE80211_IS_CHAN_HT40D(ni->ni_chan))
frm[0] |= IEEE80211_HTINFO_2NDCHAN_BELOW;
else
frm[0] |= IEEE80211_HTINFO_2NDCHAN_NONE;
if (IEEE80211_IS_CHAN_HT40(ni->ni_chan))
frm[0] |= IEEE80211_HTINFO_TXWIDTH_2040;
frm[1] = ic->ic_curhtprotmode;
frm += 5;
/* basic MCS set */
ieee80211_set_basic_htrates(frm, &ni->ni_htrates);
frm += sizeof(struct ieee80211_ie_htinfo) -
__offsetof(struct ieee80211_ie_htinfo, hi_basicmcsset);
return frm;
}
/*
* Add 802.11n HT information information element.
*/
uint8_t *
ieee80211_add_htinfo(uint8_t *frm, struct ieee80211_node *ni)
{
frm[0] = IEEE80211_ELEMID_HTINFO;
frm[1] = sizeof(struct ieee80211_ie_htinfo) - 2;
return ieee80211_add_htinfo_body(frm + 2, ni);
}
/*
* Add Broadcom OUI wrapped standard HTINFO ie; this is
* used for compatibility w/ pre-draft implementations.
*/
uint8_t *
ieee80211_add_htinfo_vendor(uint8_t *frm, struct ieee80211_node *ni)
{
frm[0] = IEEE80211_ELEMID_VENDOR;
frm[1] = 4 + sizeof(struct ieee80211_ie_htinfo) - 2;
frm[2] = (BCM_OUI >> 0) & 0xff;
frm[3] = (BCM_OUI >> 8) & 0xff;
frm[4] = (BCM_OUI >> 16) & 0xff;
frm[5] = BCM_OUI_HTINFO;
return ieee80211_add_htinfo_body(frm + 6, ni);
}
Index: head/sys/net80211/ieee80211_hwmp.c
===================================================================
--- head/sys/net80211/ieee80211_hwmp.c (revision 283290)
+++ head/sys/net80211/ieee80211_hwmp.c (revision 283291)
@@ -1,2094 +1,2094 @@
/*-
* Copyright (c) 2009 The FreeBSD Foundation
* All rights reserved.
*
* This software was developed by Rui Paulo under sponsorship from the
* FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#ifdef __FreeBSD__
__FBSDID("$FreeBSD$");
#endif
/*
* IEEE 802.11s Hybrid Wireless Mesh Protocol, HWMP.
*
* Based on March 2009, D3.0 802.11s draft spec.
*/
#include "opt_inet.h"
#include "opt_wlan.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/endian.h>
#include <sys/errno.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <net/if.h>
#include <net/if_media.h>
#include <net/if_llc.h>
#include <net/ethernet.h>
#include <net/bpf.h>
#include <net80211/ieee80211_var.h>
#include <net80211/ieee80211_action.h>
#include <net80211/ieee80211_input.h>
#include <net80211/ieee80211_mesh.h>
static void hwmp_vattach(struct ieee80211vap *);
static void hwmp_vdetach(struct ieee80211vap *);
static int hwmp_newstate(struct ieee80211vap *,
enum ieee80211_state, int);
static int hwmp_send_action(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN],
uint8_t *, size_t);
static uint8_t * hwmp_add_meshpreq(uint8_t *,
const struct ieee80211_meshpreq_ie *);
static uint8_t * hwmp_add_meshprep(uint8_t *,
const struct ieee80211_meshprep_ie *);
static uint8_t * hwmp_add_meshperr(uint8_t *,
const struct ieee80211_meshperr_ie *);
static uint8_t * hwmp_add_meshrann(uint8_t *,
const struct ieee80211_meshrann_ie *);
static void hwmp_rootmode_setup(struct ieee80211vap *);
static void hwmp_rootmode_cb(void *);
static void hwmp_rootmode_rann_cb(void *);
static void hwmp_recv_preq(struct ieee80211vap *, struct ieee80211_node *,
const struct ieee80211_frame *,
const struct ieee80211_meshpreq_ie *);
static int hwmp_send_preq(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN],
struct ieee80211_meshpreq_ie *,
struct timeval *, struct timeval *);
static void hwmp_recv_prep(struct ieee80211vap *, struct ieee80211_node *,
const struct ieee80211_frame *,
const struct ieee80211_meshprep_ie *);
static int hwmp_send_prep(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN],
struct ieee80211_meshprep_ie *);
static void hwmp_recv_perr(struct ieee80211vap *, struct ieee80211_node *,
const struct ieee80211_frame *,
const struct ieee80211_meshperr_ie *);
static int hwmp_send_perr(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN],
struct ieee80211_meshperr_ie *);
static void hwmp_senderror(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN],
struct ieee80211_mesh_route *, int);
static void hwmp_recv_rann(struct ieee80211vap *, struct ieee80211_node *,
const struct ieee80211_frame *,
const struct ieee80211_meshrann_ie *);
static int hwmp_send_rann(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN],
struct ieee80211_meshrann_ie *);
static struct ieee80211_node *
hwmp_discover(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN], struct mbuf *);
static void hwmp_peerdown(struct ieee80211_node *);
static struct timeval ieee80211_hwmp_preqminint = { 0, 100000 };
static struct timeval ieee80211_hwmp_perrminint = { 0, 100000 };
/* unalligned little endian access */
#define LE_WRITE_2(p, v) do { \
((uint8_t *)(p))[0] = (v) & 0xff; \
((uint8_t *)(p))[1] = ((v) >> 8) & 0xff; \
} while (0)
#define LE_WRITE_4(p, v) do { \
((uint8_t *)(p))[0] = (v) & 0xff; \
((uint8_t *)(p))[1] = ((v) >> 8) & 0xff; \
((uint8_t *)(p))[2] = ((v) >> 16) & 0xff; \
((uint8_t *)(p))[3] = ((v) >> 24) & 0xff; \
} while (0)
/* NB: the Target Address set in a Proactive PREQ is the broadcast address. */
static const uint8_t broadcastaddr[IEEE80211_ADDR_LEN] =
{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
typedef uint32_t ieee80211_hwmp_seq;
#define HWMP_SEQ_LT(a, b) ((int32_t)((a)-(b)) < 0)
#define HWMP_SEQ_LEQ(a, b) ((int32_t)((a)-(b)) <= 0)
#define HWMP_SEQ_EQ(a, b) ((int32_t)((a)-(b)) == 0)
#define HWMP_SEQ_GT(a, b) ((int32_t)((a)-(b)) > 0)
#define HWMP_SEQ_GEQ(a, b) ((int32_t)((a)-(b)) >= 0)
#define HWMP_SEQ_MAX(a, b) (a > b ? a : b)
/*
* Private extension of ieee80211_mesh_route.
*/
struct ieee80211_hwmp_route {
ieee80211_hwmp_seq hr_seq; /* last HWMP seq seen from dst*/
ieee80211_hwmp_seq hr_preqid; /* last PREQ ID seen from dst */
ieee80211_hwmp_seq hr_origseq; /* seq. no. on our latest PREQ*/
struct timeval hr_lastpreq; /* last time we sent a PREQ */
struct timeval hr_lastrootconf; /* last sent PREQ root conf */
int hr_preqretries; /* number of discoveries */
int hr_lastdiscovery; /* last discovery in ticks */
};
struct ieee80211_hwmp_state {
ieee80211_hwmp_seq hs_seq; /* next seq to be used */
ieee80211_hwmp_seq hs_preqid; /* next PREQ ID to be used */
int hs_rootmode; /* proactive HWMP */
struct timeval hs_lastperr; /* last time we sent a PERR */
struct callout hs_roottimer;
uint8_t hs_maxhops; /* max hop count */
};
static SYSCTL_NODE(_net_wlan, OID_AUTO, hwmp, CTLFLAG_RD, 0,
"IEEE 802.11s HWMP parameters");
static int ieee80211_hwmp_targetonly = 0;
SYSCTL_INT(_net_wlan_hwmp, OID_AUTO, targetonly, CTLFLAG_RW,
&ieee80211_hwmp_targetonly, 0, "Set TO bit on generated PREQs");
static int ieee80211_hwmp_pathtimeout = -1;
SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, pathlifetime, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_hwmp_pathtimeout, 0, ieee80211_sysctl_msecs_ticks, "I",
"path entry lifetime (ms)");
static int ieee80211_hwmp_maxpreq_retries = -1;
SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, maxpreq_retries, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_hwmp_maxpreq_retries, 0, ieee80211_sysctl_msecs_ticks, "I",
"maximum number of preq retries");
static int ieee80211_hwmp_net_diameter_traversaltime = -1;
SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, net_diameter_traversal_time,
CTLTYPE_INT | CTLFLAG_RW, &ieee80211_hwmp_net_diameter_traversaltime, 0,
ieee80211_sysctl_msecs_ticks, "I",
"estimate travelse time across the MBSS (ms)");
static int ieee80211_hwmp_roottimeout = -1;
SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, roottimeout, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_hwmp_roottimeout, 0, ieee80211_sysctl_msecs_ticks, "I",
"root PREQ timeout (ms)");
static int ieee80211_hwmp_rootint = -1;
SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, rootint, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_hwmp_rootint, 0, ieee80211_sysctl_msecs_ticks, "I",
"root interval (ms)");
static int ieee80211_hwmp_rannint = -1;
SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, rannint, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_hwmp_rannint, 0, ieee80211_sysctl_msecs_ticks, "I",
"root announcement interval (ms)");
static struct timeval ieee80211_hwmp_rootconfint = { 0, 0 };
static int ieee80211_hwmp_rootconfint_internal = -1;
SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, rootconfint, CTLTYPE_INT | CTLFLAG_RD,
&ieee80211_hwmp_rootconfint_internal, 0, ieee80211_sysctl_msecs_ticks, "I",
"root confirmation interval (ms) (read-only)");
#define IEEE80211_HWMP_DEFAULT_MAXHOPS 31
static ieee80211_recv_action_func hwmp_recv_action_meshpath;
static struct ieee80211_mesh_proto_path mesh_proto_hwmp = {
.mpp_descr = "HWMP",
.mpp_ie = IEEE80211_MESHCONF_PATH_HWMP,
.mpp_discover = hwmp_discover,
.mpp_peerdown = hwmp_peerdown,
.mpp_senderror = hwmp_senderror,
.mpp_vattach = hwmp_vattach,
.mpp_vdetach = hwmp_vdetach,
.mpp_newstate = hwmp_newstate,
.mpp_privlen = sizeof(struct ieee80211_hwmp_route),
};
SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, inact, CTLTYPE_INT | CTLFLAG_RW,
&mesh_proto_hwmp.mpp_inact, 0, ieee80211_sysctl_msecs_ticks, "I",
"mesh route inactivity timeout (ms)");
static void
ieee80211_hwmp_init(void)
{
/* Default values as per amendment */
ieee80211_hwmp_pathtimeout = msecs_to_ticks(5*1000);
ieee80211_hwmp_roottimeout = msecs_to_ticks(5*1000);
ieee80211_hwmp_rootint = msecs_to_ticks(2*1000);
ieee80211_hwmp_rannint = msecs_to_ticks(1*1000);
ieee80211_hwmp_rootconfint_internal = msecs_to_ticks(2*1000);
ieee80211_hwmp_maxpreq_retries = 3;
/*
* (TU): A measurement of time equal to 1024 μs,
* 500 TU is 512 ms.
*/
ieee80211_hwmp_net_diameter_traversaltime = msecs_to_ticks(512);
/*
* NB: I dont know how to make SYSCTL_PROC that calls ms to ticks
* and return a struct timeval...
*/
ieee80211_hwmp_rootconfint.tv_usec =
ieee80211_hwmp_rootconfint_internal * 1000;
/*
* Register action frame handler.
*/
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_MESH,
IEEE80211_ACTION_MESH_HWMP, hwmp_recv_action_meshpath);
/* NB: default is 5 secs per spec */
mesh_proto_hwmp.mpp_inact = msecs_to_ticks(5*1000);
/*
* Register HWMP.
*/
ieee80211_mesh_register_proto_path(&mesh_proto_hwmp);
}
SYSINIT(wlan_hwmp, SI_SUB_DRIVERS, SI_ORDER_SECOND, ieee80211_hwmp_init, NULL);
void
hwmp_vattach(struct ieee80211vap *vap)
{
struct ieee80211_hwmp_state *hs;
KASSERT(vap->iv_opmode == IEEE80211_M_MBSS,
("not a mesh vap, opmode %d", vap->iv_opmode));
hs = malloc(sizeof(struct ieee80211_hwmp_state), M_80211_VAP,
M_NOWAIT | M_ZERO);
if (hs == NULL) {
printf("%s: couldn't alloc HWMP state\n", __func__);
return;
}
hs->hs_maxhops = IEEE80211_HWMP_DEFAULT_MAXHOPS;
- callout_init(&hs->hs_roottimer, CALLOUT_MPSAFE);
+ callout_init(&hs->hs_roottimer, 1);
vap->iv_hwmp = hs;
}
void
hwmp_vdetach(struct ieee80211vap *vap)
{
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
callout_drain(&hs->hs_roottimer);
free(vap->iv_hwmp, M_80211_VAP);
vap->iv_hwmp = NULL;
}
int
hwmp_newstate(struct ieee80211vap *vap, enum ieee80211_state ostate, int arg)
{
enum ieee80211_state nstate = vap->iv_state;
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE, "%s: %s -> %s (%d)\n",
__func__, ieee80211_state_name[ostate],
ieee80211_state_name[nstate], arg);
if (nstate != IEEE80211_S_RUN && ostate == IEEE80211_S_RUN)
callout_drain(&hs->hs_roottimer);
if (nstate == IEEE80211_S_RUN)
hwmp_rootmode_setup(vap);
return 0;
}
/*
* Verify the length of an HWMP PREQ and return the number
* of destinations >= 1, if verification fails -1 is returned.
*/
static int
verify_mesh_preq_len(struct ieee80211vap *vap,
const struct ieee80211_frame *wh, const uint8_t *iefrm)
{
int alloc_sz = -1;
int ndest = -1;
if (iefrm[2] & IEEE80211_MESHPREQ_FLAGS_AE) {
/* Originator External Address present */
alloc_sz = IEEE80211_MESHPREQ_BASE_SZ_AE;
ndest = iefrm[IEEE80211_MESHPREQ_TCNT_OFFSET_AE];
} else {
/* w/o Originator External Address */
alloc_sz = IEEE80211_MESHPREQ_BASE_SZ;
ndest = iefrm[IEEE80211_MESHPREQ_TCNT_OFFSET];
}
alloc_sz += ndest * IEEE80211_MESHPREQ_TRGT_SZ;
if(iefrm[1] != (alloc_sz)) {
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP,
wh, NULL, "PREQ (AE=%s) with wrong len",
iefrm[2] & IEEE80211_MESHPREQ_FLAGS_AE ? "1" : "0");
return (-1);
}
return ndest;
}
/*
* Verify the length of an HWMP PREP and returns 1 on success,
* otherwise -1.
*/
static int
verify_mesh_prep_len(struct ieee80211vap *vap,
const struct ieee80211_frame *wh, const uint8_t *iefrm)
{
int alloc_sz = -1;
if (iefrm[2] & IEEE80211_MESHPREP_FLAGS_AE) {
if (iefrm[1] == IEEE80211_MESHPREP_BASE_SZ_AE)
alloc_sz = IEEE80211_MESHPREP_BASE_SZ_AE;
} else if (iefrm[1] == IEEE80211_MESHPREP_BASE_SZ)
alloc_sz = IEEE80211_MESHPREP_BASE_SZ;
if(alloc_sz < 0) {
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP,
wh, NULL, "PREP (AE=%s) with wrong len",
iefrm[2] & IEEE80211_MESHPREP_FLAGS_AE ? "1" : "0");
return (-1);
}
return (1);
}
/*
* Verify the length of an HWMP PERR and return the number
* of destinations >= 1, if verification fails -1 is returned.
*/
static int
verify_mesh_perr_len(struct ieee80211vap *vap,
const struct ieee80211_frame *wh, const uint8_t *iefrm)
{
int alloc_sz = -1;
const uint8_t *iefrm_t = iefrm;
uint8_t ndest = iefrm_t[IEEE80211_MESHPERR_NDEST_OFFSET];
int i;
if(ndest > IEEE80211_MESHPERR_MAXDEST) {
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP,
wh, NULL, "PERR with wrong number of destionat (>19), %u",
ndest);
return (-1);
}
iefrm_t += IEEE80211_MESHPERR_NDEST_OFFSET + 1; /* flag is next field */
/* We need to check each destionation flag to know size */
for(i = 0; i<ndest; i++) {
if ((*iefrm_t) & IEEE80211_MESHPERR_FLAGS_AE)
iefrm_t += IEEE80211_MESHPERR_DEST_SZ_AE;
else
iefrm_t += IEEE80211_MESHPERR_DEST_SZ;
}
alloc_sz = (iefrm_t - iefrm) - 2; /* action + code */
if(alloc_sz != iefrm[1]) {
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP,
wh, NULL, "%s", "PERR with wrong len");
return (-1);
}
return ndest;
}
static int
hwmp_recv_action_meshpath(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_meshpreq_ie *preq;
struct ieee80211_meshprep_ie *prep;
struct ieee80211_meshperr_ie *perr;
struct ieee80211_meshrann_ie rann;
const uint8_t *iefrm = frm + 2; /* action + code */
const uint8_t *iefrm_t = iefrm; /* temporary pointer */
int ndest = -1;
int found = 0;
while (efrm - iefrm > 1) {
IEEE80211_VERIFY_LENGTH(efrm - iefrm, iefrm[1] + 2, return 0);
switch (*iefrm) {
case IEEE80211_ELEMID_MESHPREQ:
{
int i = 0;
iefrm_t = iefrm;
ndest = verify_mesh_preq_len(vap, wh, iefrm_t);
if (ndest < 0) {
vap->iv_stats.is_rx_mgtdiscard++;
break;
}
preq = malloc(sizeof(*preq) +
(ndest - 1) * sizeof(*preq->preq_targets),
M_80211_MESH_PREQ, M_NOWAIT | M_ZERO);
KASSERT(preq != NULL, ("preq == NULL"));
preq->preq_ie = *iefrm_t++;
preq->preq_len = *iefrm_t++;
preq->preq_flags = *iefrm_t++;
preq->preq_hopcount = *iefrm_t++;
preq->preq_ttl = *iefrm_t++;
preq->preq_id = LE_READ_4(iefrm_t); iefrm_t += 4;
IEEE80211_ADDR_COPY(preq->preq_origaddr, iefrm_t);
iefrm_t += 6;
preq->preq_origseq = LE_READ_4(iefrm_t); iefrm_t += 4;
/* NB: may have Originator Proxied Address */
if (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AE) {
IEEE80211_ADDR_COPY(
preq->preq_orig_ext_addr, iefrm_t);
iefrm_t += 6;
}
preq->preq_lifetime = LE_READ_4(iefrm_t); iefrm_t += 4;
preq->preq_metric = LE_READ_4(iefrm_t); iefrm_t += 4;
preq->preq_tcount = *iefrm_t++;
for (i = 0; i < preq->preq_tcount; i++) {
preq->preq_targets[i].target_flags = *iefrm_t++;
IEEE80211_ADDR_COPY(
preq->preq_targets[i].target_addr, iefrm_t);
iefrm_t += 6;
preq->preq_targets[i].target_seq =
LE_READ_4(iefrm_t);
iefrm_t += 4;
}
hwmp_recv_preq(vap, ni, wh, preq);
free(preq, M_80211_MESH_PREQ);
found++;
break;
}
case IEEE80211_ELEMID_MESHPREP:
{
iefrm_t = iefrm;
ndest = verify_mesh_prep_len(vap, wh, iefrm_t);
if (ndest < 0) {
vap->iv_stats.is_rx_mgtdiscard++;
break;
}
prep = malloc(sizeof(*prep),
M_80211_MESH_PREP, M_NOWAIT | M_ZERO);
KASSERT(prep != NULL, ("prep == NULL"));
prep->prep_ie = *iefrm_t++;
prep->prep_len = *iefrm_t++;
prep->prep_flags = *iefrm_t++;
prep->prep_hopcount = *iefrm_t++;
prep->prep_ttl = *iefrm_t++;
IEEE80211_ADDR_COPY(prep->prep_targetaddr, iefrm_t);
iefrm_t += 6;
prep->prep_targetseq = LE_READ_4(iefrm_t); iefrm_t += 4;
/* NB: May have Target Proxied Address */
if (prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE) {
IEEE80211_ADDR_COPY(
prep->prep_target_ext_addr, iefrm_t);
iefrm_t += 6;
}
prep->prep_lifetime = LE_READ_4(iefrm_t); iefrm_t += 4;
prep->prep_metric = LE_READ_4(iefrm_t); iefrm_t += 4;
IEEE80211_ADDR_COPY(prep->prep_origaddr, iefrm_t);
iefrm_t += 6;
prep->prep_origseq = LE_READ_4(iefrm_t); iefrm_t += 4;
hwmp_recv_prep(vap, ni, wh, prep);
free(prep, M_80211_MESH_PREP);
found++;
break;
}
case IEEE80211_ELEMID_MESHPERR:
{
int i = 0;
iefrm_t = iefrm;
ndest = verify_mesh_perr_len(vap, wh, iefrm_t);
if (ndest < 0) {
vap->iv_stats.is_rx_mgtdiscard++;
break;
}
perr = malloc(sizeof(*perr) +
(ndest - 1) * sizeof(*perr->perr_dests),
M_80211_MESH_PERR, M_NOWAIT | M_ZERO);
KASSERT(perr != NULL, ("perr == NULL"));
perr->perr_ie = *iefrm_t++;
perr->perr_len = *iefrm_t++;
perr->perr_ttl = *iefrm_t++;
perr->perr_ndests = *iefrm_t++;
for (i = 0; i<perr->perr_ndests; i++) {
perr->perr_dests[i].dest_flags = *iefrm_t++;
IEEE80211_ADDR_COPY(
perr->perr_dests[i].dest_addr, iefrm_t);
iefrm_t += 6;
perr->perr_dests[i].dest_seq = LE_READ_4(iefrm_t);
iefrm_t += 4;
/* NB: May have Target Proxied Address */
if (perr->perr_dests[i].dest_flags &
IEEE80211_MESHPERR_FLAGS_AE) {
IEEE80211_ADDR_COPY(
perr->perr_dests[i].dest_ext_addr,
iefrm_t);
iefrm_t += 6;
}
perr->perr_dests[i].dest_rcode =
LE_READ_2(iefrm_t);
iefrm_t += 2;
}
hwmp_recv_perr(vap, ni, wh, perr);
free(perr, M_80211_MESH_PERR);
found++;
break;
}
case IEEE80211_ELEMID_MESHRANN:
{
const struct ieee80211_meshrann_ie *mrann =
(const struct ieee80211_meshrann_ie *) iefrm;
if (mrann->rann_len !=
sizeof(struct ieee80211_meshrann_ie) - 2) {
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP,
wh, NULL, "%s", "RAN with wrong len");
vap->iv_stats.is_rx_mgtdiscard++;
return 1;
}
memcpy(&rann, mrann, sizeof(rann));
rann.rann_seq = LE_READ_4(&mrann->rann_seq);
rann.rann_interval = LE_READ_4(&mrann->rann_interval);
rann.rann_metric = LE_READ_4(&mrann->rann_metric);
hwmp_recv_rann(vap, ni, wh, &rann);
found++;
break;
}
}
iefrm += iefrm[1] + 2;
}
if (!found) {
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP,
wh, NULL, "%s", "PATH SEL action without IE");
vap->iv_stats.is_rx_mgtdiscard++;
}
return 0;
}
static int
hwmp_send_action(struct ieee80211vap *vap,
const uint8_t da[IEEE80211_ADDR_LEN],
uint8_t *ie, size_t len)
{
struct ieee80211_node *ni;
struct ieee80211com *ic;
struct ieee80211_bpf_params params;
struct mbuf *m;
uint8_t *frm;
int ret;
if (IEEE80211_IS_MULTICAST(da)) {
ni = ieee80211_ref_node(vap->iv_bss);
#ifdef IEEE80211_DEBUG_REFCNT
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n",
__func__, __LINE__,
ni, ether_sprintf(ni->ni_macaddr),
ieee80211_node_refcnt(ni)+1);
#endif
ieee80211_ref_node(ni);
}
else
ni = ieee80211_mesh_find_txnode(vap, da);
if (vap->iv_state == IEEE80211_S_CAC) {
IEEE80211_NOTE(vap, IEEE80211_MSG_OUTPUT, ni,
"block %s frame in CAC state", "HWMP action");
vap->iv_stats.is_tx_badstate++;
return EIO; /* XXX */
}
KASSERT(ni != NULL, ("null node"));
ic = ni->ni_ic;
m = ieee80211_getmgtframe(&frm,
ic->ic_headroom + sizeof(struct ieee80211_frame),
sizeof(struct ieee80211_action) + len
);
if (m == NULL) {
ieee80211_free_node(ni);
vap->iv_stats.is_tx_nobuf++;
return ENOMEM;
}
*frm++ = IEEE80211_ACTION_CAT_MESH;
*frm++ = IEEE80211_ACTION_MESH_HWMP;
switch (*ie) {
case IEEE80211_ELEMID_MESHPREQ:
frm = hwmp_add_meshpreq(frm,
(struct ieee80211_meshpreq_ie *)ie);
break;
case IEEE80211_ELEMID_MESHPREP:
frm = hwmp_add_meshprep(frm,
(struct ieee80211_meshprep_ie *)ie);
break;
case IEEE80211_ELEMID_MESHPERR:
frm = hwmp_add_meshperr(frm,
(struct ieee80211_meshperr_ie *)ie);
break;
case IEEE80211_ELEMID_MESHRANN:
frm = hwmp_add_meshrann(frm,
(struct ieee80211_meshrann_ie *)ie);
break;
}
m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
M_PREPEND(m, sizeof(struct ieee80211_frame), M_NOWAIT);
if (m == NULL) {
ieee80211_free_node(ni);
vap->iv_stats.is_tx_nobuf++;
return ENOMEM;
}
IEEE80211_TX_LOCK(ic);
ieee80211_send_setup(ni, m,
IEEE80211_FC0_TYPE_MGT | IEEE80211_FC0_SUBTYPE_ACTION,
IEEE80211_NONQOS_TID, vap->iv_myaddr, da, vap->iv_myaddr);
m->m_flags |= M_ENCAP; /* mark encapsulated */
IEEE80211_NODE_STAT(ni, tx_mgmt);
memset(&params, 0, sizeof(params));
params.ibp_pri = WME_AC_VO;
params.ibp_rate0 = ni->ni_txparms->mgmtrate;
if (IEEE80211_IS_MULTICAST(da))
params.ibp_try0 = 1;
else
params.ibp_try0 = ni->ni_txparms->maxretry;
params.ibp_power = ni->ni_txpower;
ret = ieee80211_raw_output(vap, ni, m, &params);
IEEE80211_TX_UNLOCK(ic);
return (ret);
}
#define ADDSHORT(frm, v) do { \
frm[0] = (v) & 0xff; \
frm[1] = (v) >> 8; \
frm += 2; \
} while (0)
#define ADDWORD(frm, v) do { \
LE_WRITE_4(frm, v); \
frm += 4; \
} while (0)
/*
* Add a Mesh Path Request IE to a frame.
*/
#define PREQ_TFLAGS(n) preq->preq_targets[n].target_flags
#define PREQ_TADDR(n) preq->preq_targets[n].target_addr
#define PREQ_TSEQ(n) preq->preq_targets[n].target_seq
static uint8_t *
hwmp_add_meshpreq(uint8_t *frm, const struct ieee80211_meshpreq_ie *preq)
{
int i;
*frm++ = IEEE80211_ELEMID_MESHPREQ;
*frm++ = preq->preq_len; /* len already calculated */
*frm++ = preq->preq_flags;
*frm++ = preq->preq_hopcount;
*frm++ = preq->preq_ttl;
ADDWORD(frm, preq->preq_id);
IEEE80211_ADDR_COPY(frm, preq->preq_origaddr); frm += 6;
ADDWORD(frm, preq->preq_origseq);
if (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AE) {
IEEE80211_ADDR_COPY(frm, preq->preq_orig_ext_addr);
frm += 6;
}
ADDWORD(frm, preq->preq_lifetime);
ADDWORD(frm, preq->preq_metric);
*frm++ = preq->preq_tcount;
for (i = 0; i < preq->preq_tcount; i++) {
*frm++ = PREQ_TFLAGS(i);
IEEE80211_ADDR_COPY(frm, PREQ_TADDR(i));
frm += 6;
ADDWORD(frm, PREQ_TSEQ(i));
}
return frm;
}
#undef PREQ_TFLAGS
#undef PREQ_TADDR
#undef PREQ_TSEQ
/*
* Add a Mesh Path Reply IE to a frame.
*/
static uint8_t *
hwmp_add_meshprep(uint8_t *frm, const struct ieee80211_meshprep_ie *prep)
{
*frm++ = IEEE80211_ELEMID_MESHPREP;
*frm++ = prep->prep_len; /* len already calculated */
*frm++ = prep->prep_flags;
*frm++ = prep->prep_hopcount;
*frm++ = prep->prep_ttl;
IEEE80211_ADDR_COPY(frm, prep->prep_targetaddr); frm += 6;
ADDWORD(frm, prep->prep_targetseq);
if (prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE) {
IEEE80211_ADDR_COPY(frm, prep->prep_target_ext_addr);
frm += 6;
}
ADDWORD(frm, prep->prep_lifetime);
ADDWORD(frm, prep->prep_metric);
IEEE80211_ADDR_COPY(frm, prep->prep_origaddr); frm += 6;
ADDWORD(frm, prep->prep_origseq);
return frm;
}
/*
* Add a Mesh Path Error IE to a frame.
*/
#define PERR_DFLAGS(n) perr->perr_dests[n].dest_flags
#define PERR_DADDR(n) perr->perr_dests[n].dest_addr
#define PERR_DSEQ(n) perr->perr_dests[n].dest_seq
#define PERR_EXTADDR(n) perr->perr_dests[n].dest_ext_addr
#define PERR_DRCODE(n) perr->perr_dests[n].dest_rcode
static uint8_t *
hwmp_add_meshperr(uint8_t *frm, const struct ieee80211_meshperr_ie *perr)
{
int i;
*frm++ = IEEE80211_ELEMID_MESHPERR;
*frm++ = perr->perr_len; /* len already calculated */
*frm++ = perr->perr_ttl;
*frm++ = perr->perr_ndests;
for (i = 0; i < perr->perr_ndests; i++) {
*frm++ = PERR_DFLAGS(i);
IEEE80211_ADDR_COPY(frm, PERR_DADDR(i));
frm += 6;
ADDWORD(frm, PERR_DSEQ(i));
if (PERR_DFLAGS(i) & IEEE80211_MESHPERR_FLAGS_AE) {
IEEE80211_ADDR_COPY(frm, PERR_EXTADDR(i));
frm += 6;
}
ADDSHORT(frm, PERR_DRCODE(i));
}
return frm;
}
#undef PERR_DFLAGS
#undef PERR_DADDR
#undef PERR_DSEQ
#undef PERR_EXTADDR
#undef PERR_DRCODE
/*
* Add a Root Annoucement IE to a frame.
*/
static uint8_t *
hwmp_add_meshrann(uint8_t *frm, const struct ieee80211_meshrann_ie *rann)
{
*frm++ = IEEE80211_ELEMID_MESHRANN;
*frm++ = rann->rann_len;
*frm++ = rann->rann_flags;
*frm++ = rann->rann_hopcount;
*frm++ = rann->rann_ttl;
IEEE80211_ADDR_COPY(frm, rann->rann_addr); frm += 6;
ADDWORD(frm, rann->rann_seq);
ADDWORD(frm, rann->rann_interval);
ADDWORD(frm, rann->rann_metric);
return frm;
}
static void
hwmp_rootmode_setup(struct ieee80211vap *vap)
{
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
switch (hs->hs_rootmode) {
case IEEE80211_HWMP_ROOTMODE_DISABLED:
callout_drain(&hs->hs_roottimer);
ms->ms_flags &= ~IEEE80211_MESHFLAGS_ROOT;
break;
case IEEE80211_HWMP_ROOTMODE_NORMAL:
case IEEE80211_HWMP_ROOTMODE_PROACTIVE:
callout_reset(&hs->hs_roottimer, ieee80211_hwmp_rootint,
hwmp_rootmode_cb, vap);
ms->ms_flags |= IEEE80211_MESHFLAGS_ROOT;
break;
case IEEE80211_HWMP_ROOTMODE_RANN:
callout_reset(&hs->hs_roottimer, ieee80211_hwmp_rannint,
hwmp_rootmode_rann_cb, vap);
ms->ms_flags |= IEEE80211_MESHFLAGS_ROOT;
break;
}
}
/*
* Send a broadcast Path Request to find all nodes on the mesh. We are
* called when the vap is configured as a HWMP root node.
*/
#define PREQ_TFLAGS(n) preq.preq_targets[n].target_flags
#define PREQ_TADDR(n) preq.preq_targets[n].target_addr
#define PREQ_TSEQ(n) preq.preq_targets[n].target_seq
static void
hwmp_rootmode_cb(void *arg)
{
struct ieee80211vap *vap = (struct ieee80211vap *)arg;
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_meshpreq_ie preq;
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, vap->iv_bss,
"%s", "send broadcast PREQ");
preq.preq_flags = 0;
if (ms->ms_flags & IEEE80211_MESHFLAGS_GATE)
preq.preq_flags |= IEEE80211_MESHPREQ_FLAGS_GATE;
if (hs->hs_rootmode == IEEE80211_HWMP_ROOTMODE_PROACTIVE)
preq.preq_flags |= IEEE80211_MESHPREQ_FLAGS_PP;
preq.preq_hopcount = 0;
preq.preq_ttl = ms->ms_ttl;
preq.preq_id = ++hs->hs_preqid;
IEEE80211_ADDR_COPY(preq.preq_origaddr, vap->iv_myaddr);
preq.preq_origseq = ++hs->hs_seq;
preq.preq_lifetime = ticks_to_msecs(ieee80211_hwmp_roottimeout);
preq.preq_metric = IEEE80211_MESHLMETRIC_INITIALVAL;
preq.preq_tcount = 1;
IEEE80211_ADDR_COPY(PREQ_TADDR(0), broadcastaddr);
PREQ_TFLAGS(0) = IEEE80211_MESHPREQ_TFLAGS_TO |
IEEE80211_MESHPREQ_TFLAGS_USN;
PREQ_TSEQ(0) = 0;
vap->iv_stats.is_hwmp_rootreqs++;
/* NB: we enforce rate check ourself */
hwmp_send_preq(vap, broadcastaddr, &preq, NULL, NULL);
hwmp_rootmode_setup(vap);
}
#undef PREQ_TFLAGS
#undef PREQ_TADDR
#undef PREQ_TSEQ
/*
* Send a Root Annoucement (RANN) to find all the nodes on the mesh. We are
* called when the vap is configured as a HWMP RANN root node.
*/
static void
hwmp_rootmode_rann_cb(void *arg)
{
struct ieee80211vap *vap = (struct ieee80211vap *)arg;
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_meshrann_ie rann;
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, vap->iv_bss,
"%s", "send broadcast RANN");
rann.rann_flags = 0;
if (ms->ms_flags & IEEE80211_MESHFLAGS_GATE)
rann.rann_flags |= IEEE80211_MESHFLAGS_GATE;
rann.rann_hopcount = 0;
rann.rann_ttl = ms->ms_ttl;
IEEE80211_ADDR_COPY(rann.rann_addr, vap->iv_myaddr);
rann.rann_seq = ++hs->hs_seq;
rann.rann_interval = ieee80211_hwmp_rannint;
rann.rann_metric = IEEE80211_MESHLMETRIC_INITIALVAL;
vap->iv_stats.is_hwmp_rootrann++;
hwmp_send_rann(vap, broadcastaddr, &rann);
hwmp_rootmode_setup(vap);
}
/*
* Update forwarding information to TA if metric improves.
*/
static void
hwmp_update_transmitter(struct ieee80211vap *vap, struct ieee80211_node *ni,
const char *hwmp_frame)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rttran = NULL; /* Transmitter */
int metric = 0;
rttran = ieee80211_mesh_rt_find(vap, ni->ni_macaddr);
if (rttran == NULL) {
rttran = ieee80211_mesh_rt_add(vap, ni->ni_macaddr);
if (rttran == NULL) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"unable to add path to transmitter %6D of %s",
ni->ni_macaddr, ":", hwmp_frame);
vap->iv_stats.is_mesh_rtaddfailed++;
return;
}
}
metric = ms->ms_pmetric->mpm_metric(ni);
if (!(rttran->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) ||
rttran->rt_metric > metric)
{
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"%s path to transmiter %6D of %s, metric %d:%d",
rttran->rt_flags & IEEE80211_MESHRT_FLAGS_VALID ?
"prefer" : "update", ni->ni_macaddr, ":", hwmp_frame,
rttran->rt_metric, metric);
IEEE80211_ADDR_COPY(rttran->rt_nexthop, ni->ni_macaddr);
rttran->rt_metric = metric;
rttran->rt_nhops = 1;
ieee80211_mesh_rt_update(rttran, ms->ms_ppath->mpp_inact);
rttran->rt_flags = IEEE80211_MESHRT_FLAGS_VALID;
}
}
#define PREQ_TFLAGS(n) preq->preq_targets[n].target_flags
#define PREQ_TADDR(n) preq->preq_targets[n].target_addr
#define PREQ_TSEQ(n) preq->preq_targets[n].target_seq
static void
hwmp_recv_preq(struct ieee80211vap *vap, struct ieee80211_node *ni,
const struct ieee80211_frame *wh, const struct ieee80211_meshpreq_ie *preq)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rtorig = NULL;
struct ieee80211_mesh_route *rtorig_ext = NULL;
struct ieee80211_mesh_route *rttarg = NULL;
struct ieee80211_hwmp_route *hrorig = NULL;
struct ieee80211_hwmp_route *hrtarg = NULL;
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
struct ieee80211_meshprep_ie prep;
ieee80211_hwmp_seq preqid; /* last seen preqid for orig */
uint32_t metric = 0;
/*
* Ignore PREQs from us. Could happen because someone forward it
* back to us.
*/
if (IEEE80211_ADDR_EQ(vap->iv_myaddr, preq->preq_origaddr))
return;
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"received PREQ, orig %6D, targ(0) %6D", preq->preq_origaddr, ":",
PREQ_TADDR(0), ":");
/*
* Acceptance criteria: (if the PREQ is not for us or not broadcast,
* or an external mac address not proxied by us),
* AND forwarding is disabled, discard this PREQ.
*/
rttarg = ieee80211_mesh_rt_find(vap, PREQ_TADDR(0));
if (!(ms->ms_flags & IEEE80211_MESHFLAGS_FWD) &&
(!IEEE80211_ADDR_EQ(vap->iv_myaddr, PREQ_TADDR(0)) ||
!IEEE80211_IS_MULTICAST(PREQ_TADDR(0)) ||
(rttarg != NULL &&
rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY &&
IEEE80211_ADDR_EQ(vap->iv_myaddr, rttarg->rt_mesh_gate)))) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_HWMP,
preq->preq_origaddr, NULL, "%s", "not accepting PREQ");
return;
}
/*
* Acceptance criteria: if unicast addressed
* AND no valid forwarding for Target of PREQ, discard this PREQ.
*/
if(rttarg != NULL)
hrtarg = IEEE80211_MESH_ROUTE_PRIV(rttarg,
struct ieee80211_hwmp_route);
/* Address mode: ucast */
if(preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AM &&
rttarg == NULL &&
!IEEE80211_ADDR_EQ(vap->iv_myaddr, PREQ_TADDR(0))) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_HWMP,
preq->preq_origaddr, NULL,
"unicast addressed PREQ of unknown target %6D",
PREQ_TADDR(0), ":");
return;
}
/* PREQ ACCEPTED */
rtorig = ieee80211_mesh_rt_find(vap, preq->preq_origaddr);
if (rtorig == NULL) {
rtorig = ieee80211_mesh_rt_add(vap, preq->preq_origaddr);
if (rtorig == NULL) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"unable to add orig path to %6D",
preq->preq_origaddr, ":");
vap->iv_stats.is_mesh_rtaddfailed++;
return;
}
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"adding originator %6D", preq->preq_origaddr, ":");
}
hrorig = IEEE80211_MESH_ROUTE_PRIV(rtorig, struct ieee80211_hwmp_route);
/* record last seen preqid */
preqid = hrorig->hr_preqid;
hrorig->hr_preqid = HWMP_SEQ_MAX(hrorig->hr_preqid, preq->preq_id);
/* Data creation and update of forwarding information
* according to Table 11C-8 for originator mesh STA.
*/
metric = preq->preq_metric + ms->ms_pmetric->mpm_metric(ni);
if (HWMP_SEQ_GT(preq->preq_origseq, hrorig->hr_seq) ||
(HWMP_SEQ_EQ(preq->preq_origseq, hrorig->hr_seq) &&
metric < rtorig->rt_metric)) {
hrorig->hr_seq = preq->preq_origseq;
IEEE80211_ADDR_COPY(rtorig->rt_nexthop, wh->i_addr2);
rtorig->rt_metric = metric;
rtorig->rt_nhops = preq->preq_hopcount + 1;
ieee80211_mesh_rt_update(rtorig, preq->preq_lifetime);
/* Path to orig is valid now.
* NB: we know it can't be Proxy, and if it is GATE
* it will be marked below.
*/
rtorig->rt_flags = IEEE80211_MESHRT_FLAGS_VALID;
} else if ((hrtarg != NULL &&
!HWMP_SEQ_EQ(hrtarg->hr_seq, PREQ_TSEQ(0))) ||
(rtorig->rt_flags & IEEE80211_MESHRT_FLAGS_VALID &&
preqid >= preq->preq_id)) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"discard PREQ from %6D, old seqno %u <= %u,"
" or old preqid %u < %u",
preq->preq_origaddr, ":",
preq->preq_origseq, hrorig->hr_seq,
preq->preq_id, preqid);
return;
}
/* Update forwarding information to TA if metric improves. */
hwmp_update_transmitter(vap, ni, "PREQ");
/*
* Check if the PREQ is addressed to us.
* or a Proxy currently gated by us.
*/
if (IEEE80211_ADDR_EQ(vap->iv_myaddr, PREQ_TADDR(0)) ||
(ms->ms_flags & IEEE80211_MESHFLAGS_GATE &&
rttarg != NULL &&
IEEE80211_ADDR_EQ(vap->iv_myaddr, rttarg->rt_mesh_gate) &&
rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY &&
rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_VALID)) {
/*
* When we are the target we shall update our own HWMP seq
* number with max of (current and preq->seq) + 1
*/
hs->hs_seq = HWMP_SEQ_MAX(hs->hs_seq, PREQ_TSEQ(0)) + 1;
prep.prep_flags = 0;
prep.prep_hopcount = 0;
prep.prep_metric = IEEE80211_MESHLMETRIC_INITIALVAL;
IEEE80211_ADDR_COPY(prep.prep_targetaddr, vap->iv_myaddr);
if (rttarg != NULL && /* if NULL it means we are the target */
rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"reply for proxy %6D", rttarg->rt_dest, ":");
prep.prep_flags |= IEEE80211_MESHPREP_FLAGS_AE;
IEEE80211_ADDR_COPY(prep.prep_target_ext_addr,
rttarg->rt_dest);
/* update proxy seqno to HWMP seqno */
rttarg->rt_ext_seq = hs->hs_seq;
prep.prep_hopcount = rttarg->rt_nhops;
prep.prep_metric = rttarg->rt_metric;
IEEE80211_ADDR_COPY(prep.prep_targetaddr, rttarg->rt_mesh_gate);
}
/*
* Build and send a PREP frame.
*/
prep.prep_ttl = ms->ms_ttl;
prep.prep_targetseq = hs->hs_seq;
prep.prep_lifetime = preq->preq_lifetime;
IEEE80211_ADDR_COPY(prep.prep_origaddr, preq->preq_origaddr);
prep.prep_origseq = preq->preq_origseq;
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"reply to %6D", preq->preq_origaddr, ":");
hwmp_send_prep(vap, wh->i_addr2, &prep);
return;
}
/* we may update our proxy information for the orig external */
else if (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AE) {
rtorig_ext =
ieee80211_mesh_rt_find(vap, preq->preq_orig_ext_addr);
if (rtorig_ext == NULL) {
rtorig_ext = ieee80211_mesh_rt_add(vap,
preq->preq_orig_ext_addr);
if (rtorig_ext == NULL) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"unable to add orig ext proxy to %6D",
preq->preq_orig_ext_addr, ":");
vap->iv_stats.is_mesh_rtaddfailed++;
return;
}
IEEE80211_ADDR_COPY(rtorig_ext->rt_mesh_gate,
preq->preq_origaddr);
}
rtorig_ext->rt_ext_seq = preq->preq_origseq;
ieee80211_mesh_rt_update(rtorig_ext, preq->preq_lifetime);
}
/*
* Proactive PREQ: reply with a proactive PREP to the
* root STA if requested.
*/
if (IEEE80211_ADDR_EQ(PREQ_TADDR(0), broadcastaddr) &&
(PREQ_TFLAGS(0) & IEEE80211_MESHPREQ_TFLAGS_TO)) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"root mesh station @ %6D", preq->preq_origaddr, ":");
/* Check if root is a mesh gate, mark it */
if (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_GATE) {
struct ieee80211_mesh_gate_route *gr;
rtorig->rt_flags |= IEEE80211_MESHRT_FLAGS_GATE;
gr = ieee80211_mesh_mark_gate(vap, preq->preq_origaddr,
rtorig);
gr->gr_lastseq = 0; /* NOT GANN */
}
/*
* Reply with a PREP if we don't have a path to the root
* or if the root sent us a proactive PREQ.
*/
if ((rtorig->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0 ||
(preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_PP)) {
prep.prep_flags = 0;
prep.prep_hopcount = 0;
prep.prep_ttl = ms->ms_ttl;
IEEE80211_ADDR_COPY(prep.prep_origaddr,
preq->preq_origaddr);
prep.prep_origseq = preq->preq_origseq;
prep.prep_lifetime = preq->preq_lifetime;
prep.prep_metric = IEEE80211_MESHLMETRIC_INITIALVAL;
IEEE80211_ADDR_COPY(prep.prep_targetaddr,
vap->iv_myaddr);
prep.prep_targetseq = ++hs->hs_seq;
hwmp_send_prep(vap, rtorig->rt_nexthop, &prep);
}
}
/*
* Forwarding and Intermediate reply for PREQs with 1 target.
*/
if ((preq->preq_tcount == 1) && (preq->preq_ttl > 1) &&
(ms->ms_flags & IEEE80211_MESHFLAGS_FWD)) {
struct ieee80211_meshpreq_ie ppreq; /* propagated PREQ */
memcpy(&ppreq, preq, sizeof(ppreq));
/*
* We have a valid route to this node.
* NB: if target is proxy dont reply.
*/
if (rttarg != NULL &&
rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_VALID &&
!(rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY)) {
/*
* Check if we can send an intermediate Path Reply,
* i.e., Target Only bit is not set and target is not
* the MAC broadcast address.
*/
if (!(PREQ_TFLAGS(0) & IEEE80211_MESHPREQ_TFLAGS_TO) &&
!IEEE80211_ADDR_EQ(PREQ_TADDR(0), broadcastaddr)) {
struct ieee80211_meshprep_ie prep;
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"intermediate reply for PREQ from %6D",
preq->preq_origaddr, ":");
prep.prep_flags = 0;
prep.prep_hopcount = rttarg->rt_nhops;
prep.prep_ttl = ms->ms_ttl;
IEEE80211_ADDR_COPY(&prep.prep_targetaddr,
PREQ_TADDR(0));
prep.prep_targetseq = hrtarg->hr_seq;
prep.prep_lifetime = preq->preq_lifetime;
prep.prep_metric =rttarg->rt_metric;
IEEE80211_ADDR_COPY(&prep.prep_origaddr,
preq->preq_origaddr);
prep.prep_origseq = hrorig->hr_seq;
hwmp_send_prep(vap, rtorig->rt_nexthop, &prep);
/*
* Set TO and unset RF bits because we have
* sent a PREP.
*/
ppreq.preq_targets[0].target_flags |=
IEEE80211_MESHPREQ_TFLAGS_TO;
}
}
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"forward PREQ from %6D",
preq->preq_origaddr, ":");
ppreq.preq_hopcount += 1;
ppreq.preq_ttl -= 1;
ppreq.preq_metric += ms->ms_pmetric->mpm_metric(ni);
/* don't do PREQ ratecheck when we propagate */
hwmp_send_preq(vap, broadcastaddr, &ppreq, NULL, NULL);
}
}
#undef PREQ_TFLAGS
#undef PREQ_TADDR
#undef PREQ_TSEQ
static int
hwmp_send_preq(struct ieee80211vap *vap,
const uint8_t da[IEEE80211_ADDR_LEN],
struct ieee80211_meshpreq_ie *preq,
struct timeval *last, struct timeval *minint)
{
/*
* Enforce PREQ interval.
* NB: Proactive ROOT PREQs rate is handled by cb task.
*/
if (last != NULL && minint != NULL) {
if (ratecheck(last, minint) == 0)
return EALREADY; /* XXX: we should postpone */
getmicrouptime(last);
}
/*
* mesh preq action frame format
* [6] da
* [6] sa
* [6] addr3 = sa
* [1] action
* [1] category
* [tlv] mesh path request
*/
preq->preq_ie = IEEE80211_ELEMID_MESHPREQ;
preq->preq_len = (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AE ?
IEEE80211_MESHPREQ_BASE_SZ_AE : IEEE80211_MESHPREQ_BASE_SZ) +
preq->preq_tcount * IEEE80211_MESHPREQ_TRGT_SZ;
return hwmp_send_action(vap, da, (uint8_t *)preq, preq->preq_len+2);
}
static void
hwmp_recv_prep(struct ieee80211vap *vap, struct ieee80211_node *ni,
const struct ieee80211_frame *wh, const struct ieee80211_meshprep_ie *prep)
{
#define IS_PROXY(rt) (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY)
#define PROXIED_BY_US(rt) \
(IEEE80211_ADDR_EQ(vap->iv_myaddr, rt->rt_mesh_gate))
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
struct ieee80211_mesh_route *rt = NULL;
struct ieee80211_mesh_route *rtorig = NULL;
struct ieee80211_mesh_route *rtext = NULL;
struct ieee80211_hwmp_route *hr;
struct ieee80211com *ic = vap->iv_ic;
struct mbuf *m, *next;
uint32_t metric = 0;
const uint8_t *addr;
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"received PREP, orig %6D, targ %6D", prep->prep_origaddr, ":",
prep->prep_targetaddr, ":");
/*
* Acceptance criteria: (If the corresponding PREP was not generated
* by us OR not generated by an external mac that is not proxied by us)
* AND forwarding is disabled, discard this PREP.
*/
rtorig = ieee80211_mesh_rt_find(vap, prep->prep_origaddr);
if ((!IEEE80211_ADDR_EQ(vap->iv_myaddr, prep->prep_origaddr) ||
(rtorig != NULL && IS_PROXY(rtorig) && !PROXIED_BY_US(rtorig))) &&
!(ms->ms_flags & IEEE80211_MESHFLAGS_FWD)){
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"discard PREP, orig(%6D) not proxied or generated by us",
prep->prep_origaddr, ":");
return;
}
/* PREP ACCEPTED */
/*
* If accepted shall create or update the active forwarding information
* it maintains for the target mesh STA of the PREP (according to the
* rules defined in 13.10.8.4). If the conditions for creating or
* updating the forwarding information have not been met in those
* rules, no further steps are applied to the PREP.
*/
rt = ieee80211_mesh_rt_find(vap, prep->prep_targetaddr);
if (rt == NULL) {
rt = ieee80211_mesh_rt_add(vap, prep->prep_targetaddr);
if (rt == NULL) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"unable to add PREP path to %6D",
prep->prep_targetaddr, ":");
vap->iv_stats.is_mesh_rtaddfailed++;
return;
}
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"adding target %6D", prep->prep_targetaddr, ":");
}
hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route);
/* update path metric */
metric = prep->prep_metric + ms->ms_pmetric->mpm_metric(ni);
if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID)) {
if (HWMP_SEQ_LT(prep->prep_targetseq, hr->hr_seq)) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"discard PREP from %6D, old seq no %u < %u",
prep->prep_targetaddr, ":",
prep->prep_targetseq, hr->hr_seq);
return;
} else if (HWMP_SEQ_LEQ(prep->prep_targetseq, hr->hr_seq) &&
metric > rt->rt_metric) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"discard PREP from %6D, new metric %u > %u",
prep->prep_targetaddr, ":",
metric, rt->rt_metric);
return;
}
}
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"%s path to %6D, hopcount %d:%d metric %d:%d",
rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID ?
"prefer" : "update",
prep->prep_targetaddr, ":",
rt->rt_nhops, prep->prep_hopcount + 1,
rt->rt_metric, metric);
hr->hr_seq = prep->prep_targetseq;
hr->hr_preqretries = 0;
IEEE80211_ADDR_COPY(rt->rt_nexthop, ni->ni_macaddr);
rt->rt_metric = metric;
rt->rt_nhops = prep->prep_hopcount + 1;
ieee80211_mesh_rt_update(rt, prep->prep_lifetime);
if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_DISCOVER) {
/* discovery complete */
rt->rt_flags &= ~IEEE80211_MESHRT_FLAGS_DISCOVER;
}
rt->rt_flags |= IEEE80211_MESHRT_FLAGS_VALID; /* mark valid */
/* Update forwarding information to TA if metric improves */
hwmp_update_transmitter(vap, ni, "PREP");
/*
* If it's NOT for us, propagate the PREP
*/
if (!IEEE80211_ADDR_EQ(vap->iv_myaddr, prep->prep_origaddr) &&
prep->prep_ttl > 1 &&
prep->prep_hopcount < hs->hs_maxhops) {
struct ieee80211_meshprep_ie pprep; /* propagated PREP */
/*
* NB: We should already have setup the path to orig
* mesh STA when we propagated PREQ to target mesh STA,
* no PREP is generated without a corresponding PREQ.
* XXX: for now just ignore.
*/
if (rtorig == NULL) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"received PREP for an unknown orig(%6D)",
prep->prep_origaddr, ":");
return;
}
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"propagate PREP from %6D",
prep->prep_targetaddr, ":");
memcpy(&pprep, prep, sizeof(pprep));
pprep.prep_hopcount += 1;
pprep.prep_ttl -= 1;
pprep.prep_metric += ms->ms_pmetric->mpm_metric(ni);
hwmp_send_prep(vap, rtorig->rt_nexthop, &pprep);
/* precursor list for the Target Mesh STA Address is updated */
}
/*
* Check if we received a PREP w/ AE and store target external address.
* We may store target external address if recevied PREP w/ AE
* and we are not final destination
*/
if (prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE) {
rtext = ieee80211_mesh_rt_find(vap,
prep->prep_target_ext_addr);
if (rtext == NULL) {
rtext = ieee80211_mesh_rt_add(vap,
prep->prep_target_ext_addr);
if (rtext == NULL) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"unable to add PREP path to proxy %6D",
prep->prep_targetaddr, ":");
vap->iv_stats.is_mesh_rtaddfailed++;
return;
}
}
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"%s path to %6D, hopcount %d:%d metric %d:%d",
rtext->rt_flags & IEEE80211_MESHRT_FLAGS_VALID ?
"prefer" : "update",
prep->prep_target_ext_addr, ":",
rtext->rt_nhops, prep->prep_hopcount + 1,
rtext->rt_metric, metric);
rtext->rt_flags = IEEE80211_MESHRT_FLAGS_PROXY |
IEEE80211_MESHRT_FLAGS_VALID;
IEEE80211_ADDR_COPY(rtext->rt_dest,
prep->prep_target_ext_addr);
IEEE80211_ADDR_COPY(rtext->rt_mesh_gate,
prep->prep_targetaddr);
IEEE80211_ADDR_COPY(rtext->rt_nexthop, wh->i_addr2);
rtext->rt_metric = metric;
rtext->rt_lifetime = prep->prep_lifetime;
rtext->rt_nhops = prep->prep_hopcount + 1;
rtext->rt_ext_seq = prep->prep_origseq; /* new proxy seq */
/*
* XXX: proxy entries have no HWMP priv data,
* nullify them to be sure?
*/
}
/*
* Check for frames queued awaiting path discovery.
* XXX probably can tell exactly and avoid remove call
* NB: hash may have false matches, if so they will get
* stuck back on the stageq because there won't be
* a path.
*/
addr = prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE ?
prep->prep_target_ext_addr : prep->prep_targetaddr;
m = ieee80211_ageq_remove(&ic->ic_stageq,
(struct ieee80211_node *)(uintptr_t)
ieee80211_mac_hash(ic, addr)); /* either dest or ext_dest */
/*
* All frames in the stageq here should be non-M_ENCAP; or things
* will get very unhappy.
*/
for (; m != NULL; m = next) {
next = m->m_nextpkt;
m->m_nextpkt = NULL;
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"flush queued frame %p len %d", m, m->m_pkthdr.len);
/*
* If the mbuf has M_ENCAP set, ensure we free it.
* Note that after if_transmit() is called, m is invalid.
*/
(void) ieee80211_vap_xmitpkt(vap, m);
}
#undef IS_PROXY
#undef PROXIED_BY_US
}
static int
hwmp_send_prep(struct ieee80211vap *vap,
const uint8_t da[IEEE80211_ADDR_LEN],
struct ieee80211_meshprep_ie *prep)
{
/* NB: there's no PREP minimum interval. */
/*
* mesh prep action frame format
* [6] da
* [6] sa
* [6] addr3 = sa
* [1] action
* [1] category
* [tlv] mesh path reply
*/
prep->prep_ie = IEEE80211_ELEMID_MESHPREP;
prep->prep_len = prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE ?
IEEE80211_MESHPREP_BASE_SZ_AE : IEEE80211_MESHPREP_BASE_SZ;
return hwmp_send_action(vap, da, (uint8_t *)prep, prep->prep_len + 2);
}
#define PERR_DFLAGS(n) perr.perr_dests[n].dest_flags
#define PERR_DADDR(n) perr.perr_dests[n].dest_addr
#define PERR_DSEQ(n) perr.perr_dests[n].dest_seq
#define PERR_DRCODE(n) perr.perr_dests[n].dest_rcode
static void
hwmp_peerdown(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_meshperr_ie perr;
struct ieee80211_mesh_route *rt;
struct ieee80211_hwmp_route *hr;
rt = ieee80211_mesh_rt_find(vap, ni->ni_macaddr);
if (rt == NULL)
return;
hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route);
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"%s", "delete route entry");
perr.perr_ttl = ms->ms_ttl;
perr.perr_ndests = 1;
PERR_DFLAGS(0) = 0;
if (hr->hr_seq == 0)
PERR_DFLAGS(0) |= IEEE80211_MESHPERR_DFLAGS_USN;
PERR_DFLAGS(0) |= IEEE80211_MESHPERR_DFLAGS_RC;
IEEE80211_ADDR_COPY(PERR_DADDR(0), rt->rt_dest);
PERR_DSEQ(0) = ++hr->hr_seq;
PERR_DRCODE(0) = IEEE80211_REASON_MESH_PERR_DEST_UNREACH;
/* NB: flush everything passing through peer */
ieee80211_mesh_rt_flush_peer(vap, ni->ni_macaddr);
hwmp_send_perr(vap, broadcastaddr, &perr);
}
#undef PERR_DFLAGS
#undef PERR_DADDR
#undef PERR_DSEQ
#undef PERR_DRCODE
#define PERR_DFLAGS(n) perr->perr_dests[n].dest_flags
#define PERR_DADDR(n) perr->perr_dests[n].dest_addr
#define PERR_DSEQ(n) perr->perr_dests[n].dest_seq
#define PERR_DEXTADDR(n) perr->perr_dests[n].dest_ext_addr
#define PERR_DRCODE(n) perr->perr_dests[n].dest_rcode
static void
hwmp_recv_perr(struct ieee80211vap *vap, struct ieee80211_node *ni,
const struct ieee80211_frame *wh, const struct ieee80211_meshperr_ie *perr)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt = NULL;
struct ieee80211_mesh_route *rt_ext = NULL;
struct ieee80211_hwmp_route *hr;
struct ieee80211_meshperr_ie *pperr = NULL;
int i, j = 0, forward = 0;
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"received PERR from %6D", wh->i_addr2, ":");
/*
* if forwarding is true, prepare pperr
*/
if (ms->ms_flags & IEEE80211_MESHFLAGS_FWD) {
forward = 1;
pperr = malloc(sizeof(*perr) + 31*sizeof(*perr->perr_dests),
M_80211_MESH_PERR, M_NOWAIT); /* XXX: magic number, 32 err dests */
}
/*
* Acceptance criteria: check if we have forwarding information
* stored about destination, and that nexthop == TA of this PERR.
* NB: we also build a new PERR to propagate in case we should forward.
*/
for (i = 0; i < perr->perr_ndests; i++) {
rt = ieee80211_mesh_rt_find(vap, PERR_DADDR(i));
if (rt == NULL)
continue;
if (!IEEE80211_ADDR_EQ(rt->rt_nexthop, wh->i_addr2))
continue;
/* found and accepted a PERR ndest element, process it... */
if (forward)
memcpy(&pperr->perr_dests[j], &perr->perr_dests[i],
sizeof(*perr->perr_dests));
hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route);
switch(PERR_DFLAGS(i)) {
case (IEEE80211_REASON_MESH_PERR_NO_FI):
if (PERR_DSEQ(i) == 0) {
hr->hr_seq++;
if (forward) {
pperr->perr_dests[j].dest_seq =
hr->hr_seq;
}
} else {
hr->hr_seq = PERR_DSEQ(i);
}
rt->rt_flags &= ~IEEE80211_MESHRT_FLAGS_VALID;
j++;
break;
case (IEEE80211_REASON_MESH_PERR_DEST_UNREACH):
if(HWMP_SEQ_GT(PERR_DSEQ(i), hr->hr_seq)) {
hr->hr_seq = PERR_DSEQ(i);
rt->rt_flags &= ~IEEE80211_MESHRT_FLAGS_VALID;
j++;
}
break;
case (IEEE80211_REASON_MESH_PERR_NO_PROXY):
rt_ext = ieee80211_mesh_rt_find(vap, PERR_DEXTADDR(i));
if (rt_ext != NULL) {
rt_ext->rt_flags &=
~IEEE80211_MESHRT_FLAGS_VALID;
j++;
}
break;
default:
IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL,
"PERR, unknown reason code %u\n", PERR_DFLAGS(i));
goto done; /* XXX: stats?? */
}
ieee80211_mesh_rt_flush_peer(vap, PERR_DADDR(i));
KASSERT(j < 32, ("PERR, error ndest >= 32 (%u)", j));
}
if (j == 0) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL, "%s",
"PERR not accepted");
goto done; /* XXX: stats?? */
}
/*
* Propagate the PERR if we previously found it on our routing table.
*/
if (forward && perr->perr_ttl > 1) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni,
"propagate PERR from %6D", wh->i_addr2, ":");
pperr->perr_ndests = j;
pperr->perr_ttl--;
hwmp_send_perr(vap, broadcastaddr, pperr);
}
done:
if (pperr != NULL)
free(pperr, M_80211_MESH_PERR);
}
#undef PERR_DFLAGS
#undef PERR_DADDR
#undef PERR_DSEQ
#undef PERR_DEXTADDR
#undef PERR_DRCODE
static int
hwmp_send_perr(struct ieee80211vap *vap,
const uint8_t da[IEEE80211_ADDR_LEN],
struct ieee80211_meshperr_ie *perr)
{
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
int i;
uint8_t length = 0;
/*
* Enforce PERR interval.
*/
if (ratecheck(&hs->hs_lastperr, &ieee80211_hwmp_perrminint) == 0)
return EALREADY;
getmicrouptime(&hs->hs_lastperr);
/*
* mesh perr action frame format
* [6] da
* [6] sa
* [6] addr3 = sa
* [1] action
* [1] category
* [tlv] mesh path error
*/
perr->perr_ie = IEEE80211_ELEMID_MESHPERR;
length = IEEE80211_MESHPERR_BASE_SZ;
for (i = 0; i<perr->perr_ndests; i++) {
if (perr->perr_dests[i].dest_flags &
IEEE80211_MESHPERR_FLAGS_AE) {
length += IEEE80211_MESHPERR_DEST_SZ_AE;
continue ;
}
length += IEEE80211_MESHPERR_DEST_SZ;
}
perr->perr_len =length;
return hwmp_send_action(vap, da, (uint8_t *)perr, perr->perr_len+2);
}
/*
* Called from the rest of the net80211 code (mesh code for example).
* NB: IEEE80211_REASON_MESH_PERR_DEST_UNREACH can be trigger by the fact that
* a mesh STA is unable to forward an MSDU/MMPDU to a next-hop mesh STA.
*/
#define PERR_DFLAGS(n) perr.perr_dests[n].dest_flags
#define PERR_DADDR(n) perr.perr_dests[n].dest_addr
#define PERR_DSEQ(n) perr.perr_dests[n].dest_seq
#define PERR_DEXTADDR(n) perr.perr_dests[n].dest_ext_addr
#define PERR_DRCODE(n) perr.perr_dests[n].dest_rcode
static void
hwmp_senderror(struct ieee80211vap *vap,
const uint8_t addr[IEEE80211_ADDR_LEN],
struct ieee80211_mesh_route *rt, int rcode)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_hwmp_route *hr = NULL;
struct ieee80211_meshperr_ie perr;
if (rt != NULL)
hr = IEEE80211_MESH_ROUTE_PRIV(rt,
struct ieee80211_hwmp_route);
perr.perr_ndests = 1;
perr.perr_ttl = ms->ms_ttl;
PERR_DFLAGS(0) = 0;
PERR_DRCODE(0) = rcode;
switch (rcode) {
case IEEE80211_REASON_MESH_PERR_NO_FI:
IEEE80211_ADDR_COPY(PERR_DADDR(0), addr);
PERR_DSEQ(0) = 0; /* reserved */
break;
case IEEE80211_REASON_MESH_PERR_NO_PROXY:
KASSERT(rt != NULL, ("no proxy info for sending PERR"));
KASSERT(rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY,
("route is not marked proxy"));
PERR_DFLAGS(0) |= IEEE80211_MESHPERR_FLAGS_AE;
IEEE80211_ADDR_COPY(PERR_DADDR(0), vap->iv_myaddr);
PERR_DSEQ(0) = rt->rt_ext_seq;
IEEE80211_ADDR_COPY(PERR_DEXTADDR(0), addr);
break;
case IEEE80211_REASON_MESH_PERR_DEST_UNREACH:
KASSERT(rt != NULL, ("no route info for sending PERR"));
IEEE80211_ADDR_COPY(PERR_DADDR(0), addr);
PERR_DSEQ(0) = hr->hr_seq;
break;
default:
KASSERT(0, ("unknown reason code for HWMP PERR (%u)", rcode));
}
hwmp_send_perr(vap, broadcastaddr, &perr);
}
#undef PERR_DFLAGS
#undef PEER_DADDR
#undef PERR_DSEQ
#undef PERR_DEXTADDR
#undef PERR_DRCODE
static void
hwmp_recv_rann(struct ieee80211vap *vap, struct ieee80211_node *ni,
const struct ieee80211_frame *wh, const struct ieee80211_meshrann_ie *rann)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
struct ieee80211_mesh_route *rt = NULL;
struct ieee80211_hwmp_route *hr;
struct ieee80211_meshpreq_ie preq;
struct ieee80211_meshrann_ie prann;
uint32_t metric = 0;
if (IEEE80211_ADDR_EQ(rann->rann_addr, vap->iv_myaddr))
return;
rt = ieee80211_mesh_rt_find(vap, rann->rann_addr);
if (rt != NULL && rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) {
hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route);
/* Acceptance criteria: if RANN.seq < stored seq, discard RANN */
if (HWMP_SEQ_LT(rann->rann_seq, hr->hr_seq)) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL,
"RANN seq %u < %u", rann->rann_seq, hr->hr_seq);
return;
}
/* Acceptance criteria: if RANN.seq == stored seq AND
* RANN.metric > stored metric, discard RANN */
if (HWMP_SEQ_EQ(rann->rann_seq, hr->hr_seq) &&
rann->rann_metric > rt->rt_metric) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL,
"RANN metric %u > %u", rann->rann_metric, rt->rt_metric);
return;
}
}
/* RANN ACCEPTED */
ieee80211_hwmp_rannint = rann->rann_interval; /* XXX: mtx lock? */
metric = rann->rann_metric + ms->ms_pmetric->mpm_metric(ni);
if (rt == NULL) {
rt = ieee80211_mesh_rt_add(vap, rann->rann_addr);
if (rt == NULL) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL,
"unable to add mac for RANN root %6D",
rann->rann_addr, ":");
vap->iv_stats.is_mesh_rtaddfailed++;
return;
}
}
hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route);
/* Check if root is a mesh gate, mark it */
if (rann->rann_flags & IEEE80211_MESHRANN_FLAGS_GATE) {
struct ieee80211_mesh_gate_route *gr;
rt->rt_flags |= IEEE80211_MESHRT_FLAGS_GATE;
gr = ieee80211_mesh_mark_gate(vap, rann->rann_addr,
rt);
gr->gr_lastseq = 0; /* NOT GANN */
}
/* discovery timeout */
ieee80211_mesh_rt_update(rt,
ticks_to_msecs(ieee80211_hwmp_roottimeout));
preq.preq_flags = IEEE80211_MESHPREQ_FLAGS_AM;
preq.preq_hopcount = 0;
preq.preq_ttl = ms->ms_ttl;
preq.preq_id = 0; /* reserved */
IEEE80211_ADDR_COPY(preq.preq_origaddr, vap->iv_myaddr);
preq.preq_origseq = ++hs->hs_seq;
preq.preq_lifetime = ieee80211_hwmp_roottimeout;
preq.preq_metric = IEEE80211_MESHLMETRIC_INITIALVAL;
preq.preq_tcount = 1;
preq.preq_targets[0].target_flags = IEEE80211_MESHPREQ_TFLAGS_TO;
/* NB: IEEE80211_MESHPREQ_TFLAGS_USN = 0 implicitly implied */
IEEE80211_ADDR_COPY(preq.preq_targets[0].target_addr, rann->rann_addr);
preq.preq_targets[0].target_seq = rann->rann_seq;
/* XXX: if rootconfint have not passed, we built this preq in vain */
hwmp_send_preq(vap, wh->i_addr2, &preq, &hr->hr_lastrootconf,
&ieee80211_hwmp_rootconfint);
/* propagate a RANN */
if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID &&
rann->rann_ttl > 1 &&
ms->ms_flags & IEEE80211_MESHFLAGS_FWD) {
hr->hr_seq = rann->rann_seq;
memcpy(&prann, rann, sizeof(prann));
prann.rann_hopcount += 1;
prann.rann_ttl -= 1;
prann.rann_metric += ms->ms_pmetric->mpm_metric(ni);
hwmp_send_rann(vap, broadcastaddr, &prann);
}
}
static int
hwmp_send_rann(struct ieee80211vap *vap,
const uint8_t da[IEEE80211_ADDR_LEN],
struct ieee80211_meshrann_ie *rann)
{
/*
* mesh rann action frame format
* [6] da
* [6] sa
* [6] addr3 = sa
* [1] action
* [1] category
* [tlv] root annoucement
*/
rann->rann_ie = IEEE80211_ELEMID_MESHRANN;
rann->rann_len = IEEE80211_MESHRANN_BASE_SZ;
return hwmp_send_action(vap, da, (uint8_t *)rann, rann->rann_len + 2);
}
#define PREQ_TFLAGS(n) preq.preq_targets[n].target_flags
#define PREQ_TADDR(n) preq.preq_targets[n].target_addr
#define PREQ_TSEQ(n) preq.preq_targets[n].target_seq
static void
hwmp_rediscover_cb(void *arg)
{
struct ieee80211_mesh_route *rt = arg;
struct ieee80211vap *vap = rt->rt_vap;
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_hwmp_route *hr;
struct ieee80211_meshpreq_ie preq; /* Optimize: storing first preq? */
if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID))
return ; /* nothing to do */
hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route);
if (hr->hr_preqretries >=
ieee80211_hwmp_maxpreq_retries) {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_ANY,
rt->rt_dest, "%s",
"max number of discovery, send queued frames to GATE");
ieee80211_mesh_forward_to_gates(vap, rt);
vap->iv_stats.is_mesh_fwd_nopath++;
return ; /* XXX: flush queue? */
}
hr->hr_preqretries++;
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, rt->rt_dest,
"start path rediscovery , target seq %u", hr->hr_seq);
/*
* Try to discover the path for this node.
* Group addressed PREQ Case A
*/
preq.preq_flags = 0;
preq.preq_hopcount = 0;
preq.preq_ttl = ms->ms_ttl;
preq.preq_id = ++hs->hs_preqid;
IEEE80211_ADDR_COPY(preq.preq_origaddr, vap->iv_myaddr);
preq.preq_origseq = hr->hr_origseq;
preq.preq_lifetime = ticks_to_msecs(ieee80211_hwmp_pathtimeout);
preq.preq_metric = IEEE80211_MESHLMETRIC_INITIALVAL;
preq.preq_tcount = 1;
IEEE80211_ADDR_COPY(PREQ_TADDR(0), rt->rt_dest);
PREQ_TFLAGS(0) = 0;
if (ieee80211_hwmp_targetonly)
PREQ_TFLAGS(0) |= IEEE80211_MESHPREQ_TFLAGS_TO;
PREQ_TFLAGS(0) |= IEEE80211_MESHPREQ_TFLAGS_USN;
PREQ_TSEQ(0) = 0; /* RESERVED when USN flag is set */
/* XXX check return value */
hwmp_send_preq(vap, broadcastaddr, &preq, &hr->hr_lastpreq,
&ieee80211_hwmp_preqminint);
callout_reset(&rt->rt_discovery,
ieee80211_hwmp_net_diameter_traversaltime * 2,
hwmp_rediscover_cb, rt);
}
static struct ieee80211_node *
hwmp_discover(struct ieee80211vap *vap,
const uint8_t dest[IEEE80211_ADDR_LEN], struct mbuf *m)
{
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt = NULL;
struct ieee80211_hwmp_route *hr;
struct ieee80211_meshpreq_ie preq;
struct ieee80211_node *ni;
int sendpreq = 0;
KASSERT(vap->iv_opmode == IEEE80211_M_MBSS,
("not a mesh vap, opmode %d", vap->iv_opmode));
KASSERT(!IEEE80211_ADDR_EQ(vap->iv_myaddr, dest),
("%s: discovering self!", __func__));
ni = NULL;
if (!IEEE80211_IS_MULTICAST(dest)) {
rt = ieee80211_mesh_rt_find(vap, dest);
if (rt == NULL) {
rt = ieee80211_mesh_rt_add(vap, dest);
if (rt == NULL) {
IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP,
ni, "unable to add discovery path to %6D",
dest, ":");
vap->iv_stats.is_mesh_rtaddfailed++;
goto done;
}
}
hr = IEEE80211_MESH_ROUTE_PRIV(rt,
struct ieee80211_hwmp_route);
if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_DISCOVER) {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, dest,
"%s", "already discovering queue frame until path found");
sendpreq = 1;
goto done;
}
if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) {
if (hr->hr_lastdiscovery != 0 &&
(ticks - hr->hr_lastdiscovery <
(ieee80211_hwmp_net_diameter_traversaltime * 2))) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
dest, NULL, "%s",
"too frequent discovery requeust");
sendpreq = 1;
goto done;
}
hr->hr_lastdiscovery = ticks;
if (hr->hr_preqretries >=
ieee80211_hwmp_maxpreq_retries) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
dest, NULL, "%s",
"no valid path , max number of discovery");
vap->iv_stats.is_mesh_fwd_nopath++;
goto done;
}
rt->rt_flags = IEEE80211_MESHRT_FLAGS_DISCOVER;
hr->hr_preqretries++;
if (hr->hr_origseq == 0)
hr->hr_origseq = ++hs->hs_seq;
rt->rt_metric = IEEE80211_MESHLMETRIC_INITIALVAL;
sendpreq = 1;
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, dest,
"start path discovery (src %s), target seq %u",
m == NULL ? "<none>" : ether_sprintf(
mtod(m, struct ether_header *)->ether_shost),
hr->hr_seq);
/*
* Try to discover the path for this node.
* Group addressed PREQ Case A
*/
preq.preq_flags = 0;
preq.preq_hopcount = 0;
preq.preq_ttl = ms->ms_ttl;
preq.preq_id = ++hs->hs_preqid;
IEEE80211_ADDR_COPY(preq.preq_origaddr, vap->iv_myaddr);
preq.preq_origseq = hr->hr_origseq;
preq.preq_lifetime =
ticks_to_msecs(ieee80211_hwmp_pathtimeout);
preq.preq_metric = IEEE80211_MESHLMETRIC_INITIALVAL;
preq.preq_tcount = 1;
IEEE80211_ADDR_COPY(PREQ_TADDR(0), dest);
PREQ_TFLAGS(0) = 0;
if (ieee80211_hwmp_targetonly)
PREQ_TFLAGS(0) |= IEEE80211_MESHPREQ_TFLAGS_TO;
PREQ_TFLAGS(0) |= IEEE80211_MESHPREQ_TFLAGS_USN;
PREQ_TSEQ(0) = 0; /* RESERVED when USN flag is set */
/* XXX check return value */
hwmp_send_preq(vap, broadcastaddr, &preq,
&hr->hr_lastpreq, &ieee80211_hwmp_preqminint);
callout_reset(&rt->rt_discovery,
ieee80211_hwmp_net_diameter_traversaltime * 2,
hwmp_rediscover_cb, rt);
}
if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID)
ni = ieee80211_find_txnode(vap, rt->rt_nexthop);
} else {
ni = ieee80211_find_txnode(vap, dest);
/* NB: if null then we leak mbuf */
KASSERT(ni != NULL, ("leak mcast frame"));
return ni;
}
done:
if (ni == NULL && m != NULL) {
if (sendpreq) {
struct ieee80211com *ic = vap->iv_ic;
/*
* Queue packet for transmit when path discovery
* completes. If discovery never completes the
* frame will be flushed by way of the aging timer.
*/
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, dest,
"%s", "queue frame until path found");
m->m_pkthdr.rcvif = (void *)(uintptr_t)
ieee80211_mac_hash(ic, dest);
/* XXX age chosen randomly */
ieee80211_ageq_append(&ic->ic_stageq, m,
IEEE80211_INACT_WAIT);
} else {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_HWMP,
dest, NULL, "%s", "no valid path to this node");
m_freem(m);
}
}
return ni;
}
#undef PREQ_TFLAGS
#undef PREQ_TADDR
#undef PREQ_TSEQ
static int
hwmp_ioctl_get80211(struct ieee80211vap *vap, struct ieee80211req *ireq)
{
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
int error;
if (vap->iv_opmode != IEEE80211_M_MBSS)
return ENOSYS;
error = 0;
switch (ireq->i_type) {
case IEEE80211_IOC_HWMP_ROOTMODE:
ireq->i_val = hs->hs_rootmode;
break;
case IEEE80211_IOC_HWMP_MAXHOPS:
ireq->i_val = hs->hs_maxhops;
break;
default:
return ENOSYS;
}
return error;
}
IEEE80211_IOCTL_GET(hwmp, hwmp_ioctl_get80211);
static int
hwmp_ioctl_set80211(struct ieee80211vap *vap, struct ieee80211req *ireq)
{
struct ieee80211_hwmp_state *hs = vap->iv_hwmp;
int error;
if (vap->iv_opmode != IEEE80211_M_MBSS)
return ENOSYS;
error = 0;
switch (ireq->i_type) {
case IEEE80211_IOC_HWMP_ROOTMODE:
if (ireq->i_val < 0 || ireq->i_val > 3)
return EINVAL;
hs->hs_rootmode = ireq->i_val;
hwmp_rootmode_setup(vap);
break;
case IEEE80211_IOC_HWMP_MAXHOPS:
if (ireq->i_val <= 0 || ireq->i_val > 255)
return EINVAL;
hs->hs_maxhops = ireq->i_val;
break;
default:
return ENOSYS;
}
return error;
}
IEEE80211_IOCTL_SET(hwmp, hwmp_ioctl_set80211);
Index: head/sys/net80211/ieee80211_mesh.c
===================================================================
--- head/sys/net80211/ieee80211_mesh.c (revision 283290)
+++ head/sys/net80211/ieee80211_mesh.c (revision 283291)
@@ -1,3636 +1,3636 @@
/*-
* Copyright (c) 2009 The FreeBSD Foundation
* All rights reserved.
*
* This software was developed by Rui Paulo under sponsorship from the
* FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#ifdef __FreeBSD__
__FBSDID("$FreeBSD$");
#endif
/*
* IEEE 802.11s Mesh Point (MBSS) support.
*
* Based on March 2009, D3.0 802.11s draft spec.
*/
#include "opt_inet.h"
#include "opt_wlan.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/endian.h>
#include <sys/errno.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <net/bpf.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_media.h>
#include <net/if_llc.h>
#include <net/ethernet.h>
#include <net80211/ieee80211_var.h>
#include <net80211/ieee80211_action.h>
#ifdef IEEE80211_SUPPORT_SUPERG
#include <net80211/ieee80211_superg.h>
#endif
#include <net80211/ieee80211_input.h>
#include <net80211/ieee80211_mesh.h>
static void mesh_rt_flush_invalid(struct ieee80211vap *);
static int mesh_select_proto_path(struct ieee80211vap *, const char *);
static int mesh_select_proto_metric(struct ieee80211vap *, const char *);
static void mesh_vattach(struct ieee80211vap *);
static int mesh_newstate(struct ieee80211vap *, enum ieee80211_state, int);
static void mesh_rt_cleanup_cb(void *);
static void mesh_gatemode_setup(struct ieee80211vap *);
static void mesh_gatemode_cb(void *);
static void mesh_linkchange(struct ieee80211_node *,
enum ieee80211_mesh_mlstate);
static void mesh_checkid(void *, struct ieee80211_node *);
static uint32_t mesh_generateid(struct ieee80211vap *);
static int mesh_checkpseq(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN], uint32_t);
static void mesh_transmit_to_gate(struct ieee80211vap *, struct mbuf *,
struct ieee80211_mesh_route *);
static void mesh_forward(struct ieee80211vap *, struct mbuf *,
const struct ieee80211_meshcntl *);
static int mesh_input(struct ieee80211_node *, struct mbuf *, int, int);
static void mesh_recv_mgmt(struct ieee80211_node *, struct mbuf *, int,
int, int);
static void mesh_recv_ctl(struct ieee80211_node *, struct mbuf *, int);
static void mesh_peer_timeout_setup(struct ieee80211_node *);
static void mesh_peer_timeout_backoff(struct ieee80211_node *);
static void mesh_peer_timeout_cb(void *);
static __inline void
mesh_peer_timeout_stop(struct ieee80211_node *);
static int mesh_verify_meshid(struct ieee80211vap *, const uint8_t *);
static int mesh_verify_meshconf(struct ieee80211vap *, const uint8_t *);
static int mesh_verify_meshpeer(struct ieee80211vap *, uint8_t,
const uint8_t *);
uint32_t mesh_airtime_calc(struct ieee80211_node *);
/*
* Timeout values come from the specification and are in milliseconds.
*/
static SYSCTL_NODE(_net_wlan, OID_AUTO, mesh, CTLFLAG_RD, 0,
"IEEE 802.11s parameters");
static int ieee80211_mesh_gateint = -1;
SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, gateint, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_mesh_gateint, 0, ieee80211_sysctl_msecs_ticks, "I",
"mesh gate interval (ms)");
static int ieee80211_mesh_retrytimeout = -1;
SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, retrytimeout, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_mesh_retrytimeout, 0, ieee80211_sysctl_msecs_ticks, "I",
"Retry timeout (msec)");
static int ieee80211_mesh_holdingtimeout = -1;
SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, holdingtimeout, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_mesh_holdingtimeout, 0, ieee80211_sysctl_msecs_ticks, "I",
"Holding state timeout (msec)");
static int ieee80211_mesh_confirmtimeout = -1;
SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, confirmtimeout, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_mesh_confirmtimeout, 0, ieee80211_sysctl_msecs_ticks, "I",
"Confirm state timeout (msec)");
static int ieee80211_mesh_backofftimeout = -1;
SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, backofftimeout, CTLTYPE_INT | CTLFLAG_RW,
&ieee80211_mesh_backofftimeout, 0, ieee80211_sysctl_msecs_ticks, "I",
"Backoff timeout (msec). This is to throutles peering forever when "
"not receiving answer or is rejected by a neighbor");
static int ieee80211_mesh_maxretries = 2;
SYSCTL_INT(_net_wlan_mesh, OID_AUTO, maxretries, CTLFLAG_RW,
&ieee80211_mesh_maxretries, 0,
"Maximum retries during peer link establishment");
static int ieee80211_mesh_maxholding = 2;
SYSCTL_INT(_net_wlan_mesh, OID_AUTO, maxholding, CTLFLAG_RW,
&ieee80211_mesh_maxholding, 0,
"Maximum times we are allowed to transition to HOLDING state before "
"backinoff during peer link establishment");
static const uint8_t broadcastaddr[IEEE80211_ADDR_LEN] =
{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
static ieee80211_recv_action_func mesh_recv_action_meshpeering_open;
static ieee80211_recv_action_func mesh_recv_action_meshpeering_confirm;
static ieee80211_recv_action_func mesh_recv_action_meshpeering_close;
static ieee80211_recv_action_func mesh_recv_action_meshlmetric;
static ieee80211_recv_action_func mesh_recv_action_meshgate;
static ieee80211_send_action_func mesh_send_action_meshpeering_open;
static ieee80211_send_action_func mesh_send_action_meshpeering_confirm;
static ieee80211_send_action_func mesh_send_action_meshpeering_close;
static ieee80211_send_action_func mesh_send_action_meshlmetric;
static ieee80211_send_action_func mesh_send_action_meshgate;
static const struct ieee80211_mesh_proto_metric mesh_metric_airtime = {
.mpm_descr = "AIRTIME",
.mpm_ie = IEEE80211_MESHCONF_METRIC_AIRTIME,
.mpm_metric = mesh_airtime_calc,
};
static struct ieee80211_mesh_proto_path mesh_proto_paths[4];
static struct ieee80211_mesh_proto_metric mesh_proto_metrics[4];
#define RT_ENTRY_LOCK(rt) mtx_lock(&(rt)->rt_lock)
#define RT_ENTRY_LOCK_ASSERT(rt) mtx_assert(&(rt)->rt_lock, MA_OWNED)
#define RT_ENTRY_UNLOCK(rt) mtx_unlock(&(rt)->rt_lock)
#define MESH_RT_LOCK(ms) mtx_lock(&(ms)->ms_rt_lock)
#define MESH_RT_LOCK_ASSERT(ms) mtx_assert(&(ms)->ms_rt_lock, MA_OWNED)
#define MESH_RT_UNLOCK(ms) mtx_unlock(&(ms)->ms_rt_lock)
MALLOC_DEFINE(M_80211_MESH_PREQ, "80211preq", "802.11 MESH Path Request frame");
MALLOC_DEFINE(M_80211_MESH_PREP, "80211prep", "802.11 MESH Path Reply frame");
MALLOC_DEFINE(M_80211_MESH_PERR, "80211perr", "802.11 MESH Path Error frame");
/* The longer one of the lifetime should be stored as new lifetime */
#define MESH_ROUTE_LIFETIME_MAX(a, b) (a > b ? a : b)
MALLOC_DEFINE(M_80211_MESH_RT, "80211mesh_rt", "802.11s routing table");
MALLOC_DEFINE(M_80211_MESH_GT_RT, "80211mesh_gt", "802.11s known gates table");
/*
* Helper functions to manipulate the Mesh routing table.
*/
static struct ieee80211_mesh_route *
mesh_rt_find_locked(struct ieee80211_mesh_state *ms,
const uint8_t dest[IEEE80211_ADDR_LEN])
{
struct ieee80211_mesh_route *rt;
MESH_RT_LOCK_ASSERT(ms);
TAILQ_FOREACH(rt, &ms->ms_routes, rt_next) {
if (IEEE80211_ADDR_EQ(dest, rt->rt_dest))
return rt;
}
return NULL;
}
static struct ieee80211_mesh_route *
mesh_rt_add_locked(struct ieee80211vap *vap,
const uint8_t dest[IEEE80211_ADDR_LEN])
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt;
KASSERT(!IEEE80211_ADDR_EQ(broadcastaddr, dest),
("%s: adding broadcast to the routing table", __func__));
MESH_RT_LOCK_ASSERT(ms);
rt = malloc(ALIGN(sizeof(struct ieee80211_mesh_route)) +
ms->ms_ppath->mpp_privlen, M_80211_MESH_RT, M_NOWAIT | M_ZERO);
if (rt != NULL) {
rt->rt_vap = vap;
IEEE80211_ADDR_COPY(rt->rt_dest, dest);
rt->rt_priv = (void *)ALIGN(&rt[1]);
mtx_init(&rt->rt_lock, "MBSS_RT", "802.11s route entry", MTX_DEF);
- callout_init(&rt->rt_discovery, CALLOUT_MPSAFE);
+ callout_init(&rt->rt_discovery, 1);
rt->rt_updtime = ticks; /* create time */
TAILQ_INSERT_TAIL(&ms->ms_routes, rt, rt_next);
}
return rt;
}
struct ieee80211_mesh_route *
ieee80211_mesh_rt_find(struct ieee80211vap *vap,
const uint8_t dest[IEEE80211_ADDR_LEN])
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt;
MESH_RT_LOCK(ms);
rt = mesh_rt_find_locked(ms, dest);
MESH_RT_UNLOCK(ms);
return rt;
}
struct ieee80211_mesh_route *
ieee80211_mesh_rt_add(struct ieee80211vap *vap,
const uint8_t dest[IEEE80211_ADDR_LEN])
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt;
KASSERT(ieee80211_mesh_rt_find(vap, dest) == NULL,
("%s: duplicate entry in the routing table", __func__));
KASSERT(!IEEE80211_ADDR_EQ(vap->iv_myaddr, dest),
("%s: adding self to the routing table", __func__));
MESH_RT_LOCK(ms);
rt = mesh_rt_add_locked(vap, dest);
MESH_RT_UNLOCK(ms);
return rt;
}
/*
* Update the route lifetime and returns the updated lifetime.
* If new_lifetime is zero and route is timedout it will be invalidated.
* new_lifetime is in msec
*/
int
ieee80211_mesh_rt_update(struct ieee80211_mesh_route *rt, int new_lifetime)
{
int timesince, now;
uint32_t lifetime = 0;
KASSERT(rt != NULL, ("route is NULL"));
now = ticks;
RT_ENTRY_LOCK(rt);
/* dont clobber a proxy entry gated by us */
if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY && rt->rt_nhops == 0) {
RT_ENTRY_UNLOCK(rt);
return rt->rt_lifetime;
}
timesince = ticks_to_msecs(now - rt->rt_updtime);
rt->rt_updtime = now;
if (timesince >= rt->rt_lifetime) {
if (new_lifetime != 0) {
rt->rt_lifetime = new_lifetime;
}
else {
rt->rt_flags &= ~IEEE80211_MESHRT_FLAGS_VALID;
rt->rt_lifetime = 0;
}
} else {
/* update what is left of lifetime */
rt->rt_lifetime = rt->rt_lifetime - timesince;
rt->rt_lifetime = MESH_ROUTE_LIFETIME_MAX(
new_lifetime, rt->rt_lifetime);
}
lifetime = rt->rt_lifetime;
RT_ENTRY_UNLOCK(rt);
return lifetime;
}
/*
* Add a proxy route (as needed) for the specified destination.
*/
void
ieee80211_mesh_proxy_check(struct ieee80211vap *vap,
const uint8_t dest[IEEE80211_ADDR_LEN])
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt;
MESH_RT_LOCK(ms);
rt = mesh_rt_find_locked(ms, dest);
if (rt == NULL) {
rt = mesh_rt_add_locked(vap, dest);
if (rt == NULL) {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest,
"%s", "unable to add proxy entry");
vap->iv_stats.is_mesh_rtaddfailed++;
} else {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest,
"%s", "add proxy entry");
IEEE80211_ADDR_COPY(rt->rt_mesh_gate, vap->iv_myaddr);
IEEE80211_ADDR_COPY(rt->rt_nexthop, vap->iv_myaddr);
rt->rt_flags |= IEEE80211_MESHRT_FLAGS_VALID
| IEEE80211_MESHRT_FLAGS_PROXY;
}
} else if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) {
KASSERT(rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY,
("no proxy flag for poxy entry"));
struct ieee80211com *ic = vap->iv_ic;
/*
* Fix existing entry created by received frames from
* stations that have some memory of dest. We also
* flush any frames held on the staging queue; delivering
* them is too much trouble right now.
*/
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest,
"%s", "fix proxy entry");
IEEE80211_ADDR_COPY(rt->rt_nexthop, vap->iv_myaddr);
rt->rt_flags |= IEEE80211_MESHRT_FLAGS_VALID
| IEEE80211_MESHRT_FLAGS_PROXY;
/* XXX belongs in hwmp */
ieee80211_ageq_drain_node(&ic->ic_stageq,
(void *)(uintptr_t) ieee80211_mac_hash(ic, dest));
/* XXX stat? */
}
MESH_RT_UNLOCK(ms);
}
static __inline void
mesh_rt_del(struct ieee80211_mesh_state *ms, struct ieee80211_mesh_route *rt)
{
TAILQ_REMOVE(&ms->ms_routes, rt, rt_next);
/*
* Grab the lock before destroying it, to be sure no one else
* is holding the route.
*/
RT_ENTRY_LOCK(rt);
callout_drain(&rt->rt_discovery);
mtx_destroy(&rt->rt_lock);
free(rt, M_80211_MESH_RT);
}
void
ieee80211_mesh_rt_del(struct ieee80211vap *vap,
const uint8_t dest[IEEE80211_ADDR_LEN])
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt, *next;
MESH_RT_LOCK(ms);
TAILQ_FOREACH_SAFE(rt, &ms->ms_routes, rt_next, next) {
if (IEEE80211_ADDR_EQ(rt->rt_dest, dest)) {
if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) {
ms->ms_ppath->mpp_senderror(vap, dest, rt,
IEEE80211_REASON_MESH_PERR_NO_PROXY);
} else {
ms->ms_ppath->mpp_senderror(vap, dest, rt,
IEEE80211_REASON_MESH_PERR_DEST_UNREACH);
}
mesh_rt_del(ms, rt);
MESH_RT_UNLOCK(ms);
return;
}
}
MESH_RT_UNLOCK(ms);
}
void
ieee80211_mesh_rt_flush(struct ieee80211vap *vap)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt, *next;
if (ms == NULL)
return;
MESH_RT_LOCK(ms);
TAILQ_FOREACH_SAFE(rt, &ms->ms_routes, rt_next, next)
mesh_rt_del(ms, rt);
MESH_RT_UNLOCK(ms);
}
void
ieee80211_mesh_rt_flush_peer(struct ieee80211vap *vap,
const uint8_t peer[IEEE80211_ADDR_LEN])
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt, *next;
MESH_RT_LOCK(ms);
TAILQ_FOREACH_SAFE(rt, &ms->ms_routes, rt_next, next) {
if (IEEE80211_ADDR_EQ(rt->rt_nexthop, peer))
mesh_rt_del(ms, rt);
}
MESH_RT_UNLOCK(ms);
}
/*
* Flush expired routing entries, i.e. those in invalid state for
* some time.
*/
static void
mesh_rt_flush_invalid(struct ieee80211vap *vap)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt, *next;
if (ms == NULL)
return;
MESH_RT_LOCK(ms);
TAILQ_FOREACH_SAFE(rt, &ms->ms_routes, rt_next, next) {
/* Discover paths will be deleted by their own callout */
if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_DISCOVER)
continue;
ieee80211_mesh_rt_update(rt, 0);
if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0)
mesh_rt_del(ms, rt);
}
MESH_RT_UNLOCK(ms);
}
#define N(a) (sizeof(a) / sizeof(a[0]))
int
ieee80211_mesh_register_proto_path(const struct ieee80211_mesh_proto_path *mpp)
{
int i, firstempty = -1;
for (i = 0; i < N(mesh_proto_paths); i++) {
if (strncmp(mpp->mpp_descr, mesh_proto_paths[i].mpp_descr,
IEEE80211_MESH_PROTO_DSZ) == 0)
return EEXIST;
if (!mesh_proto_paths[i].mpp_active && firstempty == -1)
firstempty = i;
}
if (firstempty < 0)
return ENOSPC;
memcpy(&mesh_proto_paths[firstempty], mpp, sizeof(*mpp));
mesh_proto_paths[firstempty].mpp_active = 1;
return 0;
}
int
ieee80211_mesh_register_proto_metric(const struct
ieee80211_mesh_proto_metric *mpm)
{
int i, firstempty = -1;
for (i = 0; i < N(mesh_proto_metrics); i++) {
if (strncmp(mpm->mpm_descr, mesh_proto_metrics[i].mpm_descr,
IEEE80211_MESH_PROTO_DSZ) == 0)
return EEXIST;
if (!mesh_proto_metrics[i].mpm_active && firstempty == -1)
firstempty = i;
}
if (firstempty < 0)
return ENOSPC;
memcpy(&mesh_proto_metrics[firstempty], mpm, sizeof(*mpm));
mesh_proto_metrics[firstempty].mpm_active = 1;
return 0;
}
static int
mesh_select_proto_path(struct ieee80211vap *vap, const char *name)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
int i;
for (i = 0; i < N(mesh_proto_paths); i++) {
if (strcasecmp(mesh_proto_paths[i].mpp_descr, name) == 0) {
ms->ms_ppath = &mesh_proto_paths[i];
return 0;
}
}
return ENOENT;
}
static int
mesh_select_proto_metric(struct ieee80211vap *vap, const char *name)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
int i;
for (i = 0; i < N(mesh_proto_metrics); i++) {
if (strcasecmp(mesh_proto_metrics[i].mpm_descr, name) == 0) {
ms->ms_pmetric = &mesh_proto_metrics[i];
return 0;
}
}
return ENOENT;
}
#undef N
static void
mesh_gatemode_setup(struct ieee80211vap *vap)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
/*
* NB: When a mesh gate is running as a ROOT it shall
* not send out periodic GANNs but instead mark the
* mesh gate flag for the corresponding proactive PREQ
* and RANN frames.
*/
if (ms->ms_flags & IEEE80211_MESHFLAGS_ROOT ||
(ms->ms_flags & IEEE80211_MESHFLAGS_GATE) == 0) {
callout_drain(&ms->ms_gatetimer);
return ;
}
callout_reset(&ms->ms_gatetimer, ieee80211_mesh_gateint,
mesh_gatemode_cb, vap);
}
static void
mesh_gatemode_cb(void *arg)
{
struct ieee80211vap *vap = (struct ieee80211vap *)arg;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_meshgann_ie gann;
gann.gann_flags = 0; /* Reserved */
gann.gann_hopcount = 0;
gann.gann_ttl = ms->ms_ttl;
IEEE80211_ADDR_COPY(gann.gann_addr, vap->iv_myaddr);
gann.gann_seq = ms->ms_gateseq++;
gann.gann_interval = ieee80211_mesh_gateint;
IEEE80211_NOTE(vap, IEEE80211_MSG_MESH, vap->iv_bss,
"send broadcast GANN (seq %u)", gann.gann_seq);
ieee80211_send_action(vap->iv_bss, IEEE80211_ACTION_CAT_MESH,
IEEE80211_ACTION_MESH_GANN, &gann);
mesh_gatemode_setup(vap);
}
static void
ieee80211_mesh_init(void)
{
memset(mesh_proto_paths, 0, sizeof(mesh_proto_paths));
memset(mesh_proto_metrics, 0, sizeof(mesh_proto_metrics));
/*
* Setup mesh parameters that depends on the clock frequency.
*/
ieee80211_mesh_gateint = msecs_to_ticks(10000);
ieee80211_mesh_retrytimeout = msecs_to_ticks(40);
ieee80211_mesh_holdingtimeout = msecs_to_ticks(40);
ieee80211_mesh_confirmtimeout = msecs_to_ticks(40);
ieee80211_mesh_backofftimeout = msecs_to_ticks(5000);
/*
* Register action frame handlers.
*/
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_OPEN,
mesh_recv_action_meshpeering_open);
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CONFIRM,
mesh_recv_action_meshpeering_confirm);
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
mesh_recv_action_meshpeering_close);
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_MESH,
IEEE80211_ACTION_MESH_LMETRIC, mesh_recv_action_meshlmetric);
ieee80211_recv_action_register(IEEE80211_ACTION_CAT_MESH,
IEEE80211_ACTION_MESH_GANN, mesh_recv_action_meshgate);
ieee80211_send_action_register(IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_OPEN,
mesh_send_action_meshpeering_open);
ieee80211_send_action_register(IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CONFIRM,
mesh_send_action_meshpeering_confirm);
ieee80211_send_action_register(IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
mesh_send_action_meshpeering_close);
ieee80211_send_action_register(IEEE80211_ACTION_CAT_MESH,
IEEE80211_ACTION_MESH_LMETRIC,
mesh_send_action_meshlmetric);
ieee80211_send_action_register(IEEE80211_ACTION_CAT_MESH,
IEEE80211_ACTION_MESH_GANN,
mesh_send_action_meshgate);
/*
* Register Airtime Link Metric.
*/
ieee80211_mesh_register_proto_metric(&mesh_metric_airtime);
}
SYSINIT(wlan_mesh, SI_SUB_DRIVERS, SI_ORDER_FIRST, ieee80211_mesh_init, NULL);
void
ieee80211_mesh_attach(struct ieee80211com *ic)
{
ic->ic_vattach[IEEE80211_M_MBSS] = mesh_vattach;
}
void
ieee80211_mesh_detach(struct ieee80211com *ic)
{
}
static void
mesh_vdetach_peers(void *arg, struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
uint16_t args[3];
if (ni->ni_mlstate == IEEE80211_NODE_MESH_ESTABLISHED) {
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
args[2] = IEEE80211_REASON_PEER_LINK_CANCELED;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
}
callout_drain(&ni->ni_mltimer);
/* XXX belongs in hwmp */
ieee80211_ageq_drain_node(&ic->ic_stageq,
(void *)(uintptr_t) ieee80211_mac_hash(ic, ni->ni_macaddr));
}
static void
mesh_vdetach(struct ieee80211vap *vap)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
callout_drain(&ms->ms_cleantimer);
ieee80211_iterate_nodes(&vap->iv_ic->ic_sta, mesh_vdetach_peers,
NULL);
ieee80211_mesh_rt_flush(vap);
mtx_destroy(&ms->ms_rt_lock);
ms->ms_ppath->mpp_vdetach(vap);
free(vap->iv_mesh, M_80211_VAP);
vap->iv_mesh = NULL;
}
static void
mesh_vattach(struct ieee80211vap *vap)
{
struct ieee80211_mesh_state *ms;
vap->iv_newstate = mesh_newstate;
vap->iv_input = mesh_input;
vap->iv_opdetach = mesh_vdetach;
vap->iv_recv_mgmt = mesh_recv_mgmt;
vap->iv_recv_ctl = mesh_recv_ctl;
ms = malloc(sizeof(struct ieee80211_mesh_state), M_80211_VAP,
M_NOWAIT | M_ZERO);
if (ms == NULL) {
printf("%s: couldn't alloc MBSS state\n", __func__);
return;
}
vap->iv_mesh = ms;
ms->ms_seq = 0;
ms->ms_flags = (IEEE80211_MESHFLAGS_AP | IEEE80211_MESHFLAGS_FWD);
ms->ms_ttl = IEEE80211_MESH_DEFAULT_TTL;
TAILQ_INIT(&ms->ms_known_gates);
TAILQ_INIT(&ms->ms_routes);
mtx_init(&ms->ms_rt_lock, "MBSS", "802.11s routing table", MTX_DEF);
- callout_init(&ms->ms_cleantimer, CALLOUT_MPSAFE);
- callout_init(&ms->ms_gatetimer, CALLOUT_MPSAFE);
+ callout_init(&ms->ms_cleantimer, 1);
+ callout_init(&ms->ms_gatetimer, 1);
ms->ms_gateseq = 0;
mesh_select_proto_metric(vap, "AIRTIME");
KASSERT(ms->ms_pmetric, ("ms_pmetric == NULL"));
mesh_select_proto_path(vap, "HWMP");
KASSERT(ms->ms_ppath, ("ms_ppath == NULL"));
ms->ms_ppath->mpp_vattach(vap);
}
/*
* IEEE80211_M_MBSS vap state machine handler.
*/
static int
mesh_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_node *ni;
enum ieee80211_state ostate;
IEEE80211_LOCK_ASSERT(ic);
ostate = vap->iv_state;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE, "%s: %s -> %s (%d)\n",
__func__, ieee80211_state_name[ostate],
ieee80211_state_name[nstate], arg);
vap->iv_state = nstate; /* state transition */
if (ostate != IEEE80211_S_SCAN)
ieee80211_cancel_scan(vap); /* background scan */
ni = vap->iv_bss; /* NB: no reference held */
if (nstate != IEEE80211_S_RUN && ostate == IEEE80211_S_RUN) {
callout_drain(&ms->ms_cleantimer);
callout_drain(&ms->ms_gatetimer);
}
switch (nstate) {
case IEEE80211_S_INIT:
switch (ostate) {
case IEEE80211_S_SCAN:
ieee80211_cancel_scan(vap);
break;
case IEEE80211_S_CAC:
ieee80211_dfs_cac_stop(vap);
break;
case IEEE80211_S_RUN:
ieee80211_iterate_nodes(&ic->ic_sta,
mesh_vdetach_peers, NULL);
break;
default:
break;
}
if (ostate != IEEE80211_S_INIT) {
/* NB: optimize INIT -> INIT case */
ieee80211_reset_bss(vap);
ieee80211_mesh_rt_flush(vap);
}
break;
case IEEE80211_S_SCAN:
switch (ostate) {
case IEEE80211_S_INIT:
if (vap->iv_des_chan != IEEE80211_CHAN_ANYC &&
!IEEE80211_IS_CHAN_RADAR(vap->iv_des_chan) &&
ms->ms_idlen != 0) {
/*
* Already have a channel and a mesh ID; bypass
* the scan and startup immediately.
*/
ieee80211_create_ibss(vap, vap->iv_des_chan);
break;
}
/*
* Initiate a scan. We can come here as a result
* of an IEEE80211_IOC_SCAN_REQ too in which case
* the vap will be marked with IEEE80211_FEXT_SCANREQ
* and the scan request parameters will be present
* in iv_scanreq. Otherwise we do the default.
*/
if (vap->iv_flags_ext & IEEE80211_FEXT_SCANREQ) {
ieee80211_check_scan(vap,
vap->iv_scanreq_flags,
vap->iv_scanreq_duration,
vap->iv_scanreq_mindwell,
vap->iv_scanreq_maxdwell,
vap->iv_scanreq_nssid, vap->iv_scanreq_ssid);
vap->iv_flags_ext &= ~IEEE80211_FEXT_SCANREQ;
} else
ieee80211_check_scan_current(vap);
break;
default:
break;
}
break;
case IEEE80211_S_CAC:
/*
* Start CAC on a DFS channel. We come here when starting
* a bss on a DFS channel (see ieee80211_create_ibss).
*/
ieee80211_dfs_cac_start(vap);
break;
case IEEE80211_S_RUN:
switch (ostate) {
case IEEE80211_S_INIT:
/*
* Already have a channel; bypass the
* scan and startup immediately.
* Note that ieee80211_create_ibss will call
* back to do a RUN->RUN state change.
*/
ieee80211_create_ibss(vap,
ieee80211_ht_adjust_channel(ic,
ic->ic_curchan, vap->iv_flags_ht));
/* NB: iv_bss is changed on return */
break;
case IEEE80211_S_CAC:
/*
* NB: This is the normal state change when CAC
* expires and no radar was detected; no need to
* clear the CAC timer as it's already expired.
*/
/* fall thru... */
case IEEE80211_S_CSA:
#if 0
/*
* Shorten inactivity timer of associated stations
* to weed out sta's that don't follow a CSA.
*/
ieee80211_iterate_nodes(&ic->ic_sta, sta_csa, vap);
#endif
/*
* Update bss node channel to reflect where
* we landed after CSA.
*/
ieee80211_node_set_chan(vap->iv_bss,
ieee80211_ht_adjust_channel(ic, ic->ic_curchan,
ieee80211_htchanflags(vap->iv_bss->ni_chan)));
/* XXX bypass debug msgs */
break;
case IEEE80211_S_SCAN:
case IEEE80211_S_RUN:
#ifdef IEEE80211_DEBUG
if (ieee80211_msg_debug(vap)) {
struct ieee80211_node *ni = vap->iv_bss;
ieee80211_note(vap,
"synchronized with %s meshid ",
ether_sprintf(ni->ni_meshid));
ieee80211_print_essid(ni->ni_meshid,
ni->ni_meshidlen);
/* XXX MCS/HT */
printf(" channel %d\n",
ieee80211_chan2ieee(ic, ic->ic_curchan));
}
#endif
break;
default:
break;
}
ieee80211_node_authorize(vap->iv_bss);
callout_reset(&ms->ms_cleantimer, ms->ms_ppath->mpp_inact,
mesh_rt_cleanup_cb, vap);
mesh_gatemode_setup(vap);
break;
default:
break;
}
/* NB: ostate not nstate */
ms->ms_ppath->mpp_newstate(vap, ostate, arg);
return 0;
}
static void
mesh_rt_cleanup_cb(void *arg)
{
struct ieee80211vap *vap = arg;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
mesh_rt_flush_invalid(vap);
callout_reset(&ms->ms_cleantimer, ms->ms_ppath->mpp_inact,
mesh_rt_cleanup_cb, vap);
}
/*
* Mark a mesh STA as gate and return a pointer to it.
* If this is first time, we create a new gate route.
* Always update the path route to this mesh gate.
*/
struct ieee80211_mesh_gate_route *
ieee80211_mesh_mark_gate(struct ieee80211vap *vap, const uint8_t *addr,
struct ieee80211_mesh_route *rt)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_gate_route *gr = NULL, *next;
int found = 0;
MESH_RT_LOCK(ms);
TAILQ_FOREACH_SAFE(gr, &ms->ms_known_gates, gr_next, next) {
if (IEEE80211_ADDR_EQ(gr->gr_addr, addr)) {
found = 1;
break;
}
}
if (!found) {
/* New mesh gate add it to known table. */
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, addr,
"%s", "stored new gate information from pro-PREQ.");
gr = malloc(ALIGN(sizeof(struct ieee80211_mesh_gate_route)),
M_80211_MESH_GT_RT, M_NOWAIT | M_ZERO);
IEEE80211_ADDR_COPY(gr->gr_addr, addr);
TAILQ_INSERT_TAIL(&ms->ms_known_gates, gr, gr_next);
}
gr->gr_route = rt;
/* TODO: link from path route to gate route */
MESH_RT_UNLOCK(ms);
return gr;
}
/*
* Helper function to note the Mesh Peer Link FSM change.
*/
static void
mesh_linkchange(struct ieee80211_node *ni, enum ieee80211_mesh_mlstate state)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
#ifdef IEEE80211_DEBUG
static const char *meshlinkstates[] = {
[IEEE80211_NODE_MESH_IDLE] = "IDLE",
[IEEE80211_NODE_MESH_OPENSNT] = "OPEN SENT",
[IEEE80211_NODE_MESH_OPENRCV] = "OPEN RECEIVED",
[IEEE80211_NODE_MESH_CONFIRMRCV] = "CONFIRM RECEIVED",
[IEEE80211_NODE_MESH_ESTABLISHED] = "ESTABLISHED",
[IEEE80211_NODE_MESH_HOLDING] = "HOLDING"
};
#endif
IEEE80211_NOTE(vap, IEEE80211_MSG_MESH,
ni, "peer link: %s -> %s",
meshlinkstates[ni->ni_mlstate], meshlinkstates[state]);
/* track neighbor count */
if (state == IEEE80211_NODE_MESH_ESTABLISHED &&
ni->ni_mlstate != IEEE80211_NODE_MESH_ESTABLISHED) {
KASSERT(ms->ms_neighbors < 65535, ("neighbor count overflow"));
ms->ms_neighbors++;
ieee80211_beacon_notify(vap, IEEE80211_BEACON_MESHCONF);
} else if (ni->ni_mlstate == IEEE80211_NODE_MESH_ESTABLISHED &&
state != IEEE80211_NODE_MESH_ESTABLISHED) {
KASSERT(ms->ms_neighbors > 0, ("neighbor count 0"));
ms->ms_neighbors--;
ieee80211_beacon_notify(vap, IEEE80211_BEACON_MESHCONF);
}
ni->ni_mlstate = state;
switch (state) {
case IEEE80211_NODE_MESH_HOLDING:
ms->ms_ppath->mpp_peerdown(ni);
break;
case IEEE80211_NODE_MESH_ESTABLISHED:
ieee80211_mesh_discover(vap, ni->ni_macaddr, NULL);
break;
default:
break;
}
}
/*
* Helper function to generate a unique local ID required for mesh
* peer establishment.
*/
static void
mesh_checkid(void *arg, struct ieee80211_node *ni)
{
uint16_t *r = arg;
if (*r == ni->ni_mllid)
*(uint16_t *)arg = 0;
}
static uint32_t
mesh_generateid(struct ieee80211vap *vap)
{
int maxiter = 4;
uint16_t r;
do {
get_random_bytes(&r, 2);
ieee80211_iterate_nodes(&vap->iv_ic->ic_sta, mesh_checkid, &r);
maxiter--;
} while (r == 0 && maxiter > 0);
return r;
}
/*
* Verifies if we already received this packet by checking its
* sequence number.
* Returns 0 if the frame is to be accepted, 1 otherwise.
*/
static int
mesh_checkpseq(struct ieee80211vap *vap,
const uint8_t source[IEEE80211_ADDR_LEN], uint32_t seq)
{
struct ieee80211_mesh_route *rt;
rt = ieee80211_mesh_rt_find(vap, source);
if (rt == NULL) {
rt = ieee80211_mesh_rt_add(vap, source);
if (rt == NULL) {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, source,
"%s", "add mcast route failed");
vap->iv_stats.is_mesh_rtaddfailed++;
return 1;
}
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, source,
"add mcast route, mesh seqno %d", seq);
rt->rt_lastmseq = seq;
return 0;
}
if (IEEE80211_MESH_SEQ_GEQ(rt->rt_lastmseq, seq)) {
return 1;
} else {
rt->rt_lastmseq = seq;
return 0;
}
}
/*
* Iterate the routing table and locate the next hop.
*/
struct ieee80211_node *
ieee80211_mesh_find_txnode(struct ieee80211vap *vap,
const uint8_t dest[IEEE80211_ADDR_LEN])
{
struct ieee80211_mesh_route *rt;
rt = ieee80211_mesh_rt_find(vap, dest);
if (rt == NULL)
return NULL;
if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest,
"%s: !valid, flags 0x%x", __func__, rt->rt_flags);
/* XXX stat */
return NULL;
}
if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) {
rt = ieee80211_mesh_rt_find(vap, rt->rt_mesh_gate);
if (rt == NULL) return NULL;
if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest,
"%s: meshgate !valid, flags 0x%x", __func__,
rt->rt_flags);
/* XXX stat */
return NULL;
}
}
return ieee80211_find_txnode(vap, rt->rt_nexthop);
}
static void
mesh_transmit_to_gate(struct ieee80211vap *vap, struct mbuf *m,
struct ieee80211_mesh_route *rt_gate)
{
struct ifnet *ifp = vap->iv_ifp;
struct ieee80211_node *ni;
IEEE80211_TX_UNLOCK_ASSERT(vap->iv_ic);
ni = ieee80211_mesh_find_txnode(vap, rt_gate->rt_dest);
if (ni == NULL) {
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
m_freem(m);
return;
}
/*
* Send through the VAP packet transmit path.
* This consumes the node ref grabbed above and
* the mbuf, regardless of whether there's a problem
* or not.
*/
(void) ieee80211_vap_pkt_send_dest(vap, m, ni);
}
/*
* Forward the queued frames to known valid mesh gates.
* Assume destination to be outside the MBSS (i.e. proxy entry),
* If no valid mesh gates are known silently discard queued frames.
* After transmitting frames to all known valid mesh gates, this route
* will be marked invalid, and a new path discovery will happen in the hopes
* that (at least) one of the mesh gates have a new proxy entry for us to use.
*/
void
ieee80211_mesh_forward_to_gates(struct ieee80211vap *vap,
struct ieee80211_mesh_route *rt_dest)
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt_gate;
struct ieee80211_mesh_gate_route *gr = NULL, *gr_next;
struct mbuf *m, *mcopy, *next;
IEEE80211_TX_UNLOCK_ASSERT(ic);
KASSERT( rt_dest->rt_flags == IEEE80211_MESHRT_FLAGS_DISCOVER,
("Route is not marked with IEEE80211_MESHRT_FLAGS_DISCOVER"));
/* XXX: send to more than one valid mash gate */
MESH_RT_LOCK(ms);
m = ieee80211_ageq_remove(&ic->ic_stageq,
(struct ieee80211_node *)(uintptr_t)
ieee80211_mac_hash(ic, rt_dest->rt_dest));
TAILQ_FOREACH_SAFE(gr, &ms->ms_known_gates, gr_next, gr_next) {
rt_gate = gr->gr_route;
if (rt_gate == NULL) {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP,
rt_dest->rt_dest,
"mesh gate with no path %6D",
gr->gr_addr, ":");
continue;
}
if ((rt_gate->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0)
continue;
KASSERT(rt_gate->rt_flags & IEEE80211_MESHRT_FLAGS_GATE,
("route not marked as a mesh gate"));
KASSERT((rt_gate->rt_flags &
IEEE80211_MESHRT_FLAGS_PROXY) == 0,
("found mesh gate that is also marked porxy"));
/*
* convert route to a proxy route gated by the current
* mesh gate, this is needed so encap can built data
* frame with correct address.
*/
rt_dest->rt_flags = IEEE80211_MESHRT_FLAGS_PROXY |
IEEE80211_MESHRT_FLAGS_VALID;
rt_dest->rt_ext_seq = 1; /* random value */
IEEE80211_ADDR_COPY(rt_dest->rt_mesh_gate, rt_gate->rt_dest);
IEEE80211_ADDR_COPY(rt_dest->rt_nexthop, rt_gate->rt_nexthop);
rt_dest->rt_metric = rt_gate->rt_metric;
rt_dest->rt_nhops = rt_gate->rt_nhops;
ieee80211_mesh_rt_update(rt_dest, ms->ms_ppath->mpp_inact);
MESH_RT_UNLOCK(ms);
/* XXX: lock?? */
mcopy = m_dup(m, M_NOWAIT);
for (; mcopy != NULL; mcopy = next) {
next = mcopy->m_nextpkt;
mcopy->m_nextpkt = NULL;
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP,
rt_dest->rt_dest,
"flush queued frame %p len %d", mcopy,
mcopy->m_pkthdr.len);
mesh_transmit_to_gate(vap, mcopy, rt_gate);
}
MESH_RT_LOCK(ms);
}
rt_dest->rt_flags = 0; /* Mark invalid */
m_freem(m);
MESH_RT_UNLOCK(ms);
}
/*
* Forward the specified frame.
* Decrement the TTL and set TA to our MAC address.
*/
static void
mesh_forward(struct ieee80211vap *vap, struct mbuf *m,
const struct ieee80211_meshcntl *mc)
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ifnet *ifp = vap->iv_ifp;
const struct ieee80211_frame *wh =
mtod(m, const struct ieee80211_frame *);
struct mbuf *mcopy;
struct ieee80211_meshcntl *mccopy;
struct ieee80211_frame *whcopy;
struct ieee80211_node *ni;
int err;
/* This is called from the RX path - don't hold this lock */
IEEE80211_TX_UNLOCK_ASSERT(ic);
/*
* mesh ttl of 1 means we are the last one receving it,
* according to amendment we decrement and then check if
* 0, if so we dont forward.
*/
if (mc->mc_ttl < 1) {
IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh,
"%s", "frame not fwd'd, ttl 1");
vap->iv_stats.is_mesh_fwd_ttl++;
return;
}
if (!(ms->ms_flags & IEEE80211_MESHFLAGS_FWD)) {
IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh,
"%s", "frame not fwd'd, fwding disabled");
vap->iv_stats.is_mesh_fwd_disabled++;
return;
}
mcopy = m_dup(m, M_NOWAIT);
if (mcopy == NULL) {
IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh,
"%s", "frame not fwd'd, cannot dup");
vap->iv_stats.is_mesh_fwd_nobuf++;
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
return;
}
mcopy = m_pullup(mcopy, ieee80211_hdrspace(ic, wh) +
sizeof(struct ieee80211_meshcntl));
if (mcopy == NULL) {
IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh,
"%s", "frame not fwd'd, too short");
vap->iv_stats.is_mesh_fwd_tooshort++;
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
m_freem(mcopy);
return;
}
whcopy = mtod(mcopy, struct ieee80211_frame *);
mccopy = (struct ieee80211_meshcntl *)
(mtod(mcopy, uint8_t *) + ieee80211_hdrspace(ic, wh));
/* XXX clear other bits? */
whcopy->i_fc[1] &= ~IEEE80211_FC1_RETRY;
IEEE80211_ADDR_COPY(whcopy->i_addr2, vap->iv_myaddr);
if (IEEE80211_IS_MULTICAST(wh->i_addr1)) {
ni = ieee80211_ref_node(vap->iv_bss);
mcopy->m_flags |= M_MCAST;
} else {
ni = ieee80211_mesh_find_txnode(vap, whcopy->i_addr3);
if (ni == NULL) {
/*
* [Optional] any of the following three actions:
* o silently discard
* o trigger a path discovery
* o inform TA that meshDA is unknown.
*/
IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh,
"%s", "frame not fwd'd, no path");
ms->ms_ppath->mpp_senderror(vap, whcopy->i_addr3, NULL,
IEEE80211_REASON_MESH_PERR_NO_FI);
vap->iv_stats.is_mesh_fwd_nopath++;
m_freem(mcopy);
return;
}
IEEE80211_ADDR_COPY(whcopy->i_addr1, ni->ni_macaddr);
}
KASSERT(mccopy->mc_ttl > 0, ("%s called with wrong ttl", __func__));
mccopy->mc_ttl--;
/* XXX calculate priority so drivers can find the tx queue */
M_WME_SETAC(mcopy, WME_AC_BE);
/* XXX do we know m_nextpkt is NULL? */
mcopy->m_pkthdr.rcvif = (void *) ni;
/*
* XXX this bypasses all of the VAP TX handling; it passes frames
* directly to the parent interface.
*
* Because of this, there's no TX lock being held as there's no
* encaps state being used.
*
* Doing a direct parent transmit may not be the correct thing
* to do here; we'll have to re-think this soon.
*/
IEEE80211_TX_LOCK(ic);
err = ieee80211_parent_xmitpkt(ic, mcopy);
IEEE80211_TX_UNLOCK(ic);
if (err != 0) {
/* NB: IFQ_HANDOFF reclaims mbuf */
ieee80211_free_node(ni);
} else {
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
}
}
static struct mbuf *
mesh_decap(struct ieee80211vap *vap, struct mbuf *m, int hdrlen, int meshdrlen)
{
#define WHDIR(wh) ((wh)->i_fc[1] & IEEE80211_FC1_DIR_MASK)
#define MC01(mc) ((const struct ieee80211_meshcntl_ae01 *)mc)
uint8_t b[sizeof(struct ieee80211_qosframe_addr4) +
sizeof(struct ieee80211_meshcntl_ae10)];
const struct ieee80211_qosframe_addr4 *wh;
const struct ieee80211_meshcntl_ae10 *mc;
struct ether_header *eh;
struct llc *llc;
int ae;
if (m->m_len < hdrlen + sizeof(*llc) &&
(m = m_pullup(m, hdrlen + sizeof(*llc))) == NULL) {
IEEE80211_DPRINTF(vap, IEEE80211_MSG_ANY,
"discard data frame: %s", "m_pullup failed");
vap->iv_stats.is_rx_tooshort++;
return NULL;
}
memcpy(b, mtod(m, caddr_t), hdrlen);
wh = (const struct ieee80211_qosframe_addr4 *)&b[0];
mc = (const struct ieee80211_meshcntl_ae10 *)&b[hdrlen - meshdrlen];
KASSERT(WHDIR(wh) == IEEE80211_FC1_DIR_FROMDS ||
WHDIR(wh) == IEEE80211_FC1_DIR_DSTODS,
("bogus dir, fc 0x%x:0x%x", wh->i_fc[0], wh->i_fc[1]));
llc = (struct llc *)(mtod(m, caddr_t) + hdrlen);
if (llc->llc_dsap == LLC_SNAP_LSAP && llc->llc_ssap == LLC_SNAP_LSAP &&
llc->llc_control == LLC_UI && llc->llc_snap.org_code[0] == 0 &&
llc->llc_snap.org_code[1] == 0 && llc->llc_snap.org_code[2] == 0 &&
/* NB: preserve AppleTalk frames that have a native SNAP hdr */
!(llc->llc_snap.ether_type == htons(ETHERTYPE_AARP) ||
llc->llc_snap.ether_type == htons(ETHERTYPE_IPX))) {
m_adj(m, hdrlen + sizeof(struct llc) - sizeof(*eh));
llc = NULL;
} else {
m_adj(m, hdrlen - sizeof(*eh));
}
eh = mtod(m, struct ether_header *);
ae = mc->mc_flags & IEEE80211_MESH_AE_MASK;
if (WHDIR(wh) == IEEE80211_FC1_DIR_FROMDS) {
IEEE80211_ADDR_COPY(eh->ether_dhost, wh->i_addr1);
if (ae == IEEE80211_MESH_AE_00) {
IEEE80211_ADDR_COPY(eh->ether_shost, wh->i_addr3);
} else if (ae == IEEE80211_MESH_AE_01) {
IEEE80211_ADDR_COPY(eh->ether_shost,
MC01(mc)->mc_addr4);
} else {
IEEE80211_DISCARD(vap, IEEE80211_MSG_ANY,
(const struct ieee80211_frame *)wh, NULL,
"bad AE %d", ae);
vap->iv_stats.is_mesh_badae++;
m_freem(m);
return NULL;
}
} else {
if (ae == IEEE80211_MESH_AE_00) {
IEEE80211_ADDR_COPY(eh->ether_dhost, wh->i_addr3);
IEEE80211_ADDR_COPY(eh->ether_shost, wh->i_addr4);
} else if (ae == IEEE80211_MESH_AE_10) {
IEEE80211_ADDR_COPY(eh->ether_dhost, mc->mc_addr5);
IEEE80211_ADDR_COPY(eh->ether_shost, mc->mc_addr6);
} else {
IEEE80211_DISCARD(vap, IEEE80211_MSG_ANY,
(const struct ieee80211_frame *)wh, NULL,
"bad AE %d", ae);
vap->iv_stats.is_mesh_badae++;
m_freem(m);
return NULL;
}
}
#ifndef __NO_STRICT_ALIGNMENT
if (!ALIGNED_POINTER(mtod(m, caddr_t) + sizeof(*eh), uint32_t)) {
m = ieee80211_realign(vap, m, sizeof(*eh));
if (m == NULL)
return NULL;
}
#endif /* !__NO_STRICT_ALIGNMENT */
if (llc != NULL) {
eh = mtod(m, struct ether_header *);
eh->ether_type = htons(m->m_pkthdr.len - sizeof(*eh));
}
return m;
#undef WDIR
#undef MC01
}
/*
* Return non-zero if the unicast mesh data frame should be processed
* locally. Frames that are not proxy'd have our address, otherwise
* we need to consult the routing table to look for a proxy entry.
*/
static __inline int
mesh_isucastforme(struct ieee80211vap *vap, const struct ieee80211_frame *wh,
const struct ieee80211_meshcntl *mc)
{
int ae = mc->mc_flags & 3;
KASSERT((wh->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS,
("bad dir 0x%x:0x%x", wh->i_fc[0], wh->i_fc[1]));
KASSERT(ae == IEEE80211_MESH_AE_00 || ae == IEEE80211_MESH_AE_10,
("bad AE %d", ae));
if (ae == IEEE80211_MESH_AE_10) { /* ucast w/ proxy */
const struct ieee80211_meshcntl_ae10 *mc10 =
(const struct ieee80211_meshcntl_ae10 *) mc;
struct ieee80211_mesh_route *rt =
ieee80211_mesh_rt_find(vap, mc10->mc_addr5);
/* check for proxy route to ourself */
return (rt != NULL &&
(rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY));
} else /* ucast w/o proxy */
return IEEE80211_ADDR_EQ(wh->i_addr3, vap->iv_myaddr);
}
/*
* Verifies transmitter, updates lifetime, precursor list and forwards data.
* > 0 means we have forwarded data and no need to process locally
* == 0 means we want to process locally (and we may have forwarded data
* < 0 means there was an error and data should be discarded
*/
static int
mesh_recv_indiv_data_to_fwrd(struct ieee80211vap *vap, struct mbuf *m,
struct ieee80211_frame *wh, const struct ieee80211_meshcntl *mc)
{
struct ieee80211_qosframe_addr4 *qwh;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt_meshda, *rt_meshsa;
/* This is called from the RX path - don't hold this lock */
IEEE80211_TX_UNLOCK_ASSERT(vap->iv_ic);
qwh = (struct ieee80211_qosframe_addr4 *)wh;
/*
* TODO:
* o verify addr2 is a legitimate transmitter
* o lifetime of precursor of addr3 (addr2) is max(init, curr)
* o lifetime of precursor of addr4 (nexthop) is max(init, curr)
*/
/* set lifetime of addr3 (meshDA) to initial value */
rt_meshda = ieee80211_mesh_rt_find(vap, qwh->i_addr3);
if (rt_meshda == NULL) {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, qwh->i_addr2,
"no route to meshDA(%6D)", qwh->i_addr3, ":");
/*
* [Optional] any of the following three actions:
* o silently discard [X]
* o trigger a path discovery [ ]
* o inform TA that meshDA is unknown. [ ]
*/
/* XXX: stats */
return (-1);
}
ieee80211_mesh_rt_update(rt_meshda, ticks_to_msecs(
ms->ms_ppath->mpp_inact));
/* set lifetime of addr4 (meshSA) to initial value */
rt_meshsa = ieee80211_mesh_rt_find(vap, qwh->i_addr4);
KASSERT(rt_meshsa != NULL, ("no route"));
ieee80211_mesh_rt_update(rt_meshsa, ticks_to_msecs(
ms->ms_ppath->mpp_inact));
mesh_forward(vap, m, mc);
return (1); /* dont process locally */
}
/*
* Verifies transmitter, updates lifetime, precursor list and process data
* locally, if data is proxy with AE = 10 it could mean data should go
* on another mesh path or data should be forwarded to the DS.
*
* > 0 means we have forwarded data and no need to process locally
* == 0 means we want to process locally (and we may have forwarded data
* < 0 means there was an error and data should be discarded
*/
static int
mesh_recv_indiv_data_to_me(struct ieee80211vap *vap, struct mbuf *m,
struct ieee80211_frame *wh, const struct ieee80211_meshcntl *mc)
{
struct ieee80211_qosframe_addr4 *qwh;
const struct ieee80211_meshcntl_ae10 *mc10;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_route *rt;
int ae;
/* This is called from the RX path - don't hold this lock */
IEEE80211_TX_UNLOCK_ASSERT(vap->iv_ic);
qwh = (struct ieee80211_qosframe_addr4 *)wh;
mc10 = (const struct ieee80211_meshcntl_ae10 *)mc;
/*
* TODO:
* o verify addr2 is a legitimate transmitter
* o lifetime of precursor entry is max(init, curr)
*/
/* set lifetime of addr4 (meshSA) to initial value */
rt = ieee80211_mesh_rt_find(vap, qwh->i_addr4);
KASSERT(rt != NULL, ("no route"));
ieee80211_mesh_rt_update(rt, ticks_to_msecs(ms->ms_ppath->mpp_inact));
rt = NULL;
ae = mc10->mc_flags & IEEE80211_MESH_AE_MASK;
KASSERT(ae == IEEE80211_MESH_AE_00 ||
ae == IEEE80211_MESH_AE_10, ("bad AE %d", ae));
if (ae == IEEE80211_MESH_AE_10) {
if (IEEE80211_ADDR_EQ(mc10->mc_addr5, qwh->i_addr3)) {
return (0); /* process locally */
}
rt = ieee80211_mesh_rt_find(vap, mc10->mc_addr5);
if (rt != NULL &&
(rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) &&
(rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) == 0) {
/*
* Forward on another mesh-path, according to
* amendment as specified in 9.32.4.1
*/
IEEE80211_ADDR_COPY(qwh->i_addr3, mc10->mc_addr5);
mesh_forward(vap, m,
(const struct ieee80211_meshcntl *)mc10);
return (1); /* dont process locally */
}
/*
* All other cases: forward of MSDUs from the MBSS to DS indiv.
* addressed according to 13.11.3.2.
*/
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_OUTPUT, qwh->i_addr2,
"forward frame to DS, SA(%6D) DA(%6D)",
mc10->mc_addr6, ":", mc10->mc_addr5, ":");
}
return (0); /* process locally */
}
/*
* Try to forward the group addressed data on to other mesh STAs, and
* also to the DS.
*
* > 0 means we have forwarded data and no need to process locally
* == 0 means we want to process locally (and we may have forwarded data
* < 0 means there was an error and data should be discarded
*/
static int
mesh_recv_group_data(struct ieee80211vap *vap, struct mbuf *m,
struct ieee80211_frame *wh, const struct ieee80211_meshcntl *mc)
{
#define MC01(mc) ((const struct ieee80211_meshcntl_ae01 *)mc)
struct ieee80211_mesh_state *ms = vap->iv_mesh;
/* This is called from the RX path - don't hold this lock */
IEEE80211_TX_UNLOCK_ASSERT(vap->iv_ic);
mesh_forward(vap, m, mc);
if(mc->mc_ttl > 0) {
if (mc->mc_flags & IEEE80211_MESH_AE_01) {
/*
* Forward of MSDUs from the MBSS to DS group addressed
* (according to 13.11.3.2)
* This happens by delivering the packet, and a bridge
* will sent it on another port member.
*/
if (ms->ms_flags & IEEE80211_MESHFLAGS_GATE &&
ms->ms_flags & IEEE80211_MESHFLAGS_FWD)
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH,
MC01(mc)->mc_addr4, "%s",
"forward from MBSS to the DS");
}
}
return (0); /* process locally */
#undef MC01
}
static int
mesh_input(struct ieee80211_node *ni, struct mbuf *m, int rssi, int nf)
{
#define HAS_SEQ(type) ((type & 0x4) == 0)
#define MC01(mc) ((const struct ieee80211_meshcntl_ae01 *)mc)
#define MC10(mc) ((const struct ieee80211_meshcntl_ae10 *)mc)
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
struct ifnet *ifp = vap->iv_ifp;
struct ieee80211_frame *wh;
const struct ieee80211_meshcntl *mc;
int hdrspace, meshdrlen, need_tap, error;
uint8_t dir, type, subtype, ae;
uint32_t seq;
const uint8_t *addr;
uint8_t qos[2];
ieee80211_seq rxseq;
KASSERT(ni != NULL, ("null node"));
ni->ni_inact = ni->ni_inact_reload;
need_tap = 1; /* mbuf need to be tapped. */
type = -1; /* undefined */
/* This is called from the RX path - don't hold this lock */
IEEE80211_TX_UNLOCK_ASSERT(ic);
if (m->m_pkthdr.len < sizeof(struct ieee80211_frame_min)) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
ni->ni_macaddr, NULL,
"too short (1): len %u", m->m_pkthdr.len);
vap->iv_stats.is_rx_tooshort++;
goto out;
}
/*
* Bit of a cheat here, we use a pointer for a 3-address
* frame format but don't reference fields past outside
* ieee80211_frame_min w/o first validating the data is
* present.
*/
wh = mtod(m, struct ieee80211_frame *);
if ((wh->i_fc[0] & IEEE80211_FC0_VERSION_MASK) !=
IEEE80211_FC0_VERSION_0) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
ni->ni_macaddr, NULL, "wrong version %x", wh->i_fc[0]);
vap->iv_stats.is_rx_badversion++;
goto err;
}
dir = wh->i_fc[1] & IEEE80211_FC1_DIR_MASK;
type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK;
if ((ic->ic_flags & IEEE80211_F_SCAN) == 0) {
IEEE80211_RSSI_LPF(ni->ni_avgrssi, rssi);
ni->ni_noise = nf;
if (HAS_SEQ(type)) {
uint8_t tid = ieee80211_gettid(wh);
if (IEEE80211_QOS_HAS_SEQ(wh) &&
TID_TO_WME_AC(tid) >= WME_AC_VI)
ic->ic_wme.wme_hipri_traffic++;
rxseq = le16toh(*(uint16_t *)wh->i_seq);
if (! ieee80211_check_rxseq(ni, wh)) {
/* duplicate, discard */
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT,
wh->i_addr1, "duplicate",
"seqno <%u,%u> fragno <%u,%u> tid %u",
rxseq >> IEEE80211_SEQ_SEQ_SHIFT,
ni->ni_rxseqs[tid] >>
IEEE80211_SEQ_SEQ_SHIFT,
rxseq & IEEE80211_SEQ_FRAG_MASK,
ni->ni_rxseqs[tid] &
IEEE80211_SEQ_FRAG_MASK,
tid);
vap->iv_stats.is_rx_dup++;
IEEE80211_NODE_STAT(ni, rx_dup);
goto out;
}
ni->ni_rxseqs[tid] = rxseq;
}
}
#ifdef IEEE80211_DEBUG
/*
* It's easier, but too expensive, to simulate different mesh
* topologies by consulting the ACL policy very early, so do this
* only under DEBUG.
*
* NB: this check is also done upon peering link initiation.
*/
if (vap->iv_acl != NULL && !vap->iv_acl->iac_check(vap, wh)) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_ACL,
wh, NULL, "%s", "disallowed by ACL");
vap->iv_stats.is_rx_acl++;
goto out;
}
#endif
switch (type) {
case IEEE80211_FC0_TYPE_DATA:
if (ni == vap->iv_bss)
goto out;
if (ni->ni_mlstate != IEEE80211_NODE_MESH_ESTABLISHED) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_MESH,
ni->ni_macaddr, NULL,
"peer link not yet established (%d)",
ni->ni_mlstate);
vap->iv_stats.is_mesh_nolink++;
goto out;
}
if (dir != IEEE80211_FC1_DIR_FROMDS &&
dir != IEEE80211_FC1_DIR_DSTODS) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, "data", "incorrect dir 0x%x", dir);
vap->iv_stats.is_rx_wrongdir++;
goto err;
}
/* All Mesh data frames are QoS subtype */
if (!HAS_SEQ(type)) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, "data", "incorrect subtype 0x%x", subtype);
vap->iv_stats.is_rx_badsubtype++;
goto err;
}
/*
* Next up, any fragmentation.
* XXX: we defrag before we even try to forward,
* Mesh Control field is not present in sub-sequent
* fragmented frames. This is in contrast to Draft 4.0.
*/
hdrspace = ieee80211_hdrspace(ic, wh);
if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
m = ieee80211_defrag(ni, m, hdrspace);
if (m == NULL) {
/* Fragment dropped or frame not complete yet */
goto out;
}
}
wh = mtod(m, struct ieee80211_frame *); /* NB: after defrag */
/*
* Now we have a complete Mesh Data frame.
*/
/*
* Only fromDStoDS data frames use 4 address qos frames
* as specified in amendment. Otherwise addr4 is located
* in the Mesh Control field and a 3 address qos frame
* is used.
*/
if (IEEE80211_IS_DSTODS(wh))
*(uint16_t *)qos = *(uint16_t *)
((struct ieee80211_qosframe_addr4 *)wh)->i_qos;
else
*(uint16_t *)qos = *(uint16_t *)
((struct ieee80211_qosframe *)wh)->i_qos;
/*
* NB: The mesh STA sets the Mesh Control Present
* subfield to 1 in the Mesh Data frame containing
* an unfragmented MSDU, an A-MSDU, or the first
* fragment of an MSDU.
* After defrag it should always be present.
*/
if (!(qos[1] & IEEE80211_QOS_MC)) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_MESH,
ni->ni_macaddr, NULL,
"%s", "Mesh control field not present");
vap->iv_stats.is_rx_elem_missing++; /* XXX: kinda */
goto err;
}
/* pull up enough to get to the mesh control */
if (m->m_len < hdrspace + sizeof(struct ieee80211_meshcntl) &&
(m = m_pullup(m, hdrspace +
sizeof(struct ieee80211_meshcntl))) == NULL) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
ni->ni_macaddr, NULL,
"data too short: expecting %u", hdrspace);
vap->iv_stats.is_rx_tooshort++;
goto out; /* XXX */
}
/*
* Now calculate the full extent of the headers. Note
* mesh_decap will pull up anything we didn't get
* above when it strips the 802.11 headers.
*/
mc = (const struct ieee80211_meshcntl *)
(mtod(m, const uint8_t *) + hdrspace);
ae = mc->mc_flags & IEEE80211_MESH_AE_MASK;
meshdrlen = sizeof(struct ieee80211_meshcntl) +
ae * IEEE80211_ADDR_LEN;
hdrspace += meshdrlen;
/* pull complete hdrspace = ieee80211_hdrspace + meshcontrol */
if ((meshdrlen > sizeof(struct ieee80211_meshcntl)) &&
(m->m_len < hdrspace) &&
((m = m_pullup(m, hdrspace)) == NULL)) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
ni->ni_macaddr, NULL,
"data too short: expecting %u", hdrspace);
vap->iv_stats.is_rx_tooshort++;
goto out; /* XXX */
}
/* XXX: are we sure there is no reallocating after m_pullup? */
seq = LE_READ_4(mc->mc_seq);
if (IEEE80211_IS_MULTICAST(wh->i_addr1))
addr = wh->i_addr3;
else if (ae == IEEE80211_MESH_AE_01)
addr = MC01(mc)->mc_addr4;
else
addr = ((struct ieee80211_qosframe_addr4 *)wh)->i_addr4;
if (IEEE80211_ADDR_EQ(vap->iv_myaddr, addr)) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT,
addr, "data", "%s", "not to me");
vap->iv_stats.is_rx_wrongbss++; /* XXX kinda */
goto out;
}
if (mesh_checkpseq(vap, addr, seq) != 0) {
vap->iv_stats.is_rx_dup++;
goto out;
}
/* This code "routes" the frame to the right control path */
if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
if (IEEE80211_ADDR_EQ(vap->iv_myaddr, wh->i_addr3))
error =
mesh_recv_indiv_data_to_me(vap, m, wh, mc);
else if (IEEE80211_IS_MULTICAST(wh->i_addr3))
error = mesh_recv_group_data(vap, m, wh, mc);
else
error = mesh_recv_indiv_data_to_fwrd(vap, m,
wh, mc);
} else
error = mesh_recv_group_data(vap, m, wh, mc);
if (error < 0)
goto err;
else if (error > 0)
goto out;
if (ieee80211_radiotap_active_vap(vap))
ieee80211_radiotap_rx(vap, m);
need_tap = 0;
/*
* Finally, strip the 802.11 header.
*/
m = mesh_decap(vap, m, hdrspace, meshdrlen);
if (m == NULL) {
/* XXX mask bit to check for both */
/* don't count Null data frames as errors */
if (subtype == IEEE80211_FC0_SUBTYPE_NODATA ||
subtype == IEEE80211_FC0_SUBTYPE_QOS_NULL)
goto out;
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT,
ni->ni_macaddr, "data", "%s", "decap error");
vap->iv_stats.is_rx_decap++;
IEEE80211_NODE_STAT(ni, rx_decap);
goto err;
}
if (qos[0] & IEEE80211_QOS_AMSDU) {
m = ieee80211_decap_amsdu(ni, m);
if (m == NULL)
return IEEE80211_FC0_TYPE_DATA;
}
ieee80211_deliver_data(vap, ni, m);
return type;
case IEEE80211_FC0_TYPE_MGT:
vap->iv_stats.is_rx_mgmt++;
IEEE80211_NODE_STAT(ni, rx_mgmt);
if (dir != IEEE80211_FC1_DIR_NODS) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, "mgt", "incorrect dir 0x%x", dir);
vap->iv_stats.is_rx_wrongdir++;
goto err;
}
if (m->m_pkthdr.len < sizeof(struct ieee80211_frame)) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
ni->ni_macaddr, "mgt", "too short: len %u",
m->m_pkthdr.len);
vap->iv_stats.is_rx_tooshort++;
goto out;
}
#ifdef IEEE80211_DEBUG
if ((ieee80211_msg_debug(vap) &&
(vap->iv_ic->ic_flags & IEEE80211_F_SCAN)) ||
ieee80211_msg_dumppkts(vap)) {
if_printf(ifp, "received %s from %s rssi %d\n",
ieee80211_mgt_subtype_name[subtype >>
IEEE80211_FC0_SUBTYPE_SHIFT],
ether_sprintf(wh->i_addr2), rssi);
}
#endif
if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, NULL, "%s", "WEP set but not permitted");
vap->iv_stats.is_rx_mgtdiscard++; /* XXX */
goto out;
}
vap->iv_recv_mgmt(ni, m, subtype, rssi, nf);
goto out;
case IEEE80211_FC0_TYPE_CTL:
vap->iv_stats.is_rx_ctl++;
IEEE80211_NODE_STAT(ni, rx_ctrl);
goto out;
default:
IEEE80211_DISCARD(vap, IEEE80211_MSG_ANY,
wh, "bad", "frame type 0x%x", type);
/* should not come here */
break;
}
err:
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
out:
if (m != NULL) {
if (need_tap && ieee80211_radiotap_active_vap(vap))
ieee80211_radiotap_rx(vap, m);
m_freem(m);
}
return type;
#undef HAS_SEQ
#undef MC01
#undef MC10
}
static void
mesh_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0, int subtype,
int rssi, int nf)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211_frame *wh;
struct ieee80211_mesh_route *rt;
uint8_t *frm, *efrm;
wh = mtod(m0, struct ieee80211_frame *);
frm = (uint8_t *)&wh[1];
efrm = mtod(m0, uint8_t *) + m0->m_len;
switch (subtype) {
case IEEE80211_FC0_SUBTYPE_PROBE_RESP:
case IEEE80211_FC0_SUBTYPE_BEACON:
{
struct ieee80211_scanparams scan;
/*
* We process beacon/probe response
* frames to discover neighbors.
*/
if (ieee80211_parse_beacon(ni, m0, &scan) != 0)
return;
/*
* Count frame now that we know it's to be processed.
*/
if (subtype == IEEE80211_FC0_SUBTYPE_BEACON) {
vap->iv_stats.is_rx_beacon++; /* XXX remove */
IEEE80211_NODE_STAT(ni, rx_beacons);
} else
IEEE80211_NODE_STAT(ni, rx_proberesp);
/*
* If scanning, just pass information to the scan module.
*/
if (ic->ic_flags & IEEE80211_F_SCAN) {
if (ic->ic_flags_ext & IEEE80211_FEXT_PROBECHAN) {
/*
* Actively scanning a channel marked passive;
* send a probe request now that we know there
* is 802.11 traffic present.
*
* XXX check if the beacon we recv'd gives
* us what we need and suppress the probe req
*/
ieee80211_probe_curchan(vap, 1);
ic->ic_flags_ext &= ~IEEE80211_FEXT_PROBECHAN;
}
ieee80211_add_scan(vap, ic->ic_curchan, &scan, wh,
subtype, rssi, nf);
return;
}
/* The rest of this code assumes we are running */
if (vap->iv_state != IEEE80211_S_RUN)
return;
/*
* Ignore non-mesh STAs.
*/
if ((scan.capinfo &
(IEEE80211_CAPINFO_ESS|IEEE80211_CAPINFO_IBSS)) ||
scan.meshid == NULL || scan.meshconf == NULL) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, "beacon", "%s", "not a mesh sta");
vap->iv_stats.is_mesh_wrongmesh++;
return;
}
/*
* Ignore STAs for other mesh networks.
*/
if (memcmp(scan.meshid+2, ms->ms_id, ms->ms_idlen) != 0 ||
mesh_verify_meshconf(vap, scan.meshconf)) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, "beacon", "%s", "not for our mesh");
vap->iv_stats.is_mesh_wrongmesh++;
return;
}
/*
* Peer only based on the current ACL policy.
*/
if (vap->iv_acl != NULL && !vap->iv_acl->iac_check(vap, wh)) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_ACL,
wh, NULL, "%s", "disallowed by ACL");
vap->iv_stats.is_rx_acl++;
return;
}
/*
* Do neighbor discovery.
*/
if (!IEEE80211_ADDR_EQ(wh->i_addr2, ni->ni_macaddr)) {
/*
* Create a new entry in the neighbor table.
*/
ni = ieee80211_add_neighbor(vap, wh, &scan);
}
/*
* Automatically peer with discovered nodes if possible.
*/
if (ni != vap->iv_bss &&
(ms->ms_flags & IEEE80211_MESHFLAGS_AP)) {
switch (ni->ni_mlstate) {
case IEEE80211_NODE_MESH_IDLE:
{
uint16_t args[1];
/* Wait for backoff callout to reset counter */
if (ni->ni_mlhcnt >= ieee80211_mesh_maxholding)
return;
ni->ni_mlpid = mesh_generateid(vap);
if (ni->ni_mlpid == 0)
return;
mesh_linkchange(ni, IEEE80211_NODE_MESH_OPENSNT);
args[0] = ni->ni_mlpid;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_OPEN, args);
ni->ni_mlrcnt = 0;
mesh_peer_timeout_setup(ni);
break;
}
case IEEE80211_NODE_MESH_ESTABLISHED:
{
/*
* Valid beacon from a peer mesh STA
* bump TA lifetime
*/
rt = ieee80211_mesh_rt_find(vap, wh->i_addr2);
if(rt != NULL) {
ieee80211_mesh_rt_update(rt,
ticks_to_msecs(
ms->ms_ppath->mpp_inact));
}
break;
}
default:
break; /* ignore */
}
}
break;
}
case IEEE80211_FC0_SUBTYPE_PROBE_REQ:
{
uint8_t *ssid, *meshid, *rates, *xrates;
uint8_t *sfrm;
if (vap->iv_state != IEEE80211_S_RUN) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, NULL, "wrong state %s",
ieee80211_state_name[vap->iv_state]);
vap->iv_stats.is_rx_mgtdiscard++;
return;
}
if (IEEE80211_IS_MULTICAST(wh->i_addr2)) {
/* frame must be directed */
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, NULL, "%s", "not unicast");
vap->iv_stats.is_rx_mgtdiscard++; /* XXX stat */
return;
}
/*
* prreq frame format
* [tlv] ssid
* [tlv] supported rates
* [tlv] extended supported rates
* [tlv] mesh id
*/
ssid = meshid = rates = xrates = NULL;
sfrm = frm;
while (efrm - frm > 1) {
IEEE80211_VERIFY_LENGTH(efrm - frm, frm[1] + 2, return);
switch (*frm) {
case IEEE80211_ELEMID_SSID:
ssid = frm;
break;
case IEEE80211_ELEMID_RATES:
rates = frm;
break;
case IEEE80211_ELEMID_XRATES:
xrates = frm;
break;
case IEEE80211_ELEMID_MESHID:
meshid = frm;
break;
}
frm += frm[1] + 2;
}
IEEE80211_VERIFY_ELEMENT(ssid, IEEE80211_NWID_LEN, return);
IEEE80211_VERIFY_ELEMENT(rates, IEEE80211_RATE_MAXSIZE, return);
if (xrates != NULL)
IEEE80211_VERIFY_ELEMENT(xrates,
IEEE80211_RATE_MAXSIZE - rates[1], return);
if (meshid != NULL) {
IEEE80211_VERIFY_ELEMENT(meshid,
IEEE80211_MESHID_LEN, return);
/* NB: meshid, not ssid */
IEEE80211_VERIFY_SSID(vap->iv_bss, meshid, return);
}
/* XXX find a better class or define it's own */
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_INPUT, wh->i_addr2,
"%s", "recv probe req");
/*
* Some legacy 11b clients cannot hack a complete
* probe response frame. When the request includes
* only a bare-bones rate set, communicate this to
* the transmit side.
*/
ieee80211_send_proberesp(vap, wh->i_addr2, 0);
break;
}
case IEEE80211_FC0_SUBTYPE_ACTION:
case IEEE80211_FC0_SUBTYPE_ACTION_NOACK:
if (ni == vap->iv_bss) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, NULL, "%s", "unknown node");
vap->iv_stats.is_rx_mgtdiscard++;
} else if (!IEEE80211_ADDR_EQ(vap->iv_myaddr, wh->i_addr1) &&
!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, NULL, "%s", "not for us");
vap->iv_stats.is_rx_mgtdiscard++;
} else if (vap->iv_state != IEEE80211_S_RUN) {
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, NULL, "wrong state %s",
ieee80211_state_name[vap->iv_state]);
vap->iv_stats.is_rx_mgtdiscard++;
} else {
if (ieee80211_parse_action(ni, m0) == 0)
(void)ic->ic_recv_action(ni, wh, frm, efrm);
}
break;
case IEEE80211_FC0_SUBTYPE_ASSOC_REQ:
case IEEE80211_FC0_SUBTYPE_ASSOC_RESP:
case IEEE80211_FC0_SUBTYPE_REASSOC_REQ:
case IEEE80211_FC0_SUBTYPE_REASSOC_RESP:
case IEEE80211_FC0_SUBTYPE_ATIM:
case IEEE80211_FC0_SUBTYPE_DISASSOC:
case IEEE80211_FC0_SUBTYPE_AUTH:
case IEEE80211_FC0_SUBTYPE_DEAUTH:
IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT,
wh, NULL, "%s", "not handled");
vap->iv_stats.is_rx_mgtdiscard++;
break;
default:
IEEE80211_DISCARD(vap, IEEE80211_MSG_ANY,
wh, "mgt", "subtype 0x%x not handled", subtype);
vap->iv_stats.is_rx_badsubtype++;
break;
}
}
static void
mesh_recv_ctl(struct ieee80211_node *ni, struct mbuf *m, int subtype)
{
switch (subtype) {
case IEEE80211_FC0_SUBTYPE_BAR:
ieee80211_recv_bar(ni, m);
break;
}
}
/*
* Parse meshpeering action ie's for MPM frames
*/
static const struct ieee80211_meshpeer_ie *
mesh_parse_meshpeering_action(struct ieee80211_node *ni,
const struct ieee80211_frame *wh, /* XXX for VERIFY_LENGTH */
const uint8_t *frm, const uint8_t *efrm,
struct ieee80211_meshpeer_ie *mp, uint8_t subtype)
{
struct ieee80211vap *vap = ni->ni_vap;
const struct ieee80211_meshpeer_ie *mpie;
uint16_t args[3];
const uint8_t *meshid, *meshconf, *meshpeer;
uint8_t sendclose = 0; /* 1 = MPM frame rejected, close will be sent */
meshid = meshconf = meshpeer = NULL;
while (efrm - frm > 1) {
IEEE80211_VERIFY_LENGTH(efrm - frm, frm[1] + 2, return NULL);
switch (*frm) {
case IEEE80211_ELEMID_MESHID:
meshid = frm;
break;
case IEEE80211_ELEMID_MESHCONF:
meshconf = frm;
break;
case IEEE80211_ELEMID_MESHPEER:
meshpeer = frm;
mpie = (const struct ieee80211_meshpeer_ie *) frm;
memset(mp, 0, sizeof(*mp));
mp->peer_len = mpie->peer_len;
mp->peer_proto = LE_READ_2(&mpie->peer_proto);
mp->peer_llinkid = LE_READ_2(&mpie->peer_llinkid);
switch (subtype) {
case IEEE80211_ACTION_MESHPEERING_CONFIRM:
mp->peer_linkid =
LE_READ_2(&mpie->peer_linkid);
break;
case IEEE80211_ACTION_MESHPEERING_CLOSE:
/* NB: peer link ID is optional */
if (mpie->peer_len ==
(IEEE80211_MPM_BASE_SZ + 2)) {
mp->peer_linkid = 0;
mp->peer_rcode =
LE_READ_2(&mpie->peer_linkid);
} else {
mp->peer_linkid =
LE_READ_2(&mpie->peer_linkid);
mp->peer_rcode =
LE_READ_2(&mpie->peer_rcode);
}
break;
}
break;
}
frm += frm[1] + 2;
}
/*
* Verify the contents of the frame.
* If it fails validation, close the peer link.
*/
if (mesh_verify_meshpeer(vap, subtype, (const uint8_t *)mp)) {
sendclose = 1;
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH,
wh, NULL, "%s", "MPM validation failed");
}
/* If meshid is not the same reject any frames type. */
if (sendclose == 0 && mesh_verify_meshid(vap, meshid)) {
sendclose = 1;
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH,
wh, NULL, "%s", "not for our mesh");
if (subtype == IEEE80211_ACTION_MESHPEERING_CLOSE) {
/*
* Standard not clear about this, if we dont ignore
* there will be an endless loop between nodes sending
* CLOSE frames between each other with wrong meshid.
* Discard and timers will bring FSM to IDLE state.
*/
return NULL;
}
}
/*
* Close frames are accepted if meshid is the same.
* Verify the other two types.
*/
if (sendclose == 0 && subtype != IEEE80211_ACTION_MESHPEERING_CLOSE &&
mesh_verify_meshconf(vap, meshconf)) {
sendclose = 1;
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH,
wh, NULL, "%s", "configuration missmatch");
}
if (sendclose) {
vap->iv_stats.is_rx_mgtdiscard++;
switch (ni->ni_mlstate) {
case IEEE80211_NODE_MESH_IDLE:
case IEEE80211_NODE_MESH_ESTABLISHED:
case IEEE80211_NODE_MESH_HOLDING:
/* ignore */
break;
case IEEE80211_NODE_MESH_OPENSNT:
case IEEE80211_NODE_MESH_OPENRCV:
case IEEE80211_NODE_MESH_CONFIRMRCV:
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
/* Reason codes for rejection */
switch (subtype) {
case IEEE80211_ACTION_MESHPEERING_OPEN:
args[2] = IEEE80211_REASON_MESH_CPVIOLATION;
break;
case IEEE80211_ACTION_MESHPEERING_CONFIRM:
args[2] = IEEE80211_REASON_MESH_INCONS_PARAMS;
break;
}
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING);
mesh_peer_timeout_setup(ni);
break;
}
return NULL;
}
return (const struct ieee80211_meshpeer_ie *) mp;
}
static int
mesh_recv_action_meshpeering_open(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_meshpeer_ie ie;
const struct ieee80211_meshpeer_ie *meshpeer;
uint16_t args[3];
/* +2+2 for action + code + capabilites */
meshpeer = mesh_parse_meshpeering_action(ni, wh, frm+2+2, efrm, &ie,
IEEE80211_ACTION_MESHPEERING_OPEN);
if (meshpeer == NULL) {
return 0;
}
/* XXX move up */
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni,
"recv PEER OPEN, lid 0x%x", meshpeer->peer_llinkid);
switch (ni->ni_mlstate) {
case IEEE80211_NODE_MESH_IDLE:
/* Reject open request if reached our maximum neighbor count */
if (ms->ms_neighbors >= IEEE80211_MESH_MAX_NEIGHBORS) {
args[0] = meshpeer->peer_llinkid;
args[1] = 0;
args[2] = IEEE80211_REASON_MESH_MAX_PEERS;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
/* stay in IDLE state */
return (0);
}
/* Open frame accepted */
mesh_linkchange(ni, IEEE80211_NODE_MESH_OPENRCV);
ni->ni_mllid = meshpeer->peer_llinkid;
ni->ni_mlpid = mesh_generateid(vap);
if (ni->ni_mlpid == 0)
return 0; /* XXX */
args[0] = ni->ni_mlpid;
/* Announce we're open too... */
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_OPEN, args);
/* ...and confirm the link. */
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CONFIRM,
args);
mesh_peer_timeout_setup(ni);
break;
case IEEE80211_NODE_MESH_OPENRCV:
/* Wrong Link ID */
if (ni->ni_mllid != meshpeer->peer_llinkid) {
args[0] = ni->ni_mllid;
args[1] = ni->ni_mlpid;
args[2] = IEEE80211_REASON_PEER_LINK_CANCELED;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING);
mesh_peer_timeout_setup(ni);
break;
}
/* Duplicate open, confirm again. */
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CONFIRM,
args);
break;
case IEEE80211_NODE_MESH_OPENSNT:
ni->ni_mllid = meshpeer->peer_llinkid;
mesh_linkchange(ni, IEEE80211_NODE_MESH_OPENRCV);
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CONFIRM,
args);
/* NB: don't setup/clear any timeout */
break;
case IEEE80211_NODE_MESH_CONFIRMRCV:
if (ni->ni_mlpid != meshpeer->peer_linkid ||
ni->ni_mllid != meshpeer->peer_llinkid) {
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
args[2] = IEEE80211_REASON_PEER_LINK_CANCELED;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
mesh_linkchange(ni,
IEEE80211_NODE_MESH_HOLDING);
mesh_peer_timeout_setup(ni);
break;
}
mesh_linkchange(ni, IEEE80211_NODE_MESH_ESTABLISHED);
ni->ni_mllid = meshpeer->peer_llinkid;
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CONFIRM,
args);
mesh_peer_timeout_stop(ni);
break;
case IEEE80211_NODE_MESH_ESTABLISHED:
if (ni->ni_mllid != meshpeer->peer_llinkid) {
args[0] = ni->ni_mllid;
args[1] = ni->ni_mlpid;
args[2] = IEEE80211_REASON_PEER_LINK_CANCELED;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING);
mesh_peer_timeout_setup(ni);
break;
}
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CONFIRM,
args);
break;
case IEEE80211_NODE_MESH_HOLDING:
args[0] = ni->ni_mlpid;
args[1] = meshpeer->peer_llinkid;
/* Standard not clear about what the reaason code should be */
args[2] = IEEE80211_REASON_PEER_LINK_CANCELED;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
break;
}
return 0;
}
static int
mesh_recv_action_meshpeering_confirm(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_meshpeer_ie ie;
const struct ieee80211_meshpeer_ie *meshpeer;
uint16_t args[3];
/* +2+2+2+2 for action + code + capabilites + status code + AID */
meshpeer = mesh_parse_meshpeering_action(ni, wh, frm+2+2+2+2, efrm, &ie,
IEEE80211_ACTION_MESHPEERING_CONFIRM);
if (meshpeer == NULL) {
return 0;
}
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni,
"recv PEER CONFIRM, local id 0x%x, peer id 0x%x",
meshpeer->peer_llinkid, meshpeer->peer_linkid);
switch (ni->ni_mlstate) {
case IEEE80211_NODE_MESH_OPENRCV:
mesh_linkchange(ni, IEEE80211_NODE_MESH_ESTABLISHED);
mesh_peer_timeout_stop(ni);
break;
case IEEE80211_NODE_MESH_OPENSNT:
mesh_linkchange(ni, IEEE80211_NODE_MESH_CONFIRMRCV);
mesh_peer_timeout_setup(ni);
break;
case IEEE80211_NODE_MESH_HOLDING:
args[0] = ni->ni_mlpid;
args[1] = meshpeer->peer_llinkid;
/* Standard not clear about what the reaason code should be */
args[2] = IEEE80211_REASON_PEER_LINK_CANCELED;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
break;
case IEEE80211_NODE_MESH_CONFIRMRCV:
if (ni->ni_mllid != meshpeer->peer_llinkid) {
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
args[2] = IEEE80211_REASON_PEER_LINK_CANCELED;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING);
mesh_peer_timeout_setup(ni);
}
break;
default:
IEEE80211_DISCARD(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH,
wh, NULL, "received confirm in invalid state %d",
ni->ni_mlstate);
vap->iv_stats.is_rx_mgtdiscard++;
break;
}
return 0;
}
static int
mesh_recv_action_meshpeering_close(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
struct ieee80211_meshpeer_ie ie;
const struct ieee80211_meshpeer_ie *meshpeer;
uint16_t args[3];
/* +2 for action + code */
meshpeer = mesh_parse_meshpeering_action(ni, wh, frm+2, efrm, &ie,
IEEE80211_ACTION_MESHPEERING_CLOSE);
if (meshpeer == NULL) {
return 0;
}
/*
* XXX: check reason code, for example we could receive
* IEEE80211_REASON_MESH_MAX_PEERS then we should not attempt
* to peer again.
*/
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH,
ni, "%s", "recv PEER CLOSE");
switch (ni->ni_mlstate) {
case IEEE80211_NODE_MESH_IDLE:
/* ignore */
break;
case IEEE80211_NODE_MESH_OPENRCV:
case IEEE80211_NODE_MESH_OPENSNT:
case IEEE80211_NODE_MESH_CONFIRMRCV:
case IEEE80211_NODE_MESH_ESTABLISHED:
args[0] = ni->ni_mlpid;
args[1] = ni->ni_mllid;
args[2] = IEEE80211_REASON_MESH_CLOSE_RCVD;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args);
mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING);
mesh_peer_timeout_setup(ni);
break;
case IEEE80211_NODE_MESH_HOLDING:
mesh_linkchange(ni, IEEE80211_NODE_MESH_IDLE);
mesh_peer_timeout_stop(ni);
break;
}
return 0;
}
/*
* Link Metric handling.
*/
static int
mesh_recv_action_meshlmetric(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
const struct ieee80211_meshlmetric_ie *ie =
(const struct ieee80211_meshlmetric_ie *)
(frm+2); /* action + code */
struct ieee80211_meshlmetric_ie lm_rep;
if (ie->lm_flags & IEEE80211_MESH_LMETRIC_FLAGS_REQ) {
lm_rep.lm_flags = 0;
lm_rep.lm_metric = mesh_airtime_calc(ni);
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_MESH,
IEEE80211_ACTION_MESH_LMETRIC,
&lm_rep);
}
/* XXX: else do nothing for now */
return 0;
}
/*
* Parse meshgate action ie's for GANN frames.
* Returns -1 if parsing fails, otherwise 0.
*/
static int
mesh_parse_meshgate_action(struct ieee80211_node *ni,
const struct ieee80211_frame *wh, /* XXX for VERIFY_LENGTH */
struct ieee80211_meshgann_ie *ie, const uint8_t *frm, const uint8_t *efrm)
{
struct ieee80211vap *vap = ni->ni_vap;
const struct ieee80211_meshgann_ie *gannie;
while (efrm - frm > 1) {
IEEE80211_VERIFY_LENGTH(efrm - frm, frm[1] + 2, return -1);
switch (*frm) {
case IEEE80211_ELEMID_MESHGANN:
gannie = (const struct ieee80211_meshgann_ie *) frm;
memset(ie, 0, sizeof(*ie));
ie->gann_ie = gannie->gann_ie;
ie->gann_len = gannie->gann_len;
ie->gann_flags = gannie->gann_flags;
ie->gann_hopcount = gannie->gann_hopcount;
ie->gann_ttl = gannie->gann_ttl;
IEEE80211_ADDR_COPY(ie->gann_addr, gannie->gann_addr);
ie->gann_seq = LE_READ_4(&gannie->gann_seq);
ie->gann_interval = LE_READ_2(&gannie->gann_interval);
break;
}
frm += frm[1] + 2;
}
return 0;
}
/*
* Mesh Gate Announcement handling.
*/
static int
mesh_recv_action_meshgate(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const uint8_t *frm, const uint8_t *efrm)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
struct ieee80211_mesh_gate_route *gr, *next;
struct ieee80211_mesh_route *rt_gate;
struct ieee80211_meshgann_ie pgann;
struct ieee80211_meshgann_ie ie;
int found = 0;
/* +2 for action + code */
if (mesh_parse_meshgate_action(ni, wh, &ie, frm+2, efrm) != 0) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_MESH,
ni->ni_macaddr, NULL, "%s",
"GANN parsing failed");
vap->iv_stats.is_rx_mgtdiscard++;
return (0);
}
if (IEEE80211_ADDR_EQ(vap->iv_myaddr, ie.gann_addr))
return 0;
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, ni->ni_macaddr,
"received GANN, meshgate: %6D (seq %u)", ie.gann_addr, ":",
ie.gann_seq);
if (ms == NULL)
return (0);
MESH_RT_LOCK(ms);
TAILQ_FOREACH_SAFE(gr, &ms->ms_known_gates, gr_next, next) {
if (!IEEE80211_ADDR_EQ(gr->gr_addr, ie.gann_addr))
continue;
if (ie.gann_seq <= gr->gr_lastseq) {
IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_MESH,
ni->ni_macaddr, NULL,
"GANN old seqno %u <= %u",
ie.gann_seq, gr->gr_lastseq);
MESH_RT_UNLOCK(ms);
return (0);
}
/* corresponding mesh gate found & GANN accepted */
found = 1;
break;
}
if (found == 0) {
/* this GANN is from a new mesh Gate add it to known table. */
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, ie.gann_addr,
"stored new GANN information, seq %u.", ie.gann_seq);
gr = malloc(ALIGN(sizeof(struct ieee80211_mesh_gate_route)),
M_80211_MESH_GT_RT, M_NOWAIT | M_ZERO);
IEEE80211_ADDR_COPY(gr->gr_addr, ie.gann_addr);
TAILQ_INSERT_TAIL(&ms->ms_known_gates, gr, gr_next);
}
gr->gr_lastseq = ie.gann_seq;
/* check if we have a path to this gate */
rt_gate = mesh_rt_find_locked(ms, gr->gr_addr);
if (rt_gate != NULL &&
rt_gate->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) {
gr->gr_route = rt_gate;
rt_gate->rt_flags |= IEEE80211_MESHRT_FLAGS_GATE;
}
MESH_RT_UNLOCK(ms);
/* popagate only if decremented ttl >= 1 && forwarding is enabled */
if ((ie.gann_ttl - 1) < 1 && !(ms->ms_flags & IEEE80211_MESHFLAGS_FWD))
return 0;
pgann.gann_flags = ie.gann_flags; /* Reserved */
pgann.gann_hopcount = ie.gann_hopcount + 1;
pgann.gann_ttl = ie.gann_ttl - 1;
IEEE80211_ADDR_COPY(pgann.gann_addr, ie.gann_addr);
pgann.gann_seq = ie.gann_seq;
pgann.gann_interval = ie.gann_interval;
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, ie.gann_addr,
"%s", "propagate GANN");
ieee80211_send_action(vap->iv_bss, IEEE80211_ACTION_CAT_MESH,
IEEE80211_ACTION_MESH_GANN, &pgann);
return 0;
}
static int
mesh_send_action(struct ieee80211_node *ni,
const uint8_t sa[IEEE80211_ADDR_LEN],
const uint8_t da[IEEE80211_ADDR_LEN],
struct mbuf *m)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211_bpf_params params;
struct ieee80211_frame *wh;
int ret;
KASSERT(ni != NULL, ("null node"));
if (vap->iv_state == IEEE80211_S_CAC) {
IEEE80211_NOTE(vap, IEEE80211_MSG_OUTPUT, ni,
"block %s frame in CAC state", "Mesh action");
vap->iv_stats.is_tx_badstate++;
ieee80211_free_node(ni);
m_freem(m);
return EIO; /* XXX */
}
M_PREPEND(m, sizeof(struct ieee80211_frame), M_NOWAIT);
if (m == NULL) {
ieee80211_free_node(ni);
return ENOMEM;
}
IEEE80211_TX_LOCK(ic);
wh = mtod(m, struct ieee80211_frame *);
ieee80211_send_setup(ni, m,
IEEE80211_FC0_TYPE_MGT | IEEE80211_FC0_SUBTYPE_ACTION,
IEEE80211_NONQOS_TID, sa, da, sa);
m->m_flags |= M_ENCAP; /* mark encapsulated */
memset(&params, 0, sizeof(params));
params.ibp_pri = WME_AC_VO;
params.ibp_rate0 = ni->ni_txparms->mgmtrate;
if (IEEE80211_IS_MULTICAST(da))
params.ibp_try0 = 1;
else
params.ibp_try0 = ni->ni_txparms->maxretry;
params.ibp_power = ni->ni_txpower;
IEEE80211_NODE_STAT(ni, tx_mgmt);
ret = ieee80211_raw_output(vap, ni, m, &params);
IEEE80211_TX_UNLOCK(ic);
return (ret);
}
#define ADDSHORT(frm, v) do { \
frm[0] = (v) & 0xff; \
frm[1] = (v) >> 8; \
frm += 2; \
} while (0)
#define ADDWORD(frm, v) do { \
frm[0] = (v) & 0xff; \
frm[1] = ((v) >> 8) & 0xff; \
frm[2] = ((v) >> 16) & 0xff; \
frm[3] = ((v) >> 24) & 0xff; \
frm += 4; \
} while (0)
static int
mesh_send_action_meshpeering_open(struct ieee80211_node *ni,
int category, int action, void *args0)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
uint16_t *args = args0;
const struct ieee80211_rateset *rs;
struct mbuf *m;
uint8_t *frm;
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni,
"send PEER OPEN action: localid 0x%x", args[0]);
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__,
ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1);
ieee80211_ref_node(ni);
m = ieee80211_getmgtframe(&frm,
ic->ic_headroom + sizeof(struct ieee80211_frame),
sizeof(uint16_t) /* action+category */
+ sizeof(uint16_t) /* capabilites */
+ 2 + IEEE80211_RATE_SIZE
+ 2 + (IEEE80211_RATE_MAXSIZE - IEEE80211_RATE_SIZE)
+ 2 + IEEE80211_MESHID_LEN
+ sizeof(struct ieee80211_meshconf_ie)
+ sizeof(struct ieee80211_meshpeer_ie)
);
if (m != NULL) {
/*
* mesh peer open action frame format:
* [1] category
* [1] action
* [2] capabilities
* [tlv] rates
* [tlv] xrates
* [tlv] mesh id
* [tlv] mesh conf
* [tlv] mesh peer link mgmt
*/
*frm++ = category;
*frm++ = action;
ADDSHORT(frm, ieee80211_getcapinfo(vap, ni->ni_chan));
rs = ieee80211_get_suprates(ic, ic->ic_curchan);
frm = ieee80211_add_rates(frm, rs);
frm = ieee80211_add_xrates(frm, rs);
frm = ieee80211_add_meshid(frm, vap);
frm = ieee80211_add_meshconf(frm, vap);
frm = ieee80211_add_meshpeer(frm, IEEE80211_ACTION_MESHPEERING_OPEN,
args[0], 0, 0);
m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
return mesh_send_action(ni, vap->iv_myaddr, ni->ni_macaddr, m);
} else {
vap->iv_stats.is_tx_nobuf++;
ieee80211_free_node(ni);
return ENOMEM;
}
}
static int
mesh_send_action_meshpeering_confirm(struct ieee80211_node *ni,
int category, int action, void *args0)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
uint16_t *args = args0;
const struct ieee80211_rateset *rs;
struct mbuf *m;
uint8_t *frm;
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni,
"send PEER CONFIRM action: localid 0x%x, peerid 0x%x",
args[0], args[1]);
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__,
ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1);
ieee80211_ref_node(ni);
m = ieee80211_getmgtframe(&frm,
ic->ic_headroom + sizeof(struct ieee80211_frame),
sizeof(uint16_t) /* action+category */
+ sizeof(uint16_t) /* capabilites */
+ sizeof(uint16_t) /* status code */
+ sizeof(uint16_t) /* AID */
+ 2 + IEEE80211_RATE_SIZE
+ 2 + (IEEE80211_RATE_MAXSIZE - IEEE80211_RATE_SIZE)
+ 2 + IEEE80211_MESHID_LEN
+ sizeof(struct ieee80211_meshconf_ie)
+ sizeof(struct ieee80211_meshpeer_ie)
);
if (m != NULL) {
/*
* mesh peer confirm action frame format:
* [1] category
* [1] action
* [2] capabilities
* [2] status code
* [2] association id (peer ID)
* [tlv] rates
* [tlv] xrates
* [tlv] mesh id
* [tlv] mesh conf
* [tlv] mesh peer link mgmt
*/
*frm++ = category;
*frm++ = action;
ADDSHORT(frm, ieee80211_getcapinfo(vap, ni->ni_chan));
ADDSHORT(frm, 0); /* status code */
ADDSHORT(frm, args[1]); /* AID */
rs = ieee80211_get_suprates(ic, ic->ic_curchan);
frm = ieee80211_add_rates(frm, rs);
frm = ieee80211_add_xrates(frm, rs);
frm = ieee80211_add_meshid(frm, vap);
frm = ieee80211_add_meshconf(frm, vap);
frm = ieee80211_add_meshpeer(frm,
IEEE80211_ACTION_MESHPEERING_CONFIRM,
args[0], args[1], 0);
m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
return mesh_send_action(ni, vap->iv_myaddr, ni->ni_macaddr, m);
} else {
vap->iv_stats.is_tx_nobuf++;
ieee80211_free_node(ni);
return ENOMEM;
}
}
static int
mesh_send_action_meshpeering_close(struct ieee80211_node *ni,
int category, int action, void *args0)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
uint16_t *args = args0;
struct mbuf *m;
uint8_t *frm;
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni,
"send PEER CLOSE action: localid 0x%x, peerid 0x%x reason %d",
args[0], args[1], args[2]);
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__,
ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1);
ieee80211_ref_node(ni);
m = ieee80211_getmgtframe(&frm,
ic->ic_headroom + sizeof(struct ieee80211_frame),
sizeof(uint16_t) /* action+category */
+ sizeof(uint16_t) /* reason code */
+ 2 + IEEE80211_MESHID_LEN
+ sizeof(struct ieee80211_meshpeer_ie)
);
if (m != NULL) {
/*
* mesh peer close action frame format:
* [1] category
* [1] action
* [tlv] mesh id
* [tlv] mesh peer link mgmt
*/
*frm++ = category;
*frm++ = action;
frm = ieee80211_add_meshid(frm, vap);
frm = ieee80211_add_meshpeer(frm,
IEEE80211_ACTION_MESHPEERING_CLOSE,
args[0], args[1], args[2]);
m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
return mesh_send_action(ni, vap->iv_myaddr, ni->ni_macaddr, m);
} else {
vap->iv_stats.is_tx_nobuf++;
ieee80211_free_node(ni);
return ENOMEM;
}
}
static int
mesh_send_action_meshlmetric(struct ieee80211_node *ni,
int category, int action, void *arg0)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211_meshlmetric_ie *ie = arg0;
struct mbuf *m;
uint8_t *frm;
if (ie->lm_flags & IEEE80211_MESH_LMETRIC_FLAGS_REQ) {
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH,
ni, "%s", "send LINK METRIC REQUEST action");
} else {
IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH,
ni, "send LINK METRIC REPLY action: metric 0x%x",
ie->lm_metric);
}
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__,
ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1);
ieee80211_ref_node(ni);
m = ieee80211_getmgtframe(&frm,
ic->ic_headroom + sizeof(struct ieee80211_frame),
sizeof(uint16_t) + /* action+category */
sizeof(struct ieee80211_meshlmetric_ie)
);
if (m != NULL) {
/*
* mesh link metric
* [1] category
* [1] action
* [tlv] mesh link metric
*/
*frm++ = category;
*frm++ = action;
frm = ieee80211_add_meshlmetric(frm,
ie->lm_flags, ie->lm_metric);
m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
return mesh_send_action(ni, vap->iv_myaddr, ni->ni_macaddr, m);
} else {
vap->iv_stats.is_tx_nobuf++;
ieee80211_free_node(ni);
return ENOMEM;
}
}
static int
mesh_send_action_meshgate(struct ieee80211_node *ni,
int category, int action, void *arg0)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211_meshgann_ie *ie = arg0;
struct mbuf *m;
uint8_t *frm;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__,
ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1);
ieee80211_ref_node(ni);
m = ieee80211_getmgtframe(&frm,
ic->ic_headroom + sizeof(struct ieee80211_frame),
sizeof(uint16_t) + /* action+category */
IEEE80211_MESHGANN_BASE_SZ
);
if (m != NULL) {
/*
* mesh link metric
* [1] category
* [1] action
* [tlv] mesh gate annoucement
*/
*frm++ = category;
*frm++ = action;
frm = ieee80211_add_meshgate(frm, ie);
m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
return mesh_send_action(ni, vap->iv_myaddr, broadcastaddr, m);
} else {
vap->iv_stats.is_tx_nobuf++;
ieee80211_free_node(ni);
return ENOMEM;
}
}
static void
mesh_peer_timeout_setup(struct ieee80211_node *ni)
{
switch (ni->ni_mlstate) {
case IEEE80211_NODE_MESH_HOLDING:
ni->ni_mltval = ieee80211_mesh_holdingtimeout;
break;
case IEEE80211_NODE_MESH_CONFIRMRCV:
ni->ni_mltval = ieee80211_mesh_confirmtimeout;
break;
case IEEE80211_NODE_MESH_IDLE:
ni->ni_mltval = 0;
break;
default:
ni->ni_mltval = ieee80211_mesh_retrytimeout;
break;
}
if (ni->ni_mltval)
callout_reset(&ni->ni_mltimer, ni->ni_mltval,
mesh_peer_timeout_cb, ni);
}
/*
* Same as above but backoffs timer statisically 50%.
*/
static void
mesh_peer_timeout_backoff(struct ieee80211_node *ni)
{
uint32_t r;
r = arc4random();
ni->ni_mltval += r % ni->ni_mltval;
callout_reset(&ni->ni_mltimer, ni->ni_mltval, mesh_peer_timeout_cb,
ni);
}
static __inline void
mesh_peer_timeout_stop(struct ieee80211_node *ni)
{
callout_drain(&ni->ni_mltimer);
}
static void
mesh_peer_backoff_cb(void *arg)
{
struct ieee80211_node *ni = (struct ieee80211_node *)arg;
/* After backoff timeout, try to peer automatically again. */
ni->ni_mlhcnt = 0;
}
/*
* Mesh Peer Link Management FSM timeout handling.
*/
static void
mesh_peer_timeout_cb(void *arg)
{
struct ieee80211_node *ni = (struct ieee80211_node *)arg;
uint16_t args[3];
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_MESH,
ni, "mesh link timeout, state %d, retry counter %d",
ni->ni_mlstate, ni->ni_mlrcnt);
switch (ni->ni_mlstate) {
case IEEE80211_NODE_MESH_IDLE:
case IEEE80211_NODE_MESH_ESTABLISHED:
break;
case IEEE80211_NODE_MESH_OPENSNT:
case IEEE80211_NODE_MESH_OPENRCV:
if (ni->ni_mlrcnt == ieee80211_mesh_maxretries) {
args[0] = ni->ni_mlpid;
args[2] = IEEE80211_REASON_MESH_MAX_RETRIES;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE, args);
ni->ni_mlrcnt = 0;
mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING);
mesh_peer_timeout_setup(ni);
} else {
args[0] = ni->ni_mlpid;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_OPEN, args);
ni->ni_mlrcnt++;
mesh_peer_timeout_backoff(ni);
}
break;
case IEEE80211_NODE_MESH_CONFIRMRCV:
args[0] = ni->ni_mlpid;
args[2] = IEEE80211_REASON_MESH_CONFIRM_TIMEOUT;
ieee80211_send_action(ni,
IEEE80211_ACTION_CAT_SELF_PROT,
IEEE80211_ACTION_MESHPEERING_CLOSE, args);
mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING);
mesh_peer_timeout_setup(ni);
break;
case IEEE80211_NODE_MESH_HOLDING:
ni->ni_mlhcnt++;
if (ni->ni_mlhcnt >= ieee80211_mesh_maxholding)
callout_reset(&ni->ni_mlhtimer,
ieee80211_mesh_backofftimeout,
mesh_peer_backoff_cb, ni);
mesh_linkchange(ni, IEEE80211_NODE_MESH_IDLE);
break;
}
}
static int
mesh_verify_meshid(struct ieee80211vap *vap, const uint8_t *ie)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
if (ie == NULL || ie[1] != ms->ms_idlen)
return 1;
return memcmp(ms->ms_id, ie + 2, ms->ms_idlen);
}
/*
* Check if we are using the same algorithms for this mesh.
*/
static int
mesh_verify_meshconf(struct ieee80211vap *vap, const uint8_t *ie)
{
const struct ieee80211_meshconf_ie *meshconf =
(const struct ieee80211_meshconf_ie *) ie;
const struct ieee80211_mesh_state *ms = vap->iv_mesh;
if (meshconf == NULL)
return 1;
if (meshconf->conf_pselid != ms->ms_ppath->mpp_ie) {
IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH,
"unknown path selection algorithm: 0x%x\n",
meshconf->conf_pselid);
return 1;
}
if (meshconf->conf_pmetid != ms->ms_pmetric->mpm_ie) {
IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH,
"unknown path metric algorithm: 0x%x\n",
meshconf->conf_pmetid);
return 1;
}
if (meshconf->conf_ccid != 0) {
IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH,
"unknown congestion control algorithm: 0x%x\n",
meshconf->conf_ccid);
return 1;
}
if (meshconf->conf_syncid != IEEE80211_MESHCONF_SYNC_NEIGHOFF) {
IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH,
"unknown sync algorithm: 0x%x\n",
meshconf->conf_syncid);
return 1;
}
if (meshconf->conf_authid != 0) {
IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH,
"unknown auth auth algorithm: 0x%x\n",
meshconf->conf_pselid);
return 1;
}
/* Not accepting peers */
if (!(meshconf->conf_cap & IEEE80211_MESHCONF_CAP_AP)) {
IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH,
"not accepting peers: 0x%x\n", meshconf->conf_cap);
return 1;
}
return 0;
}
static int
mesh_verify_meshpeer(struct ieee80211vap *vap, uint8_t subtype,
const uint8_t *ie)
{
const struct ieee80211_meshpeer_ie *meshpeer =
(const struct ieee80211_meshpeer_ie *) ie;
if (meshpeer == NULL ||
meshpeer->peer_len < IEEE80211_MPM_BASE_SZ ||
meshpeer->peer_len > IEEE80211_MPM_MAX_SZ)
return 1;
if (meshpeer->peer_proto != IEEE80211_MPPID_MPM) {
IEEE80211_DPRINTF(vap,
IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH,
"Only MPM protocol is supported (proto: 0x%02X)",
meshpeer->peer_proto);
return 1;
}
switch (subtype) {
case IEEE80211_ACTION_MESHPEERING_OPEN:
if (meshpeer->peer_len != IEEE80211_MPM_BASE_SZ)
return 1;
break;
case IEEE80211_ACTION_MESHPEERING_CONFIRM:
if (meshpeer->peer_len != IEEE80211_MPM_BASE_SZ + 2)
return 1;
break;
case IEEE80211_ACTION_MESHPEERING_CLOSE:
if (meshpeer->peer_len < IEEE80211_MPM_BASE_SZ + 2)
return 1;
if (meshpeer->peer_len == (IEEE80211_MPM_BASE_SZ + 2) &&
meshpeer->peer_linkid != 0)
return 1;
if (meshpeer->peer_rcode == 0)
return 1;
break;
}
return 0;
}
/*
* Add a Mesh ID IE to a frame.
*/
uint8_t *
ieee80211_add_meshid(uint8_t *frm, struct ieee80211vap *vap)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
KASSERT(vap->iv_opmode == IEEE80211_M_MBSS, ("not a mbss vap"));
*frm++ = IEEE80211_ELEMID_MESHID;
*frm++ = ms->ms_idlen;
memcpy(frm, ms->ms_id, ms->ms_idlen);
return frm + ms->ms_idlen;
}
/*
* Add a Mesh Configuration IE to a frame.
* For now just use HWMP routing, Airtime link metric, Null Congestion
* Signaling, Null Sync Protocol and Null Authentication.
*/
uint8_t *
ieee80211_add_meshconf(uint8_t *frm, struct ieee80211vap *vap)
{
const struct ieee80211_mesh_state *ms = vap->iv_mesh;
uint16_t caps;
KASSERT(vap->iv_opmode == IEEE80211_M_MBSS, ("not a MBSS vap"));
*frm++ = IEEE80211_ELEMID_MESHCONF;
*frm++ = IEEE80211_MESH_CONF_SZ;
*frm++ = ms->ms_ppath->mpp_ie; /* path selection */
*frm++ = ms->ms_pmetric->mpm_ie; /* link metric */
*frm++ = IEEE80211_MESHCONF_CC_DISABLED;
*frm++ = IEEE80211_MESHCONF_SYNC_NEIGHOFF;
*frm++ = IEEE80211_MESHCONF_AUTH_DISABLED;
/* NB: set the number of neighbors before the rest */
*frm = (ms->ms_neighbors > IEEE80211_MESH_MAX_NEIGHBORS ?
IEEE80211_MESH_MAX_NEIGHBORS : ms->ms_neighbors) << 1;
if (ms->ms_flags & IEEE80211_MESHFLAGS_GATE)
*frm |= IEEE80211_MESHCONF_FORM_GATE;
frm += 1;
caps = 0;
if (ms->ms_flags & IEEE80211_MESHFLAGS_AP)
caps |= IEEE80211_MESHCONF_CAP_AP;
if (ms->ms_flags & IEEE80211_MESHFLAGS_FWD)
caps |= IEEE80211_MESHCONF_CAP_FWRD;
*frm++ = caps;
return frm;
}
/*
* Add a Mesh Peer Management IE to a frame.
*/
uint8_t *
ieee80211_add_meshpeer(uint8_t *frm, uint8_t subtype, uint16_t localid,
uint16_t peerid, uint16_t reason)
{
KASSERT(localid != 0, ("localid == 0"));
*frm++ = IEEE80211_ELEMID_MESHPEER;
switch (subtype) {
case IEEE80211_ACTION_MESHPEERING_OPEN:
*frm++ = IEEE80211_MPM_BASE_SZ; /* length */
ADDSHORT(frm, IEEE80211_MPPID_MPM); /* proto */
ADDSHORT(frm, localid); /* local ID */
break;
case IEEE80211_ACTION_MESHPEERING_CONFIRM:
KASSERT(peerid != 0, ("sending peer confirm without peer id"));
*frm++ = IEEE80211_MPM_BASE_SZ + 2; /* length */
ADDSHORT(frm, IEEE80211_MPPID_MPM); /* proto */
ADDSHORT(frm, localid); /* local ID */
ADDSHORT(frm, peerid); /* peer ID */
break;
case IEEE80211_ACTION_MESHPEERING_CLOSE:
if (peerid)
*frm++ = IEEE80211_MPM_MAX_SZ; /* length */
else
*frm++ = IEEE80211_MPM_BASE_SZ + 2; /* length */
ADDSHORT(frm, IEEE80211_MPPID_MPM); /* proto */
ADDSHORT(frm, localid); /* local ID */
if (peerid)
ADDSHORT(frm, peerid); /* peer ID */
ADDSHORT(frm, reason);
break;
}
return frm;
}
/*
* Compute an Airtime Link Metric for the link with this node.
*
* Based on Draft 3.0 spec (11B.10, p.149).
*/
/*
* Max 802.11s overhead.
*/
#define IEEE80211_MESH_MAXOVERHEAD \
(sizeof(struct ieee80211_qosframe_addr4) \
+ sizeof(struct ieee80211_meshcntl_ae10) \
+ sizeof(struct llc) \
+ IEEE80211_ADDR_LEN \
+ IEEE80211_WEP_IVLEN \
+ IEEE80211_WEP_KIDLEN \
+ IEEE80211_WEP_CRCLEN \
+ IEEE80211_WEP_MICLEN \
+ IEEE80211_CRC_LEN)
uint32_t
mesh_airtime_calc(struct ieee80211_node *ni)
{
#define M_BITS 8
#define S_FACTOR (2 * M_BITS)
struct ieee80211com *ic = ni->ni_ic;
struct ifnet *ifp = ni->ni_vap->iv_ifp;
const static int nbits = 8192 << M_BITS;
uint32_t overhead, rate, errrate;
uint64_t res;
/* Time to transmit a frame */
rate = ni->ni_txrate;
overhead = ieee80211_compute_duration(ic->ic_rt,
ifp->if_mtu + IEEE80211_MESH_MAXOVERHEAD, rate, 0) << M_BITS;
/* Error rate in percentage */
/* XXX assuming small failures are ok */
errrate = (((ifp->if_get_counter(ifp, IFCOUNTER_OERRORS) +
ifp->if_get_counter(ifp, IFCOUNTER_IERRORS)) / 100) << M_BITS)
/ 100;
res = (overhead + (nbits / rate)) *
((1 << S_FACTOR) / ((1 << M_BITS) - errrate));
return (uint32_t)(res >> S_FACTOR);
#undef M_BITS
#undef S_FACTOR
}
/*
* Add a Mesh Link Metric report IE to a frame.
*/
uint8_t *
ieee80211_add_meshlmetric(uint8_t *frm, uint8_t flags, uint32_t metric)
{
*frm++ = IEEE80211_ELEMID_MESHLINK;
*frm++ = 5;
*frm++ = flags;
ADDWORD(frm, metric);
return frm;
}
/*
* Add a Mesh Gate Announcement IE to a frame.
*/
uint8_t *
ieee80211_add_meshgate(uint8_t *frm, struct ieee80211_meshgann_ie *ie)
{
*frm++ = IEEE80211_ELEMID_MESHGANN; /* ie */
*frm++ = IEEE80211_MESHGANN_BASE_SZ; /* len */
*frm++ = ie->gann_flags;
*frm++ = ie->gann_hopcount;
*frm++ = ie->gann_ttl;
IEEE80211_ADDR_COPY(frm, ie->gann_addr);
frm += 6;
ADDWORD(frm, ie->gann_seq);
ADDSHORT(frm, ie->gann_interval);
return frm;
}
#undef ADDSHORT
#undef ADDWORD
/*
* Initialize any mesh-specific node state.
*/
void
ieee80211_mesh_node_init(struct ieee80211vap *vap, struct ieee80211_node *ni)
{
ni->ni_flags |= IEEE80211_NODE_QOS;
- callout_init(&ni->ni_mltimer, CALLOUT_MPSAFE);
- callout_init(&ni->ni_mlhtimer, CALLOUT_MPSAFE);
+ callout_init(&ni->ni_mltimer, 1);
+ callout_init(&ni->ni_mlhtimer, 1);
}
/*
* Cleanup any mesh-specific node state.
*/
void
ieee80211_mesh_node_cleanup(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_mesh_state *ms = vap->iv_mesh;
callout_drain(&ni->ni_mltimer);
callout_drain(&ni->ni_mlhtimer);
/* NB: short-circuit callbacks after mesh_vdetach */
if (vap->iv_mesh != NULL)
ms->ms_ppath->mpp_peerdown(ni);
}
void
ieee80211_parse_meshid(struct ieee80211_node *ni, const uint8_t *ie)
{
ni->ni_meshidlen = ie[1];
memcpy(ni->ni_meshid, ie + 2, ie[1]);
}
/*
* Setup mesh-specific node state on neighbor discovery.
*/
void
ieee80211_mesh_init_neighbor(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const struct ieee80211_scanparams *sp)
{
ieee80211_parse_meshid(ni, sp->meshid);
}
void
ieee80211_mesh_update_beacon(struct ieee80211vap *vap,
struct ieee80211_beacon_offsets *bo)
{
KASSERT(vap->iv_opmode == IEEE80211_M_MBSS, ("not a MBSS vap"));
if (isset(bo->bo_flags, IEEE80211_BEACON_MESHCONF)) {
(void)ieee80211_add_meshconf(bo->bo_meshconf, vap);
clrbit(bo->bo_flags, IEEE80211_BEACON_MESHCONF);
}
}
static int
mesh_ioctl_get80211(struct ieee80211vap *vap, struct ieee80211req *ireq)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
uint8_t tmpmeshid[IEEE80211_NWID_LEN];
struct ieee80211_mesh_route *rt;
struct ieee80211req_mesh_route *imr;
size_t len, off;
uint8_t *p;
int error;
if (vap->iv_opmode != IEEE80211_M_MBSS)
return ENOSYS;
error = 0;
switch (ireq->i_type) {
case IEEE80211_IOC_MESH_ID:
ireq->i_len = ms->ms_idlen;
memcpy(tmpmeshid, ms->ms_id, ireq->i_len);
error = copyout(tmpmeshid, ireq->i_data, ireq->i_len);
break;
case IEEE80211_IOC_MESH_AP:
ireq->i_val = (ms->ms_flags & IEEE80211_MESHFLAGS_AP) != 0;
break;
case IEEE80211_IOC_MESH_FWRD:
ireq->i_val = (ms->ms_flags & IEEE80211_MESHFLAGS_FWD) != 0;
break;
case IEEE80211_IOC_MESH_GATE:
ireq->i_val = (ms->ms_flags & IEEE80211_MESHFLAGS_GATE) != 0;
break;
case IEEE80211_IOC_MESH_TTL:
ireq->i_val = ms->ms_ttl;
break;
case IEEE80211_IOC_MESH_RTCMD:
switch (ireq->i_val) {
case IEEE80211_MESH_RTCMD_LIST:
len = 0;
MESH_RT_LOCK(ms);
TAILQ_FOREACH(rt, &ms->ms_routes, rt_next) {
len += sizeof(*imr);
}
MESH_RT_UNLOCK(ms);
if (len > ireq->i_len || ireq->i_len < sizeof(*imr)) {
ireq->i_len = len;
return ENOMEM;
}
ireq->i_len = len;
/* XXX M_WAIT? */
p = malloc(len, M_TEMP, M_NOWAIT | M_ZERO);
if (p == NULL)
return ENOMEM;
off = 0;
MESH_RT_LOCK(ms);
TAILQ_FOREACH(rt, &ms->ms_routes, rt_next) {
if (off >= len)
break;
imr = (struct ieee80211req_mesh_route *)
(p + off);
IEEE80211_ADDR_COPY(imr->imr_dest,
rt->rt_dest);
IEEE80211_ADDR_COPY(imr->imr_nexthop,
rt->rt_nexthop);
imr->imr_metric = rt->rt_metric;
imr->imr_nhops = rt->rt_nhops;
imr->imr_lifetime =
ieee80211_mesh_rt_update(rt, 0);
imr->imr_lastmseq = rt->rt_lastmseq;
imr->imr_flags = rt->rt_flags; /* last */
off += sizeof(*imr);
}
MESH_RT_UNLOCK(ms);
error = copyout(p, (uint8_t *)ireq->i_data,
ireq->i_len);
free(p, M_TEMP);
break;
case IEEE80211_MESH_RTCMD_FLUSH:
case IEEE80211_MESH_RTCMD_ADD:
case IEEE80211_MESH_RTCMD_DELETE:
return EINVAL;
default:
return ENOSYS;
}
break;
case IEEE80211_IOC_MESH_PR_METRIC:
len = strlen(ms->ms_pmetric->mpm_descr);
if (ireq->i_len < len)
return EINVAL;
ireq->i_len = len;
error = copyout(ms->ms_pmetric->mpm_descr,
(uint8_t *)ireq->i_data, len);
break;
case IEEE80211_IOC_MESH_PR_PATH:
len = strlen(ms->ms_ppath->mpp_descr);
if (ireq->i_len < len)
return EINVAL;
ireq->i_len = len;
error = copyout(ms->ms_ppath->mpp_descr,
(uint8_t *)ireq->i_data, len);
break;
default:
return ENOSYS;
}
return error;
}
IEEE80211_IOCTL_GET(mesh, mesh_ioctl_get80211);
static int
mesh_ioctl_set80211(struct ieee80211vap *vap, struct ieee80211req *ireq)
{
struct ieee80211_mesh_state *ms = vap->iv_mesh;
uint8_t tmpmeshid[IEEE80211_NWID_LEN];
uint8_t tmpaddr[IEEE80211_ADDR_LEN];
char tmpproto[IEEE80211_MESH_PROTO_DSZ];
int error;
if (vap->iv_opmode != IEEE80211_M_MBSS)
return ENOSYS;
error = 0;
switch (ireq->i_type) {
case IEEE80211_IOC_MESH_ID:
if (ireq->i_val != 0 || ireq->i_len > IEEE80211_MESHID_LEN)
return EINVAL;
error = copyin(ireq->i_data, tmpmeshid, ireq->i_len);
if (error != 0)
break;
memset(ms->ms_id, 0, IEEE80211_NWID_LEN);
ms->ms_idlen = ireq->i_len;
memcpy(ms->ms_id, tmpmeshid, ireq->i_len);
error = ENETRESET;
break;
case IEEE80211_IOC_MESH_AP:
if (ireq->i_val)
ms->ms_flags |= IEEE80211_MESHFLAGS_AP;
else
ms->ms_flags &= ~IEEE80211_MESHFLAGS_AP;
error = ENETRESET;
break;
case IEEE80211_IOC_MESH_FWRD:
if (ireq->i_val)
ms->ms_flags |= IEEE80211_MESHFLAGS_FWD;
else
ms->ms_flags &= ~IEEE80211_MESHFLAGS_FWD;
mesh_gatemode_setup(vap);
break;
case IEEE80211_IOC_MESH_GATE:
if (ireq->i_val)
ms->ms_flags |= IEEE80211_MESHFLAGS_GATE;
else
ms->ms_flags &= ~IEEE80211_MESHFLAGS_GATE;
break;
case IEEE80211_IOC_MESH_TTL:
ms->ms_ttl = (uint8_t) ireq->i_val;
break;
case IEEE80211_IOC_MESH_RTCMD:
switch (ireq->i_val) {
case IEEE80211_MESH_RTCMD_LIST:
return EINVAL;
case IEEE80211_MESH_RTCMD_FLUSH:
ieee80211_mesh_rt_flush(vap);
break;
case IEEE80211_MESH_RTCMD_ADD:
if (IEEE80211_ADDR_EQ(vap->iv_myaddr, ireq->i_data) ||
IEEE80211_ADDR_EQ(broadcastaddr, ireq->i_data))
return EINVAL;
error = copyin(ireq->i_data, &tmpaddr,
IEEE80211_ADDR_LEN);
if (error == 0)
ieee80211_mesh_discover(vap, tmpaddr, NULL);
break;
case IEEE80211_MESH_RTCMD_DELETE:
ieee80211_mesh_rt_del(vap, ireq->i_data);
break;
default:
return ENOSYS;
}
break;
case IEEE80211_IOC_MESH_PR_METRIC:
error = copyin(ireq->i_data, tmpproto, sizeof(tmpproto));
if (error == 0) {
error = mesh_select_proto_metric(vap, tmpproto);
if (error == 0)
error = ENETRESET;
}
break;
case IEEE80211_IOC_MESH_PR_PATH:
error = copyin(ireq->i_data, tmpproto, sizeof(tmpproto));
if (error == 0) {
error = mesh_select_proto_path(vap, tmpproto);
if (error == 0)
error = ENETRESET;
}
break;
default:
return ENOSYS;
}
return error;
}
IEEE80211_IOCTL_SET(mesh, mesh_ioctl_set80211);
Index: head/sys/net80211/ieee80211_node.c
===================================================================
--- head/sys/net80211/ieee80211_node.c (revision 283290)
+++ head/sys/net80211/ieee80211_node.c (revision 283291)
@@ -1,2827 +1,2827 @@
/*-
* Copyright (c) 2001 Atsushi Onoe
* Copyright (c) 2002-2009 Sam Leffler, Errno Consulting
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_wlan.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_media.h>
#include <net/ethernet.h>
#include <net80211/ieee80211_var.h>
#include <net80211/ieee80211_input.h>
#ifdef IEEE80211_SUPPORT_SUPERG
#include <net80211/ieee80211_superg.h>
#endif
#ifdef IEEE80211_SUPPORT_TDMA
#include <net80211/ieee80211_tdma.h>
#endif
#include <net80211/ieee80211_wds.h>
#include <net80211/ieee80211_mesh.h>
#include <net80211/ieee80211_ratectl.h>
#include <net/bpf.h>
/*
* IEEE80211_NODE_HASHSIZE must be a power of 2.
*/
CTASSERT((IEEE80211_NODE_HASHSIZE & (IEEE80211_NODE_HASHSIZE-1)) == 0);
/*
* Association id's are managed with a bit vector.
*/
#define IEEE80211_AID_SET(_vap, b) \
((_vap)->iv_aid_bitmap[IEEE80211_AID(b) / 32] |= \
(1 << (IEEE80211_AID(b) % 32)))
#define IEEE80211_AID_CLR(_vap, b) \
((_vap)->iv_aid_bitmap[IEEE80211_AID(b) / 32] &= \
~(1 << (IEEE80211_AID(b) % 32)))
#define IEEE80211_AID_ISSET(_vap, b) \
((_vap)->iv_aid_bitmap[IEEE80211_AID(b) / 32] & (1 << (IEEE80211_AID(b) % 32)))
#ifdef IEEE80211_DEBUG_REFCNT
#define REFCNT_LOC "%s (%s:%u) %p<%s> refcnt %d\n", __func__, func, line
#else
#define REFCNT_LOC "%s %p<%s> refcnt %d\n", __func__
#endif
static int ieee80211_sta_join1(struct ieee80211_node *);
static struct ieee80211_node *node_alloc(struct ieee80211vap *,
const uint8_t [IEEE80211_ADDR_LEN]);
static void node_cleanup(struct ieee80211_node *);
static void node_free(struct ieee80211_node *);
static void node_age(struct ieee80211_node *);
static int8_t node_getrssi(const struct ieee80211_node *);
static void node_getsignal(const struct ieee80211_node *, int8_t *, int8_t *);
static void node_getmimoinfo(const struct ieee80211_node *,
struct ieee80211_mimo_info *);
static void _ieee80211_free_node(struct ieee80211_node *);
static void node_reclaim(struct ieee80211_node_table *nt,
struct ieee80211_node *ni);
static void ieee80211_node_table_init(struct ieee80211com *ic,
struct ieee80211_node_table *nt, const char *name,
int inact, int keymaxix);
static void ieee80211_node_table_reset(struct ieee80211_node_table *,
struct ieee80211vap *);
static void ieee80211_node_table_cleanup(struct ieee80211_node_table *nt);
static void ieee80211_erp_timeout(struct ieee80211com *);
MALLOC_DEFINE(M_80211_NODE, "80211node", "802.11 node state");
MALLOC_DEFINE(M_80211_NODE_IE, "80211nodeie", "802.11 node ie");
void
ieee80211_node_attach(struct ieee80211com *ic)
{
/* XXX really want maxlen enforced per-sta */
ieee80211_ageq_init(&ic->ic_stageq, ic->ic_max_keyix * 8,
"802.11 staging q");
ieee80211_node_table_init(ic, &ic->ic_sta, "station",
IEEE80211_INACT_INIT, ic->ic_max_keyix);
- callout_init(&ic->ic_inact, CALLOUT_MPSAFE);
+ callout_init(&ic->ic_inact, 1);
callout_reset(&ic->ic_inact, IEEE80211_INACT_WAIT*hz,
ieee80211_node_timeout, ic);
ic->ic_node_alloc = node_alloc;
ic->ic_node_free = node_free;
ic->ic_node_cleanup = node_cleanup;
ic->ic_node_age = node_age;
ic->ic_node_drain = node_age; /* NB: same as age */
ic->ic_node_getrssi = node_getrssi;
ic->ic_node_getsignal = node_getsignal;
ic->ic_node_getmimoinfo = node_getmimoinfo;
/*
* Set flags to be propagated to all vap's;
* these define default behaviour/configuration.
*/
ic->ic_flags_ext |= IEEE80211_FEXT_INACT; /* inactivity processing */
}
void
ieee80211_node_detach(struct ieee80211com *ic)
{
callout_drain(&ic->ic_inact);
ieee80211_node_table_cleanup(&ic->ic_sta);
ieee80211_ageq_cleanup(&ic->ic_stageq);
}
void
ieee80211_node_vattach(struct ieee80211vap *vap)
{
/* NB: driver can override */
vap->iv_max_aid = IEEE80211_AID_DEF;
/* default station inactivity timer setings */
vap->iv_inact_init = IEEE80211_INACT_INIT;
vap->iv_inact_auth = IEEE80211_INACT_AUTH;
vap->iv_inact_run = IEEE80211_INACT_RUN;
vap->iv_inact_probe = IEEE80211_INACT_PROBE;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_INACT,
"%s: init %u auth %u run %u probe %u\n", __func__,
vap->iv_inact_init, vap->iv_inact_auth,
vap->iv_inact_run, vap->iv_inact_probe);
}
void
ieee80211_node_latevattach(struct ieee80211vap *vap)
{
if (vap->iv_opmode == IEEE80211_M_HOSTAP) {
/* XXX should we allow max aid to be zero? */
if (vap->iv_max_aid < IEEE80211_AID_MIN) {
vap->iv_max_aid = IEEE80211_AID_MIN;
if_printf(vap->iv_ifp,
"WARNING: max aid too small, changed to %d\n",
vap->iv_max_aid);
}
vap->iv_aid_bitmap = (uint32_t *) malloc(
howmany(vap->iv_max_aid, 32) * sizeof(uint32_t),
M_80211_NODE, M_NOWAIT | M_ZERO);
if (vap->iv_aid_bitmap == NULL) {
/* XXX no way to recover */
printf("%s: no memory for AID bitmap, max aid %d!\n",
__func__, vap->iv_max_aid);
vap->iv_max_aid = 0;
}
}
ieee80211_reset_bss(vap);
vap->iv_auth = ieee80211_authenticator_get(vap->iv_bss->ni_authmode);
}
void
ieee80211_node_vdetach(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
ieee80211_node_table_reset(&ic->ic_sta, vap);
if (vap->iv_bss != NULL) {
ieee80211_free_node(vap->iv_bss);
vap->iv_bss = NULL;
}
if (vap->iv_aid_bitmap != NULL) {
free(vap->iv_aid_bitmap, M_80211_NODE);
vap->iv_aid_bitmap = NULL;
}
}
/*
* Port authorize/unauthorize interfaces for use by an authenticator.
*/
void
ieee80211_node_authorize(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
ni->ni_flags |= IEEE80211_NODE_AUTH;
ni->ni_inact_reload = vap->iv_inact_run;
ni->ni_inact = ni->ni_inact_reload;
IEEE80211_NOTE(vap, IEEE80211_MSG_INACT, ni,
"%s: inact_reload %u", __func__, ni->ni_inact_reload);
}
void
ieee80211_node_unauthorize(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
ni->ni_flags &= ~IEEE80211_NODE_AUTH;
ni->ni_inact_reload = vap->iv_inact_auth;
if (ni->ni_inact > ni->ni_inact_reload)
ni->ni_inact = ni->ni_inact_reload;
IEEE80211_NOTE(vap, IEEE80211_MSG_INACT, ni,
"%s: inact_reload %u inact %u", __func__,
ni->ni_inact_reload, ni->ni_inact);
}
/*
* Fix tx parameters for a node according to ``association state''.
*/
void
ieee80211_node_setuptxparms(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
enum ieee80211_phymode mode;
if (ni->ni_flags & IEEE80211_NODE_HT) {
if (IEEE80211_IS_CHAN_5GHZ(ni->ni_chan))
mode = IEEE80211_MODE_11NA;
else
mode = IEEE80211_MODE_11NG;
} else { /* legacy rate handling */
if (IEEE80211_IS_CHAN_ST(ni->ni_chan))
mode = IEEE80211_MODE_STURBO_A;
else if (IEEE80211_IS_CHAN_HALF(ni->ni_chan))
mode = IEEE80211_MODE_HALF;
else if (IEEE80211_IS_CHAN_QUARTER(ni->ni_chan))
mode = IEEE80211_MODE_QUARTER;
/* NB: 108A should be handled as 11a */
else if (IEEE80211_IS_CHAN_A(ni->ni_chan))
mode = IEEE80211_MODE_11A;
else if (IEEE80211_IS_CHAN_108G(ni->ni_chan) ||
(ni->ni_flags & IEEE80211_NODE_ERP))
mode = IEEE80211_MODE_11G;
else
mode = IEEE80211_MODE_11B;
}
ni->ni_txparms = &vap->iv_txparms[mode];
}
/*
* Set/change the channel. The rate set is also updated as
* to insure a consistent view by drivers.
* XXX should be private but hostap needs it to deal with CSA
*/
void
ieee80211_node_set_chan(struct ieee80211_node *ni,
struct ieee80211_channel *chan)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211vap *vap = ni->ni_vap;
enum ieee80211_phymode mode;
KASSERT(chan != IEEE80211_CHAN_ANYC, ("no channel"));
ni->ni_chan = chan;
mode = ieee80211_chan2mode(chan);
if (IEEE80211_IS_CHAN_HT(chan)) {
/*
* We must install the legacy rate est in ni_rates and the
* HT rate set in ni_htrates.
*/
ni->ni_htrates = *ieee80211_get_suphtrates(ic, chan);
/*
* Setup bss tx parameters based on operating mode. We
* use legacy rates when operating in a mixed HT+non-HT bss
* and non-ERP rates in 11g for mixed ERP+non-ERP bss.
*/
if (mode == IEEE80211_MODE_11NA &&
(vap->iv_flags_ht & IEEE80211_FHT_PUREN) == 0)
mode = IEEE80211_MODE_11A;
else if (mode == IEEE80211_MODE_11NG &&
(vap->iv_flags_ht & IEEE80211_FHT_PUREN) == 0)
mode = IEEE80211_MODE_11G;
if (mode == IEEE80211_MODE_11G &&
(vap->iv_flags & IEEE80211_F_PUREG) == 0)
mode = IEEE80211_MODE_11B;
}
ni->ni_txparms = &vap->iv_txparms[mode];
ni->ni_rates = *ieee80211_get_suprates(ic, chan);
}
static __inline void
copy_bss(struct ieee80211_node *nbss, const struct ieee80211_node *obss)
{
/* propagate useful state */
nbss->ni_authmode = obss->ni_authmode;
nbss->ni_txpower = obss->ni_txpower;
nbss->ni_vlan = obss->ni_vlan;
/* XXX statistics? */
/* XXX legacy WDS bssid? */
}
void
ieee80211_create_ibss(struct ieee80211vap* vap, struct ieee80211_channel *chan)
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_node *ni;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_SCAN,
"%s: creating %s on channel %u\n", __func__,
ieee80211_opmode_name[vap->iv_opmode],
ieee80211_chan2ieee(ic, chan));
ni = ieee80211_alloc_node(&ic->ic_sta, vap, vap->iv_myaddr);
if (ni == NULL) {
/* XXX recovery? */
return;
}
IEEE80211_ADDR_COPY(ni->ni_bssid, vap->iv_myaddr);
ni->ni_esslen = vap->iv_des_ssid[0].len;
memcpy(ni->ni_essid, vap->iv_des_ssid[0].ssid, ni->ni_esslen);
if (vap->iv_bss != NULL)
copy_bss(ni, vap->iv_bss);
ni->ni_intval = ic->ic_bintval;
if (vap->iv_flags & IEEE80211_F_PRIVACY)
ni->ni_capinfo |= IEEE80211_CAPINFO_PRIVACY;
if (ic->ic_phytype == IEEE80211_T_FH) {
ni->ni_fhdwell = 200; /* XXX */
ni->ni_fhindex = 1;
}
if (vap->iv_opmode == IEEE80211_M_IBSS) {
vap->iv_flags |= IEEE80211_F_SIBSS;
ni->ni_capinfo |= IEEE80211_CAPINFO_IBSS; /* XXX */
if (vap->iv_flags & IEEE80211_F_DESBSSID)
IEEE80211_ADDR_COPY(ni->ni_bssid, vap->iv_des_bssid);
else {
get_random_bytes(ni->ni_bssid, IEEE80211_ADDR_LEN);
/* clear group bit, add local bit */
ni->ni_bssid[0] = (ni->ni_bssid[0] &~ 0x01) | 0x02;
}
} else if (vap->iv_opmode == IEEE80211_M_AHDEMO) {
if (vap->iv_flags & IEEE80211_F_DESBSSID)
IEEE80211_ADDR_COPY(ni->ni_bssid, vap->iv_des_bssid);
else
#ifdef IEEE80211_SUPPORT_TDMA
if ((vap->iv_caps & IEEE80211_C_TDMA) == 0)
#endif
memset(ni->ni_bssid, 0, IEEE80211_ADDR_LEN);
#ifdef IEEE80211_SUPPORT_MESH
} else if (vap->iv_opmode == IEEE80211_M_MBSS) {
ni->ni_meshidlen = vap->iv_mesh->ms_idlen;
memcpy(ni->ni_meshid, vap->iv_mesh->ms_id, ni->ni_meshidlen);
#endif
}
/*
* Fix the channel and related attributes.
*/
/* clear DFS CAC state on previous channel */
if (ic->ic_bsschan != IEEE80211_CHAN_ANYC &&
ic->ic_bsschan->ic_freq != chan->ic_freq &&
IEEE80211_IS_CHAN_CACDONE(ic->ic_bsschan))
ieee80211_dfs_cac_clear(ic, ic->ic_bsschan);
ic->ic_bsschan = chan;
ieee80211_node_set_chan(ni, chan);
ic->ic_curmode = ieee80211_chan2mode(chan);
/*
* Do mode-specific setup.
*/
if (IEEE80211_IS_CHAN_FULL(chan)) {
if (IEEE80211_IS_CHAN_ANYG(chan)) {
/*
* Use a mixed 11b/11g basic rate set.
*/
ieee80211_setbasicrates(&ni->ni_rates,
IEEE80211_MODE_11G);
if (vap->iv_flags & IEEE80211_F_PUREG) {
/*
* Also mark OFDM rates basic so 11b
* stations do not join (WiFi compliance).
*/
ieee80211_addbasicrates(&ni->ni_rates,
IEEE80211_MODE_11A);
}
} else if (IEEE80211_IS_CHAN_B(chan)) {
/*
* Force pure 11b rate set.
*/
ieee80211_setbasicrates(&ni->ni_rates,
IEEE80211_MODE_11B);
}
}
(void) ieee80211_sta_join1(ieee80211_ref_node(ni));
}
/*
* Reset bss state on transition to the INIT state.
* Clear any stations from the table (they have been
* deauth'd) and reset the bss node (clears key, rate
* etc. state).
*/
void
ieee80211_reset_bss(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_node *ni, *obss;
ieee80211_node_table_reset(&ic->ic_sta, vap);
/* XXX multi-bss: wrong */
ieee80211_reset_erp(ic);
ni = ieee80211_alloc_node(&ic->ic_sta, vap, vap->iv_myaddr);
KASSERT(ni != NULL, ("unable to setup initial BSS node"));
obss = vap->iv_bss;
vap->iv_bss = ieee80211_ref_node(ni);
if (obss != NULL) {
copy_bss(ni, obss);
ni->ni_intval = ic->ic_bintval;
ieee80211_free_node(obss);
} else
IEEE80211_ADDR_COPY(ni->ni_bssid, vap->iv_myaddr);
}
static int
match_ssid(const struct ieee80211_node *ni,
int nssid, const struct ieee80211_scan_ssid ssids[])
{
int i;
for (i = 0; i < nssid; i++) {
if (ni->ni_esslen == ssids[i].len &&
memcmp(ni->ni_essid, ssids[i].ssid, ni->ni_esslen) == 0)
return 1;
}
return 0;
}
/*
* Test a node for suitability/compatibility.
*/
static int
check_bss(struct ieee80211vap *vap, struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
uint8_t rate;
if (isclr(ic->ic_chan_active, ieee80211_chan2ieee(ic, ni->ni_chan)))
return 0;
if (vap->iv_opmode == IEEE80211_M_IBSS) {
if ((ni->ni_capinfo & IEEE80211_CAPINFO_IBSS) == 0)
return 0;
} else {
if ((ni->ni_capinfo & IEEE80211_CAPINFO_ESS) == 0)
return 0;
}
if (vap->iv_flags & IEEE80211_F_PRIVACY) {
if ((ni->ni_capinfo & IEEE80211_CAPINFO_PRIVACY) == 0)
return 0;
} else {
/* XXX does this mean privacy is supported or required? */
if (ni->ni_capinfo & IEEE80211_CAPINFO_PRIVACY)
return 0;
}
rate = ieee80211_fix_rate(ni, &ni->ni_rates,
IEEE80211_F_JOIN | IEEE80211_F_DONEGO | IEEE80211_F_DOFRATE);
if (rate & IEEE80211_RATE_BASIC)
return 0;
if (vap->iv_des_nssid != 0 &&
!match_ssid(ni, vap->iv_des_nssid, vap->iv_des_ssid))
return 0;
if ((vap->iv_flags & IEEE80211_F_DESBSSID) &&
!IEEE80211_ADDR_EQ(vap->iv_des_bssid, ni->ni_bssid))
return 0;
return 1;
}
#ifdef IEEE80211_DEBUG
/*
* Display node suitability/compatibility.
*/
static void
check_bss_debug(struct ieee80211vap *vap, struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
uint8_t rate;
int fail;
fail = 0;
if (isclr(ic->ic_chan_active, ieee80211_chan2ieee(ic, ni->ni_chan)))
fail |= 0x01;
if (vap->iv_opmode == IEEE80211_M_IBSS) {
if ((ni->ni_capinfo & IEEE80211_CAPINFO_IBSS) == 0)
fail |= 0x02;
} else {
if ((ni->ni_capinfo & IEEE80211_CAPINFO_ESS) == 0)
fail |= 0x02;
}
if (vap->iv_flags & IEEE80211_F_PRIVACY) {
if ((ni->ni_capinfo & IEEE80211_CAPINFO_PRIVACY) == 0)
fail |= 0x04;
} else {
/* XXX does this mean privacy is supported or required? */
if (ni->ni_capinfo & IEEE80211_CAPINFO_PRIVACY)
fail |= 0x04;
}
rate = ieee80211_fix_rate(ni, &ni->ni_rates,
IEEE80211_F_JOIN | IEEE80211_F_DONEGO | IEEE80211_F_DOFRATE);
if (rate & IEEE80211_RATE_BASIC)
fail |= 0x08;
if (vap->iv_des_nssid != 0 &&
!match_ssid(ni, vap->iv_des_nssid, vap->iv_des_ssid))
fail |= 0x10;
if ((vap->iv_flags & IEEE80211_F_DESBSSID) &&
!IEEE80211_ADDR_EQ(vap->iv_des_bssid, ni->ni_bssid))
fail |= 0x20;
printf(" %c %s", fail ? '-' : '+', ether_sprintf(ni->ni_macaddr));
printf(" %s%c", ether_sprintf(ni->ni_bssid), fail & 0x20 ? '!' : ' ');
printf(" %3d%c",
ieee80211_chan2ieee(ic, ni->ni_chan), fail & 0x01 ? '!' : ' ');
printf(" %2dM%c", (rate & IEEE80211_RATE_VAL) / 2,
fail & 0x08 ? '!' : ' ');
printf(" %4s%c",
(ni->ni_capinfo & IEEE80211_CAPINFO_ESS) ? "ess" :
(ni->ni_capinfo & IEEE80211_CAPINFO_IBSS) ? "ibss" :
"????",
fail & 0x02 ? '!' : ' ');
printf(" %3s%c ",
(ni->ni_capinfo & IEEE80211_CAPINFO_PRIVACY) ? "wep" : "no",
fail & 0x04 ? '!' : ' ');
ieee80211_print_essid(ni->ni_essid, ni->ni_esslen);
printf("%s\n", fail & 0x10 ? "!" : "");
}
#endif /* IEEE80211_DEBUG */
/*
* Handle 802.11 ad hoc network merge. The
* convention, set by the Wireless Ethernet Compatibility Alliance
* (WECA), is that an 802.11 station will change its BSSID to match
* the "oldest" 802.11 ad hoc network, on the same channel, that
* has the station's desired SSID. The "oldest" 802.11 network
* sends beacons with the greatest TSF timestamp.
*
* The caller is assumed to validate TSF's before attempting a merge.
*
* Return !0 if the BSSID changed, 0 otherwise.
*/
int
ieee80211_ibss_merge(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
#ifdef IEEE80211_DEBUG
struct ieee80211com *ic = ni->ni_ic;
#endif
if (ni == vap->iv_bss ||
IEEE80211_ADDR_EQ(ni->ni_bssid, vap->iv_bss->ni_bssid)) {
/* unchanged, nothing to do */
return 0;
}
if (!check_bss(vap, ni)) {
/* capabilities mismatch */
IEEE80211_DPRINTF(vap, IEEE80211_MSG_ASSOC,
"%s: merge failed, capabilities mismatch\n", __func__);
#ifdef IEEE80211_DEBUG
if (ieee80211_msg_assoc(vap))
check_bss_debug(vap, ni);
#endif
vap->iv_stats.is_ibss_capmismatch++;
return 0;
}
IEEE80211_DPRINTF(vap, IEEE80211_MSG_ASSOC,
"%s: new bssid %s: %s preamble, %s slot time%s\n", __func__,
ether_sprintf(ni->ni_bssid),
ic->ic_flags&IEEE80211_F_SHPREAMBLE ? "short" : "long",
ic->ic_flags&IEEE80211_F_SHSLOT ? "short" : "long",
ic->ic_flags&IEEE80211_F_USEPROT ? ", protection" : ""
);
return ieee80211_sta_join1(ieee80211_ref_node(ni));
}
/*
* Calculate HT channel promotion flags for all vaps.
* This assumes ni_chan have been setup for each vap.
*/
static int
gethtadjustflags(struct ieee80211com *ic)
{
struct ieee80211vap *vap;
int flags;
flags = 0;
/* XXX locking */
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
if (vap->iv_state < IEEE80211_S_RUN)
continue;
switch (vap->iv_opmode) {
case IEEE80211_M_WDS:
case IEEE80211_M_STA:
case IEEE80211_M_AHDEMO:
case IEEE80211_M_HOSTAP:
case IEEE80211_M_IBSS:
case IEEE80211_M_MBSS:
flags |= ieee80211_htchanflags(vap->iv_bss->ni_chan);
break;
default:
break;
}
}
return flags;
}
/*
* Check if the current channel needs to change based on whether
* any vap's are using HT20/HT40. This is used to sync the state
* of ic_curchan after a channel width change on a running vap.
*/
void
ieee80211_sync_curchan(struct ieee80211com *ic)
{
struct ieee80211_channel *c;
c = ieee80211_ht_adjust_channel(ic, ic->ic_curchan, gethtadjustflags(ic));
if (c != ic->ic_curchan) {
ic->ic_curchan = c;
ic->ic_curmode = ieee80211_chan2mode(ic->ic_curchan);
ic->ic_rt = ieee80211_get_ratetable(ic->ic_curchan);
IEEE80211_UNLOCK(ic);
ic->ic_set_channel(ic);
ieee80211_radiotap_chan_change(ic);
IEEE80211_LOCK(ic);
}
}
/*
* Setup the current channel. The request channel may be
* promoted if other vap's are operating with HT20/HT40.
*/
void
ieee80211_setupcurchan(struct ieee80211com *ic, struct ieee80211_channel *c)
{
if (ic->ic_htcaps & IEEE80211_HTC_HT) {
int flags = gethtadjustflags(ic);
/*
* Check for channel promotion required to support the
* set of running vap's. This assumes we are called
* after ni_chan is setup for each vap.
*/
/* NB: this assumes IEEE80211_FHT_USEHT40 > IEEE80211_FHT_HT */
if (flags > ieee80211_htchanflags(c))
c = ieee80211_ht_adjust_channel(ic, c, flags);
}
ic->ic_bsschan = ic->ic_curchan = c;
ic->ic_curmode = ieee80211_chan2mode(ic->ic_curchan);
ic->ic_rt = ieee80211_get_ratetable(ic->ic_curchan);
}
/*
* Change the current channel. The channel change is guaranteed to have
* happened before the next state change.
*/
void
ieee80211_setcurchan(struct ieee80211com *ic, struct ieee80211_channel *c)
{
ieee80211_setupcurchan(ic, c);
ieee80211_runtask(ic, &ic->ic_chan_task);
}
void
ieee80211_update_chw(struct ieee80211com *ic)
{
ieee80211_setupcurchan(ic, ic->ic_curchan);
ieee80211_runtask(ic, &ic->ic_chw_task);
}
/*
* Join the specified IBSS/BSS network. The node is assumed to
* be passed in with a held reference.
*/
static int
ieee80211_sta_join1(struct ieee80211_node *selbs)
{
struct ieee80211vap *vap = selbs->ni_vap;
struct ieee80211com *ic = selbs->ni_ic;
struct ieee80211_node *obss;
int canreassoc;
/*
* Committed to selbs, setup state.
*/
obss = vap->iv_bss;
/*
* Check if old+new node have the same address in which
* case we can reassociate when operating in sta mode.
*/
canreassoc = (obss != NULL &&
vap->iv_state == IEEE80211_S_RUN &&
IEEE80211_ADDR_EQ(obss->ni_macaddr, selbs->ni_macaddr));
vap->iv_bss = selbs; /* NB: caller assumed to bump refcnt */
if (obss != NULL) {
struct ieee80211_node_table *nt = obss->ni_table;
copy_bss(selbs, obss);
ieee80211_node_decref(obss); /* iv_bss reference */
IEEE80211_NODE_LOCK(nt);
node_reclaim(nt, obss); /* station table reference */
IEEE80211_NODE_UNLOCK(nt);
obss = NULL; /* NB: guard against later use */
}
/*
* Delete unusable rates; we've already checked
* that the negotiated rate set is acceptable.
*/
ieee80211_fix_rate(vap->iv_bss, &vap->iv_bss->ni_rates,
IEEE80211_F_DODEL | IEEE80211_F_JOIN);
ieee80211_setcurchan(ic, selbs->ni_chan);
/*
* Set the erp state (mostly the slot time) to deal with
* the auto-select case; this should be redundant if the
* mode is locked.
*/
ieee80211_reset_erp(ic);
ieee80211_wme_initparams(vap);
if (vap->iv_opmode == IEEE80211_M_STA) {
if (canreassoc) {
/* Reassociate */
ieee80211_new_state(vap, IEEE80211_S_ASSOC, 1);
} else {
/*
* Act as if we received a DEAUTH frame in case we
* are invoked from the RUN state. This will cause
* us to try to re-authenticate if we are operating
* as a station.
*/
ieee80211_new_state(vap, IEEE80211_S_AUTH,
IEEE80211_FC0_SUBTYPE_DEAUTH);
}
} else
ieee80211_new_state(vap, IEEE80211_S_RUN, -1);
return 1;
}
int
ieee80211_sta_join(struct ieee80211vap *vap, struct ieee80211_channel *chan,
const struct ieee80211_scan_entry *se)
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_node *ni;
ni = ieee80211_alloc_node(&ic->ic_sta, vap, se->se_macaddr);
if (ni == NULL) {
/* XXX msg */
return 0;
}
/*
* Expand scan state into node's format.
* XXX may not need all this stuff
*/
IEEE80211_ADDR_COPY(ni->ni_bssid, se->se_bssid);
ni->ni_esslen = se->se_ssid[1];
memcpy(ni->ni_essid, se->se_ssid+2, ni->ni_esslen);
ni->ni_tstamp.tsf = se->se_tstamp.tsf;
ni->ni_intval = se->se_intval;
ni->ni_capinfo = se->se_capinfo;
ni->ni_chan = chan;
ni->ni_timoff = se->se_timoff;
ni->ni_fhdwell = se->se_fhdwell;
ni->ni_fhindex = se->se_fhindex;
ni->ni_erp = se->se_erp;
IEEE80211_RSSI_LPF(ni->ni_avgrssi, se->se_rssi);
ni->ni_noise = se->se_noise;
if (vap->iv_opmode == IEEE80211_M_STA) {
/* NB: only infrastructure mode requires an associd */
ni->ni_flags |= IEEE80211_NODE_ASSOCID;
}
if (ieee80211_ies_init(&ni->ni_ies, se->se_ies.data, se->se_ies.len)) {
ieee80211_ies_expand(&ni->ni_ies);
#ifdef IEEE80211_SUPPORT_SUPERG
if (ni->ni_ies.ath_ie != NULL)
ieee80211_parse_ath(ni, ni->ni_ies.ath_ie);
#endif
if (ni->ni_ies.htcap_ie != NULL)
ieee80211_parse_htcap(ni, ni->ni_ies.htcap_ie);
if (ni->ni_ies.htinfo_ie != NULL)
ieee80211_parse_htinfo(ni, ni->ni_ies.htinfo_ie);
#ifdef IEEE80211_SUPPORT_MESH
if (ni->ni_ies.meshid_ie != NULL)
ieee80211_parse_meshid(ni, ni->ni_ies.meshid_ie);
#endif
#ifdef IEEE80211_SUPPORT_TDMA
if (ni->ni_ies.tdma_ie != NULL)
ieee80211_parse_tdma(ni, ni->ni_ies.tdma_ie);
#endif
}
vap->iv_dtim_period = se->se_dtimperiod;
vap->iv_dtim_count = 0;
/* NB: must be after ni_chan is setup */
ieee80211_setup_rates(ni, se->se_rates, se->se_xrates,
IEEE80211_F_DOSORT);
if (ieee80211_iserp_rateset(&ni->ni_rates))
ni->ni_flags |= IEEE80211_NODE_ERP;
/*
* Setup HT state for this node if it's available, otherwise
* non-STA modes won't pick this state up.
*
* For IBSS and related modes that don't go through an
* association request/response, the only appropriate place
* to setup the HT state is here.
*/
if (ni->ni_ies.htinfo_ie != NULL &&
ni->ni_ies.htcap_ie != NULL &&
vap->iv_flags_ht & IEEE80211_FHT_HT) {
ieee80211_ht_node_init(ni);
ieee80211_ht_updateparams(ni,
ni->ni_ies.htcap_ie,
ni->ni_ies.htinfo_ie);
ieee80211_setup_htrates(ni, ni->ni_ies.htcap_ie,
IEEE80211_F_JOIN | IEEE80211_F_DOBRS);
ieee80211_setup_basic_htrates(ni, ni->ni_ies.htinfo_ie);
}
/* XXX else check for ath FF? */
/* XXX QoS? Difficult given that WME config is specific to a master */
ieee80211_node_setuptxparms(ni);
ieee80211_ratectl_node_init(ni);
return ieee80211_sta_join1(ieee80211_ref_node(ni));
}
/*
* Leave the specified IBSS/BSS network. The node is assumed to
* be passed in with a held reference.
*/
void
ieee80211_sta_leave(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
ic->ic_node_cleanup(ni);
ieee80211_notify_node_leave(ni);
}
/*
* Send a deauthenticate frame and drop the station.
*/
void
ieee80211_node_deauth(struct ieee80211_node *ni, int reason)
{
/* NB: bump the refcnt to be sure temporary nodes are not reclaimed */
ieee80211_ref_node(ni);
if (ni->ni_associd != 0)
IEEE80211_SEND_MGMT(ni, IEEE80211_FC0_SUBTYPE_DEAUTH, reason);
ieee80211_node_leave(ni);
ieee80211_free_node(ni);
}
static struct ieee80211_node *
node_alloc(struct ieee80211vap *vap, const uint8_t macaddr[IEEE80211_ADDR_LEN])
{
struct ieee80211_node *ni;
ni = (struct ieee80211_node *) malloc(sizeof(struct ieee80211_node),
M_80211_NODE, M_NOWAIT | M_ZERO);
return ni;
}
/*
* Initialize an ie blob with the specified data. If previous
* data exists re-use the data block. As a side effect we clear
* all references to specific ie's; the caller is required to
* recalculate them.
*/
int
ieee80211_ies_init(struct ieee80211_ies *ies, const uint8_t *data, int len)
{
/* NB: assumes data+len are the last fields */
memset(ies, 0, offsetof(struct ieee80211_ies, data));
if (ies->data != NULL && ies->len != len) {
/* data size changed */
free(ies->data, M_80211_NODE_IE);
ies->data = NULL;
}
if (ies->data == NULL) {
ies->data = (uint8_t *) malloc(len, M_80211_NODE_IE, M_NOWAIT);
if (ies->data == NULL) {
ies->len = 0;
/* NB: pointers have already been zero'd above */
return 0;
}
}
memcpy(ies->data, data, len);
ies->len = len;
return 1;
}
/*
* Reclaim storage for an ie blob.
*/
void
ieee80211_ies_cleanup(struct ieee80211_ies *ies)
{
if (ies->data != NULL)
free(ies->data, M_80211_NODE_IE);
}
/*
* Expand an ie blob data contents and to fillin individual
* ie pointers. The data blob is assumed to be well-formed;
* we don't do any validity checking of ie lengths.
*/
void
ieee80211_ies_expand(struct ieee80211_ies *ies)
{
uint8_t *ie;
int ielen;
ie = ies->data;
ielen = ies->len;
while (ielen > 0) {
switch (ie[0]) {
case IEEE80211_ELEMID_VENDOR:
if (iswpaoui(ie))
ies->wpa_ie = ie;
else if (iswmeoui(ie))
ies->wme_ie = ie;
#ifdef IEEE80211_SUPPORT_SUPERG
else if (isatherosoui(ie))
ies->ath_ie = ie;
#endif
#ifdef IEEE80211_SUPPORT_TDMA
else if (istdmaoui(ie))
ies->tdma_ie = ie;
#endif
break;
case IEEE80211_ELEMID_RSN:
ies->rsn_ie = ie;
break;
case IEEE80211_ELEMID_HTCAP:
ies->htcap_ie = ie;
break;
case IEEE80211_ELEMID_HTINFO:
ies->htinfo_ie = ie;
break;
#ifdef IEEE80211_SUPPORT_MESH
case IEEE80211_ELEMID_MESHID:
ies->meshid_ie = ie;
break;
#endif
}
ielen -= 2 + ie[1];
ie += 2 + ie[1];
}
}
/*
* Reclaim any resources in a node and reset any critical
* state. Typically nodes are free'd immediately after,
* but in some cases the storage may be reused so we need
* to insure consistent state (should probably fix that).
*/
static void
node_cleanup(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
int i;
/* NB: preserve ni_table */
if (ni->ni_flags & IEEE80211_NODE_PWR_MGT) {
if (vap->iv_opmode != IEEE80211_M_STA)
vap->iv_ps_sta--;
ni->ni_flags &= ~IEEE80211_NODE_PWR_MGT;
IEEE80211_NOTE(vap, IEEE80211_MSG_POWER, ni,
"power save mode off, %u sta's in ps mode", vap->iv_ps_sta);
}
/*
* Cleanup any HT-related state.
*/
if (ni->ni_flags & IEEE80211_NODE_HT)
ieee80211_ht_node_cleanup(ni);
#ifdef IEEE80211_SUPPORT_SUPERG
else if (ni->ni_ath_flags & IEEE80211_NODE_ATH)
ieee80211_ff_node_cleanup(ni);
#endif
#ifdef IEEE80211_SUPPORT_MESH
/*
* Cleanup any mesh-related state.
*/
if (vap->iv_opmode == IEEE80211_M_MBSS)
ieee80211_mesh_node_cleanup(ni);
#endif
/*
* Clear any staging queue entries.
*/
ieee80211_ageq_drain_node(&ic->ic_stageq, ni);
/*
* Clear AREF flag that marks the authorization refcnt bump
* has happened. This is probably not needed as the node
* should always be removed from the table so not found but
* do it just in case.
* Likewise clear the ASSOCID flag as these flags are intended
* to be managed in tandem.
*/
ni->ni_flags &= ~(IEEE80211_NODE_AREF | IEEE80211_NODE_ASSOCID);
/*
* Drain power save queue and, if needed, clear TIM.
*/
if (ieee80211_node_psq_drain(ni) != 0 && vap->iv_set_tim != NULL)
vap->iv_set_tim(ni, 0);
ni->ni_associd = 0;
if (ni->ni_challenge != NULL) {
free(ni->ni_challenge, M_80211_NODE);
ni->ni_challenge = NULL;
}
/*
* Preserve SSID, WPA, and WME ie's so the bss node is
* reusable during a re-auth/re-assoc state transition.
* If we remove these data they will not be recreated
* because they come from a probe-response or beacon frame
* which cannot be expected prior to the association-response.
* This should not be an issue when operating in other modes
* as stations leaving always go through a full state transition
* which will rebuild this state.
*
* XXX does this leave us open to inheriting old state?
*/
for (i = 0; i < nitems(ni->ni_rxfrag); i++)
if (ni->ni_rxfrag[i] != NULL) {
m_freem(ni->ni_rxfrag[i]);
ni->ni_rxfrag[i] = NULL;
}
/*
* Must be careful here to remove any key map entry w/o a LOR.
*/
ieee80211_node_delucastkey(ni);
}
static void
node_free(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
ieee80211_ratectl_node_deinit(ni);
ic->ic_node_cleanup(ni);
ieee80211_ies_cleanup(&ni->ni_ies);
ieee80211_psq_cleanup(&ni->ni_psq);
free(ni, M_80211_NODE);
}
static void
node_age(struct ieee80211_node *ni)
{
struct ieee80211vap *vap = ni->ni_vap;
IEEE80211_NODE_LOCK_ASSERT(&vap->iv_ic->ic_sta);
/*
* Age frames on the power save queue.
*/
if (ieee80211_node_psq_age(ni) != 0 &&
ni->ni_psq.psq_len == 0 && vap->iv_set_tim != NULL)
vap->iv_set_tim(ni, 0);
/*
* Age out HT resources (e.g. frames on the
* A-MPDU reorder queues).
*/
if (ni->ni_associd != 0 && (ni->ni_flags & IEEE80211_NODE_HT))
ieee80211_ht_node_age(ni);
}
static int8_t
node_getrssi(const struct ieee80211_node *ni)
{
uint32_t avgrssi = ni->ni_avgrssi;
int32_t rssi;
if (avgrssi == IEEE80211_RSSI_DUMMY_MARKER)
return 0;
rssi = IEEE80211_RSSI_GET(avgrssi);
return rssi < 0 ? 0 : rssi > 127 ? 127 : rssi;
}
static void
node_getsignal(const struct ieee80211_node *ni, int8_t *rssi, int8_t *noise)
{
*rssi = node_getrssi(ni);
*noise = ni->ni_noise;
}
static void
node_getmimoinfo(const struct ieee80211_node *ni,
struct ieee80211_mimo_info *info)
{
int i;
uint32_t avgrssi;
int32_t rssi;
bzero(info, sizeof(*info));
for (i = 0; i < ni->ni_mimo_chains; i++) {
avgrssi = ni->ni_mimo_rssi_ctl[i];
if (avgrssi == IEEE80211_RSSI_DUMMY_MARKER) {
info->rssi[i] = 0;
} else {
rssi = IEEE80211_RSSI_GET(avgrssi);
info->rssi[i] = rssi < 0 ? 0 : rssi > 127 ? 127 : rssi;
}
info->noise[i] = ni->ni_mimo_noise_ctl[i];
}
/* XXX ext radios? */
/* XXX EVM? */
}
struct ieee80211_node *
ieee80211_alloc_node(struct ieee80211_node_table *nt,
struct ieee80211vap *vap, const uint8_t macaddr[IEEE80211_ADDR_LEN])
{
struct ieee80211com *ic = nt->nt_ic;
struct ieee80211_node *ni;
int hash;
ni = ic->ic_node_alloc(vap, macaddr);
if (ni == NULL) {
vap->iv_stats.is_rx_nodealloc++;
return NULL;
}
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"%s %p<%s> in %s table\n", __func__, ni,
ether_sprintf(macaddr), nt->nt_name);
IEEE80211_ADDR_COPY(ni->ni_macaddr, macaddr);
hash = IEEE80211_NODE_HASH(ic, macaddr);
ieee80211_node_initref(ni); /* mark referenced */
ni->ni_chan = IEEE80211_CHAN_ANYC;
ni->ni_authmode = IEEE80211_AUTH_OPEN;
ni->ni_txpower = ic->ic_txpowlimit; /* max power */
ni->ni_txparms = &vap->iv_txparms[ieee80211_chan2mode(ic->ic_curchan)];
ieee80211_crypto_resetkey(vap, &ni->ni_ucastkey, IEEE80211_KEYIX_NONE);
ni->ni_avgrssi = IEEE80211_RSSI_DUMMY_MARKER;
ni->ni_inact_reload = nt->nt_inact_init;
ni->ni_inact = ni->ni_inact_reload;
ni->ni_ath_defkeyix = 0x7fff;
ieee80211_psq_init(&ni->ni_psq, "unknown");
#ifdef IEEE80211_SUPPORT_MESH
if (vap->iv_opmode == IEEE80211_M_MBSS)
ieee80211_mesh_node_init(vap, ni);
#endif
IEEE80211_NODE_LOCK(nt);
TAILQ_INSERT_TAIL(&nt->nt_node, ni, ni_list);
LIST_INSERT_HEAD(&nt->nt_hash[hash], ni, ni_hash);
ni->ni_table = nt;
ni->ni_vap = vap;
ni->ni_ic = ic;
IEEE80211_NODE_UNLOCK(nt);
IEEE80211_NOTE(vap, IEEE80211_MSG_INACT, ni,
"%s: inact_reload %u", __func__, ni->ni_inact_reload);
ieee80211_ratectl_node_init(ni);
return ni;
}
/*
* Craft a temporary node suitable for sending a management frame
* to the specified station. We craft only as much state as we
* need to do the work since the node will be immediately reclaimed
* once the send completes.
*/
struct ieee80211_node *
ieee80211_tmp_node(struct ieee80211vap *vap,
const uint8_t macaddr[IEEE80211_ADDR_LEN])
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_node *ni;
ni = ic->ic_node_alloc(vap, macaddr);
if (ni != NULL) {
struct ieee80211_node *bss = vap->iv_bss;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"%s %p<%s>\n", __func__, ni, ether_sprintf(macaddr));
ni->ni_table = NULL; /* NB: pedantic */
ni->ni_ic = ic; /* NB: needed to set channel */
ni->ni_vap = vap;
IEEE80211_ADDR_COPY(ni->ni_macaddr, macaddr);
IEEE80211_ADDR_COPY(ni->ni_bssid, bss->ni_bssid);
ieee80211_node_initref(ni); /* mark referenced */
/* NB: required by ieee80211_fix_rate */
ieee80211_node_set_chan(ni, bss->ni_chan);
ieee80211_crypto_resetkey(vap, &ni->ni_ucastkey,
IEEE80211_KEYIX_NONE);
ni->ni_txpower = bss->ni_txpower;
/* XXX optimize away */
ieee80211_psq_init(&ni->ni_psq, "unknown");
ieee80211_ratectl_node_init(ni);
} else {
/* XXX msg */
vap->iv_stats.is_rx_nodealloc++;
}
return ni;
}
struct ieee80211_node *
ieee80211_dup_bss(struct ieee80211vap *vap,
const uint8_t macaddr[IEEE80211_ADDR_LEN])
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_node *ni;
ni = ieee80211_alloc_node(&ic->ic_sta, vap, macaddr);
if (ni != NULL) {
struct ieee80211_node *bss = vap->iv_bss;
/*
* Inherit from iv_bss.
*/
copy_bss(ni, bss);
IEEE80211_ADDR_COPY(ni->ni_bssid, bss->ni_bssid);
ieee80211_node_set_chan(ni, bss->ni_chan);
}
return ni;
}
/*
* Create a bss node for a legacy WDS vap. The far end does
* not associate so we just create create a new node and
* simulate an association. The caller is responsible for
* installing the node as the bss node and handling any further
* setup work like authorizing the port.
*/
struct ieee80211_node *
ieee80211_node_create_wds(struct ieee80211vap *vap,
const uint8_t bssid[IEEE80211_ADDR_LEN], struct ieee80211_channel *chan)
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_node *ni;
/* XXX check if node already in sta table? */
ni = ieee80211_alloc_node(&ic->ic_sta, vap, bssid);
if (ni != NULL) {
ni->ni_wdsvap = vap;
IEEE80211_ADDR_COPY(ni->ni_bssid, bssid);
/*
* Inherit any manually configured settings.
*/
copy_bss(ni, vap->iv_bss);
ieee80211_node_set_chan(ni, chan);
/* NB: propagate ssid so available to WPA supplicant */
ni->ni_esslen = vap->iv_des_ssid[0].len;
memcpy(ni->ni_essid, vap->iv_des_ssid[0].ssid, ni->ni_esslen);
/* NB: no associd for peer */
/*
* There are no management frames to use to
* discover neighbor capabilities, so blindly
* propagate the local configuration.
*/
if (vap->iv_flags & IEEE80211_F_WME)
ni->ni_flags |= IEEE80211_NODE_QOS;
#ifdef IEEE80211_SUPPORT_SUPERG
if (vap->iv_flags & IEEE80211_F_FF)
ni->ni_flags |= IEEE80211_NODE_FF;
#endif
if ((ic->ic_htcaps & IEEE80211_HTC_HT) &&
(vap->iv_flags_ht & IEEE80211_FHT_HT)) {
/*
* Device is HT-capable and HT is enabled for
* the vap; setup HT operation. On return
* ni_chan will be adjusted to an HT channel.
*/
ieee80211_ht_wds_init(ni);
} else {
struct ieee80211_channel *c = ni->ni_chan;
/*
* Force a legacy channel to be used.
*/
c = ieee80211_find_channel(ic,
c->ic_freq, c->ic_flags &~ IEEE80211_CHAN_HT);
KASSERT(c != NULL, ("no legacy channel, %u/%x",
ni->ni_chan->ic_freq, ni->ni_chan->ic_flags));
ni->ni_chan = c;
}
}
return ni;
}
struct ieee80211_node *
#ifdef IEEE80211_DEBUG_REFCNT
ieee80211_find_node_locked_debug(struct ieee80211_node_table *nt,
const uint8_t macaddr[IEEE80211_ADDR_LEN], const char *func, int line)
#else
ieee80211_find_node_locked(struct ieee80211_node_table *nt,
const uint8_t macaddr[IEEE80211_ADDR_LEN])
#endif
{
struct ieee80211_node *ni;
int hash;
IEEE80211_NODE_LOCK_ASSERT(nt);
hash = IEEE80211_NODE_HASH(nt->nt_ic, macaddr);
LIST_FOREACH(ni, &nt->nt_hash[hash], ni_hash) {
if (IEEE80211_ADDR_EQ(ni->ni_macaddr, macaddr)) {
ieee80211_ref_node(ni); /* mark referenced */
#ifdef IEEE80211_DEBUG_REFCNT
IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_NODE,
"%s (%s:%u) %p<%s> refcnt %d\n", __func__,
func, line,
ni, ether_sprintf(ni->ni_macaddr),
ieee80211_node_refcnt(ni));
#endif
return ni;
}
}
return NULL;
}
struct ieee80211_node *
#ifdef IEEE80211_DEBUG_REFCNT
ieee80211_find_node_debug(struct ieee80211_node_table *nt,
const uint8_t macaddr[IEEE80211_ADDR_LEN], const char *func, int line)
#else
ieee80211_find_node(struct ieee80211_node_table *nt,
const uint8_t macaddr[IEEE80211_ADDR_LEN])
#endif
{
struct ieee80211_node *ni;
IEEE80211_NODE_LOCK(nt);
ni = ieee80211_find_node_locked(nt, macaddr);
IEEE80211_NODE_UNLOCK(nt);
return ni;
}
struct ieee80211_node *
#ifdef IEEE80211_DEBUG_REFCNT
ieee80211_find_vap_node_locked_debug(struct ieee80211_node_table *nt,
const struct ieee80211vap *vap,
const uint8_t macaddr[IEEE80211_ADDR_LEN], const char *func, int line)
#else
ieee80211_find_vap_node_locked(struct ieee80211_node_table *nt,
const struct ieee80211vap *vap,
const uint8_t macaddr[IEEE80211_ADDR_LEN])
#endif
{
struct ieee80211_node *ni;
int hash;
IEEE80211_NODE_LOCK_ASSERT(nt);
hash = IEEE80211_NODE_HASH(nt->nt_ic, macaddr);
LIST_FOREACH(ni, &nt->nt_hash[hash], ni_hash) {
if (ni->ni_vap == vap &&
IEEE80211_ADDR_EQ(ni->ni_macaddr, macaddr)) {
ieee80211_ref_node(ni); /* mark referenced */
#ifdef IEEE80211_DEBUG_REFCNT
IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_NODE,
"%s (%s:%u) %p<%s> refcnt %d\n", __func__,
func, line,
ni, ether_sprintf(ni->ni_macaddr),
ieee80211_node_refcnt(ni));
#endif
return ni;
}
}
return NULL;
}
struct ieee80211_node *
#ifdef IEEE80211_DEBUG_REFCNT
ieee80211_find_vap_node_debug(struct ieee80211_node_table *nt,
const struct ieee80211vap *vap,
const uint8_t macaddr[IEEE80211_ADDR_LEN], const char *func, int line)
#else
ieee80211_find_vap_node(struct ieee80211_node_table *nt,
const struct ieee80211vap *vap,
const uint8_t macaddr[IEEE80211_ADDR_LEN])
#endif
{
struct ieee80211_node *ni;
IEEE80211_NODE_LOCK(nt);
ni = ieee80211_find_vap_node_locked(nt, vap, macaddr);
IEEE80211_NODE_UNLOCK(nt);
return ni;
}
/*
* Fake up a node; this handles node discovery in adhoc mode.
* Note that for the driver's benefit we we treat this like
* an association so the driver has an opportunity to setup
* it's private state.
*/
struct ieee80211_node *
ieee80211_fakeup_adhoc_node(struct ieee80211vap *vap,
const uint8_t macaddr[IEEE80211_ADDR_LEN])
{
struct ieee80211_node *ni;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE | IEEE80211_MSG_ASSOC,
"%s: mac<%s>\n", __func__, ether_sprintf(macaddr));
ni = ieee80211_dup_bss(vap, macaddr);
if (ni != NULL) {
struct ieee80211com *ic = vap->iv_ic;
/* XXX no rate negotiation; just dup */
ni->ni_rates = vap->iv_bss->ni_rates;
if (ieee80211_iserp_rateset(&ni->ni_rates))
ni->ni_flags |= IEEE80211_NODE_ERP;
if (vap->iv_opmode == IEEE80211_M_AHDEMO) {
/*
* In adhoc demo mode there are no management
* frames to use to discover neighbor capabilities,
* so blindly propagate the local configuration
* so we can do interesting things (e.g. use
* WME to disable ACK's).
*/
if (vap->iv_flags & IEEE80211_F_WME)
ni->ni_flags |= IEEE80211_NODE_QOS;
#ifdef IEEE80211_SUPPORT_SUPERG
if (vap->iv_flags & IEEE80211_F_FF)
ni->ni_flags |= IEEE80211_NODE_FF;
#endif
}
ieee80211_node_setuptxparms(ni);
ieee80211_ratectl_node_init(ni);
if (ic->ic_newassoc != NULL)
ic->ic_newassoc(ni, 1);
/* XXX not right for 802.1x/WPA */
ieee80211_node_authorize(ni);
}
return ni;
}
void
ieee80211_init_neighbor(struct ieee80211_node *ni,
const struct ieee80211_frame *wh,
const struct ieee80211_scanparams *sp)
{
int do_ht_setup = 0;
ni->ni_esslen = sp->ssid[1];
memcpy(ni->ni_essid, sp->ssid + 2, sp->ssid[1]);
IEEE80211_ADDR_COPY(ni->ni_bssid, wh->i_addr3);
memcpy(ni->ni_tstamp.data, sp->tstamp, sizeof(ni->ni_tstamp));
ni->ni_intval = sp->bintval;
ni->ni_capinfo = sp->capinfo;
ni->ni_chan = ni->ni_ic->ic_curchan;
ni->ni_fhdwell = sp->fhdwell;
ni->ni_fhindex = sp->fhindex;
ni->ni_erp = sp->erp;
ni->ni_timoff = sp->timoff;
#ifdef IEEE80211_SUPPORT_MESH
if (ni->ni_vap->iv_opmode == IEEE80211_M_MBSS)
ieee80211_mesh_init_neighbor(ni, wh, sp);
#endif
if (ieee80211_ies_init(&ni->ni_ies, sp->ies, sp->ies_len)) {
ieee80211_ies_expand(&ni->ni_ies);
if (ni->ni_ies.wme_ie != NULL)
ni->ni_flags |= IEEE80211_NODE_QOS;
else
ni->ni_flags &= ~IEEE80211_NODE_QOS;
#ifdef IEEE80211_SUPPORT_SUPERG
if (ni->ni_ies.ath_ie != NULL)
ieee80211_parse_ath(ni, ni->ni_ies.ath_ie);
#endif
if (ni->ni_ies.htcap_ie != NULL)
ieee80211_parse_htcap(ni, ni->ni_ies.htcap_ie);
if (ni->ni_ies.htinfo_ie != NULL)
ieee80211_parse_htinfo(ni, ni->ni_ies.htinfo_ie);
if ((ni->ni_ies.htcap_ie != NULL) &&
(ni->ni_ies.htinfo_ie != NULL) &&
(ni->ni_vap->iv_flags_ht & IEEE80211_FHT_HT)) {
do_ht_setup = 1;
}
}
/* NB: must be after ni_chan is setup */
ieee80211_setup_rates(ni, sp->rates, sp->xrates,
IEEE80211_F_DOSORT | IEEE80211_F_DOFRATE |
IEEE80211_F_DONEGO | IEEE80211_F_DODEL);
/*
* If the neighbor is HT compatible, flip that on.
*/
if (do_ht_setup) {
IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_ASSOC,
"%s: doing HT setup\n", __func__);
ieee80211_ht_node_init(ni);
ieee80211_ht_updateparams(ni,
ni->ni_ies.htcap_ie,
ni->ni_ies.htinfo_ie);
ieee80211_setup_htrates(ni,
ni->ni_ies.htcap_ie,
IEEE80211_F_JOIN | IEEE80211_F_DOBRS);
ieee80211_setup_basic_htrates(ni,
ni->ni_ies.htinfo_ie);
ieee80211_node_setuptxparms(ni);
ieee80211_ratectl_node_init(ni);
}
}
/*
* Do node discovery in adhoc mode on receipt of a beacon
* or probe response frame. Note that for the driver's
* benefit we we treat this like an association so the
* driver has an opportunity to setup it's private state.
*/
struct ieee80211_node *
ieee80211_add_neighbor(struct ieee80211vap *vap,
const struct ieee80211_frame *wh,
const struct ieee80211_scanparams *sp)
{
struct ieee80211_node *ni;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_ASSOC,
"%s: mac<%s>\n", __func__, ether_sprintf(wh->i_addr2));
ni = ieee80211_dup_bss(vap, wh->i_addr2);/* XXX alloc_node? */
if (ni != NULL) {
struct ieee80211com *ic = vap->iv_ic;
ieee80211_init_neighbor(ni, wh, sp);
if (ieee80211_iserp_rateset(&ni->ni_rates))
ni->ni_flags |= IEEE80211_NODE_ERP;
ieee80211_node_setuptxparms(ni);
ieee80211_ratectl_node_init(ni);
if (ic->ic_newassoc != NULL)
ic->ic_newassoc(ni, 1);
/* XXX not right for 802.1x/WPA */
ieee80211_node_authorize(ni);
}
return ni;
}
#define IS_PROBEREQ(wh) \
((wh->i_fc[0] & (IEEE80211_FC0_TYPE_MASK|IEEE80211_FC0_SUBTYPE_MASK)) \
== (IEEE80211_FC0_TYPE_MGT | IEEE80211_FC0_SUBTYPE_PROBE_REQ))
#define IS_BCAST_PROBEREQ(wh) \
(IS_PROBEREQ(wh) && IEEE80211_IS_MULTICAST( \
((const struct ieee80211_frame *)(wh))->i_addr3))
static __inline struct ieee80211_node *
_find_rxnode(struct ieee80211_node_table *nt,
const struct ieee80211_frame_min *wh)
{
if (IS_BCAST_PROBEREQ(wh))
return NULL; /* spam bcast probe req to all vap's */
return ieee80211_find_node_locked(nt, wh->i_addr2);
}
/*
* Locate the node for sender, track state, and then pass the
* (referenced) node up to the 802.11 layer for its use. Note
* we can return NULL if the sender is not in the table.
*/
struct ieee80211_node *
#ifdef IEEE80211_DEBUG_REFCNT
ieee80211_find_rxnode_debug(struct ieee80211com *ic,
const struct ieee80211_frame_min *wh, const char *func, int line)
#else
ieee80211_find_rxnode(struct ieee80211com *ic,
const struct ieee80211_frame_min *wh)
#endif
{
struct ieee80211_node_table *nt;
struct ieee80211_node *ni;
nt = &ic->ic_sta;
IEEE80211_NODE_LOCK(nt);
ni = _find_rxnode(nt, wh);
IEEE80211_NODE_UNLOCK(nt);
return ni;
}
/*
* Like ieee80211_find_rxnode but use the supplied h/w
* key index as a hint to locate the node in the key
* mapping table. If an entry is present at the key
* index we return it; otherwise do a normal lookup and
* update the mapping table if the station has a unicast
* key assigned to it.
*/
struct ieee80211_node *
#ifdef IEEE80211_DEBUG_REFCNT
ieee80211_find_rxnode_withkey_debug(struct ieee80211com *ic,
const struct ieee80211_frame_min *wh, ieee80211_keyix keyix,
const char *func, int line)
#else
ieee80211_find_rxnode_withkey(struct ieee80211com *ic,
const struct ieee80211_frame_min *wh, ieee80211_keyix keyix)
#endif
{
struct ieee80211_node_table *nt;
struct ieee80211_node *ni;
nt = &ic->ic_sta;
IEEE80211_NODE_LOCK(nt);
if (nt->nt_keyixmap != NULL && keyix < nt->nt_keyixmax)
ni = nt->nt_keyixmap[keyix];
else
ni = NULL;
if (ni == NULL) {
ni = _find_rxnode(nt, wh);
if (ni != NULL && nt->nt_keyixmap != NULL) {
/*
* If the station has a unicast key cache slot
* assigned update the key->node mapping table.
*/
keyix = ni->ni_ucastkey.wk_rxkeyix;
/* XXX can keyixmap[keyix] != NULL? */
if (keyix < nt->nt_keyixmax &&
nt->nt_keyixmap[keyix] == NULL) {
IEEE80211_DPRINTF(ni->ni_vap,
IEEE80211_MSG_NODE,
"%s: add key map entry %p<%s> refcnt %d\n",
__func__, ni, ether_sprintf(ni->ni_macaddr),
ieee80211_node_refcnt(ni)+1);
nt->nt_keyixmap[keyix] = ieee80211_ref_node(ni);
}
}
} else {
if (IS_BCAST_PROBEREQ(wh))
ni = NULL; /* spam bcast probe req to all vap's */
else
ieee80211_ref_node(ni);
}
IEEE80211_NODE_UNLOCK(nt);
return ni;
}
#undef IS_BCAST_PROBEREQ
#undef IS_PROBEREQ
/*
* Return a reference to the appropriate node for sending
* a data frame. This handles node discovery in adhoc networks.
*/
struct ieee80211_node *
#ifdef IEEE80211_DEBUG_REFCNT
ieee80211_find_txnode_debug(struct ieee80211vap *vap,
const uint8_t macaddr[IEEE80211_ADDR_LEN],
const char *func, int line)
#else
ieee80211_find_txnode(struct ieee80211vap *vap,
const uint8_t macaddr[IEEE80211_ADDR_LEN])
#endif
{
struct ieee80211_node_table *nt = &vap->iv_ic->ic_sta;
struct ieee80211_node *ni;
/*
* The destination address should be in the node table
* unless this is a multicast/broadcast frame. We can
* also optimize station mode operation, all frames go
* to the bss node.
*/
/* XXX can't hold lock across dup_bss 'cuz of recursive locking */
IEEE80211_NODE_LOCK(nt);
if (vap->iv_opmode == IEEE80211_M_STA ||
vap->iv_opmode == IEEE80211_M_WDS ||
IEEE80211_IS_MULTICAST(macaddr))
ni = ieee80211_ref_node(vap->iv_bss);
else
ni = ieee80211_find_node_locked(nt, macaddr);
IEEE80211_NODE_UNLOCK(nt);
if (ni == NULL) {
if (vap->iv_opmode == IEEE80211_M_IBSS ||
vap->iv_opmode == IEEE80211_M_AHDEMO) {
/*
* In adhoc mode cons up a node for the destination.
* Note that we need an additional reference for the
* caller to be consistent with
* ieee80211_find_node_locked.
*/
ni = ieee80211_fakeup_adhoc_node(vap, macaddr);
if (ni != NULL)
(void) ieee80211_ref_node(ni);
} else {
IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_OUTPUT, macaddr,
"no node, discard frame (%s)", __func__);
vap->iv_stats.is_tx_nonode++;
}
}
return ni;
}
static void
_ieee80211_free_node(struct ieee80211_node *ni)
{
struct ieee80211_node_table *nt = ni->ni_table;
/*
* NB: careful about referencing the vap as it may be
* gone if the last reference was held by a driver.
* We know the com will always be present so it's safe
* to use ni_ic below to reclaim resources.
*/
#if 0
IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
"%s %p<%s> in %s table\n", __func__, ni,
ether_sprintf(ni->ni_macaddr),
nt != NULL ? nt->nt_name : "<gone>");
#endif
if (ni->ni_associd != 0) {
struct ieee80211vap *vap = ni->ni_vap;
if (vap->iv_aid_bitmap != NULL)
IEEE80211_AID_CLR(vap, ni->ni_associd);
}
if (nt != NULL) {
TAILQ_REMOVE(&nt->nt_node, ni, ni_list);
LIST_REMOVE(ni, ni_hash);
}
ni->ni_ic->ic_node_free(ni);
}
/*
* Clear any entry in the unicast key mapping table.
*/
static int
node_clear_keyixmap(struct ieee80211_node_table *nt, struct ieee80211_node *ni)
{
ieee80211_keyix keyix;
keyix = ni->ni_ucastkey.wk_rxkeyix;
if (nt->nt_keyixmap != NULL && keyix < nt->nt_keyixmax &&
nt->nt_keyixmap[keyix] == ni) {
IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_NODE,
"%s: %p<%s> clear key map entry %u\n",
__func__, ni, ether_sprintf(ni->ni_macaddr), keyix);
nt->nt_keyixmap[keyix] = NULL;
ieee80211_node_decref(ni);
return 1;
}
return 0;
}
void
#ifdef IEEE80211_DEBUG_REFCNT
ieee80211_free_node_debug(struct ieee80211_node *ni, const char *func, int line)
#else
ieee80211_free_node(struct ieee80211_node *ni)
#endif
{
struct ieee80211_node_table *nt = ni->ni_table;
#ifdef IEEE80211_DEBUG_REFCNT
IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_NODE,
"%s (%s:%u) %p<%s> refcnt %d\n", __func__, func, line, ni,
ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)-1);
#endif
if (nt != NULL) {
IEEE80211_NODE_LOCK(nt);
if (ieee80211_node_dectestref(ni)) {
/*
* Last reference, reclaim state.
*/
_ieee80211_free_node(ni);
} else if (ieee80211_node_refcnt(ni) == 1)
if (node_clear_keyixmap(nt, ni))
_ieee80211_free_node(ni);
IEEE80211_NODE_UNLOCK(nt);
} else {
if (ieee80211_node_dectestref(ni))
_ieee80211_free_node(ni);
}
}
/*
* Reclaim a unicast key and clear any key cache state.
*/
int
ieee80211_node_delucastkey(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211_node_table *nt = &ic->ic_sta;
struct ieee80211_node *nikey;
ieee80211_keyix keyix;
int isowned, status;
/*
* NB: We must beware of LOR here; deleting the key
* can cause the crypto layer to block traffic updates
* which can generate a LOR against the node table lock;
* grab it here and stash the key index for our use below.
*
* Must also beware of recursion on the node table lock.
* When called from node_cleanup we may already have
* the node table lock held. Unfortunately there's no
* way to separate out this path so we must do this
* conditionally.
*/
isowned = IEEE80211_NODE_IS_LOCKED(nt);
if (!isowned)
IEEE80211_NODE_LOCK(nt);
nikey = NULL;
status = 1; /* NB: success */
if (ni->ni_ucastkey.wk_keyix != IEEE80211_KEYIX_NONE) {
keyix = ni->ni_ucastkey.wk_rxkeyix;
status = ieee80211_crypto_delkey(ni->ni_vap, &ni->ni_ucastkey);
if (nt->nt_keyixmap != NULL && keyix < nt->nt_keyixmax) {
nikey = nt->nt_keyixmap[keyix];
nt->nt_keyixmap[keyix] = NULL;
}
}
if (!isowned)
IEEE80211_NODE_UNLOCK(nt);
if (nikey != NULL) {
KASSERT(nikey == ni,
("key map out of sync, ni %p nikey %p", ni, nikey));
IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_NODE,
"%s: delete key map entry %p<%s> refcnt %d\n",
__func__, ni, ether_sprintf(ni->ni_macaddr),
ieee80211_node_refcnt(ni)-1);
ieee80211_free_node(ni);
}
return status;
}
/*
* Reclaim a node. If this is the last reference count then
* do the normal free work. Otherwise remove it from the node
* table and mark it gone by clearing the back-reference.
*/
static void
node_reclaim(struct ieee80211_node_table *nt, struct ieee80211_node *ni)
{
IEEE80211_NODE_LOCK_ASSERT(nt);
IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_NODE,
"%s: remove %p<%s> from %s table, refcnt %d\n",
__func__, ni, ether_sprintf(ni->ni_macaddr),
nt->nt_name, ieee80211_node_refcnt(ni)-1);
/*
* Clear any entry in the unicast key mapping table.
* We need to do it here so rx lookups don't find it
* in the mapping table even if it's not in the hash
* table. We cannot depend on the mapping table entry
* being cleared because the node may not be free'd.
*/
(void)node_clear_keyixmap(nt, ni);
if (!ieee80211_node_dectestref(ni)) {
/*
* Other references are present, just remove the
* node from the table so it cannot be found. When
* the references are dropped storage will be
* reclaimed.
*/
TAILQ_REMOVE(&nt->nt_node, ni, ni_list);
LIST_REMOVE(ni, ni_hash);
ni->ni_table = NULL; /* clear reference */
} else
_ieee80211_free_node(ni);
}
/*
* Node table support.
*/
static void
ieee80211_node_table_init(struct ieee80211com *ic,
struct ieee80211_node_table *nt,
const char *name, int inact, int keyixmax)
{
struct ifnet *ifp = ic->ic_ifp;
nt->nt_ic = ic;
IEEE80211_NODE_LOCK_INIT(nt, ifp->if_xname);
IEEE80211_NODE_ITERATE_LOCK_INIT(nt, ifp->if_xname);
TAILQ_INIT(&nt->nt_node);
nt->nt_name = name;
nt->nt_scangen = 1;
nt->nt_inact_init = inact;
nt->nt_keyixmax = keyixmax;
if (nt->nt_keyixmax > 0) {
nt->nt_keyixmap = (struct ieee80211_node **) malloc(
keyixmax * sizeof(struct ieee80211_node *),
M_80211_NODE, M_NOWAIT | M_ZERO);
if (nt->nt_keyixmap == NULL)
if_printf(ic->ic_ifp,
"Cannot allocate key index map with %u entries\n",
keyixmax);
} else
nt->nt_keyixmap = NULL;
}
static void
ieee80211_node_table_reset(struct ieee80211_node_table *nt,
struct ieee80211vap *match)
{
struct ieee80211_node *ni, *next;
IEEE80211_NODE_LOCK(nt);
TAILQ_FOREACH_SAFE(ni, &nt->nt_node, ni_list, next) {
if (match != NULL && ni->ni_vap != match)
continue;
/* XXX can this happen? if so need's work */
if (ni->ni_associd != 0) {
struct ieee80211vap *vap = ni->ni_vap;
if (vap->iv_auth->ia_node_leave != NULL)
vap->iv_auth->ia_node_leave(ni);
if (vap->iv_aid_bitmap != NULL)
IEEE80211_AID_CLR(vap, ni->ni_associd);
}
ni->ni_wdsvap = NULL; /* clear reference */
node_reclaim(nt, ni);
}
if (match != NULL && match->iv_opmode == IEEE80211_M_WDS) {
/*
* Make a separate pass to clear references to this vap
* held by DWDS entries. They will not be matched above
* because ni_vap will point to the ap vap but we still
* need to clear ni_wdsvap when the WDS vap is destroyed
* and/or reset.
*/
TAILQ_FOREACH_SAFE(ni, &nt->nt_node, ni_list, next)
if (ni->ni_wdsvap == match)
ni->ni_wdsvap = NULL;
}
IEEE80211_NODE_UNLOCK(nt);
}
static void
ieee80211_node_table_cleanup(struct ieee80211_node_table *nt)
{
ieee80211_node_table_reset(nt, NULL);
if (nt->nt_keyixmap != NULL) {
#ifdef DIAGNOSTIC
/* XXX verify all entries are NULL */
int i;
for (i = 0; i < nt->nt_keyixmax; i++)
if (nt->nt_keyixmap[i] != NULL)
printf("%s: %s[%u] still active\n", __func__,
nt->nt_name, i);
#endif
free(nt->nt_keyixmap, M_80211_NODE);
nt->nt_keyixmap = NULL;
}
IEEE80211_NODE_ITERATE_LOCK_DESTROY(nt);
IEEE80211_NODE_LOCK_DESTROY(nt);
}
/*
* Timeout inactive stations and do related housekeeping.
* Note that we cannot hold the node lock while sending a
* frame as this would lead to a LOR. Instead we use a
* generation number to mark nodes that we've scanned and
* drop the lock and restart a scan if we have to time out
* a node. Since we are single-threaded by virtue of
* controlling the inactivity timer we can be sure this will
* process each node only once.
*/
static void
ieee80211_timeout_stations(struct ieee80211com *ic)
{
struct ieee80211_node_table *nt = &ic->ic_sta;
struct ieee80211vap *vap;
struct ieee80211_node *ni;
int gen = 0;
IEEE80211_NODE_ITERATE_LOCK(nt);
gen = ++nt->nt_scangen;
restart:
IEEE80211_NODE_LOCK(nt);
TAILQ_FOREACH(ni, &nt->nt_node, ni_list) {
if (ni->ni_scangen == gen) /* previously handled */
continue;
ni->ni_scangen = gen;
/*
* Ignore entries for which have yet to receive an
* authentication frame. These are transient and
* will be reclaimed when the last reference to them
* goes away (when frame xmits complete).
*/
vap = ni->ni_vap;
/*
* Only process stations when in RUN state. This
* insures, for example, that we don't timeout an
* inactive station during CAC. Note that CSA state
* is actually handled in ieee80211_node_timeout as
* it applies to more than timeout processing.
*/
if (vap->iv_state != IEEE80211_S_RUN)
continue;
/* XXX can vap be NULL? */
if ((vap->iv_opmode == IEEE80211_M_HOSTAP ||
vap->iv_opmode == IEEE80211_M_STA) &&
(ni->ni_flags & IEEE80211_NODE_AREF) == 0)
continue;
/*
* Free fragment if not needed anymore
* (last fragment older than 1s).
* XXX doesn't belong here, move to node_age
*/
if (ni->ni_rxfrag[0] != NULL &&
ticks > ni->ni_rxfragstamp + hz) {
m_freem(ni->ni_rxfrag[0]);
ni->ni_rxfrag[0] = NULL;
}
if (ni->ni_inact > 0) {
ni->ni_inact--;
IEEE80211_NOTE(vap, IEEE80211_MSG_INACT, ni,
"%s: inact %u inact_reload %u nrates %u",
__func__, ni->ni_inact, ni->ni_inact_reload,
ni->ni_rates.rs_nrates);
}
/*
* Special case ourself; we may be idle for extended periods
* of time and regardless reclaiming our state is wrong.
* XXX run ic_node_age
*/
if (ni == vap->iv_bss)
continue;
if (ni->ni_associd != 0 ||
(vap->iv_opmode == IEEE80211_M_IBSS ||
vap->iv_opmode == IEEE80211_M_AHDEMO)) {
/*
* Age/drain resources held by the station.
*/
ic->ic_node_age(ni);
/*
* Probe the station before time it out. We
* send a null data frame which may not be
* universally supported by drivers (need it
* for ps-poll support so it should be...).
*
* XXX don't probe the station unless we've
* received a frame from them (and have
* some idea of the rates they are capable
* of); this will get fixed more properly
* soon with better handling of the rate set.
*/
if ((vap->iv_flags_ext & IEEE80211_FEXT_INACT) &&
(0 < ni->ni_inact &&
ni->ni_inact <= vap->iv_inact_probe) &&
ni->ni_rates.rs_nrates != 0) {
IEEE80211_NOTE(vap,
IEEE80211_MSG_INACT | IEEE80211_MSG_NODE,
ni, "%s",
"probe station due to inactivity");
/*
* Grab a reference before unlocking the table
* so the node cannot be reclaimed before we
* send the frame. ieee80211_send_nulldata
* understands we've done this and reclaims the
* ref for us as needed.
*/
ieee80211_ref_node(ni);
IEEE80211_NODE_UNLOCK(nt);
ieee80211_send_nulldata(ni);
/* XXX stat? */
goto restart;
}
}
if ((vap->iv_flags_ext & IEEE80211_FEXT_INACT) &&
ni->ni_inact <= 0) {
IEEE80211_NOTE(vap,
IEEE80211_MSG_INACT | IEEE80211_MSG_NODE, ni,
"station timed out due to inactivity "
"(refcnt %u)", ieee80211_node_refcnt(ni));
/*
* Send a deauthenticate frame and drop the station.
* This is somewhat complicated due to reference counts
* and locking. At this point a station will typically
* have a reference count of 1. ieee80211_node_leave
* will do a "free" of the node which will drop the
* reference count. But in the meantime a reference
* wil be held by the deauth frame. The actual reclaim
* of the node will happen either after the tx is
* completed or by ieee80211_node_leave.
*
* Separately we must drop the node lock before sending
* in case the driver takes a lock, as this can result
* in a LOR between the node lock and the driver lock.
*/
ieee80211_ref_node(ni);
IEEE80211_NODE_UNLOCK(nt);
if (ni->ni_associd != 0) {
IEEE80211_SEND_MGMT(ni,
IEEE80211_FC0_SUBTYPE_DEAUTH,
IEEE80211_REASON_AUTH_EXPIRE);
}
ieee80211_node_leave(ni);
ieee80211_free_node(ni);
vap->iv_stats.is_node_timeout++;
goto restart;
}
}
IEEE80211_NODE_UNLOCK(nt);
IEEE80211_NODE_ITERATE_UNLOCK(nt);
}
/*
* Aggressively reclaim resources. This should be used
* only in a critical situation to reclaim mbuf resources.
*/
void
ieee80211_drain(struct ieee80211com *ic)
{
struct ieee80211_node_table *nt = &ic->ic_sta;
struct ieee80211vap *vap;
struct ieee80211_node *ni;
IEEE80211_NODE_LOCK(nt);
TAILQ_FOREACH(ni, &nt->nt_node, ni_list) {
/*
* Ignore entries for which have yet to receive an
* authentication frame. These are transient and
* will be reclaimed when the last reference to them
* goes away (when frame xmits complete).
*/
vap = ni->ni_vap;
/*
* Only process stations when in RUN state. This
* insures, for example, that we don't timeout an
* inactive station during CAC. Note that CSA state
* is actually handled in ieee80211_node_timeout as
* it applies to more than timeout processing.
*/
if (vap->iv_state != IEEE80211_S_RUN)
continue;
/* XXX can vap be NULL? */
if ((vap->iv_opmode == IEEE80211_M_HOSTAP ||
vap->iv_opmode == IEEE80211_M_STA) &&
(ni->ni_flags & IEEE80211_NODE_AREF) == 0)
continue;
/*
* Free fragments.
* XXX doesn't belong here, move to node_drain
*/
if (ni->ni_rxfrag[0] != NULL) {
m_freem(ni->ni_rxfrag[0]);
ni->ni_rxfrag[0] = NULL;
}
/*
* Drain resources held by the station.
*/
ic->ic_node_drain(ni);
}
IEEE80211_NODE_UNLOCK(nt);
}
/*
* Per-ieee80211com inactivity timer callback.
*/
void
ieee80211_node_timeout(void *arg)
{
struct ieee80211com *ic = arg;
/*
* Defer timeout processing if a channel switch is pending.
* We typically need to be mute so not doing things that
* might generate frames is good to handle in one place.
* Supressing the station timeout processing may extend the
* lifetime of inactive stations (by not decrementing their
* idle counters) but this should be ok unless the CSA is
* active for an unusually long time.
*/
if ((ic->ic_flags & IEEE80211_F_CSAPENDING) == 0) {
ieee80211_scan_timeout(ic);
ieee80211_timeout_stations(ic);
ieee80211_ageq_age(&ic->ic_stageq, IEEE80211_INACT_WAIT);
IEEE80211_LOCK(ic);
ieee80211_erp_timeout(ic);
ieee80211_ht_timeout(ic);
IEEE80211_UNLOCK(ic);
}
callout_reset(&ic->ic_inact, IEEE80211_INACT_WAIT*hz,
ieee80211_node_timeout, ic);
}
/*
* Iterate over the node table and return an array of ref'ed nodes.
*
* This is separated out from calling the actual node function so that
* no LORs will occur.
*
* If there are too many nodes (ie, the number of nodes doesn't fit
* within 'max_aid' entries) then the node references will be freed
* and an error will be returned.
*
* The responsibility of allocating and freeing "ni_arr" is up to
* the caller.
*/
int
ieee80211_iterate_nt(struct ieee80211_node_table *nt,
struct ieee80211_node **ni_arr, uint16_t max_aid)
{
u_int gen;
int i, j, ret;
struct ieee80211_node *ni;
IEEE80211_NODE_ITERATE_LOCK(nt);
IEEE80211_NODE_LOCK(nt);
gen = ++nt->nt_scangen;
i = ret = 0;
/*
* We simply assume here that since the node
* scan generation doesn't change (as
* we are holding both the node table and
* node table iteration locks), we can simply
* assign it to the node here.
*/
TAILQ_FOREACH(ni, &nt->nt_node, ni_list) {
if (i >= max_aid) {
ret = E2BIG;
if_printf(nt->nt_ic->ic_ifp,
"Node array overflow: max=%u", max_aid);
break;
}
ni_arr[i] = ieee80211_ref_node(ni);
ni_arr[i]->ni_scangen = gen;
i++;
}
/*
* It's safe to unlock here.
*
* If we're successful, the list is returned.
* If we're unsuccessful, the list is ignored
* and we remove our references.
*
* This avoids any potential LOR with
* ieee80211_free_node().
*/
IEEE80211_NODE_UNLOCK(nt);
IEEE80211_NODE_ITERATE_UNLOCK(nt);
/*
* If ret is non-zero, we hit some kind of error.
* Rather than walking some nodes, we'll walk none
* of them.
*/
if (ret) {
for (j = 0; j < i; j++) {
/* ieee80211_free_node() locks by itself */
ieee80211_free_node(ni_arr[j]);
}
}
return (ret);
}
/*
* Just a wrapper, so we don't have to change every ieee80211_iterate_nodes()
* reference in the source.
*
* Note that this fetches 'max_aid' from the first VAP, rather than finding
* the largest max_aid from all VAPs.
*/
void
ieee80211_iterate_nodes(struct ieee80211_node_table *nt,
ieee80211_iter_func *f, void *arg)
{
struct ieee80211_node **ni_arr;
size_t size;
int i;
uint16_t max_aid;
struct ieee80211vap *vap;
/* Overdoing it default */
max_aid = IEEE80211_AID_MAX;
/* Handle the case of there being no vaps just yet */
vap = TAILQ_FIRST(&nt->nt_ic->ic_vaps);
if (vap != NULL)
max_aid = vap->iv_max_aid;
size = max_aid * sizeof(struct ieee80211_node *);
ni_arr = (struct ieee80211_node **) malloc(size, M_80211_NODE,
M_NOWAIT | M_ZERO);
if (ni_arr == NULL)
return;
/*
* If this fails, the node table won't have any
* valid entries - ieee80211_iterate_nt() frees
* the references to them. So don't try walking
* the table; just skip to the end and free the
* temporary memory.
*/
if (ieee80211_iterate_nt(nt, ni_arr, max_aid) != 0)
goto done;
for (i = 0; i < max_aid; i++) {
if (ni_arr[i] == NULL) /* end of the list */
break;
(*f)(arg, ni_arr[i]);
/* ieee80211_free_node() locks by itself */
ieee80211_free_node(ni_arr[i]);
}
done:
free(ni_arr, M_80211_NODE);
}
void
ieee80211_dump_node(struct ieee80211_node_table *nt, struct ieee80211_node *ni)
{
printf("0x%p: mac %s refcnt %d\n", ni,
ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni));
printf("\tscangen %u authmode %u flags 0x%x\n",
ni->ni_scangen, ni->ni_authmode, ni->ni_flags);
printf("\tassocid 0x%x txpower %u vlan %u\n",
ni->ni_associd, ni->ni_txpower, ni->ni_vlan);
printf("\ttxseq %u rxseq %u fragno %u rxfragstamp %u\n",
ni->ni_txseqs[IEEE80211_NONQOS_TID],
ni->ni_rxseqs[IEEE80211_NONQOS_TID] >> IEEE80211_SEQ_SEQ_SHIFT,
ni->ni_rxseqs[IEEE80211_NONQOS_TID] & IEEE80211_SEQ_FRAG_MASK,
ni->ni_rxfragstamp);
printf("\trssi %d noise %d intval %u capinfo 0x%x\n",
node_getrssi(ni), ni->ni_noise,
ni->ni_intval, ni->ni_capinfo);
printf("\tbssid %s essid \"%.*s\" channel %u:0x%x\n",
ether_sprintf(ni->ni_bssid),
ni->ni_esslen, ni->ni_essid,
ni->ni_chan->ic_freq, ni->ni_chan->ic_flags);
printf("\tinact %u inact_reload %u txrate %u\n",
ni->ni_inact, ni->ni_inact_reload, ni->ni_txrate);
printf("\thtcap %x htparam %x htctlchan %u ht2ndchan %u\n",
ni->ni_htcap, ni->ni_htparam,
ni->ni_htctlchan, ni->ni_ht2ndchan);
printf("\thtopmode %x htstbc %x chw %u\n",
ni->ni_htopmode, ni->ni_htstbc, ni->ni_chw);
}
void
ieee80211_dump_nodes(struct ieee80211_node_table *nt)
{
ieee80211_iterate_nodes(nt,
(ieee80211_iter_func *) ieee80211_dump_node, nt);
}
static void
ieee80211_notify_erp_locked(struct ieee80211com *ic)
{
struct ieee80211vap *vap;
IEEE80211_LOCK_ASSERT(ic);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next)
if (vap->iv_opmode == IEEE80211_M_HOSTAP)
ieee80211_beacon_notify(vap, IEEE80211_BEACON_ERP);
}
void
ieee80211_notify_erp(struct ieee80211com *ic)
{
IEEE80211_LOCK(ic);
ieee80211_notify_erp_locked(ic);
IEEE80211_UNLOCK(ic);
}
/*
* Handle a station joining an 11g network.
*/
static void
ieee80211_node_join_11g(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
IEEE80211_LOCK_ASSERT(ic);
/*
* Station isn't capable of short slot time. Bump
* the count of long slot time stations and disable
* use of short slot time. Note that the actual switch
* over to long slot time use may not occur until the
* next beacon transmission (per sec. 7.3.1.4 of 11g).
*/
if ((ni->ni_capinfo & IEEE80211_CAPINFO_SHORT_SLOTTIME) == 0) {
ic->ic_longslotsta++;
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ASSOC, ni,
"station needs long slot time, count %d",
ic->ic_longslotsta);
/* XXX vap's w/ conflicting needs won't work */
if (!IEEE80211_IS_CHAN_108G(ic->ic_bsschan)) {
/*
* Don't force slot time when switched to turbo
* mode as non-ERP stations won't be present; this
* need only be done when on the normal G channel.
*/
ieee80211_set_shortslottime(ic, 0);
}
}
/*
* If the new station is not an ERP station
* then bump the counter and enable protection
* if configured.
*/
if (!ieee80211_iserp_rateset(&ni->ni_rates)) {
ic->ic_nonerpsta++;
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ASSOC, ni,
"station is !ERP, %d non-ERP stations associated",
ic->ic_nonerpsta);
/*
* If station does not support short preamble
* then we must enable use of Barker preamble.
*/
if ((ni->ni_capinfo & IEEE80211_CAPINFO_SHORT_PREAMBLE) == 0) {
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ASSOC, ni,
"%s", "station needs long preamble");
ic->ic_flags |= IEEE80211_F_USEBARKER;
ic->ic_flags &= ~IEEE80211_F_SHPREAMBLE;
}
/*
* If protection is configured and this is the first
* indication we should use protection, enable it.
*/
if (ic->ic_protmode != IEEE80211_PROT_NONE &&
ic->ic_nonerpsta == 1 &&
(ic->ic_flags_ext & IEEE80211_FEXT_NONERP_PR) == 0) {
IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_ASSOC,
"%s: enable use of protection\n", __func__);
ic->ic_flags |= IEEE80211_F_USEPROT;
ieee80211_notify_erp_locked(ic);
}
} else
ni->ni_flags |= IEEE80211_NODE_ERP;
}
void
ieee80211_node_join(struct ieee80211_node *ni, int resp)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211vap *vap = ni->ni_vap;
int newassoc;
if (ni->ni_associd == 0) {
uint16_t aid;
KASSERT(vap->iv_aid_bitmap != NULL, ("no aid bitmap"));
/*
* It would be good to search the bitmap
* more efficiently, but this will do for now.
*/
for (aid = 1; aid < vap->iv_max_aid; aid++) {
if (!IEEE80211_AID_ISSET(vap, aid))
break;
}
if (aid >= vap->iv_max_aid) {
IEEE80211_SEND_MGMT(ni, resp, IEEE80211_STATUS_TOOMANY);
ieee80211_node_leave(ni);
return;
}
ni->ni_associd = aid | 0xc000;
ni->ni_jointime = time_uptime;
IEEE80211_LOCK(ic);
IEEE80211_AID_SET(vap, ni->ni_associd);
vap->iv_sta_assoc++;
ic->ic_sta_assoc++;
if (IEEE80211_IS_CHAN_HT(ic->ic_bsschan))
ieee80211_ht_node_join(ni);
if (IEEE80211_IS_CHAN_ANYG(ic->ic_bsschan) &&
IEEE80211_IS_CHAN_FULL(ic->ic_bsschan))
ieee80211_node_join_11g(ni);
IEEE80211_UNLOCK(ic);
newassoc = 1;
} else
newassoc = 0;
IEEE80211_NOTE(vap, IEEE80211_MSG_ASSOC | IEEE80211_MSG_DEBUG, ni,
"station associated at aid %d: %s preamble, %s slot time%s%s%s%s%s%s%s%s",
IEEE80211_NODE_AID(ni),
ic->ic_flags & IEEE80211_F_SHPREAMBLE ? "short" : "long",
ic->ic_flags & IEEE80211_F_SHSLOT ? "short" : "long",
ic->ic_flags & IEEE80211_F_USEPROT ? ", protection" : "",
ni->ni_flags & IEEE80211_NODE_QOS ? ", QoS" : "",
ni->ni_flags & IEEE80211_NODE_HT ?
(ni->ni_chw == 40 ? ", HT40" : ", HT20") : "",
ni->ni_flags & IEEE80211_NODE_AMPDU ? " (+AMPDU)" : "",
ni->ni_flags & IEEE80211_NODE_MIMO_RTS ? " (+SMPS-DYN)" :
ni->ni_flags & IEEE80211_NODE_MIMO_PS ? " (+SMPS)" : "",
ni->ni_flags & IEEE80211_NODE_RIFS ? " (+RIFS)" : "",
IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF) ?
", fast-frames" : "",
IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_TURBOP) ?
", turbo" : ""
);
ieee80211_node_setuptxparms(ni);
ieee80211_ratectl_node_init(ni);
/* give driver a chance to setup state like ni_txrate */
if (ic->ic_newassoc != NULL)
ic->ic_newassoc(ni, newassoc);
IEEE80211_SEND_MGMT(ni, resp, IEEE80211_STATUS_SUCCESS);
/* tell the authenticator about new station */
if (vap->iv_auth->ia_node_join != NULL)
vap->iv_auth->ia_node_join(ni);
ieee80211_notify_node_join(ni,
resp == IEEE80211_FC0_SUBTYPE_ASSOC_RESP);
}
static void
disable_protection(struct ieee80211com *ic)
{
KASSERT(ic->ic_nonerpsta == 0 &&
(ic->ic_flags_ext & IEEE80211_FEXT_NONERP_PR) == 0,
("%d non ERP stations, flags 0x%x", ic->ic_nonerpsta,
ic->ic_flags_ext));
ic->ic_flags &= ~IEEE80211_F_USEPROT;
/* XXX verify mode? */
if (ic->ic_caps & IEEE80211_C_SHPREAMBLE) {
ic->ic_flags |= IEEE80211_F_SHPREAMBLE;
ic->ic_flags &= ~IEEE80211_F_USEBARKER;
}
ieee80211_notify_erp_locked(ic);
}
/*
* Handle a station leaving an 11g network.
*/
static void
ieee80211_node_leave_11g(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
IEEE80211_LOCK_ASSERT(ic);
KASSERT(IEEE80211_IS_CHAN_ANYG(ic->ic_bsschan),
("not in 11g, bss %u:0x%x", ic->ic_bsschan->ic_freq,
ic->ic_bsschan->ic_flags));
/*
* If a long slot station do the slot time bookkeeping.
*/
if ((ni->ni_capinfo & IEEE80211_CAPINFO_SHORT_SLOTTIME) == 0) {
KASSERT(ic->ic_longslotsta > 0,
("bogus long slot station count %d", ic->ic_longslotsta));
ic->ic_longslotsta--;
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ASSOC, ni,
"long slot time station leaves, count now %d",
ic->ic_longslotsta);
if (ic->ic_longslotsta == 0) {
/*
* Re-enable use of short slot time if supported
* and not operating in IBSS mode (per spec).
*/
if ((ic->ic_caps & IEEE80211_C_SHSLOT) &&
ic->ic_opmode != IEEE80211_M_IBSS) {
IEEE80211_DPRINTF(ni->ni_vap,
IEEE80211_MSG_ASSOC,
"%s: re-enable use of short slot time\n",
__func__);
ieee80211_set_shortslottime(ic, 1);
}
}
}
/*
* If a non-ERP station do the protection-related bookkeeping.
*/
if ((ni->ni_flags & IEEE80211_NODE_ERP) == 0) {
KASSERT(ic->ic_nonerpsta > 0,
("bogus non-ERP station count %d", ic->ic_nonerpsta));
ic->ic_nonerpsta--;
IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ASSOC, ni,
"non-ERP station leaves, count now %d%s", ic->ic_nonerpsta,
(ic->ic_flags_ext & IEEE80211_FEXT_NONERP_PR) ?
" (non-ERP sta present)" : "");
if (ic->ic_nonerpsta == 0 &&
(ic->ic_flags_ext & IEEE80211_FEXT_NONERP_PR) == 0) {
IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_ASSOC,
"%s: disable use of protection\n", __func__);
disable_protection(ic);
}
}
}
/*
* Time out presence of an overlapping bss with non-ERP
* stations. When operating in hostap mode we listen for
* beacons from other stations and if we identify a non-ERP
* station is present we enable protection. To identify
* when all non-ERP stations are gone we time out this
* condition.
*/
static void
ieee80211_erp_timeout(struct ieee80211com *ic)
{
IEEE80211_LOCK_ASSERT(ic);
if ((ic->ic_flags_ext & IEEE80211_FEXT_NONERP_PR) &&
time_after(ticks, ic->ic_lastnonerp + IEEE80211_NONERP_PRESENT_AGE)) {
#if 0
IEEE80211_NOTE(vap, IEEE80211_MSG_ASSOC, ni,
"%s", "age out non-ERP sta present on channel");
#endif
ic->ic_flags_ext &= ~IEEE80211_FEXT_NONERP_PR;
if (ic->ic_nonerpsta == 0)
disable_protection(ic);
}
}
/*
* Handle bookkeeping for station deauthentication/disassociation
* when operating as an ap.
*/
void
ieee80211_node_leave(struct ieee80211_node *ni)
{
struct ieee80211com *ic = ni->ni_ic;
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211_node_table *nt = ni->ni_table;
IEEE80211_NOTE(vap, IEEE80211_MSG_ASSOC | IEEE80211_MSG_DEBUG, ni,
"station with aid %d leaves", IEEE80211_NODE_AID(ni));
KASSERT(vap->iv_opmode != IEEE80211_M_STA,
("unexpected operating mode %u", vap->iv_opmode));
/*
* If node wasn't previously associated all
* we need to do is reclaim the reference.
*/
/* XXX ibss mode bypasses 11g and notification */
if (ni->ni_associd == 0)
goto done;
/*
* Tell the authenticator the station is leaving.
* Note that we must do this before yanking the
* association id as the authenticator uses the
* associd to locate it's state block.
*/
if (vap->iv_auth->ia_node_leave != NULL)
vap->iv_auth->ia_node_leave(ni);
IEEE80211_LOCK(ic);
IEEE80211_AID_CLR(vap, ni->ni_associd);
ni->ni_associd = 0;
vap->iv_sta_assoc--;
ic->ic_sta_assoc--;
if (IEEE80211_IS_CHAN_HT(ic->ic_bsschan))
ieee80211_ht_node_leave(ni);
if (IEEE80211_IS_CHAN_ANYG(ic->ic_bsschan) &&
IEEE80211_IS_CHAN_FULL(ic->ic_bsschan))
ieee80211_node_leave_11g(ni);
IEEE80211_UNLOCK(ic);
/*
* Cleanup station state. In particular clear various
* state that might otherwise be reused if the node
* is reused before the reference count goes to zero
* (and memory is reclaimed).
*/
ieee80211_sta_leave(ni);
done:
/*
* Remove the node from any table it's recorded in and
* drop the caller's reference. Removal from the table
* is important to insure the node is not reprocessed
* for inactivity.
*/
if (nt != NULL) {
IEEE80211_NODE_LOCK(nt);
node_reclaim(nt, ni);
IEEE80211_NODE_UNLOCK(nt);
} else
ieee80211_free_node(ni);
}
struct rssiinfo {
struct ieee80211vap *vap;
int rssi_samples;
uint32_t rssi_total;
};
static void
get_hostap_rssi(void *arg, struct ieee80211_node *ni)
{
struct rssiinfo *info = arg;
struct ieee80211vap *vap = ni->ni_vap;
int8_t rssi;
if (info->vap != vap)
return;
/* only associated stations */
if (ni->ni_associd == 0)
return;
rssi = vap->iv_ic->ic_node_getrssi(ni);
if (rssi != 0) {
info->rssi_samples++;
info->rssi_total += rssi;
}
}
static void
get_adhoc_rssi(void *arg, struct ieee80211_node *ni)
{
struct rssiinfo *info = arg;
struct ieee80211vap *vap = ni->ni_vap;
int8_t rssi;
if (info->vap != vap)
return;
/* only neighbors */
/* XXX check bssid */
if ((ni->ni_capinfo & IEEE80211_CAPINFO_IBSS) == 0)
return;
rssi = vap->iv_ic->ic_node_getrssi(ni);
if (rssi != 0) {
info->rssi_samples++;
info->rssi_total += rssi;
}
}
#ifdef IEEE80211_SUPPORT_MESH
static void
get_mesh_rssi(void *arg, struct ieee80211_node *ni)
{
struct rssiinfo *info = arg;
struct ieee80211vap *vap = ni->ni_vap;
int8_t rssi;
if (info->vap != vap)
return;
/* only neighbors that peered successfully */
if (ni->ni_mlstate != IEEE80211_NODE_MESH_ESTABLISHED)
return;
rssi = vap->iv_ic->ic_node_getrssi(ni);
if (rssi != 0) {
info->rssi_samples++;
info->rssi_total += rssi;
}
}
#endif /* IEEE80211_SUPPORT_MESH */
int8_t
ieee80211_getrssi(struct ieee80211vap *vap)
{
#define NZ(x) ((x) == 0 ? 1 : (x))
struct ieee80211com *ic = vap->iv_ic;
struct rssiinfo info;
info.rssi_total = 0;
info.rssi_samples = 0;
info.vap = vap;
switch (vap->iv_opmode) {
case IEEE80211_M_IBSS: /* average of all ibss neighbors */
case IEEE80211_M_AHDEMO: /* average of all neighbors */
ieee80211_iterate_nodes(&ic->ic_sta, get_adhoc_rssi, &info);
break;
case IEEE80211_M_HOSTAP: /* average of all associated stations */
ieee80211_iterate_nodes(&ic->ic_sta, get_hostap_rssi, &info);
break;
#ifdef IEEE80211_SUPPORT_MESH
case IEEE80211_M_MBSS: /* average of all mesh neighbors */
ieee80211_iterate_nodes(&ic->ic_sta, get_mesh_rssi, &info);
break;
#endif
case IEEE80211_M_MONITOR: /* XXX */
case IEEE80211_M_STA: /* use stats from associated ap */
default:
if (vap->iv_bss != NULL)
info.rssi_total = ic->ic_node_getrssi(vap->iv_bss);
info.rssi_samples = 1;
break;
}
return info.rssi_total / NZ(info.rssi_samples);
#undef NZ
}
void
ieee80211_getsignal(struct ieee80211vap *vap, int8_t *rssi, int8_t *noise)
{
if (vap->iv_bss == NULL) /* NB: shouldn't happen */
return;
vap->iv_ic->ic_node_getsignal(vap->iv_bss, rssi, noise);
/* for non-station mode return avg'd rssi accounting */
if (vap->iv_opmode != IEEE80211_M_STA)
*rssi = ieee80211_getrssi(vap);
}
Index: head/sys/net80211/ieee80211_proto.c
===================================================================
--- head/sys/net80211/ieee80211_proto.c (revision 283290)
+++ head/sys/net80211/ieee80211_proto.c (revision 283291)
@@ -1,1996 +1,1996 @@
/*-
* Copyright (c) 2001 Atsushi Onoe
* Copyright (c) 2002-2008 Sam Leffler, Errno Consulting
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* IEEE 802.11 protocol support.
*/
#include "opt_inet.h"
#include "opt_wlan.h"
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_media.h>
#include <net/ethernet.h> /* XXX for ether_sprintf */
#include <net80211/ieee80211_var.h>
#include <net80211/ieee80211_adhoc.h>
#include <net80211/ieee80211_sta.h>
#include <net80211/ieee80211_hostap.h>
#include <net80211/ieee80211_wds.h>
#ifdef IEEE80211_SUPPORT_MESH
#include <net80211/ieee80211_mesh.h>
#endif
#include <net80211/ieee80211_monitor.h>
#include <net80211/ieee80211_input.h>
/* XXX tunables */
#define AGGRESSIVE_MODE_SWITCH_HYSTERESIS 3 /* pkts / 100ms */
#define HIGH_PRI_SWITCH_THRESH 10 /* pkts / 100ms */
const char *ieee80211_mgt_subtype_name[] = {
"assoc_req", "assoc_resp", "reassoc_req", "reassoc_resp",
"probe_req", "probe_resp", "reserved#6", "reserved#7",
"beacon", "atim", "disassoc", "auth",
"deauth", "action", "action_noack", "reserved#15"
};
const char *ieee80211_ctl_subtype_name[] = {
"reserved#0", "reserved#1", "reserved#2", "reserved#3",
"reserved#3", "reserved#5", "reserved#6", "reserved#7",
"reserved#8", "reserved#9", "ps_poll", "rts",
"cts", "ack", "cf_end", "cf_end_ack"
};
const char *ieee80211_opmode_name[IEEE80211_OPMODE_MAX] = {
"IBSS", /* IEEE80211_M_IBSS */
"STA", /* IEEE80211_M_STA */
"WDS", /* IEEE80211_M_WDS */
"AHDEMO", /* IEEE80211_M_AHDEMO */
"HOSTAP", /* IEEE80211_M_HOSTAP */
"MONITOR", /* IEEE80211_M_MONITOR */
"MBSS" /* IEEE80211_M_MBSS */
};
const char *ieee80211_state_name[IEEE80211_S_MAX] = {
"INIT", /* IEEE80211_S_INIT */
"SCAN", /* IEEE80211_S_SCAN */
"AUTH", /* IEEE80211_S_AUTH */
"ASSOC", /* IEEE80211_S_ASSOC */
"CAC", /* IEEE80211_S_CAC */
"RUN", /* IEEE80211_S_RUN */
"CSA", /* IEEE80211_S_CSA */
"SLEEP", /* IEEE80211_S_SLEEP */
};
const char *ieee80211_wme_acnames[] = {
"WME_AC_BE",
"WME_AC_BK",
"WME_AC_VI",
"WME_AC_VO",
"WME_UPSD",
};
static void beacon_miss(void *, int);
static void beacon_swmiss(void *, int);
static void parent_updown(void *, int);
static void update_mcast(void *, int);
static void update_promisc(void *, int);
static void update_channel(void *, int);
static void update_chw(void *, int);
static void ieee80211_newstate_cb(void *, int);
static int
null_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
const struct ieee80211_bpf_params *params)
{
struct ifnet *ifp = ni->ni_ic->ic_ifp;
if_printf(ifp, "missing ic_raw_xmit callback, drop frame\n");
m_freem(m);
return ENETDOWN;
}
void
ieee80211_proto_attach(struct ieee80211com *ic)
{
struct ifnet *ifp = ic->ic_ifp;
/* override the 802.3 setting */
ifp->if_hdrlen = ic->ic_headroom
+ sizeof(struct ieee80211_qosframe_addr4)
+ IEEE80211_WEP_IVLEN + IEEE80211_WEP_KIDLEN
+ IEEE80211_WEP_EXTIVLEN;
/* XXX no way to recalculate on ifdetach */
if (ALIGN(ifp->if_hdrlen) > max_linkhdr) {
/* XXX sanity check... */
max_linkhdr = ALIGN(ifp->if_hdrlen);
max_hdr = max_linkhdr + max_protohdr;
max_datalen = MHLEN - max_hdr;
}
ic->ic_protmode = IEEE80211_PROT_CTSONLY;
TASK_INIT(&ic->ic_parent_task, 0, parent_updown, ifp);
TASK_INIT(&ic->ic_mcast_task, 0, update_mcast, ic);
TASK_INIT(&ic->ic_promisc_task, 0, update_promisc, ic);
TASK_INIT(&ic->ic_chan_task, 0, update_channel, ic);
TASK_INIT(&ic->ic_bmiss_task, 0, beacon_miss, ic);
TASK_INIT(&ic->ic_chw_task, 0, update_chw, ic);
ic->ic_wme.wme_hipri_switch_hysteresis =
AGGRESSIVE_MODE_SWITCH_HYSTERESIS;
/* initialize management frame handlers */
ic->ic_send_mgmt = ieee80211_send_mgmt;
ic->ic_raw_xmit = null_raw_xmit;
ieee80211_adhoc_attach(ic);
ieee80211_sta_attach(ic);
ieee80211_wds_attach(ic);
ieee80211_hostap_attach(ic);
#ifdef IEEE80211_SUPPORT_MESH
ieee80211_mesh_attach(ic);
#endif
ieee80211_monitor_attach(ic);
}
void
ieee80211_proto_detach(struct ieee80211com *ic)
{
ieee80211_monitor_detach(ic);
#ifdef IEEE80211_SUPPORT_MESH
ieee80211_mesh_detach(ic);
#endif
ieee80211_hostap_detach(ic);
ieee80211_wds_detach(ic);
ieee80211_adhoc_detach(ic);
ieee80211_sta_detach(ic);
}
static void
null_update_beacon(struct ieee80211vap *vap, int item)
{
}
void
ieee80211_proto_vattach(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
struct ifnet *ifp = vap->iv_ifp;
int i;
/* override the 802.3 setting */
ifp->if_hdrlen = ic->ic_ifp->if_hdrlen;
vap->iv_rtsthreshold = IEEE80211_RTS_DEFAULT;
vap->iv_fragthreshold = IEEE80211_FRAG_DEFAULT;
vap->iv_bmiss_max = IEEE80211_BMISS_MAX;
callout_init_mtx(&vap->iv_swbmiss, IEEE80211_LOCK_OBJ(ic), 0);
- callout_init(&vap->iv_mgtsend, CALLOUT_MPSAFE);
+ callout_init(&vap->iv_mgtsend, 1);
TASK_INIT(&vap->iv_nstate_task, 0, ieee80211_newstate_cb, vap);
TASK_INIT(&vap->iv_swbmiss_task, 0, beacon_swmiss, vap);
/*
* Install default tx rate handling: no fixed rate, lowest
* supported rate for mgmt and multicast frames. Default
* max retry count. These settings can be changed by the
* driver and/or user applications.
*/
for (i = IEEE80211_MODE_11A; i < IEEE80211_MODE_MAX; i++) {
const struct ieee80211_rateset *rs = &ic->ic_sup_rates[i];
vap->iv_txparms[i].ucastrate = IEEE80211_FIXED_RATE_NONE;
/*
* Setting the management rate to MCS 0 assumes that the
* BSS Basic rate set is empty and the BSS Basic MCS set
* is not.
*
* Since we're not checking this, default to the lowest
* defined rate for this mode.
*
* At least one 11n AP (DLINK DIR-825) is reported to drop
* some MCS management traffic (eg BA response frames.)
*
* See also: 9.6.0 of the 802.11n-2009 specification.
*/
#ifdef NOTYET
if (i == IEEE80211_MODE_11NA || i == IEEE80211_MODE_11NG) {
vap->iv_txparms[i].mgmtrate = 0 | IEEE80211_RATE_MCS;
vap->iv_txparms[i].mcastrate = 0 | IEEE80211_RATE_MCS;
} else {
vap->iv_txparms[i].mgmtrate =
rs->rs_rates[0] & IEEE80211_RATE_VAL;
vap->iv_txparms[i].mcastrate =
rs->rs_rates[0] & IEEE80211_RATE_VAL;
}
#endif
vap->iv_txparms[i].mgmtrate = rs->rs_rates[0] & IEEE80211_RATE_VAL;
vap->iv_txparms[i].mcastrate = rs->rs_rates[0] & IEEE80211_RATE_VAL;
vap->iv_txparms[i].maxretry = IEEE80211_TXMAX_DEFAULT;
}
vap->iv_roaming = IEEE80211_ROAMING_AUTO;
vap->iv_update_beacon = null_update_beacon;
vap->iv_deliver_data = ieee80211_deliver_data;
/* attach support for operating mode */
ic->ic_vattach[vap->iv_opmode](vap);
}
void
ieee80211_proto_vdetach(struct ieee80211vap *vap)
{
#define FREEAPPIE(ie) do { \
if (ie != NULL) \
free(ie, M_80211_NODE_IE); \
} while (0)
/*
* Detach operating mode module.
*/
if (vap->iv_opdetach != NULL)
vap->iv_opdetach(vap);
/*
* This should not be needed as we detach when reseting
* the state but be conservative here since the
* authenticator may do things like spawn kernel threads.
*/
if (vap->iv_auth->ia_detach != NULL)
vap->iv_auth->ia_detach(vap);
/*
* Detach any ACL'ator.
*/
if (vap->iv_acl != NULL)
vap->iv_acl->iac_detach(vap);
FREEAPPIE(vap->iv_appie_beacon);
FREEAPPIE(vap->iv_appie_probereq);
FREEAPPIE(vap->iv_appie_proberesp);
FREEAPPIE(vap->iv_appie_assocreq);
FREEAPPIE(vap->iv_appie_assocresp);
FREEAPPIE(vap->iv_appie_wpa);
#undef FREEAPPIE
}
/*
* Simple-minded authenticator module support.
*/
#define IEEE80211_AUTH_MAX (IEEE80211_AUTH_WPA+1)
/* XXX well-known names */
static const char *auth_modnames[IEEE80211_AUTH_MAX] = {
"wlan_internal", /* IEEE80211_AUTH_NONE */
"wlan_internal", /* IEEE80211_AUTH_OPEN */
"wlan_internal", /* IEEE80211_AUTH_SHARED */
"wlan_xauth", /* IEEE80211_AUTH_8021X */
"wlan_internal", /* IEEE80211_AUTH_AUTO */
"wlan_xauth", /* IEEE80211_AUTH_WPA */
};
static const struct ieee80211_authenticator *authenticators[IEEE80211_AUTH_MAX];
static const struct ieee80211_authenticator auth_internal = {
.ia_name = "wlan_internal",
.ia_attach = NULL,
.ia_detach = NULL,
.ia_node_join = NULL,
.ia_node_leave = NULL,
};
/*
* Setup internal authenticators once; they are never unregistered.
*/
static void
ieee80211_auth_setup(void)
{
ieee80211_authenticator_register(IEEE80211_AUTH_OPEN, &auth_internal);
ieee80211_authenticator_register(IEEE80211_AUTH_SHARED, &auth_internal);
ieee80211_authenticator_register(IEEE80211_AUTH_AUTO, &auth_internal);
}
SYSINIT(wlan_auth, SI_SUB_DRIVERS, SI_ORDER_FIRST, ieee80211_auth_setup, NULL);
const struct ieee80211_authenticator *
ieee80211_authenticator_get(int auth)
{
if (auth >= IEEE80211_AUTH_MAX)
return NULL;
if (authenticators[auth] == NULL)
ieee80211_load_module(auth_modnames[auth]);
return authenticators[auth];
}
void
ieee80211_authenticator_register(int type,
const struct ieee80211_authenticator *auth)
{
if (type >= IEEE80211_AUTH_MAX)
return;
authenticators[type] = auth;
}
void
ieee80211_authenticator_unregister(int type)
{
if (type >= IEEE80211_AUTH_MAX)
return;
authenticators[type] = NULL;
}
/*
* Very simple-minded ACL module support.
*/
/* XXX just one for now */
static const struct ieee80211_aclator *acl = NULL;
void
ieee80211_aclator_register(const struct ieee80211_aclator *iac)
{
printf("wlan: %s acl policy registered\n", iac->iac_name);
acl = iac;
}
void
ieee80211_aclator_unregister(const struct ieee80211_aclator *iac)
{
if (acl == iac)
acl = NULL;
printf("wlan: %s acl policy unregistered\n", iac->iac_name);
}
const struct ieee80211_aclator *
ieee80211_aclator_get(const char *name)
{
if (acl == NULL)
ieee80211_load_module("wlan_acl");
return acl != NULL && strcmp(acl->iac_name, name) == 0 ? acl : NULL;
}
void
ieee80211_print_essid(const uint8_t *essid, int len)
{
const uint8_t *p;
int i;
if (len > IEEE80211_NWID_LEN)
len = IEEE80211_NWID_LEN;
/* determine printable or not */
for (i = 0, p = essid; i < len; i++, p++) {
if (*p < ' ' || *p > 0x7e)
break;
}
if (i == len) {
printf("\"");
for (i = 0, p = essid; i < len; i++, p++)
printf("%c", *p);
printf("\"");
} else {
printf("0x");
for (i = 0, p = essid; i < len; i++, p++)
printf("%02x", *p);
}
}
void
ieee80211_dump_pkt(struct ieee80211com *ic,
const uint8_t *buf, int len, int rate, int rssi)
{
const struct ieee80211_frame *wh;
int i;
wh = (const struct ieee80211_frame *)buf;
switch (wh->i_fc[1] & IEEE80211_FC1_DIR_MASK) {
case IEEE80211_FC1_DIR_NODS:
printf("NODS %s", ether_sprintf(wh->i_addr2));
printf("->%s", ether_sprintf(wh->i_addr1));
printf("(%s)", ether_sprintf(wh->i_addr3));
break;
case IEEE80211_FC1_DIR_TODS:
printf("TODS %s", ether_sprintf(wh->i_addr2));
printf("->%s", ether_sprintf(wh->i_addr3));
printf("(%s)", ether_sprintf(wh->i_addr1));
break;
case IEEE80211_FC1_DIR_FROMDS:
printf("FRDS %s", ether_sprintf(wh->i_addr3));
printf("->%s", ether_sprintf(wh->i_addr1));
printf("(%s)", ether_sprintf(wh->i_addr2));
break;
case IEEE80211_FC1_DIR_DSTODS:
printf("DSDS %s", ether_sprintf((const uint8_t *)&wh[1]));
printf("->%s", ether_sprintf(wh->i_addr3));
printf("(%s", ether_sprintf(wh->i_addr2));
printf("->%s)", ether_sprintf(wh->i_addr1));
break;
}
switch (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) {
case IEEE80211_FC0_TYPE_DATA:
printf(" data");
break;
case IEEE80211_FC0_TYPE_MGT:
printf(" %s", ieee80211_mgt_subtype_name[
(wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK)
>> IEEE80211_FC0_SUBTYPE_SHIFT]);
break;
default:
printf(" type#%d", wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK);
break;
}
if (IEEE80211_QOS_HAS_SEQ(wh)) {
const struct ieee80211_qosframe *qwh =
(const struct ieee80211_qosframe *)buf;
printf(" QoS [TID %u%s]", qwh->i_qos[0] & IEEE80211_QOS_TID,
qwh->i_qos[0] & IEEE80211_QOS_ACKPOLICY ? " ACM" : "");
}
if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
int off;
off = ieee80211_anyhdrspace(ic, wh);
printf(" WEP [IV %.02x %.02x %.02x",
buf[off+0], buf[off+1], buf[off+2]);
if (buf[off+IEEE80211_WEP_IVLEN] & IEEE80211_WEP_EXTIV)
printf(" %.02x %.02x %.02x",
buf[off+4], buf[off+5], buf[off+6]);
printf(" KID %u]", buf[off+IEEE80211_WEP_IVLEN] >> 6);
}
if (rate >= 0)
printf(" %dM", rate / 2);
if (rssi >= 0)
printf(" +%d", rssi);
printf("\n");
if (len > 0) {
for (i = 0; i < len; i++) {
if ((i & 1) == 0)
printf(" ");
printf("%02x", buf[i]);
}
printf("\n");
}
}
static __inline int
findrix(const struct ieee80211_rateset *rs, int r)
{
int i;
for (i = 0; i < rs->rs_nrates; i++)
if ((rs->rs_rates[i] & IEEE80211_RATE_VAL) == r)
return i;
return -1;
}
int
ieee80211_fix_rate(struct ieee80211_node *ni,
struct ieee80211_rateset *nrs, int flags)
{
#define RV(v) ((v) & IEEE80211_RATE_VAL)
struct ieee80211vap *vap = ni->ni_vap;
struct ieee80211com *ic = ni->ni_ic;
int i, j, rix, error;
int okrate, badrate, fixedrate, ucastrate;
const struct ieee80211_rateset *srs;
uint8_t r;
error = 0;
okrate = badrate = 0;
ucastrate = vap->iv_txparms[ieee80211_chan2mode(ni->ni_chan)].ucastrate;
if (ucastrate != IEEE80211_FIXED_RATE_NONE) {
/*
* Workaround awkwardness with fixed rate. We are called
* to check both the legacy rate set and the HT rate set
* but we must apply any legacy fixed rate check only to the
* legacy rate set and vice versa. We cannot tell what type
* of rate set we've been given (legacy or HT) but we can
* distinguish the fixed rate type (MCS have 0x80 set).
* So to deal with this the caller communicates whether to
* check MCS or legacy rate using the flags and we use the
* type of any fixed rate to avoid applying an MCS to a
* legacy rate and vice versa.
*/
if (ucastrate & 0x80) {
if (flags & IEEE80211_F_DOFRATE)
flags &= ~IEEE80211_F_DOFRATE;
} else if ((ucastrate & 0x80) == 0) {
if (flags & IEEE80211_F_DOFMCS)
flags &= ~IEEE80211_F_DOFMCS;
}
/* NB: required to make MCS match below work */
ucastrate &= IEEE80211_RATE_VAL;
}
fixedrate = IEEE80211_FIXED_RATE_NONE;
/*
* XXX we are called to process both MCS and legacy rates;
* we must use the appropriate basic rate set or chaos will
* ensue; for now callers that want MCS must supply
* IEEE80211_F_DOBRS; at some point we'll need to split this
* function so there are two variants, one for MCS and one
* for legacy rates.
*/
if (flags & IEEE80211_F_DOBRS)
srs = (const struct ieee80211_rateset *)
ieee80211_get_suphtrates(ic, ni->ni_chan);
else
srs = ieee80211_get_suprates(ic, ni->ni_chan);
for (i = 0; i < nrs->rs_nrates; ) {
if (flags & IEEE80211_F_DOSORT) {
/*
* Sort rates.
*/
for (j = i + 1; j < nrs->rs_nrates; j++) {
if (RV(nrs->rs_rates[i]) > RV(nrs->rs_rates[j])) {
r = nrs->rs_rates[i];
nrs->rs_rates[i] = nrs->rs_rates[j];
nrs->rs_rates[j] = r;
}
}
}
r = nrs->rs_rates[i] & IEEE80211_RATE_VAL;
badrate = r;
/*
* Check for fixed rate.
*/
if (r == ucastrate)
fixedrate = r;
/*
* Check against supported rates.
*/
rix = findrix(srs, r);
if (flags & IEEE80211_F_DONEGO) {
if (rix < 0) {
/*
* A rate in the node's rate set is not
* supported. If this is a basic rate and we
* are operating as a STA then this is an error.
* Otherwise we just discard/ignore the rate.
*/
if ((flags & IEEE80211_F_JOIN) &&
(nrs->rs_rates[i] & IEEE80211_RATE_BASIC))
error++;
} else if ((flags & IEEE80211_F_JOIN) == 0) {
/*
* Overwrite with the supported rate
* value so any basic rate bit is set.
*/
nrs->rs_rates[i] = srs->rs_rates[rix];
}
}
if ((flags & IEEE80211_F_DODEL) && rix < 0) {
/*
* Delete unacceptable rates.
*/
nrs->rs_nrates--;
for (j = i; j < nrs->rs_nrates; j++)
nrs->rs_rates[j] = nrs->rs_rates[j + 1];
nrs->rs_rates[j] = 0;
continue;
}
if (rix >= 0)
okrate = nrs->rs_rates[i];
i++;
}
if (okrate == 0 || error != 0 ||
((flags & (IEEE80211_F_DOFRATE|IEEE80211_F_DOFMCS)) &&
fixedrate != ucastrate)) {
IEEE80211_NOTE(vap, IEEE80211_MSG_XRATE | IEEE80211_MSG_11N, ni,
"%s: flags 0x%x okrate %d error %d fixedrate 0x%x "
"ucastrate %x\n", __func__, fixedrate, ucastrate, flags);
return badrate | IEEE80211_RATE_BASIC;
} else
return RV(okrate);
#undef RV
}
/*
* Reset 11g-related state.
*/
void
ieee80211_reset_erp(struct ieee80211com *ic)
{
ic->ic_flags &= ~IEEE80211_F_USEPROT;
ic->ic_nonerpsta = 0;
ic->ic_longslotsta = 0;
/*
* Short slot time is enabled only when operating in 11g
* and not in an IBSS. We must also honor whether or not
* the driver is capable of doing it.
*/
ieee80211_set_shortslottime(ic,
IEEE80211_IS_CHAN_A(ic->ic_curchan) ||
IEEE80211_IS_CHAN_HT(ic->ic_curchan) ||
(IEEE80211_IS_CHAN_ANYG(ic->ic_curchan) &&
ic->ic_opmode == IEEE80211_M_HOSTAP &&
(ic->ic_caps & IEEE80211_C_SHSLOT)));
/*
* Set short preamble and ERP barker-preamble flags.
*/
if (IEEE80211_IS_CHAN_A(ic->ic_curchan) ||
(ic->ic_caps & IEEE80211_C_SHPREAMBLE)) {
ic->ic_flags |= IEEE80211_F_SHPREAMBLE;
ic->ic_flags &= ~IEEE80211_F_USEBARKER;
} else {
ic->ic_flags &= ~IEEE80211_F_SHPREAMBLE;
ic->ic_flags |= IEEE80211_F_USEBARKER;
}
}
/*
* Set the short slot time state and notify the driver.
*/
void
ieee80211_set_shortslottime(struct ieee80211com *ic, int onoff)
{
if (onoff)
ic->ic_flags |= IEEE80211_F_SHSLOT;
else
ic->ic_flags &= ~IEEE80211_F_SHSLOT;
/* notify driver */
if (ic->ic_updateslot != NULL)
ic->ic_updateslot(ic->ic_ifp);
}
/*
* Check if the specified rate set supports ERP.
* NB: the rate set is assumed to be sorted.
*/
int
ieee80211_iserp_rateset(const struct ieee80211_rateset *rs)
{
static const int rates[] = { 2, 4, 11, 22, 12, 24, 48 };
int i, j;
if (rs->rs_nrates < nitems(rates))
return 0;
for (i = 0; i < nitems(rates); i++) {
for (j = 0; j < rs->rs_nrates; j++) {
int r = rs->rs_rates[j] & IEEE80211_RATE_VAL;
if (rates[i] == r)
goto next;
if (r > rates[i])
return 0;
}
return 0;
next:
;
}
return 1;
}
/*
* Mark the basic rates for the rate table based on the
* operating mode. For real 11g we mark all the 11b rates
* and 6, 12, and 24 OFDM. For 11b compatibility we mark only
* 11b rates. There's also a pseudo 11a-mode used to mark only
* the basic OFDM rates.
*/
static void
setbasicrates(struct ieee80211_rateset *rs,
enum ieee80211_phymode mode, int add)
{
static const struct ieee80211_rateset basic[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_11A] = { 3, { 12, 24, 48 } },
[IEEE80211_MODE_11B] = { 2, { 2, 4 } },
/* NB: mixed b/g */
[IEEE80211_MODE_11G] = { 4, { 2, 4, 11, 22 } },
[IEEE80211_MODE_TURBO_A] = { 3, { 12, 24, 48 } },
[IEEE80211_MODE_TURBO_G] = { 4, { 2, 4, 11, 22 } },
[IEEE80211_MODE_STURBO_A] = { 3, { 12, 24, 48 } },
[IEEE80211_MODE_HALF] = { 3, { 6, 12, 24 } },
[IEEE80211_MODE_QUARTER] = { 3, { 3, 6, 12 } },
[IEEE80211_MODE_11NA] = { 3, { 12, 24, 48 } },
/* NB: mixed b/g */
[IEEE80211_MODE_11NG] = { 4, { 2, 4, 11, 22 } },
};
int i, j;
for (i = 0; i < rs->rs_nrates; i++) {
if (!add)
rs->rs_rates[i] &= IEEE80211_RATE_VAL;
for (j = 0; j < basic[mode].rs_nrates; j++)
if (basic[mode].rs_rates[j] == rs->rs_rates[i]) {
rs->rs_rates[i] |= IEEE80211_RATE_BASIC;
break;
}
}
}
/*
* Set the basic rates in a rate set.
*/
void
ieee80211_setbasicrates(struct ieee80211_rateset *rs,
enum ieee80211_phymode mode)
{
setbasicrates(rs, mode, 0);
}
/*
* Add basic rates to a rate set.
*/
void
ieee80211_addbasicrates(struct ieee80211_rateset *rs,
enum ieee80211_phymode mode)
{
setbasicrates(rs, mode, 1);
}
/*
* WME protocol support.
*
* The default 11a/b/g/n parameters come from the WiFi Alliance WMM
* System Interopability Test Plan (v1.4, Appendix F) and the 802.11n
* Draft 2.0 Test Plan (Appendix D).
*
* Static/Dynamic Turbo mode settings come from Atheros.
*/
typedef struct phyParamType {
uint8_t aifsn;
uint8_t logcwmin;
uint8_t logcwmax;
uint16_t txopLimit;
uint8_t acm;
} paramType;
static const struct phyParamType phyParamForAC_BE[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_AUTO] = { 3, 4, 6, 0, 0 },
[IEEE80211_MODE_11A] = { 3, 4, 6, 0, 0 },
[IEEE80211_MODE_11B] = { 3, 4, 6, 0, 0 },
[IEEE80211_MODE_11G] = { 3, 4, 6, 0, 0 },
[IEEE80211_MODE_FH] = { 3, 4, 6, 0, 0 },
[IEEE80211_MODE_TURBO_A]= { 2, 3, 5, 0, 0 },
[IEEE80211_MODE_TURBO_G]= { 2, 3, 5, 0, 0 },
[IEEE80211_MODE_STURBO_A]={ 2, 3, 5, 0, 0 },
[IEEE80211_MODE_HALF] = { 3, 4, 6, 0, 0 },
[IEEE80211_MODE_QUARTER]= { 3, 4, 6, 0, 0 },
[IEEE80211_MODE_11NA] = { 3, 4, 6, 0, 0 },
[IEEE80211_MODE_11NG] = { 3, 4, 6, 0, 0 },
};
static const struct phyParamType phyParamForAC_BK[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_AUTO] = { 7, 4, 10, 0, 0 },
[IEEE80211_MODE_11A] = { 7, 4, 10, 0, 0 },
[IEEE80211_MODE_11B] = { 7, 4, 10, 0, 0 },
[IEEE80211_MODE_11G] = { 7, 4, 10, 0, 0 },
[IEEE80211_MODE_FH] = { 7, 4, 10, 0, 0 },
[IEEE80211_MODE_TURBO_A]= { 7, 3, 10, 0, 0 },
[IEEE80211_MODE_TURBO_G]= { 7, 3, 10, 0, 0 },
[IEEE80211_MODE_STURBO_A]={ 7, 3, 10, 0, 0 },
[IEEE80211_MODE_HALF] = { 7, 4, 10, 0, 0 },
[IEEE80211_MODE_QUARTER]= { 7, 4, 10, 0, 0 },
[IEEE80211_MODE_11NA] = { 7, 4, 10, 0, 0 },
[IEEE80211_MODE_11NG] = { 7, 4, 10, 0, 0 },
};
static const struct phyParamType phyParamForAC_VI[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_AUTO] = { 1, 3, 4, 94, 0 },
[IEEE80211_MODE_11A] = { 1, 3, 4, 94, 0 },
[IEEE80211_MODE_11B] = { 1, 3, 4, 188, 0 },
[IEEE80211_MODE_11G] = { 1, 3, 4, 94, 0 },
[IEEE80211_MODE_FH] = { 1, 3, 4, 188, 0 },
[IEEE80211_MODE_TURBO_A]= { 1, 2, 3, 94, 0 },
[IEEE80211_MODE_TURBO_G]= { 1, 2, 3, 94, 0 },
[IEEE80211_MODE_STURBO_A]={ 1, 2, 3, 94, 0 },
[IEEE80211_MODE_HALF] = { 1, 3, 4, 94, 0 },
[IEEE80211_MODE_QUARTER]= { 1, 3, 4, 94, 0 },
[IEEE80211_MODE_11NA] = { 1, 3, 4, 94, 0 },
[IEEE80211_MODE_11NG] = { 1, 3, 4, 94, 0 },
};
static const struct phyParamType phyParamForAC_VO[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_AUTO] = { 1, 2, 3, 47, 0 },
[IEEE80211_MODE_11A] = { 1, 2, 3, 47, 0 },
[IEEE80211_MODE_11B] = { 1, 2, 3, 102, 0 },
[IEEE80211_MODE_11G] = { 1, 2, 3, 47, 0 },
[IEEE80211_MODE_FH] = { 1, 2, 3, 102, 0 },
[IEEE80211_MODE_TURBO_A]= { 1, 2, 2, 47, 0 },
[IEEE80211_MODE_TURBO_G]= { 1, 2, 2, 47, 0 },
[IEEE80211_MODE_STURBO_A]={ 1, 2, 2, 47, 0 },
[IEEE80211_MODE_HALF] = { 1, 2, 3, 47, 0 },
[IEEE80211_MODE_QUARTER]= { 1, 2, 3, 47, 0 },
[IEEE80211_MODE_11NA] = { 1, 2, 3, 47, 0 },
[IEEE80211_MODE_11NG] = { 1, 2, 3, 47, 0 },
};
static const struct phyParamType bssPhyParamForAC_BE[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_AUTO] = { 3, 4, 10, 0, 0 },
[IEEE80211_MODE_11A] = { 3, 4, 10, 0, 0 },
[IEEE80211_MODE_11B] = { 3, 4, 10, 0, 0 },
[IEEE80211_MODE_11G] = { 3, 4, 10, 0, 0 },
[IEEE80211_MODE_FH] = { 3, 4, 10, 0, 0 },
[IEEE80211_MODE_TURBO_A]= { 2, 3, 10, 0, 0 },
[IEEE80211_MODE_TURBO_G]= { 2, 3, 10, 0, 0 },
[IEEE80211_MODE_STURBO_A]={ 2, 3, 10, 0, 0 },
[IEEE80211_MODE_HALF] = { 3, 4, 10, 0, 0 },
[IEEE80211_MODE_QUARTER]= { 3, 4, 10, 0, 0 },
[IEEE80211_MODE_11NA] = { 3, 4, 10, 0, 0 },
[IEEE80211_MODE_11NG] = { 3, 4, 10, 0, 0 },
};
static const struct phyParamType bssPhyParamForAC_VI[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_AUTO] = { 2, 3, 4, 94, 0 },
[IEEE80211_MODE_11A] = { 2, 3, 4, 94, 0 },
[IEEE80211_MODE_11B] = { 2, 3, 4, 188, 0 },
[IEEE80211_MODE_11G] = { 2, 3, 4, 94, 0 },
[IEEE80211_MODE_FH] = { 2, 3, 4, 188, 0 },
[IEEE80211_MODE_TURBO_A]= { 2, 2, 3, 94, 0 },
[IEEE80211_MODE_TURBO_G]= { 2, 2, 3, 94, 0 },
[IEEE80211_MODE_STURBO_A]={ 2, 2, 3, 94, 0 },
[IEEE80211_MODE_HALF] = { 2, 3, 4, 94, 0 },
[IEEE80211_MODE_QUARTER]= { 2, 3, 4, 94, 0 },
[IEEE80211_MODE_11NA] = { 2, 3, 4, 94, 0 },
[IEEE80211_MODE_11NG] = { 2, 3, 4, 94, 0 },
};
static const struct phyParamType bssPhyParamForAC_VO[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_AUTO] = { 2, 2, 3, 47, 0 },
[IEEE80211_MODE_11A] = { 2, 2, 3, 47, 0 },
[IEEE80211_MODE_11B] = { 2, 2, 3, 102, 0 },
[IEEE80211_MODE_11G] = { 2, 2, 3, 47, 0 },
[IEEE80211_MODE_FH] = { 2, 2, 3, 102, 0 },
[IEEE80211_MODE_TURBO_A]= { 1, 2, 2, 47, 0 },
[IEEE80211_MODE_TURBO_G]= { 1, 2, 2, 47, 0 },
[IEEE80211_MODE_STURBO_A]={ 1, 2, 2, 47, 0 },
[IEEE80211_MODE_HALF] = { 2, 2, 3, 47, 0 },
[IEEE80211_MODE_QUARTER]= { 2, 2, 3, 47, 0 },
[IEEE80211_MODE_11NA] = { 2, 2, 3, 47, 0 },
[IEEE80211_MODE_11NG] = { 2, 2, 3, 47, 0 },
};
static void
_setifsparams(struct wmeParams *wmep, const paramType *phy)
{
wmep->wmep_aifsn = phy->aifsn;
wmep->wmep_logcwmin = phy->logcwmin;
wmep->wmep_logcwmax = phy->logcwmax;
wmep->wmep_txopLimit = phy->txopLimit;
}
static void
setwmeparams(struct ieee80211vap *vap, const char *type, int ac,
struct wmeParams *wmep, const paramType *phy)
{
wmep->wmep_acm = phy->acm;
_setifsparams(wmep, phy);
IEEE80211_DPRINTF(vap, IEEE80211_MSG_WME,
"set %s (%s) [acm %u aifsn %u logcwmin %u logcwmax %u txop %u]\n",
ieee80211_wme_acnames[ac], type,
wmep->wmep_acm, wmep->wmep_aifsn, wmep->wmep_logcwmin,
wmep->wmep_logcwmax, wmep->wmep_txopLimit);
}
static void
ieee80211_wme_initparams_locked(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_wme_state *wme = &ic->ic_wme;
const paramType *pPhyParam, *pBssPhyParam;
struct wmeParams *wmep;
enum ieee80211_phymode mode;
int i;
IEEE80211_LOCK_ASSERT(ic);
if ((ic->ic_caps & IEEE80211_C_WME) == 0 || ic->ic_nrunning > 1)
return;
/*
* Clear the wme cap_info field so a qoscount from a previous
* vap doesn't confuse later code which only parses the beacon
* field and updates hardware when said field changes.
* Otherwise the hardware is programmed with defaults, not what
* the beacon actually announces.
*/
wme->wme_wmeChanParams.cap_info = 0;
/*
* Select mode; we can be called early in which case we
* always use auto mode. We know we'll be called when
* entering the RUN state with bsschan setup properly
* so state will eventually get set correctly
*/
if (ic->ic_bsschan != IEEE80211_CHAN_ANYC)
mode = ieee80211_chan2mode(ic->ic_bsschan);
else
mode = IEEE80211_MODE_AUTO;
for (i = 0; i < WME_NUM_AC; i++) {
switch (i) {
case WME_AC_BK:
pPhyParam = &phyParamForAC_BK[mode];
pBssPhyParam = &phyParamForAC_BK[mode];
break;
case WME_AC_VI:
pPhyParam = &phyParamForAC_VI[mode];
pBssPhyParam = &bssPhyParamForAC_VI[mode];
break;
case WME_AC_VO:
pPhyParam = &phyParamForAC_VO[mode];
pBssPhyParam = &bssPhyParamForAC_VO[mode];
break;
case WME_AC_BE:
default:
pPhyParam = &phyParamForAC_BE[mode];
pBssPhyParam = &bssPhyParamForAC_BE[mode];
break;
}
wmep = &wme->wme_wmeChanParams.cap_wmeParams[i];
if (ic->ic_opmode == IEEE80211_M_HOSTAP) {
setwmeparams(vap, "chan", i, wmep, pPhyParam);
} else {
setwmeparams(vap, "chan", i, wmep, pBssPhyParam);
}
wmep = &wme->wme_wmeBssChanParams.cap_wmeParams[i];
setwmeparams(vap, "bss ", i, wmep, pBssPhyParam);
}
/* NB: check ic_bss to avoid NULL deref on initial attach */
if (vap->iv_bss != NULL) {
/*
* Calculate agressive mode switching threshold based
* on beacon interval. This doesn't need locking since
* we're only called before entering the RUN state at
* which point we start sending beacon frames.
*/
wme->wme_hipri_switch_thresh =
(HIGH_PRI_SWITCH_THRESH * vap->iv_bss->ni_intval) / 100;
wme->wme_flags &= ~WME_F_AGGRMODE;
ieee80211_wme_updateparams(vap);
}
}
void
ieee80211_wme_initparams(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
IEEE80211_LOCK(ic);
ieee80211_wme_initparams_locked(vap);
IEEE80211_UNLOCK(ic);
}
/*
* Update WME parameters for ourself and the BSS.
*/
void
ieee80211_wme_updateparams_locked(struct ieee80211vap *vap)
{
static const paramType aggrParam[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_AUTO] = { 2, 4, 10, 64, 0 },
[IEEE80211_MODE_11A] = { 2, 4, 10, 64, 0 },
[IEEE80211_MODE_11B] = { 2, 5, 10, 64, 0 },
[IEEE80211_MODE_11G] = { 2, 4, 10, 64, 0 },
[IEEE80211_MODE_FH] = { 2, 5, 10, 64, 0 },
[IEEE80211_MODE_TURBO_A] = { 1, 3, 10, 64, 0 },
[IEEE80211_MODE_TURBO_G] = { 1, 3, 10, 64, 0 },
[IEEE80211_MODE_STURBO_A] = { 1, 3, 10, 64, 0 },
[IEEE80211_MODE_HALF] = { 2, 4, 10, 64, 0 },
[IEEE80211_MODE_QUARTER] = { 2, 4, 10, 64, 0 },
[IEEE80211_MODE_11NA] = { 2, 4, 10, 64, 0 }, /* XXXcheck*/
[IEEE80211_MODE_11NG] = { 2, 4, 10, 64, 0 }, /* XXXcheck*/
};
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211_wme_state *wme = &ic->ic_wme;
const struct wmeParams *wmep;
struct wmeParams *chanp, *bssp;
enum ieee80211_phymode mode;
int i;
int do_aggrmode = 0;
/*
* Set up the channel access parameters for the physical
* device. First populate the configured settings.
*/
for (i = 0; i < WME_NUM_AC; i++) {
chanp = &wme->wme_chanParams.cap_wmeParams[i];
wmep = &wme->wme_wmeChanParams.cap_wmeParams[i];
chanp->wmep_aifsn = wmep->wmep_aifsn;
chanp->wmep_logcwmin = wmep->wmep_logcwmin;
chanp->wmep_logcwmax = wmep->wmep_logcwmax;
chanp->wmep_txopLimit = wmep->wmep_txopLimit;
chanp = &wme->wme_bssChanParams.cap_wmeParams[i];
wmep = &wme->wme_wmeBssChanParams.cap_wmeParams[i];
chanp->wmep_aifsn = wmep->wmep_aifsn;
chanp->wmep_logcwmin = wmep->wmep_logcwmin;
chanp->wmep_logcwmax = wmep->wmep_logcwmax;
chanp->wmep_txopLimit = wmep->wmep_txopLimit;
}
/*
* Select mode; we can be called early in which case we
* always use auto mode. We know we'll be called when
* entering the RUN state with bsschan setup properly
* so state will eventually get set correctly
*/
if (ic->ic_bsschan != IEEE80211_CHAN_ANYC)
mode = ieee80211_chan2mode(ic->ic_bsschan);
else
mode = IEEE80211_MODE_AUTO;
/*
* This implements agressive mode as found in certain
* vendors' AP's. When there is significant high
* priority (VI/VO) traffic in the BSS throttle back BE
* traffic by using conservative parameters. Otherwise
* BE uses agressive params to optimize performance of
* legacy/non-QoS traffic.
*/
/* Hostap? Only if aggressive mode is enabled */
if (vap->iv_opmode == IEEE80211_M_HOSTAP &&
(wme->wme_flags & WME_F_AGGRMODE) != 0)
do_aggrmode = 1;
/*
* Station? Only if we're in a non-QoS BSS.
*/
else if ((vap->iv_opmode == IEEE80211_M_STA &&
(vap->iv_bss->ni_flags & IEEE80211_NODE_QOS) == 0))
do_aggrmode = 1;
/*
* IBSS? Only if we we have WME enabled.
*/
else if ((vap->iv_opmode == IEEE80211_M_IBSS) &&
(vap->iv_flags & IEEE80211_F_WME))
do_aggrmode = 1;
/*
* If WME is disabled on this VAP, default to aggressive mode
* regardless of the configuration.
*/
if ((vap->iv_flags & IEEE80211_F_WME) == 0)
do_aggrmode = 1;
/* XXX WDS? */
/* XXX MBSS? */
if (do_aggrmode) {
chanp = &wme->wme_chanParams.cap_wmeParams[WME_AC_BE];
bssp = &wme->wme_bssChanParams.cap_wmeParams[WME_AC_BE];
chanp->wmep_aifsn = bssp->wmep_aifsn = aggrParam[mode].aifsn;
chanp->wmep_logcwmin = bssp->wmep_logcwmin =
aggrParam[mode].logcwmin;
chanp->wmep_logcwmax = bssp->wmep_logcwmax =
aggrParam[mode].logcwmax;
chanp->wmep_txopLimit = bssp->wmep_txopLimit =
(vap->iv_flags & IEEE80211_F_BURST) ?
aggrParam[mode].txopLimit : 0;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_WME,
"update %s (chan+bss) [acm %u aifsn %u logcwmin %u "
"logcwmax %u txop %u]\n", ieee80211_wme_acnames[WME_AC_BE],
chanp->wmep_acm, chanp->wmep_aifsn, chanp->wmep_logcwmin,
chanp->wmep_logcwmax, chanp->wmep_txopLimit);
}
/*
* Change the contention window based on the number of associated
* stations. If the number of associated stations is 1 and
* aggressive mode is enabled, lower the contention window even
* further.
*/
if (vap->iv_opmode == IEEE80211_M_HOSTAP &&
ic->ic_sta_assoc < 2 && (wme->wme_flags & WME_F_AGGRMODE) != 0) {
static const uint8_t logCwMin[IEEE80211_MODE_MAX] = {
[IEEE80211_MODE_AUTO] = 3,
[IEEE80211_MODE_11A] = 3,
[IEEE80211_MODE_11B] = 4,
[IEEE80211_MODE_11G] = 3,
[IEEE80211_MODE_FH] = 4,
[IEEE80211_MODE_TURBO_A] = 3,
[IEEE80211_MODE_TURBO_G] = 3,
[IEEE80211_MODE_STURBO_A] = 3,
[IEEE80211_MODE_HALF] = 3,
[IEEE80211_MODE_QUARTER] = 3,
[IEEE80211_MODE_11NA] = 3,
[IEEE80211_MODE_11NG] = 3,
};
chanp = &wme->wme_chanParams.cap_wmeParams[WME_AC_BE];
bssp = &wme->wme_bssChanParams.cap_wmeParams[WME_AC_BE];
chanp->wmep_logcwmin = bssp->wmep_logcwmin = logCwMin[mode];
IEEE80211_DPRINTF(vap, IEEE80211_MSG_WME,
"update %s (chan+bss) logcwmin %u\n",
ieee80211_wme_acnames[WME_AC_BE], chanp->wmep_logcwmin);
}
/*
* Arrange for the beacon update.
*
* XXX what about MBSS, WDS?
*/
if (vap->iv_opmode == IEEE80211_M_HOSTAP
|| vap->iv_opmode == IEEE80211_M_IBSS) {
/*
* Arrange for a beacon update and bump the parameter
* set number so associated stations load the new values.
*/
wme->wme_bssChanParams.cap_info =
(wme->wme_bssChanParams.cap_info+1) & WME_QOSINFO_COUNT;
ieee80211_beacon_notify(vap, IEEE80211_BEACON_WME);
}
wme->wme_update(ic);
IEEE80211_DPRINTF(vap, IEEE80211_MSG_WME,
"%s: WME params updated, cap_info 0x%x\n", __func__,
vap->iv_opmode == IEEE80211_M_STA ?
wme->wme_wmeChanParams.cap_info :
wme->wme_bssChanParams.cap_info);
}
void
ieee80211_wme_updateparams(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
if (ic->ic_caps & IEEE80211_C_WME) {
IEEE80211_LOCK(ic);
ieee80211_wme_updateparams_locked(vap);
IEEE80211_UNLOCK(ic);
}
}
static void
parent_updown(void *arg, int npending)
{
struct ifnet *parent = arg;
parent->if_ioctl(parent, SIOCSIFFLAGS, NULL);
}
static void
update_mcast(void *arg, int npending)
{
struct ieee80211com *ic = arg;
struct ifnet *parent = ic->ic_ifp;
ic->ic_update_mcast(parent);
}
static void
update_promisc(void *arg, int npending)
{
struct ieee80211com *ic = arg;
struct ifnet *parent = ic->ic_ifp;
ic->ic_update_promisc(parent);
}
static void
update_channel(void *arg, int npending)
{
struct ieee80211com *ic = arg;
ic->ic_set_channel(ic);
ieee80211_radiotap_chan_change(ic);
}
static void
update_chw(void *arg, int npending)
{
struct ieee80211com *ic = arg;
/*
* XXX should we defer the channel width _config_ update until now?
*/
ic->ic_update_chw(ic);
}
/*
* Block until the parent is in a known state. This is
* used after any operations that dispatch a task (e.g.
* to auto-configure the parent device up/down).
*/
void
ieee80211_waitfor_parent(struct ieee80211com *ic)
{
taskqueue_block(ic->ic_tq);
ieee80211_draintask(ic, &ic->ic_parent_task);
ieee80211_draintask(ic, &ic->ic_mcast_task);
ieee80211_draintask(ic, &ic->ic_promisc_task);
ieee80211_draintask(ic, &ic->ic_chan_task);
ieee80211_draintask(ic, &ic->ic_bmiss_task);
ieee80211_draintask(ic, &ic->ic_chw_task);
taskqueue_unblock(ic->ic_tq);
}
/*
* Start a vap running. If this is the first vap to be
* set running on the underlying device then we
* automatically bring the device up.
*/
void
ieee80211_start_locked(struct ieee80211vap *vap)
{
struct ifnet *ifp = vap->iv_ifp;
struct ieee80211com *ic = vap->iv_ic;
struct ifnet *parent = ic->ic_ifp;
IEEE80211_LOCK_ASSERT(ic);
IEEE80211_DPRINTF(vap,
IEEE80211_MSG_STATE | IEEE80211_MSG_DEBUG,
"start running, %d vaps running\n", ic->ic_nrunning);
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
/*
* Mark us running. Note that it's ok to do this first;
* if we need to bring the parent device up we defer that
* to avoid dropping the com lock. We expect the device
* to respond to being marked up by calling back into us
* through ieee80211_start_all at which point we'll come
* back in here and complete the work.
*/
ifp->if_drv_flags |= IFF_DRV_RUNNING;
/*
* We are not running; if this we are the first vap
* to be brought up auto-up the parent if necessary.
*/
if (ic->ic_nrunning++ == 0 &&
(parent->if_drv_flags & IFF_DRV_RUNNING) == 0) {
IEEE80211_DPRINTF(vap,
IEEE80211_MSG_STATE | IEEE80211_MSG_DEBUG,
"%s: up parent %s\n", __func__, parent->if_xname);
parent->if_flags |= IFF_UP;
ieee80211_runtask(ic, &ic->ic_parent_task);
return;
}
}
/*
* If the parent is up and running, then kick the
* 802.11 state machine as appropriate.
*/
if ((parent->if_drv_flags & IFF_DRV_RUNNING) &&
vap->iv_roaming != IEEE80211_ROAMING_MANUAL) {
if (vap->iv_opmode == IEEE80211_M_STA) {
#if 0
/* XXX bypasses scan too easily; disable for now */
/*
* Try to be intelligent about clocking the state
* machine. If we're currently in RUN state then
* we should be able to apply any new state/parameters
* simply by re-associating. Otherwise we need to
* re-scan to select an appropriate ap.
*/
if (vap->iv_state >= IEEE80211_S_RUN)
ieee80211_new_state_locked(vap,
IEEE80211_S_ASSOC, 1);
else
#endif
ieee80211_new_state_locked(vap,
IEEE80211_S_SCAN, 0);
} else {
/*
* For monitor+wds mode there's nothing to do but
* start running. Otherwise if this is the first
* vap to be brought up, start a scan which may be
* preempted if the station is locked to a particular
* channel.
*/
vap->iv_flags_ext |= IEEE80211_FEXT_REINIT;
if (vap->iv_opmode == IEEE80211_M_MONITOR ||
vap->iv_opmode == IEEE80211_M_WDS)
ieee80211_new_state_locked(vap,
IEEE80211_S_RUN, -1);
else
ieee80211_new_state_locked(vap,
IEEE80211_S_SCAN, 0);
}
}
}
/*
* Start a single vap.
*/
void
ieee80211_init(void *arg)
{
struct ieee80211vap *vap = arg;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE | IEEE80211_MSG_DEBUG,
"%s\n", __func__);
IEEE80211_LOCK(vap->iv_ic);
ieee80211_start_locked(vap);
IEEE80211_UNLOCK(vap->iv_ic);
}
/*
* Start all runnable vap's on a device.
*/
void
ieee80211_start_all(struct ieee80211com *ic)
{
struct ieee80211vap *vap;
IEEE80211_LOCK(ic);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
struct ifnet *ifp = vap->iv_ifp;
if (IFNET_IS_UP_RUNNING(ifp)) /* NB: avoid recursion */
ieee80211_start_locked(vap);
}
IEEE80211_UNLOCK(ic);
}
/*
* Stop a vap. We force it down using the state machine
* then mark it's ifnet not running. If this is the last
* vap running on the underlying device then we close it
* too to insure it will be properly initialized when the
* next vap is brought up.
*/
void
ieee80211_stop_locked(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
struct ifnet *ifp = vap->iv_ifp;
struct ifnet *parent = ic->ic_ifp;
IEEE80211_LOCK_ASSERT(ic);
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE | IEEE80211_MSG_DEBUG,
"stop running, %d vaps running\n", ic->ic_nrunning);
ieee80211_new_state_locked(vap, IEEE80211_S_INIT, -1);
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
ifp->if_drv_flags &= ~IFF_DRV_RUNNING; /* mark us stopped */
if (--ic->ic_nrunning == 0 &&
(parent->if_drv_flags & IFF_DRV_RUNNING)) {
IEEE80211_DPRINTF(vap,
IEEE80211_MSG_STATE | IEEE80211_MSG_DEBUG,
"down parent %s\n", parent->if_xname);
parent->if_flags &= ~IFF_UP;
ieee80211_runtask(ic, &ic->ic_parent_task);
}
}
}
void
ieee80211_stop(struct ieee80211vap *vap)
{
struct ieee80211com *ic = vap->iv_ic;
IEEE80211_LOCK(ic);
ieee80211_stop_locked(vap);
IEEE80211_UNLOCK(ic);
}
/*
* Stop all vap's running on a device.
*/
void
ieee80211_stop_all(struct ieee80211com *ic)
{
struct ieee80211vap *vap;
IEEE80211_LOCK(ic);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
struct ifnet *ifp = vap->iv_ifp;
if (IFNET_IS_UP_RUNNING(ifp)) /* NB: avoid recursion */
ieee80211_stop_locked(vap);
}
IEEE80211_UNLOCK(ic);
ieee80211_waitfor_parent(ic);
}
/*
* Stop all vap's running on a device and arrange
* for those that were running to be resumed.
*/
void
ieee80211_suspend_all(struct ieee80211com *ic)
{
struct ieee80211vap *vap;
IEEE80211_LOCK(ic);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
struct ifnet *ifp = vap->iv_ifp;
if (IFNET_IS_UP_RUNNING(ifp)) { /* NB: avoid recursion */
vap->iv_flags_ext |= IEEE80211_FEXT_RESUME;
ieee80211_stop_locked(vap);
}
}
IEEE80211_UNLOCK(ic);
ieee80211_waitfor_parent(ic);
}
/*
* Start all vap's marked for resume.
*/
void
ieee80211_resume_all(struct ieee80211com *ic)
{
struct ieee80211vap *vap;
IEEE80211_LOCK(ic);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
struct ifnet *ifp = vap->iv_ifp;
if (!IFNET_IS_UP_RUNNING(ifp) &&
(vap->iv_flags_ext & IEEE80211_FEXT_RESUME)) {
vap->iv_flags_ext &= ~IEEE80211_FEXT_RESUME;
ieee80211_start_locked(vap);
}
}
IEEE80211_UNLOCK(ic);
}
void
ieee80211_beacon_miss(struct ieee80211com *ic)
{
IEEE80211_LOCK(ic);
if ((ic->ic_flags & IEEE80211_F_SCAN) == 0) {
/* Process in a taskq, the handler may reenter the driver */
ieee80211_runtask(ic, &ic->ic_bmiss_task);
}
IEEE80211_UNLOCK(ic);
}
static void
beacon_miss(void *arg, int npending)
{
struct ieee80211com *ic = arg;
struct ieee80211vap *vap;
IEEE80211_LOCK(ic);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
/*
* We only pass events through for sta vap's in RUN state;
* may be too restrictive but for now this saves all the
* handlers duplicating these checks.
*/
if (vap->iv_opmode == IEEE80211_M_STA &&
vap->iv_state >= IEEE80211_S_RUN &&
vap->iv_bmiss != NULL)
vap->iv_bmiss(vap);
}
IEEE80211_UNLOCK(ic);
}
static void
beacon_swmiss(void *arg, int npending)
{
struct ieee80211vap *vap = arg;
struct ieee80211com *ic = vap->iv_ic;
IEEE80211_LOCK(ic);
if (vap->iv_state == IEEE80211_S_RUN) {
/* XXX Call multiple times if npending > zero? */
vap->iv_bmiss(vap);
}
IEEE80211_UNLOCK(ic);
}
/*
* Software beacon miss handling. Check if any beacons
* were received in the last period. If not post a
* beacon miss; otherwise reset the counter.
*/
void
ieee80211_swbmiss(void *arg)
{
struct ieee80211vap *vap = arg;
struct ieee80211com *ic = vap->iv_ic;
IEEE80211_LOCK_ASSERT(ic);
/* XXX sleep state? */
KASSERT(vap->iv_state == IEEE80211_S_RUN,
("wrong state %d", vap->iv_state));
if (ic->ic_flags & IEEE80211_F_SCAN) {
/*
* If scanning just ignore and reset state. If we get a
* bmiss after coming out of scan because we haven't had
* time to receive a beacon then we should probe the AP
* before posting a real bmiss (unless iv_bmiss_max has
* been artifiically lowered). A cleaner solution might
* be to disable the timer on scan start/end but to handle
* case of multiple sta vap's we'd need to disable the
* timers of all affected vap's.
*/
vap->iv_swbmiss_count = 0;
} else if (vap->iv_swbmiss_count == 0) {
if (vap->iv_bmiss != NULL)
ieee80211_runtask(ic, &vap->iv_swbmiss_task);
} else
vap->iv_swbmiss_count = 0;
callout_reset(&vap->iv_swbmiss, vap->iv_swbmiss_period,
ieee80211_swbmiss, vap);
}
/*
* Start an 802.11h channel switch. We record the parameters,
* mark the operation pending, notify each vap through the
* beacon update mechanism so it can update the beacon frame
* contents, and then switch vap's to CSA state to block outbound
* traffic. Devices that handle CSA directly can use the state
* switch to do the right thing so long as they call
* ieee80211_csa_completeswitch when it's time to complete the
* channel change. Devices that depend on the net80211 layer can
* use ieee80211_beacon_update to handle the countdown and the
* channel switch.
*/
void
ieee80211_csa_startswitch(struct ieee80211com *ic,
struct ieee80211_channel *c, int mode, int count)
{
struct ieee80211vap *vap;
IEEE80211_LOCK_ASSERT(ic);
ic->ic_csa_newchan = c;
ic->ic_csa_mode = mode;
ic->ic_csa_count = count;
ic->ic_flags |= IEEE80211_F_CSAPENDING;
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
if (vap->iv_opmode == IEEE80211_M_HOSTAP ||
vap->iv_opmode == IEEE80211_M_IBSS ||
vap->iv_opmode == IEEE80211_M_MBSS)
ieee80211_beacon_notify(vap, IEEE80211_BEACON_CSA);
/* switch to CSA state to block outbound traffic */
if (vap->iv_state == IEEE80211_S_RUN)
ieee80211_new_state_locked(vap, IEEE80211_S_CSA, 0);
}
ieee80211_notify_csa(ic, c, mode, count);
}
/*
* Complete the channel switch by transitioning all CSA VAPs to RUN.
* This is called by both the completion and cancellation functions
* so each VAP is placed back in the RUN state and can thus transmit.
*/
static void
csa_completeswitch(struct ieee80211com *ic)
{
struct ieee80211vap *vap;
ic->ic_csa_newchan = NULL;
ic->ic_flags &= ~IEEE80211_F_CSAPENDING;
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next)
if (vap->iv_state == IEEE80211_S_CSA)
ieee80211_new_state_locked(vap, IEEE80211_S_RUN, 0);
}
/*
* Complete an 802.11h channel switch started by ieee80211_csa_startswitch.
* We clear state and move all vap's in CSA state to RUN state
* so they can again transmit.
*
* Although this may not be completely correct, update the BSS channel
* for each VAP to the newly configured channel. The setcurchan sets
* the current operating channel for the interface (so the radio does
* switch over) but the VAP BSS isn't updated, leading to incorrectly
* reported information via ioctl.
*/
void
ieee80211_csa_completeswitch(struct ieee80211com *ic)
{
struct ieee80211vap *vap;
IEEE80211_LOCK_ASSERT(ic);
KASSERT(ic->ic_flags & IEEE80211_F_CSAPENDING, ("csa not pending"));
ieee80211_setcurchan(ic, ic->ic_csa_newchan);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next)
if (vap->iv_state == IEEE80211_S_CSA)
vap->iv_bss->ni_chan = ic->ic_curchan;
csa_completeswitch(ic);
}
/*
* Cancel an 802.11h channel switch started by ieee80211_csa_startswitch.
* We clear state and move all vap's in CSA state to RUN state
* so they can again transmit.
*/
void
ieee80211_csa_cancelswitch(struct ieee80211com *ic)
{
IEEE80211_LOCK_ASSERT(ic);
csa_completeswitch(ic);
}
/*
* Complete a DFS CAC started by ieee80211_dfs_cac_start.
* We clear state and move all vap's in CAC state to RUN state.
*/
void
ieee80211_cac_completeswitch(struct ieee80211vap *vap0)
{
struct ieee80211com *ic = vap0->iv_ic;
struct ieee80211vap *vap;
IEEE80211_LOCK(ic);
/*
* Complete CAC state change for lead vap first; then
* clock all the other vap's waiting.
*/
KASSERT(vap0->iv_state == IEEE80211_S_CAC,
("wrong state %d", vap0->iv_state));
ieee80211_new_state_locked(vap0, IEEE80211_S_RUN, 0);
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next)
if (vap->iv_state == IEEE80211_S_CAC)
ieee80211_new_state_locked(vap, IEEE80211_S_RUN, 0);
IEEE80211_UNLOCK(ic);
}
/*
* Force all vap's other than the specified vap to the INIT state
* and mark them as waiting for a scan to complete. These vaps
* will be brought up when the scan completes and the scanning vap
* reaches RUN state by wakeupwaiting.
*/
static void
markwaiting(struct ieee80211vap *vap0)
{
struct ieee80211com *ic = vap0->iv_ic;
struct ieee80211vap *vap;
IEEE80211_LOCK_ASSERT(ic);
/*
* A vap list entry can not disappear since we are running on the
* taskqueue and a vap destroy will queue and drain another state
* change task.
*/
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
if (vap == vap0)
continue;
if (vap->iv_state != IEEE80211_S_INIT) {
/* NB: iv_newstate may drop the lock */
vap->iv_newstate(vap, IEEE80211_S_INIT, 0);
IEEE80211_LOCK_ASSERT(ic);
vap->iv_flags_ext |= IEEE80211_FEXT_SCANWAIT;
}
}
}
/*
* Wakeup all vap's waiting for a scan to complete. This is the
* companion to markwaiting (above) and is used to coordinate
* multiple vaps scanning.
* This is called from the state taskqueue.
*/
static void
wakeupwaiting(struct ieee80211vap *vap0)
{
struct ieee80211com *ic = vap0->iv_ic;
struct ieee80211vap *vap;
IEEE80211_LOCK_ASSERT(ic);
/*
* A vap list entry can not disappear since we are running on the
* taskqueue and a vap destroy will queue and drain another state
* change task.
*/
TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
if (vap == vap0)
continue;
if (vap->iv_flags_ext & IEEE80211_FEXT_SCANWAIT) {
vap->iv_flags_ext &= ~IEEE80211_FEXT_SCANWAIT;
/* NB: sta's cannot go INIT->RUN */
/* NB: iv_newstate may drop the lock */
vap->iv_newstate(vap,
vap->iv_opmode == IEEE80211_M_STA ?
IEEE80211_S_SCAN : IEEE80211_S_RUN, 0);
IEEE80211_LOCK_ASSERT(ic);
}
}
}
/*
* Handle post state change work common to all operating modes.
*/
static void
ieee80211_newstate_cb(void *xvap, int npending)
{
struct ieee80211vap *vap = xvap;
struct ieee80211com *ic = vap->iv_ic;
enum ieee80211_state nstate, ostate;
int arg, rc;
IEEE80211_LOCK(ic);
nstate = vap->iv_nstate;
arg = vap->iv_nstate_arg;
if (vap->iv_flags_ext & IEEE80211_FEXT_REINIT) {
/*
* We have been requested to drop back to the INIT before
* proceeding to the new state.
*/
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
"%s: %s -> %s arg %d\n", __func__,
ieee80211_state_name[vap->iv_state],
ieee80211_state_name[IEEE80211_S_INIT], arg);
vap->iv_newstate(vap, IEEE80211_S_INIT, arg);
IEEE80211_LOCK_ASSERT(ic);
vap->iv_flags_ext &= ~IEEE80211_FEXT_REINIT;
}
ostate = vap->iv_state;
if (nstate == IEEE80211_S_SCAN && ostate != IEEE80211_S_INIT) {
/*
* SCAN was forced; e.g. on beacon miss. Force other running
* vap's to INIT state and mark them as waiting for the scan to
* complete. This insures they don't interfere with our
* scanning. Since we are single threaded the vaps can not
* transition again while we are executing.
*
* XXX not always right, assumes ap follows sta
*/
markwaiting(vap);
}
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
"%s: %s -> %s arg %d\n", __func__,
ieee80211_state_name[ostate], ieee80211_state_name[nstate], arg);
rc = vap->iv_newstate(vap, nstate, arg);
IEEE80211_LOCK_ASSERT(ic);
vap->iv_flags_ext &= ~IEEE80211_FEXT_STATEWAIT;
if (rc != 0) {
/* State transition failed */
KASSERT(rc != EINPROGRESS, ("iv_newstate was deferred"));
KASSERT(nstate != IEEE80211_S_INIT,
("INIT state change failed"));
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
"%s: %s returned error %d\n", __func__,
ieee80211_state_name[nstate], rc);
goto done;
}
/* No actual transition, skip post processing */
if (ostate == nstate)
goto done;
if (nstate == IEEE80211_S_RUN) {
/*
* OACTIVE may be set on the vap if the upper layer
* tried to transmit (e.g. IPv6 NDP) before we reach
* RUN state. Clear it and restart xmit.
*
* Note this can also happen as a result of SLEEP->RUN
* (i.e. coming out of power save mode).
*/
vap->iv_ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
/*
* XXX TODO Kick-start a VAP queue - this should be a method!
*/
/* bring up any vaps waiting on us */
wakeupwaiting(vap);
} else if (nstate == IEEE80211_S_INIT) {
/*
* Flush the scan cache if we did the last scan (XXX?)
* and flush any frames on send queues from this vap.
* Note the mgt q is used only for legacy drivers and
* will go away shortly.
*/
ieee80211_scan_flush(vap);
/*
* XXX TODO: ic/vap queue flush
*/
}
done:
IEEE80211_UNLOCK(ic);
}
/*
* Public interface for initiating a state machine change.
* This routine single-threads the request and coordinates
* the scheduling of multiple vaps for the purpose of selecting
* an operating channel. Specifically the following scenarios
* are handled:
* o only one vap can be selecting a channel so on transition to
* SCAN state if another vap is already scanning then
* mark the caller for later processing and return without
* doing anything (XXX? expectations by caller of synchronous operation)
* o only one vap can be doing CAC of a channel so on transition to
* CAC state if another vap is already scanning for radar then
* mark the caller for later processing and return without
* doing anything (XXX? expectations by caller of synchronous operation)
* o if another vap is already running when a request is made
* to SCAN then an operating channel has been chosen; bypass
* the scan and just join the channel
*
* Note that the state change call is done through the iv_newstate
* method pointer so any driver routine gets invoked. The driver
* will normally call back into operating mode-specific
* ieee80211_newstate routines (below) unless it needs to completely
* bypass the state machine (e.g. because the firmware has it's
* own idea how things should work). Bypassing the net80211 layer
* is usually a mistake and indicates lack of proper integration
* with the net80211 layer.
*/
int
ieee80211_new_state_locked(struct ieee80211vap *vap,
enum ieee80211_state nstate, int arg)
{
struct ieee80211com *ic = vap->iv_ic;
struct ieee80211vap *vp;
enum ieee80211_state ostate;
int nrunning, nscanning;
IEEE80211_LOCK_ASSERT(ic);
if (vap->iv_flags_ext & IEEE80211_FEXT_STATEWAIT) {
if (vap->iv_nstate == IEEE80211_S_INIT) {
/*
* XXX The vap is being stopped, do no allow any other
* state changes until this is completed.
*/
return -1;
} else if (vap->iv_state != vap->iv_nstate) {
#if 0
/* Warn if the previous state hasn't completed. */
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
"%s: pending %s -> %s transition lost\n", __func__,
ieee80211_state_name[vap->iv_state],
ieee80211_state_name[vap->iv_nstate]);
#else
/* XXX temporarily enable to identify issues */
if_printf(vap->iv_ifp,
"%s: pending %s -> %s transition lost\n",
__func__, ieee80211_state_name[vap->iv_state],
ieee80211_state_name[vap->iv_nstate]);
#endif
}
}
nrunning = nscanning = 0;
/* XXX can track this state instead of calculating */
TAILQ_FOREACH(vp, &ic->ic_vaps, iv_next) {
if (vp != vap) {
if (vp->iv_state >= IEEE80211_S_RUN)
nrunning++;
/* XXX doesn't handle bg scan */
/* NB: CAC+AUTH+ASSOC treated like SCAN */
else if (vp->iv_state > IEEE80211_S_INIT)
nscanning++;
}
}
ostate = vap->iv_state;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
"%s: %s -> %s (nrunning %d nscanning %d)\n", __func__,
ieee80211_state_name[ostate], ieee80211_state_name[nstate],
nrunning, nscanning);
switch (nstate) {
case IEEE80211_S_SCAN:
if (ostate == IEEE80211_S_INIT) {
/*
* INIT -> SCAN happens on initial bringup.
*/
KASSERT(!(nscanning && nrunning),
("%d scanning and %d running", nscanning, nrunning));
if (nscanning) {
/*
* Someone is scanning, defer our state
* change until the work has completed.
*/
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
"%s: defer %s -> %s\n",
__func__, ieee80211_state_name[ostate],
ieee80211_state_name[nstate]);
vap->iv_flags_ext |= IEEE80211_FEXT_SCANWAIT;
return 0;
}
if (nrunning) {
/*
* Someone is operating; just join the channel
* they have chosen.
*/
/* XXX kill arg? */
/* XXX check each opmode, adhoc? */
if (vap->iv_opmode == IEEE80211_M_STA)
nstate = IEEE80211_S_SCAN;
else
nstate = IEEE80211_S_RUN;
#ifdef IEEE80211_DEBUG
if (nstate != IEEE80211_S_SCAN) {
IEEE80211_DPRINTF(vap,
IEEE80211_MSG_STATE,
"%s: override, now %s -> %s\n",
__func__,
ieee80211_state_name[ostate],
ieee80211_state_name[nstate]);
}
#endif
}
}
break;
case IEEE80211_S_RUN:
if (vap->iv_opmode == IEEE80211_M_WDS &&
(vap->iv_flags_ext & IEEE80211_FEXT_WDSLEGACY) &&
nscanning) {
/*
* Legacy WDS with someone else scanning; don't
* go online until that completes as we should
* follow the other vap to the channel they choose.
*/
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
"%s: defer %s -> %s (legacy WDS)\n", __func__,
ieee80211_state_name[ostate],
ieee80211_state_name[nstate]);
vap->iv_flags_ext |= IEEE80211_FEXT_SCANWAIT;
return 0;
}
if (vap->iv_opmode == IEEE80211_M_HOSTAP &&
IEEE80211_IS_CHAN_DFS(ic->ic_bsschan) &&
(vap->iv_flags_ext & IEEE80211_FEXT_DFS) &&
!IEEE80211_IS_CHAN_CACDONE(ic->ic_bsschan)) {
/*
* This is a DFS channel, transition to CAC state
* instead of RUN. This allows us to initiate
* Channel Availability Check (CAC) as specified
* by 11h/DFS.
*/
nstate = IEEE80211_S_CAC;
IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
"%s: override %s -> %s (DFS)\n", __func__,
ieee80211_state_name[ostate],
ieee80211_state_name[nstate]);
}
break;
case IEEE80211_S_INIT:
/* cancel any scan in progress */
ieee80211_cancel_scan(vap);
if (ostate == IEEE80211_S_INIT ) {
/* XXX don't believe this */
/* INIT -> INIT. nothing to do */
vap->iv_flags_ext &= ~IEEE80211_FEXT_SCANWAIT;
}
/* fall thru... */
default:
break;
}
/* defer the state change to a thread */
vap->iv_nstate = nstate;
vap->iv_nstate_arg = arg;
vap->iv_flags_ext |= IEEE80211_FEXT_STATEWAIT;
ieee80211_runtask(ic, &vap->iv_nstate_task);
return EINPROGRESS;
}
int
ieee80211_new_state(struct ieee80211vap *vap,
enum ieee80211_state nstate, int arg)
{
struct ieee80211com *ic = vap->iv_ic;
int rc;
IEEE80211_LOCK(ic);
rc = ieee80211_new_state_locked(vap, nstate, arg);
IEEE80211_UNLOCK(ic);
return rc;
}
Index: head/sys/netgraph/netflow/ng_netflow.c
===================================================================
--- head/sys/netgraph/netflow/ng_netflow.c (revision 283290)
+++ head/sys/netgraph/netflow/ng_netflow.c (revision 283291)
@@ -1,1036 +1,1036 @@
/*-
* Copyright (c) 2010-2011 Alexander V. Chernikov <melifaro@ipfw.ru>
* Copyright (c) 2004-2005 Gleb Smirnoff <glebius@FreeBSD.org>
* Copyright (c) 2001-2003 Roman V. Palagin <romanp@unshadow.net>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $SourceForge: ng_netflow.c,v 1.30 2004/09/05 11:37:43 glebius Exp $
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet6.h"
#include "opt_route.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/counter.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/mbuf.h>
#include <sys/socket.h>
#include <sys/syslog.h>
#include <sys/ctype.h>
#include <net/if.h>
#include <net/ethernet.h>
#include <net/route.h>
#include <net/if_arp.h>
#include <net/if_var.h>
#include <net/if_vlan_var.h>
#include <net/bpf.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>
#include <netinet/tcp.h>
#include <netinet/udp.h>
#include <netinet/sctp.h>
#include <netgraph/ng_message.h>
#include <netgraph/ng_parse.h>
#include <netgraph/netgraph.h>
#include <netgraph/netflow/netflow.h>
#include <netgraph/netflow/netflow_v9.h>
#include <netgraph/netflow/ng_netflow.h>
/* Netgraph methods */
static ng_constructor_t ng_netflow_constructor;
static ng_rcvmsg_t ng_netflow_rcvmsg;
static ng_close_t ng_netflow_close;
static ng_shutdown_t ng_netflow_rmnode;
static ng_newhook_t ng_netflow_newhook;
static ng_rcvdata_t ng_netflow_rcvdata;
static ng_disconnect_t ng_netflow_disconnect;
/* Parse type for struct ng_netflow_info */
static const struct ng_parse_struct_field ng_netflow_info_type_fields[]
= NG_NETFLOW_INFO_TYPE;
static const struct ng_parse_type ng_netflow_info_type = {
&ng_parse_struct_type,
&ng_netflow_info_type_fields
};
/* Parse type for struct ng_netflow_ifinfo */
static const struct ng_parse_struct_field ng_netflow_ifinfo_type_fields[]
= NG_NETFLOW_IFINFO_TYPE;
static const struct ng_parse_type ng_netflow_ifinfo_type = {
&ng_parse_struct_type,
&ng_netflow_ifinfo_type_fields
};
/* Parse type for struct ng_netflow_setdlt */
static const struct ng_parse_struct_field ng_netflow_setdlt_type_fields[]
= NG_NETFLOW_SETDLT_TYPE;
static const struct ng_parse_type ng_netflow_setdlt_type = {
&ng_parse_struct_type,
&ng_netflow_setdlt_type_fields
};
/* Parse type for ng_netflow_setifindex */
static const struct ng_parse_struct_field ng_netflow_setifindex_type_fields[]
= NG_NETFLOW_SETIFINDEX_TYPE;
static const struct ng_parse_type ng_netflow_setifindex_type = {
&ng_parse_struct_type,
&ng_netflow_setifindex_type_fields
};
/* Parse type for ng_netflow_settimeouts */
static const struct ng_parse_struct_field ng_netflow_settimeouts_type_fields[]
= NG_NETFLOW_SETTIMEOUTS_TYPE;
static const struct ng_parse_type ng_netflow_settimeouts_type = {
&ng_parse_struct_type,
&ng_netflow_settimeouts_type_fields
};
/* Parse type for ng_netflow_setconfig */
static const struct ng_parse_struct_field ng_netflow_setconfig_type_fields[]
= NG_NETFLOW_SETCONFIG_TYPE;
static const struct ng_parse_type ng_netflow_setconfig_type = {
&ng_parse_struct_type,
&ng_netflow_setconfig_type_fields
};
/* Parse type for ng_netflow_settemplate */
static const struct ng_parse_struct_field ng_netflow_settemplate_type_fields[]
= NG_NETFLOW_SETTEMPLATE_TYPE;
static const struct ng_parse_type ng_netflow_settemplate_type = {
&ng_parse_struct_type,
&ng_netflow_settemplate_type_fields
};
/* Parse type for ng_netflow_setmtu */
static const struct ng_parse_struct_field ng_netflow_setmtu_type_fields[]
= NG_NETFLOW_SETMTU_TYPE;
static const struct ng_parse_type ng_netflow_setmtu_type = {
&ng_parse_struct_type,
&ng_netflow_setmtu_type_fields
};
/* Parse type for struct ng_netflow_v9info */
static const struct ng_parse_struct_field ng_netflow_v9info_type_fields[]
= NG_NETFLOW_V9INFO_TYPE;
static const struct ng_parse_type ng_netflow_v9info_type = {
&ng_parse_struct_type,
&ng_netflow_v9info_type_fields
};
/* List of commands and how to convert arguments to/from ASCII */
static const struct ng_cmdlist ng_netflow_cmds[] = {
{
NGM_NETFLOW_COOKIE,
NGM_NETFLOW_INFO,
"info",
NULL,
&ng_netflow_info_type
},
{
NGM_NETFLOW_COOKIE,
NGM_NETFLOW_IFINFO,
"ifinfo",
&ng_parse_uint16_type,
&ng_netflow_ifinfo_type
},
{
NGM_NETFLOW_COOKIE,
NGM_NETFLOW_SETDLT,
"setdlt",
&ng_netflow_setdlt_type,
NULL
},
{
NGM_NETFLOW_COOKIE,
NGM_NETFLOW_SETIFINDEX,
"setifindex",
&ng_netflow_setifindex_type,
NULL
},
{
NGM_NETFLOW_COOKIE,
NGM_NETFLOW_SETTIMEOUTS,
"settimeouts",
&ng_netflow_settimeouts_type,
NULL
},
{
NGM_NETFLOW_COOKIE,
NGM_NETFLOW_SETCONFIG,
"setconfig",
&ng_netflow_setconfig_type,
NULL
},
{
NGM_NETFLOW_COOKIE,
NGM_NETFLOW_SETTEMPLATE,
"settemplate",
&ng_netflow_settemplate_type,
NULL
},
{
NGM_NETFLOW_COOKIE,
NGM_NETFLOW_SETMTU,
"setmtu",
&ng_netflow_setmtu_type,
NULL
},
{
NGM_NETFLOW_COOKIE,
NGM_NETFLOW_V9INFO,
"v9info",
NULL,
&ng_netflow_v9info_type
},
{ 0 }
};
/* Netgraph node type descriptor */
static struct ng_type ng_netflow_typestruct = {
.version = NG_ABI_VERSION,
.name = NG_NETFLOW_NODE_TYPE,
.constructor = ng_netflow_constructor,
.rcvmsg = ng_netflow_rcvmsg,
.close = ng_netflow_close,
.shutdown = ng_netflow_rmnode,
.newhook = ng_netflow_newhook,
.rcvdata = ng_netflow_rcvdata,
.disconnect = ng_netflow_disconnect,
.cmdlist = ng_netflow_cmds,
};
NETGRAPH_INIT(netflow, &ng_netflow_typestruct);
/* Called at node creation */
static int
ng_netflow_constructor(node_p node)
{
priv_p priv;
int i;
/* Initialize private data */
priv = malloc(sizeof(*priv), M_NETGRAPH, M_WAITOK | M_ZERO);
/* Initialize fib data */
priv->maxfibs = rt_numfibs;
priv->fib_data = malloc(sizeof(fib_export_p) * priv->maxfibs,
M_NETGRAPH, M_WAITOK | M_ZERO);
/* Make node and its data point at each other */
NG_NODE_SET_PRIVATE(node, priv);
priv->node = node;
/* Initialize timeouts to default values */
priv->nfinfo_inact_t = INACTIVE_TIMEOUT;
priv->nfinfo_act_t = ACTIVE_TIMEOUT;
/* Set default config */
for (i = 0; i < NG_NETFLOW_MAXIFACES; i++)
priv->ifaces[i].info.conf = NG_NETFLOW_CONF_INGRESS;
/* Initialize callout handle */
- callout_init(&priv->exp_callout, CALLOUT_MPSAFE);
+ callout_init(&priv->exp_callout, 1);
/* Allocate memory and set up flow cache */
ng_netflow_cache_init(priv);
return (0);
}
/*
* ng_netflow supports two hooks: data and export.
* Incoming traffic is expected on data, and expired
* netflow datagrams are sent to export.
*/
static int
ng_netflow_newhook(node_p node, hook_p hook, const char *name)
{
const priv_p priv = NG_NODE_PRIVATE(node);
if (strncmp(name, NG_NETFLOW_HOOK_DATA, /* an iface hook? */
strlen(NG_NETFLOW_HOOK_DATA)) == 0) {
iface_p iface;
int ifnum = -1;
const char *cp;
char *eptr;
cp = name + strlen(NG_NETFLOW_HOOK_DATA);
if (!isdigit(*cp) || (cp[0] == '0' && cp[1] != '\0'))
return (EINVAL);
ifnum = (int)strtoul(cp, &eptr, 10);
if (*eptr != '\0' || ifnum < 0 || ifnum >= NG_NETFLOW_MAXIFACES)
return (EINVAL);
/* See if hook is already connected */
if (priv->ifaces[ifnum].hook != NULL)
return (EISCONN);
iface = &priv->ifaces[ifnum];
/* Link private info and hook together */
NG_HOOK_SET_PRIVATE(hook, iface);
iface->hook = hook;
/*
* In most cases traffic accounting is done on an
* Ethernet interface, so default data link type
* will be DLT_EN10MB.
*/
iface->info.ifinfo_dlt = DLT_EN10MB;
} else if (strncmp(name, NG_NETFLOW_HOOK_OUT,
strlen(NG_NETFLOW_HOOK_OUT)) == 0) {
iface_p iface;
int ifnum = -1;
const char *cp;
char *eptr;
cp = name + strlen(NG_NETFLOW_HOOK_OUT);
if (!isdigit(*cp) || (cp[0] == '0' && cp[1] != '\0'))
return (EINVAL);
ifnum = (int)strtoul(cp, &eptr, 10);
if (*eptr != '\0' || ifnum < 0 || ifnum >= NG_NETFLOW_MAXIFACES)
return (EINVAL);
/* See if hook is already connected */
if (priv->ifaces[ifnum].out != NULL)
return (EISCONN);
iface = &priv->ifaces[ifnum];
/* Link private info and hook together */
NG_HOOK_SET_PRIVATE(hook, iface);
iface->out = hook;
} else if (strcmp(name, NG_NETFLOW_HOOK_EXPORT) == 0) {
if (priv->export != NULL)
return (EISCONN);
/* Netflow version 5 supports 32-bit counters only */
if (CNTR_MAX == UINT64_MAX)
return (EINVAL);
priv->export = hook;
/* Exporter is ready. Let's schedule expiry. */
callout_reset(&priv->exp_callout, (1*hz), &ng_netflow_expire,
(void *)priv);
} else if (strcmp(name, NG_NETFLOW_HOOK_EXPORT9) == 0) {
if (priv->export9 != NULL)
return (EISCONN);
priv->export9 = hook;
/* Exporter is ready. Let's schedule expiry. */
callout_reset(&priv->exp_callout, (1*hz), &ng_netflow_expire,
(void *)priv);
} else
return (EINVAL);
return (0);
}
/* Get a netgraph control message. */
static int
ng_netflow_rcvmsg (node_p node, item_p item, hook_p lasthook)
{
const priv_p priv = NG_NODE_PRIVATE(node);
struct ng_mesg *resp = NULL;
int error = 0;
struct ng_mesg *msg;
NGI_GET_MSG(item, msg);
/* Deal with message according to cookie and command */
switch (msg->header.typecookie) {
case NGM_NETFLOW_COOKIE:
switch (msg->header.cmd) {
case NGM_NETFLOW_INFO:
{
struct ng_netflow_info *i;
NG_MKRESPONSE(resp, msg, sizeof(struct ng_netflow_info),
M_NOWAIT);
i = (struct ng_netflow_info *)resp->data;
ng_netflow_copyinfo(priv, i);
break;
}
case NGM_NETFLOW_IFINFO:
{
struct ng_netflow_ifinfo *i;
const uint16_t *index;
if (msg->header.arglen != sizeof(uint16_t))
ERROUT(EINVAL);
index = (uint16_t *)msg->data;
if (*index >= NG_NETFLOW_MAXIFACES)
ERROUT(EINVAL);
/* connected iface? */
if (priv->ifaces[*index].hook == NULL)
ERROUT(EINVAL);
NG_MKRESPONSE(resp, msg,
sizeof(struct ng_netflow_ifinfo), M_NOWAIT);
i = (struct ng_netflow_ifinfo *)resp->data;
memcpy((void *)i, (void *)&priv->ifaces[*index].info,
sizeof(priv->ifaces[*index].info));
break;
}
case NGM_NETFLOW_SETDLT:
{
struct ng_netflow_setdlt *set;
struct ng_netflow_iface *iface;
if (msg->header.arglen !=
sizeof(struct ng_netflow_setdlt))
ERROUT(EINVAL);
set = (struct ng_netflow_setdlt *)msg->data;
if (set->iface >= NG_NETFLOW_MAXIFACES)
ERROUT(EINVAL);
iface = &priv->ifaces[set->iface];
/* connected iface? */
if (iface->hook == NULL)
ERROUT(EINVAL);
switch (set->dlt) {
case DLT_EN10MB:
iface->info.ifinfo_dlt = DLT_EN10MB;
break;
case DLT_RAW:
iface->info.ifinfo_dlt = DLT_RAW;
break;
default:
ERROUT(EINVAL);
}
break;
}
case NGM_NETFLOW_SETIFINDEX:
{
struct ng_netflow_setifindex *set;
struct ng_netflow_iface *iface;
if (msg->header.arglen !=
sizeof(struct ng_netflow_setifindex))
ERROUT(EINVAL);
set = (struct ng_netflow_setifindex *)msg->data;
if (set->iface >= NG_NETFLOW_MAXIFACES)
ERROUT(EINVAL);
iface = &priv->ifaces[set->iface];
/* connected iface? */
if (iface->hook == NULL)
ERROUT(EINVAL);
iface->info.ifinfo_index = set->index;
break;
}
case NGM_NETFLOW_SETTIMEOUTS:
{
struct ng_netflow_settimeouts *set;
if (msg->header.arglen !=
sizeof(struct ng_netflow_settimeouts))
ERROUT(EINVAL);
set = (struct ng_netflow_settimeouts *)msg->data;
priv->nfinfo_inact_t = set->inactive_timeout;
priv->nfinfo_act_t = set->active_timeout;
break;
}
case NGM_NETFLOW_SETCONFIG:
{
struct ng_netflow_setconfig *set;
if (msg->header.arglen !=
sizeof(struct ng_netflow_setconfig))
ERROUT(EINVAL);
set = (struct ng_netflow_setconfig *)msg->data;
if (set->iface >= NG_NETFLOW_MAXIFACES)
ERROUT(EINVAL);
priv->ifaces[set->iface].info.conf = set->conf;
break;
}
case NGM_NETFLOW_SETTEMPLATE:
{
struct ng_netflow_settemplate *set;
if (msg->header.arglen !=
sizeof(struct ng_netflow_settemplate))
ERROUT(EINVAL);
set = (struct ng_netflow_settemplate *)msg->data;
priv->templ_packets = set->packets;
priv->templ_time = set->time;
break;
}
case NGM_NETFLOW_SETMTU:
{
struct ng_netflow_setmtu *set;
if (msg->header.arglen !=
sizeof(struct ng_netflow_setmtu))
ERROUT(EINVAL);
set = (struct ng_netflow_setmtu *)msg->data;
if ((set->mtu < MIN_MTU) || (set->mtu > MAX_MTU))
ERROUT(EINVAL);
priv->mtu = set->mtu;
break;
}
case NGM_NETFLOW_SHOW:
if (msg->header.arglen !=
sizeof(struct ngnf_show_header))
ERROUT(EINVAL);
NG_MKRESPONSE(resp, msg, NGRESP_SIZE, M_NOWAIT);
if (!resp)
ERROUT(ENOMEM);
error = ng_netflow_flow_show(priv,
(struct ngnf_show_header *)msg->data,
(struct ngnf_show_header *)resp->data);
if (error)
NG_FREE_MSG(resp);
break;
case NGM_NETFLOW_V9INFO:
{
struct ng_netflow_v9info *i;
NG_MKRESPONSE(resp, msg,
sizeof(struct ng_netflow_v9info), M_NOWAIT);
i = (struct ng_netflow_v9info *)resp->data;
ng_netflow_copyv9info(priv, i);
break;
}
default:
ERROUT(EINVAL); /* unknown command */
break;
}
break;
default:
ERROUT(EINVAL); /* incorrect cookie */
break;
}
/*
* Take care of synchronous response, if any.
* Free memory and return.
*/
done:
NG_RESPOND_MSG(error, node, item, resp);
NG_FREE_MSG(msg);
return (error);
}
/* Receive data on hook. */
static int
ng_netflow_rcvdata (hook_p hook, item_p item)
{
const node_p node = NG_HOOK_NODE(hook);
const priv_p priv = NG_NODE_PRIVATE(node);
const iface_p iface = NG_HOOK_PRIVATE(hook);
hook_p out;
struct mbuf *m = NULL, *m_old = NULL;
struct ip *ip = NULL;
struct ip6_hdr *ip6 = NULL;
struct m_tag *mtag;
int pullup_len = 0, off;
uint8_t acct = 0, bypass = 0, flags = 0, upper_proto = 0;
int error = 0, l3_off = 0;
unsigned int src_if_index;
caddr_t upper_ptr = NULL;
fib_export_p fe;
uint32_t fib;
if ((hook == priv->export) || (hook == priv->export9)) {
/*
* Data arrived on export hook.
* This must not happen.
*/
log(LOG_ERR, "ng_netflow: incoming data on export hook!\n");
ERROUT(EINVAL);
};
if (hook == iface->hook) {
if ((iface->info.conf & NG_NETFLOW_CONF_INGRESS) == 0)
bypass = 1;
out = iface->out;
} else if (hook == iface->out) {
if ((iface->info.conf & NG_NETFLOW_CONF_EGRESS) == 0)
bypass = 1;
out = iface->hook;
} else
ERROUT(EINVAL);
if ((!bypass) && (iface->info.conf &
(NG_NETFLOW_CONF_ONCE | NG_NETFLOW_CONF_THISONCE))) {
mtag = m_tag_locate(NGI_M(item), MTAG_NETFLOW,
MTAG_NETFLOW_CALLED, NULL);
while (mtag != NULL) {
if ((iface->info.conf & NG_NETFLOW_CONF_ONCE) ||
((ng_ID_t *)(mtag + 1))[0] == NG_NODE_ID(node)) {
bypass = 1;
break;
}
mtag = m_tag_locate(NGI_M(item), MTAG_NETFLOW,
MTAG_NETFLOW_CALLED, mtag);
}
}
if (bypass) {
if (out == NULL)
ERROUT(ENOTCONN);
NG_FWD_ITEM_HOOK(error, item, out);
return (error);
}
if (iface->info.conf &
(NG_NETFLOW_CONF_ONCE | NG_NETFLOW_CONF_THISONCE)) {
mtag = m_tag_alloc(MTAG_NETFLOW, MTAG_NETFLOW_CALLED,
sizeof(ng_ID_t), M_NOWAIT);
if (mtag) {
((ng_ID_t *)(mtag + 1))[0] = NG_NODE_ID(node);
m_tag_prepend(NGI_M(item), mtag);
}
}
/* Import configuration flags related to flow creation */
flags = iface->info.conf & NG_NETFLOW_FLOW_FLAGS;
NGI_GET_M(item, m);
m_old = m;
/* Increase counters. */
iface->info.ifinfo_packets++;
/*
* Depending on interface data link type and packet contents
* we pullup enough data, so that ng_netflow_flow_add() does not
* need to know about mbuf at all. We keep current length of data
* needed to be contiguous in pullup_len. mtod() is done at the
* very end one more time, since m can had changed after pulluping.
*
* In case of unrecognized data we don't return error, but just
* pass data to downstream hook, if it is available.
*/
#define M_CHECK(length) do { \
pullup_len += length; \
if (((m)->m_pkthdr.len < (pullup_len)) || \
((pullup_len) > MHLEN)) { \
error = EINVAL; \
goto bypass; \
} \
if ((m)->m_len < (pullup_len) && \
(((m) = m_pullup((m),(pullup_len))) == NULL)) { \
error = ENOBUFS; \
goto done; \
} \
} while (0)
switch (iface->info.ifinfo_dlt) {
case DLT_EN10MB: /* Ethernet */
{
struct ether_header *eh;
uint16_t etype;
M_CHECK(sizeof(struct ether_header));
eh = mtod(m, struct ether_header *);
/* Make sure this is IP frame. */
etype = ntohs(eh->ether_type);
switch (etype) {
case ETHERTYPE_IP:
M_CHECK(sizeof(struct ip));
eh = mtod(m, struct ether_header *);
ip = (struct ip *)(eh + 1);
l3_off = sizeof(struct ether_header);
break;
#ifdef INET6
case ETHERTYPE_IPV6:
/*
* m_pullup() called by M_CHECK() pullups
* kern.ipc.max_protohdr (default 60 bytes)
* which is enough.
*/
M_CHECK(sizeof(struct ip6_hdr));
eh = mtod(m, struct ether_header *);
ip6 = (struct ip6_hdr *)(eh + 1);
l3_off = sizeof(struct ether_header);
break;
#endif
case ETHERTYPE_VLAN:
{
struct ether_vlan_header *evh;
M_CHECK(sizeof(struct ether_vlan_header) -
sizeof(struct ether_header));
evh = mtod(m, struct ether_vlan_header *);
etype = ntohs(evh->evl_proto);
l3_off = sizeof(struct ether_vlan_header);
if (etype == ETHERTYPE_IP) {
M_CHECK(sizeof(struct ip));
ip = (struct ip *)(evh + 1);
break;
#ifdef INET6
} else if (etype == ETHERTYPE_IPV6) {
M_CHECK(sizeof(struct ip6_hdr));
ip6 = (struct ip6_hdr *)(evh + 1);
break;
#endif
}
}
default:
goto bypass; /* pass this frame */
}
break;
}
case DLT_RAW: /* IP packets */
M_CHECK(sizeof(struct ip));
ip = mtod(m, struct ip *);
/* l3_off is already zero */
#ifdef INET6
/*
* If INET6 is not defined IPv6 packets
* will be discarded in ng_netflow_flow_add().
*/
if (ip->ip_v == IP6VERSION) {
ip = NULL;
M_CHECK(sizeof(struct ip6_hdr) - sizeof(struct ip));
ip6 = mtod(m, struct ip6_hdr *);
}
#endif
break;
default:
goto bypass;
break;
}
off = pullup_len;
if ((ip != NULL) && ((ip->ip_off & htons(IP_OFFMASK)) == 0)) {
if ((ip->ip_v != IPVERSION) ||
((ip->ip_hl << 2) < sizeof(struct ip)))
goto bypass;
/*
* In case of IPv4 header with options, we haven't pulled
* up enough, yet.
*/
M_CHECK((ip->ip_hl << 2) - sizeof(struct ip));
/* Save upper layer offset and proto */
off = pullup_len;
upper_proto = ip->ip_p;
/*
* XXX: in case of wrong upper layer header we will
* forward this packet but skip this record in netflow.
*/
switch (ip->ip_p) {
case IPPROTO_TCP:
M_CHECK(sizeof(struct tcphdr));
break;
case IPPROTO_UDP:
M_CHECK(sizeof(struct udphdr));
break;
case IPPROTO_SCTP:
M_CHECK(sizeof(struct sctphdr));
break;
}
} else if (ip != NULL) {
/*
* Nothing to save except upper layer proto,
* since this is a packet fragment.
*/
flags |= NG_NETFLOW_IS_FRAG;
upper_proto = ip->ip_p;
if ((ip->ip_v != IPVERSION) ||
((ip->ip_hl << 2) < sizeof(struct ip)))
goto bypass;
#ifdef INET6
} else if (ip6 != NULL) {
int cur = ip6->ip6_nxt, hdr_off = 0;
struct ip6_ext *ip6e;
struct ip6_frag *ip6f;
if (priv->export9 == NULL)
goto bypass;
/* Save upper layer info. */
off = pullup_len;
upper_proto = cur;
if ((ip6->ip6_vfc & IPV6_VERSION_MASK) != IPV6_VERSION)
goto bypass;
/*
* Loop thru IPv6 extended headers to get upper
* layer header / frag.
*/
for (;;) {
switch (cur) {
/*
* Same as in IPv4, we can forward a 'bad'
* packet without accounting.
*/
case IPPROTO_TCP:
M_CHECK(sizeof(struct tcphdr));
goto loopend;
case IPPROTO_UDP:
M_CHECK(sizeof(struct udphdr));
goto loopend;
case IPPROTO_SCTP:
M_CHECK(sizeof(struct sctphdr));
goto loopend;
/* Loop until 'real' upper layer headers */
case IPPROTO_HOPOPTS:
case IPPROTO_ROUTING:
case IPPROTO_DSTOPTS:
M_CHECK(sizeof(struct ip6_ext));
ip6e = (struct ip6_ext *)(mtod(m, caddr_t) +
off);
upper_proto = ip6e->ip6e_nxt;
hdr_off = (ip6e->ip6e_len + 1) << 3;
break;
/* RFC4302, can be before DSTOPTS */
case IPPROTO_AH:
M_CHECK(sizeof(struct ip6_ext));
ip6e = (struct ip6_ext *)(mtod(m, caddr_t) +
off);
upper_proto = ip6e->ip6e_nxt;
hdr_off = (ip6e->ip6e_len + 2) << 2;
break;
case IPPROTO_FRAGMENT:
M_CHECK(sizeof(struct ip6_frag));
ip6f = (struct ip6_frag *)(mtod(m, caddr_t) +
off);
upper_proto = ip6f->ip6f_nxt;
hdr_off = sizeof(struct ip6_frag);
off += hdr_off;
flags |= NG_NETFLOW_IS_FRAG;
goto loopend;
#if 0
case IPPROTO_NONE:
goto loopend;
#endif
/*
* Any unknown header (new extension or IPv6/IPv4
* header for tunnels) ends loop.
*/
default:
goto loopend;
}
off += hdr_off;
cur = upper_proto;
}
#endif
}
#undef M_CHECK
#ifdef INET6
loopend:
#endif
/* Just in case of real reallocation in M_CHECK() / m_pullup() */
if (m != m_old) {
priv->nfinfo_realloc_mbuf++;
/* Restore ip/ipv6 pointer */
if (ip != NULL)
ip = (struct ip *)(mtod(m, caddr_t) + l3_off);
else if (ip6 != NULL)
ip6 = (struct ip6_hdr *)(mtod(m, caddr_t) + l3_off);
}
upper_ptr = (caddr_t)(mtod(m, caddr_t) + off);
/* Determine packet input interface. Prefer configured. */
src_if_index = 0;
if (hook == iface->out || iface->info.ifinfo_index == 0) {
if (m->m_pkthdr.rcvif != NULL)
src_if_index = m->m_pkthdr.rcvif->if_index;
} else
src_if_index = iface->info.ifinfo_index;
/* Check packet FIB */
fib = M_GETFIB(m);
if (fib >= priv->maxfibs) {
CTR2(KTR_NET, "ng_netflow_rcvdata(): packet fib %d is out of "
"range of available fibs: 0 .. %d",
fib, priv->maxfibs);
goto bypass;
}
if ((fe = priv_to_fib(priv, fib)) == NULL) {
/* Setup new FIB */
if (ng_netflow_fib_init(priv, fib) != 0) {
/* malloc() failed */
goto bypass;
}
fe = priv_to_fib(priv, fib);
}
if (ip != NULL)
error = ng_netflow_flow_add(priv, fe, ip, upper_ptr,
upper_proto, flags, src_if_index);
#ifdef INET6
else if (ip6 != NULL)
error = ng_netflow_flow6_add(priv, fe, ip6, upper_ptr,
upper_proto, flags, src_if_index);
#endif
else
goto bypass;
acct = 1;
bypass:
if (out != NULL) {
if (acct == 0) {
/* Accounting failure */
if (ip != NULL) {
counter_u64_add(priv->nfinfo_spackets, 1);
counter_u64_add(priv->nfinfo_sbytes,
m->m_pkthdr.len);
} else if (ip6 != NULL) {
counter_u64_add(priv->nfinfo_spackets6, 1);
counter_u64_add(priv->nfinfo_sbytes6,
m->m_pkthdr.len);
}
}
/* XXX: error gets overwritten here */
NG_FWD_NEW_DATA(error, item, out, m);
return (error);
}
done:
if (item)
NG_FREE_ITEM(item);
if (m)
NG_FREE_M(m);
return (error);
}
/* We will be shut down in a moment */
static int
ng_netflow_close(node_p node)
{
const priv_p priv = NG_NODE_PRIVATE(node);
callout_drain(&priv->exp_callout);
ng_netflow_cache_flush(priv);
return (0);
}
/* Do local shutdown processing. */
static int
ng_netflow_rmnode(node_p node)
{
const priv_p priv = NG_NODE_PRIVATE(node);
NG_NODE_SET_PRIVATE(node, NULL);
NG_NODE_UNREF(priv->node);
free(priv->fib_data, M_NETGRAPH);
free(priv, M_NETGRAPH);
return (0);
}
/* Hook disconnection. */
static int
ng_netflow_disconnect(hook_p hook)
{
node_p node = NG_HOOK_NODE(hook);
priv_p priv = NG_NODE_PRIVATE(node);
iface_p iface = NG_HOOK_PRIVATE(hook);
if (iface != NULL) {
if (iface->hook == hook)
iface->hook = NULL;
if (iface->out == hook)
iface->out = NULL;
}
/* if export hook disconnected stop running expire(). */
if (hook == priv->export) {
if (priv->export9 == NULL)
callout_drain(&priv->exp_callout);
priv->export = NULL;
}
if (hook == priv->export9) {
if (priv->export == NULL)
callout_drain(&priv->exp_callout);
priv->export9 = NULL;
}
/* Removal of the last link destroys the node. */
if (NG_NODE_NUMHOOKS(node) == 0)
ng_rmnode_self(node);
return (0);
}
Index: head/sys/netgraph/netgraph.h
===================================================================
--- head/sys/netgraph/netgraph.h (revision 283290)
+++ head/sys/netgraph/netgraph.h (revision 283291)
@@ -1,1219 +1,1219 @@
/*
* netgraph.h
*/
/*-
* Copyright (c) 1996-1999 Whistle Communications, Inc.
* All rights reserved.
*
* Subject to the following obligations and disclaimer of warranty, use and
* redistribution of this software, in source or object code forms, with or
* without modifications are expressly permitted by Whistle Communications;
* provided, however, that:
* 1. Any and all reproductions of the source or object code must include the
* copyright notice above and the following disclaimer of warranties; and
* 2. No rights are granted, in any manner or form, to use Whistle
* Communications, Inc. trademarks, including the mark "WHISTLE
* COMMUNICATIONS" on advertising, endorsements, or otherwise except as
* such appears in the above copyright notice or in the software.
*
* THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND
* TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO
* REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE,
* INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
* WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY
* REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS
* SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE.
* IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES
* RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING
* WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY
* OF SUCH DAMAGE.
*
* Author: Julian Elischer <julian@freebsd.org>
*
* $FreeBSD$
* $Whistle: netgraph.h,v 1.29 1999/11/01 07:56:13 julian Exp $
*/
#ifndef _NETGRAPH_NETGRAPH_H_
#define _NETGRAPH_NETGRAPH_H_
#ifndef _KERNEL
#error "This file should not be included in user level programs"
#endif
#include <sys/queue.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/refcount.h>
#ifdef HAVE_KERNEL_OPTION_HEADERS
#include "opt_netgraph.h"
#include "opt_kdb.h"
#endif
/* debugging options */
#define NG_SEPARATE_MALLOC /* make modules use their own malloc types */
/*
* This defines the in-kernel binary interface version.
* It is possible to change this but leave the external message
* API the same. Each type also has it's own cookies for versioning as well.
* Change it for NETGRAPH_DEBUG version so we cannot mix debug and non debug
* modules.
*/
#define _NG_ABI_VERSION 12
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
#define NG_ABI_VERSION (_NG_ABI_VERSION + 0x10000)
#else /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
#define NG_ABI_VERSION _NG_ABI_VERSION
#endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
/*
* Forward references for the basic structures so we can
* define the typedefs and use them in the structures themselves.
*/
struct ng_hook ;
struct ng_node ;
struct ng_item ;
typedef struct ng_item *item_p;
typedef struct ng_node *node_p;
typedef struct ng_hook *hook_p;
/* node method definitions */
typedef int ng_constructor_t(node_p node);
typedef int ng_close_t(node_p node);
typedef int ng_shutdown_t(node_p node);
typedef int ng_newhook_t(node_p node, hook_p hook, const char *name);
typedef hook_p ng_findhook_t(node_p node, const char *name);
typedef int ng_connect_t(hook_p hook);
typedef int ng_rcvmsg_t(node_p node, item_p item, hook_p lasthook);
typedef int ng_rcvdata_t(hook_p hook, item_p item);
typedef int ng_disconnect_t(hook_p hook);
typedef int ng_rcvitem (node_p node, hook_p hook, item_p item);
/***********************************************************************
***************** Hook Structure and Methods **************************
***********************************************************************
*
* Structure of a hook
*/
struct ng_hook {
char hk_name[NG_HOOKSIZ]; /* what this node knows this link as */
void *hk_private; /* node dependant ID for this hook */
int hk_flags; /* info about this hook/link */
int hk_type; /* tbd: hook data link type */
struct ng_hook *hk_peer; /* the other end of this link */
struct ng_node *hk_node; /* The node this hook is attached to */
LIST_ENTRY(ng_hook) hk_hooks; /* linked list of all hooks on node */
ng_rcvmsg_t *hk_rcvmsg; /* control messages come here */
ng_rcvdata_t *hk_rcvdata; /* data comes here */
int hk_refs; /* dont actually free this till 0 */
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
#define HK_MAGIC 0x78573011
int hk_magic;
char *lastfile;
int lastline;
SLIST_ENTRY(ng_hook) hk_all; /* all existing items */
#endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
};
/* Flags for a hook */
#define HK_INVALID 0x0001 /* don't trust it! */
#define HK_QUEUE 0x0002 /* queue for later delivery */
#define HK_FORCE_WRITER 0x0004 /* Incoming data queued as a writer */
#define HK_DEAD 0x0008 /* This is the dead hook.. don't free */
#define HK_HI_STACK 0x0010 /* Hook has hi stack usage */
#define HK_TO_INBOUND 0x0020 /* Hook on ntw. stack inbound path. */
/*
* Public Methods for hook
* If you can't do it with these you probably shouldn;t be doing it.
*/
void ng_unref_hook(hook_p hook); /* don't move this */
#define _NG_HOOK_REF(hook) refcount_acquire(&(hook)->hk_refs)
#define _NG_HOOK_NAME(hook) ((hook)->hk_name)
#define _NG_HOOK_UNREF(hook) ng_unref_hook(hook)
#define _NG_HOOK_SET_PRIVATE(hook, val) do {(hook)->hk_private = val;} while (0)
#define _NG_HOOK_SET_RCVMSG(hook, val) do {(hook)->hk_rcvmsg = val;} while (0)
#define _NG_HOOK_SET_RCVDATA(hook, val) do {(hook)->hk_rcvdata = val;} while (0)
#define _NG_HOOK_PRIVATE(hook) ((hook)->hk_private)
#define _NG_HOOK_NOT_VALID(hook) ((hook)->hk_flags & HK_INVALID)
#define _NG_HOOK_IS_VALID(hook) (!((hook)->hk_flags & HK_INVALID))
#define _NG_HOOK_NODE(hook) ((hook)->hk_node) /* only rvalue! */
#define _NG_HOOK_PEER(hook) ((hook)->hk_peer) /* only rvalue! */
#define _NG_HOOK_FORCE_WRITER(hook) \
do { hook->hk_flags |= HK_FORCE_WRITER; } while (0)
#define _NG_HOOK_FORCE_QUEUE(hook) do { hook->hk_flags |= HK_QUEUE; } while (0)
#define _NG_HOOK_SET_TO_INBOUND(hook) \
do { hook->hk_flags |= HK_TO_INBOUND; } while (0)
#define _NG_HOOK_HI_STACK(hook) do { hook->hk_flags |= HK_HI_STACK; } while (0)
/* Some shortcuts */
#define NG_PEER_NODE(hook) NG_HOOK_NODE(NG_HOOK_PEER(hook))
#define NG_PEER_HOOK_NAME(hook) NG_HOOK_NAME(NG_HOOK_PEER(hook))
#define NG_PEER_NODE_NAME(hook) NG_NODE_NAME(NG_PEER_NODE(hook))
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
#define _NN_ __FILE__,__LINE__
void dumphook (hook_p hook, char *file, int line);
static __inline void _chkhook(hook_p hook, char *file, int line);
static __inline void _ng_hook_ref(hook_p hook, char * file, int line);
static __inline char * _ng_hook_name(hook_p hook, char * file, int line);
static __inline void _ng_hook_unref(hook_p hook, char * file, int line);
static __inline void _ng_hook_set_private(hook_p hook,
void * val, char * file, int line);
static __inline void _ng_hook_set_rcvmsg(hook_p hook,
ng_rcvmsg_t *val, char * file, int line);
static __inline void _ng_hook_set_rcvdata(hook_p hook,
ng_rcvdata_t *val, char * file, int line);
static __inline void * _ng_hook_private(hook_p hook, char * file, int line);
static __inline int _ng_hook_not_valid(hook_p hook, char * file, int line);
static __inline int _ng_hook_is_valid(hook_p hook, char * file, int line);
static __inline node_p _ng_hook_node(hook_p hook, char * file, int line);
static __inline hook_p _ng_hook_peer(hook_p hook, char * file, int line);
static __inline void _ng_hook_force_writer(hook_p hook, char * file,
int line);
static __inline void _ng_hook_force_queue(hook_p hook, char * file,
int line);
static __inline void _ng_hook_set_to_inbound(hook_p hook, char * file,
int line);
static __inline void
_chkhook(hook_p hook, char *file, int line)
{
if (hook->hk_magic != HK_MAGIC) {
printf("Accessing freed ");
dumphook(hook, file, line);
}
hook->lastline = line;
hook->lastfile = file;
}
static __inline void
_ng_hook_ref(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
_NG_HOOK_REF(hook);
}
static __inline char *
_ng_hook_name(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
return (_NG_HOOK_NAME(hook));
}
static __inline void
_ng_hook_unref(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
_NG_HOOK_UNREF(hook);
}
static __inline void
_ng_hook_set_private(hook_p hook, void *val, char * file, int line)
{
_chkhook(hook, file, line);
_NG_HOOK_SET_PRIVATE(hook, val);
}
static __inline void
_ng_hook_set_rcvmsg(hook_p hook, ng_rcvmsg_t *val, char * file, int line)
{
_chkhook(hook, file, line);
_NG_HOOK_SET_RCVMSG(hook, val);
}
static __inline void
_ng_hook_set_rcvdata(hook_p hook, ng_rcvdata_t *val, char * file, int line)
{
_chkhook(hook, file, line);
_NG_HOOK_SET_RCVDATA(hook, val);
}
static __inline void *
_ng_hook_private(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
return (_NG_HOOK_PRIVATE(hook));
}
static __inline int
_ng_hook_not_valid(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
return (_NG_HOOK_NOT_VALID(hook));
}
static __inline int
_ng_hook_is_valid(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
return (_NG_HOOK_IS_VALID(hook));
}
static __inline node_p
_ng_hook_node(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
return (_NG_HOOK_NODE(hook));
}
static __inline hook_p
_ng_hook_peer(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
return (_NG_HOOK_PEER(hook));
}
static __inline void
_ng_hook_force_writer(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
_NG_HOOK_FORCE_WRITER(hook);
}
static __inline void
_ng_hook_force_queue(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
_NG_HOOK_FORCE_QUEUE(hook);
}
static __inline void
_ng_hook_set_to_inbound(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
_NG_HOOK_SET_TO_INBOUND(hook);
}
static __inline void
_ng_hook_hi_stack(hook_p hook, char * file, int line)
{
_chkhook(hook, file, line);
_NG_HOOK_HI_STACK(hook);
}
#define NG_HOOK_REF(hook) _ng_hook_ref(hook, _NN_)
#define NG_HOOK_NAME(hook) _ng_hook_name(hook, _NN_)
#define NG_HOOK_UNREF(hook) _ng_hook_unref(hook, _NN_)
#define NG_HOOK_SET_PRIVATE(hook, val) _ng_hook_set_private(hook, val, _NN_)
#define NG_HOOK_SET_RCVMSG(hook, val) _ng_hook_set_rcvmsg(hook, val, _NN_)
#define NG_HOOK_SET_RCVDATA(hook, val) _ng_hook_set_rcvdata(hook, val, _NN_)
#define NG_HOOK_PRIVATE(hook) _ng_hook_private(hook, _NN_)
#define NG_HOOK_NOT_VALID(hook) _ng_hook_not_valid(hook, _NN_)
#define NG_HOOK_IS_VALID(hook) _ng_hook_is_valid(hook, _NN_)
#define NG_HOOK_NODE(hook) _ng_hook_node(hook, _NN_)
#define NG_HOOK_PEER(hook) _ng_hook_peer(hook, _NN_)
#define NG_HOOK_FORCE_WRITER(hook) _ng_hook_force_writer(hook, _NN_)
#define NG_HOOK_FORCE_QUEUE(hook) _ng_hook_force_queue(hook, _NN_)
#define NG_HOOK_SET_TO_INBOUND(hook) _ng_hook_set_to_inbound(hook, _NN_)
#define NG_HOOK_HI_STACK(hook) _ng_hook_hi_stack(hook, _NN_)
#else /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
#define NG_HOOK_REF(hook) _NG_HOOK_REF(hook)
#define NG_HOOK_NAME(hook) _NG_HOOK_NAME(hook)
#define NG_HOOK_UNREF(hook) _NG_HOOK_UNREF(hook)
#define NG_HOOK_SET_PRIVATE(hook, val) _NG_HOOK_SET_PRIVATE(hook, val)
#define NG_HOOK_SET_RCVMSG(hook, val) _NG_HOOK_SET_RCVMSG(hook, val)
#define NG_HOOK_SET_RCVDATA(hook, val) _NG_HOOK_SET_RCVDATA(hook, val)
#define NG_HOOK_PRIVATE(hook) _NG_HOOK_PRIVATE(hook)
#define NG_HOOK_NOT_VALID(hook) _NG_HOOK_NOT_VALID(hook)
#define NG_HOOK_IS_VALID(hook) _NG_HOOK_IS_VALID(hook)
#define NG_HOOK_NODE(hook) _NG_HOOK_NODE(hook)
#define NG_HOOK_PEER(hook) _NG_HOOK_PEER(hook)
#define NG_HOOK_FORCE_WRITER(hook) _NG_HOOK_FORCE_WRITER(hook)
#define NG_HOOK_FORCE_QUEUE(hook) _NG_HOOK_FORCE_QUEUE(hook)
#define NG_HOOK_SET_TO_INBOUND(hook) _NG_HOOK_SET_TO_INBOUND(hook)
#define NG_HOOK_HI_STACK(hook) _NG_HOOK_HI_STACK(hook)
#endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
/***********************************************************************
***************** Node Structure and Methods **************************
***********************************************************************
* Structure of a node
* including the eembedded queue structure.
*
* The structure for queueing Netgraph request items
* embedded in the node structure
*/
struct ng_queue {
u_int q_flags; /* Current r/w/q lock flags */
u_int q_flags2; /* Other queue flags */
struct mtx q_mtx;
STAILQ_ENTRY(ng_node) q_work; /* nodes with work to do */
STAILQ_HEAD(, ng_item) queue; /* actually items queue */
};
struct ng_node {
char nd_name[NG_NODESIZ]; /* optional globally unique name */
struct ng_type *nd_type; /* the installed 'type' */
int nd_flags; /* see below for bit definitions */
int nd_numhooks; /* number of hooks */
void *nd_private; /* node type dependant node ID */
ng_ID_t nd_ID; /* Unique per node */
LIST_HEAD(hooks, ng_hook) nd_hooks; /* linked list of node hooks */
LIST_ENTRY(ng_node) nd_nodes; /* name hash collision list */
LIST_ENTRY(ng_node) nd_idnodes; /* ID hash collision list */
struct ng_queue nd_input_queue; /* input queue for locking */
int nd_refs; /* # of references to this node */
struct vnet *nd_vnet; /* network stack instance */
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
#define ND_MAGIC 0x59264837
int nd_magic;
char *lastfile;
int lastline;
SLIST_ENTRY(ng_node) nd_all; /* all existing nodes */
#endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
};
/* Flags for a node */
#define NGF_INVALID 0x00000001 /* free when refs go to 0 */
#define NG_INVALID NGF_INVALID /* compat for old code */
#define NGF_FORCE_WRITER 0x00000004 /* Never multithread this node */
#define NG_FORCE_WRITER NGF_FORCE_WRITER /* compat for old code */
#define NGF_CLOSING 0x00000008 /* ng_rmnode() at work */
#define NG_CLOSING NGF_CLOSING /* compat for old code */
#define NGF_REALLY_DIE 0x00000010 /* "persistent" node is unloading */
#define NG_REALLY_DIE NGF_REALLY_DIE /* compat for old code */
#define NGF_HI_STACK 0x00000020 /* node has hi stack usage */
#define NGF_TYPE1 0x10000000 /* reserved for type specific storage */
#define NGF_TYPE2 0x20000000 /* reserved for type specific storage */
#define NGF_TYPE3 0x40000000 /* reserved for type specific storage */
#define NGF_TYPE4 0x80000000 /* reserved for type specific storage */
/*
* Public methods for nodes.
* If you can't do it with these you probably shouldn't be doing it.
*/
void ng_unref_node(node_p node); /* don't move this */
#define _NG_NODE_NAME(node) ((node)->nd_name + 0)
#define _NG_NODE_HAS_NAME(node) ((node)->nd_name[0] + 0)
#define _NG_NODE_ID(node) ((node)->nd_ID + 0)
#define _NG_NODE_REF(node) refcount_acquire(&(node)->nd_refs)
#define _NG_NODE_UNREF(node) ng_unref_node(node)
#define _NG_NODE_SET_PRIVATE(node, val) do {(node)->nd_private = val;} while (0)
#define _NG_NODE_PRIVATE(node) ((node)->nd_private)
#define _NG_NODE_IS_VALID(node) (!((node)->nd_flags & NGF_INVALID))
#define _NG_NODE_NOT_VALID(node) ((node)->nd_flags & NGF_INVALID)
#define _NG_NODE_NUMHOOKS(node) ((node)->nd_numhooks + 0) /* rvalue */
#define _NG_NODE_FORCE_WRITER(node) \
do{ node->nd_flags |= NGF_FORCE_WRITER; }while (0)
#define _NG_NODE_HI_STACK(node) \
do{ node->nd_flags |= NGF_HI_STACK; }while (0)
#define _NG_NODE_REALLY_DIE(node) \
do{ node->nd_flags |= (NGF_REALLY_DIE|NGF_INVALID); }while (0)
#define _NG_NODE_REVIVE(node) \
do { node->nd_flags &= ~NGF_INVALID; } while (0)
/*
* The hook iterator.
* This macro will call a function of type ng_fn_eachhook for each
* hook attached to the node. If the function returns 0, then the
* iterator will stop and return a pointer to the hook that returned 0.
*/
typedef int ng_fn_eachhook(hook_p hook, void* arg);
#define _NG_NODE_FOREACH_HOOK(node, fn, arg, rethook) \
do { \
hook_p _hook; \
(rethook) = NULL; \
LIST_FOREACH(_hook, &((node)->nd_hooks), hk_hooks) { \
if ((fn)(_hook, arg) == 0) { \
(rethook) = _hook; \
break; \
} \
} \
} while (0)
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
void dumpnode(node_p node, char *file, int line);
static __inline void _chknode(node_p node, char *file, int line);
static __inline char * _ng_node_name(node_p node, char *file, int line);
static __inline int _ng_node_has_name(node_p node, char *file, int line);
static __inline ng_ID_t _ng_node_id(node_p node, char *file, int line);
static __inline void _ng_node_ref(node_p node, char *file, int line);
static __inline void _ng_node_unref(node_p node, char *file, int line);
static __inline void _ng_node_set_private(node_p node, void * val,
char *file, int line);
static __inline void * _ng_node_private(node_p node, char *file, int line);
static __inline int _ng_node_is_valid(node_p node, char *file, int line);
static __inline int _ng_node_not_valid(node_p node, char *file, int line);
static __inline int _ng_node_numhooks(node_p node, char *file, int line);
static __inline void _ng_node_force_writer(node_p node, char *file, int line);
static __inline hook_p _ng_node_foreach_hook(node_p node,
ng_fn_eachhook *fn, void *arg, char *file, int line);
static __inline void _ng_node_revive(node_p node, char *file, int line);
static __inline void
_chknode(node_p node, char *file, int line)
{
if (node->nd_magic != ND_MAGIC) {
printf("Accessing freed ");
dumpnode(node, file, line);
}
node->lastline = line;
node->lastfile = file;
}
static __inline char *
_ng_node_name(node_p node, char *file, int line)
{
_chknode(node, file, line);
return(_NG_NODE_NAME(node));
}
static __inline int
_ng_node_has_name(node_p node, char *file, int line)
{
_chknode(node, file, line);
return(_NG_NODE_HAS_NAME(node));
}
static __inline ng_ID_t
_ng_node_id(node_p node, char *file, int line)
{
_chknode(node, file, line);
return(_NG_NODE_ID(node));
}
static __inline void
_ng_node_ref(node_p node, char *file, int line)
{
_chknode(node, file, line);
_NG_NODE_REF(node);
}
static __inline void
_ng_node_unref(node_p node, char *file, int line)
{
_chknode(node, file, line);
_NG_NODE_UNREF(node);
}
static __inline void
_ng_node_set_private(node_p node, void * val, char *file, int line)
{
_chknode(node, file, line);
_NG_NODE_SET_PRIVATE(node, val);
}
static __inline void *
_ng_node_private(node_p node, char *file, int line)
{
_chknode(node, file, line);
return (_NG_NODE_PRIVATE(node));
}
static __inline int
_ng_node_is_valid(node_p node, char *file, int line)
{
_chknode(node, file, line);
return(_NG_NODE_IS_VALID(node));
}
static __inline int
_ng_node_not_valid(node_p node, char *file, int line)
{
_chknode(node, file, line);
return(_NG_NODE_NOT_VALID(node));
}
static __inline int
_ng_node_numhooks(node_p node, char *file, int line)
{
_chknode(node, file, line);
return(_NG_NODE_NUMHOOKS(node));
}
static __inline void
_ng_node_force_writer(node_p node, char *file, int line)
{
_chknode(node, file, line);
_NG_NODE_FORCE_WRITER(node);
}
static __inline void
_ng_node_hi_stack(node_p node, char *file, int line)
{
_chknode(node, file, line);
_NG_NODE_HI_STACK(node);
}
static __inline void
_ng_node_really_die(node_p node, char *file, int line)
{
_chknode(node, file, line);
_NG_NODE_REALLY_DIE(node);
}
static __inline void
_ng_node_revive(node_p node, char *file, int line)
{
_chknode(node, file, line);
_NG_NODE_REVIVE(node);
}
static __inline hook_p
_ng_node_foreach_hook(node_p node, ng_fn_eachhook *fn, void *arg,
char *file, int line)
{
hook_p hook;
_chknode(node, file, line);
_NG_NODE_FOREACH_HOOK(node, fn, arg, hook);
return (hook);
}
#define NG_NODE_NAME(node) _ng_node_name(node, _NN_)
#define NG_NODE_HAS_NAME(node) _ng_node_has_name(node, _NN_)
#define NG_NODE_ID(node) _ng_node_id(node, _NN_)
#define NG_NODE_REF(node) _ng_node_ref(node, _NN_)
#define NG_NODE_UNREF(node) _ng_node_unref(node, _NN_)
#define NG_NODE_SET_PRIVATE(node, val) _ng_node_set_private(node, val, _NN_)
#define NG_NODE_PRIVATE(node) _ng_node_private(node, _NN_)
#define NG_NODE_IS_VALID(node) _ng_node_is_valid(node, _NN_)
#define NG_NODE_NOT_VALID(node) _ng_node_not_valid(node, _NN_)
#define NG_NODE_FORCE_WRITER(node) _ng_node_force_writer(node, _NN_)
#define NG_NODE_HI_STACK(node) _ng_node_hi_stack(node, _NN_)
#define NG_NODE_REALLY_DIE(node) _ng_node_really_die(node, _NN_)
#define NG_NODE_NUMHOOKS(node) _ng_node_numhooks(node, _NN_)
#define NG_NODE_REVIVE(node) _ng_node_revive(node, _NN_)
#define NG_NODE_FOREACH_HOOK(node, fn, arg, rethook) \
do { \
rethook = _ng_node_foreach_hook(node, fn, (void *)arg, _NN_); \
} while (0)
#else /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
#define NG_NODE_NAME(node) _NG_NODE_NAME(node)
#define NG_NODE_HAS_NAME(node) _NG_NODE_HAS_NAME(node)
#define NG_NODE_ID(node) _NG_NODE_ID(node)
#define NG_NODE_REF(node) _NG_NODE_REF(node)
#define NG_NODE_UNREF(node) _NG_NODE_UNREF(node)
#define NG_NODE_SET_PRIVATE(node, val) _NG_NODE_SET_PRIVATE(node, val)
#define NG_NODE_PRIVATE(node) _NG_NODE_PRIVATE(node)
#define NG_NODE_IS_VALID(node) _NG_NODE_IS_VALID(node)
#define NG_NODE_NOT_VALID(node) _NG_NODE_NOT_VALID(node)
#define NG_NODE_FORCE_WRITER(node) _NG_NODE_FORCE_WRITER(node)
#define NG_NODE_HI_STACK(node) _NG_NODE_HI_STACK(node)
#define NG_NODE_REALLY_DIE(node) _NG_NODE_REALLY_DIE(node)
#define NG_NODE_NUMHOOKS(node) _NG_NODE_NUMHOOKS(node)
#define NG_NODE_REVIVE(node) _NG_NODE_REVIVE(node)
#define NG_NODE_FOREACH_HOOK(node, fn, arg, rethook) \
_NG_NODE_FOREACH_HOOK(node, fn, arg, rethook)
#endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
/***********************************************************************
************* Node Queue and Item Structures and Methods **************
***********************************************************************
*
*/
typedef void ng_item_fn(node_p node, hook_p hook, void *arg1, int arg2);
typedef int ng_item_fn2(node_p node, struct ng_item *item, hook_p hook);
typedef void ng_apply_t(void *context, int error);
struct ng_apply_info {
ng_apply_t *apply;
void *context;
int refs;
int error;
};
struct ng_item {
u_long el_flags;
STAILQ_ENTRY(ng_item) el_next;
node_p el_dest; /* The node it will be applied against (or NULL) */
hook_p el_hook; /* Entering hook. Optional in Control messages */
union {
struct mbuf *da_m;
struct {
struct ng_mesg *msg_msg;
ng_ID_t msg_retaddr;
} msg;
struct {
union {
ng_item_fn *fn_fn;
ng_item_fn2 *fn_fn2;
} fn_fn;
void *fn_arg1;
int fn_arg2;
} fn;
} body;
/*
* Optional callback called when item is being applied,
* and its context.
*/
struct ng_apply_info *apply;
u_int depth;
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
char *lastfile;
int lastline;
TAILQ_ENTRY(ng_item) all; /* all existing items */
#endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
};
#define NGQF_TYPE 0x03 /* MASK of content definition */
#define NGQF_MESG 0x00 /* the queue element is a message */
#define NGQF_DATA 0x01 /* the queue element is data */
#define NGQF_FN 0x02 /* the queue element is a function */
#define NGQF_FN2 0x03 /* the queue element is a new function */
#define NGQF_RW 0x04 /* MASK for wanted queue mode */
#define NGQF_READER 0x04 /* wants to be a reader */
#define NGQF_WRITER 0x00 /* wants to be a writer */
#define NGQF_QMODE 0x08 /* MASK for how it was queued */
#define NGQF_QREADER 0x08 /* was queued as a reader */
#define NGQF_QWRITER 0x00 /* was queued as a writer */
/*
* Get the mbuf (etc) out of an item.
* Sets the value in the item to NULL in case we need to call NG_FREE_ITEM()
* with it, (to avoid freeing the things twice).
* If you don't want to zero out the item then realise that the
* item still owns it.
* Retaddr is different. There are no references on that. It's just a number.
* The debug versions must be either all used everywhere or not at all.
*/
#define _NGI_M(i) ((i)->body.da_m)
#define _NGI_MSG(i) ((i)->body.msg.msg_msg)
#define _NGI_RETADDR(i) ((i)->body.msg.msg_retaddr)
#define _NGI_FN(i) ((i)->body.fn.fn_fn.fn_fn)
#define _NGI_FN2(i) ((i)->body.fn.fn_fn.fn_fn2)
#define _NGI_ARG1(i) ((i)->body.fn.fn_arg1)
#define _NGI_ARG2(i) ((i)->body.fn.fn_arg2)
#define _NGI_NODE(i) ((i)->el_dest)
#define _NGI_HOOK(i) ((i)->el_hook)
#define _NGI_SET_HOOK(i,h) do { _NGI_HOOK(i) = h; h = NULL;} while (0)
#define _NGI_CLR_HOOK(i) do { \
hook_p _hook = _NGI_HOOK(i); \
if (_hook) { \
_NG_HOOK_UNREF(_hook); \
_NGI_HOOK(i) = NULL; \
} \
} while (0)
#define _NGI_SET_NODE(i,n) do { _NGI_NODE(i) = n; n = NULL;} while (0)
#define _NGI_CLR_NODE(i) do { \
node_p _node = _NGI_NODE(i); \
if (_node) { \
_NG_NODE_UNREF(_node); \
_NGI_NODE(i) = NULL; \
} \
} while (0)
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
void dumpitem(item_p item, char *file, int line);
static __inline void _ngi_check(item_p item, char *file, int line) ;
static __inline struct mbuf ** _ngi_m(item_p item, char *file, int line) ;
static __inline ng_ID_t * _ngi_retaddr(item_p item, char *file, int line);
static __inline struct ng_mesg ** _ngi_msg(item_p item, char *file, int line) ;
static __inline ng_item_fn ** _ngi_fn(item_p item, char *file, int line) ;
static __inline ng_item_fn2 ** _ngi_fn2(item_p item, char *file, int line) ;
static __inline void ** _ngi_arg1(item_p item, char *file, int line) ;
static __inline int * _ngi_arg2(item_p item, char *file, int line) ;
static __inline node_p _ngi_node(item_p item, char *file, int line);
static __inline hook_p _ngi_hook(item_p item, char *file, int line);
static __inline void
_ngi_check(item_p item, char *file, int line)
{
(item)->lastline = line;
(item)->lastfile = file;
}
static __inline struct mbuf **
_ngi_m(item_p item, char *file, int line)
{
_ngi_check(item, file, line);
return (&_NGI_M(item));
}
static __inline struct ng_mesg **
_ngi_msg(item_p item, char *file, int line)
{
_ngi_check(item, file, line);
return (&_NGI_MSG(item));
}
static __inline ng_ID_t *
_ngi_retaddr(item_p item, char *file, int line)
{
_ngi_check(item, file, line);
return (&_NGI_RETADDR(item));
}
static __inline ng_item_fn **
_ngi_fn(item_p item, char *file, int line)
{
_ngi_check(item, file, line);
return (&_NGI_FN(item));
}
static __inline ng_item_fn2 **
_ngi_fn2(item_p item, char *file, int line)
{
_ngi_check(item, file, line);
return (&_NGI_FN2(item));
}
static __inline void **
_ngi_arg1(item_p item, char *file, int line)
{
_ngi_check(item, file, line);
return (&_NGI_ARG1(item));
}
static __inline int *
_ngi_arg2(item_p item, char *file, int line)
{
_ngi_check(item, file, line);
return (&_NGI_ARG2(item));
}
static __inline node_p
_ngi_node(item_p item, char *file, int line)
{
_ngi_check(item, file, line);
return (_NGI_NODE(item));
}
static __inline hook_p
_ngi_hook(item_p item, char *file, int line)
{
_ngi_check(item, file, line);
return (_NGI_HOOK(item));
}
#define NGI_M(i) (*_ngi_m(i, _NN_))
#define NGI_MSG(i) (*_ngi_msg(i, _NN_))
#define NGI_RETADDR(i) (*_ngi_retaddr(i, _NN_))
#define NGI_FN(i) (*_ngi_fn(i, _NN_))
#define NGI_FN2(i) (*_ngi_fn2(i, _NN_))
#define NGI_ARG1(i) (*_ngi_arg1(i, _NN_))
#define NGI_ARG2(i) (*_ngi_arg2(i, _NN_))
#define NGI_HOOK(i) _ngi_hook(i, _NN_)
#define NGI_NODE(i) _ngi_node(i, _NN_)
#define NGI_SET_HOOK(i,h) \
do { _ngi_check(i, _NN_); _NGI_SET_HOOK(i, h); } while (0)
#define NGI_CLR_HOOK(i) \
do { _ngi_check(i, _NN_); _NGI_CLR_HOOK(i); } while (0)
#define NGI_SET_NODE(i,n) \
do { _ngi_check(i, _NN_); _NGI_SET_NODE(i, n); } while (0)
#define NGI_CLR_NODE(i) \
do { _ngi_check(i, _NN_); _NGI_CLR_NODE(i); } while (0)
#define NG_FREE_ITEM(item) \
do { \
_ngi_check(item, _NN_); \
ng_free_item((item)); \
} while (0)
#define SAVE_LINE(item) \
do { \
(item)->lastline = __LINE__; \
(item)->lastfile = __FILE__; \
} while (0)
#else /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
#define NGI_M(i) _NGI_M(i)
#define NGI_MSG(i) _NGI_MSG(i)
#define NGI_RETADDR(i) _NGI_RETADDR(i)
#define NGI_FN(i) _NGI_FN(i)
#define NGI_FN2(i) _NGI_FN2(i)
#define NGI_ARG1(i) _NGI_ARG1(i)
#define NGI_ARG2(i) _NGI_ARG2(i)
#define NGI_NODE(i) _NGI_NODE(i)
#define NGI_HOOK(i) _NGI_HOOK(i)
#define NGI_SET_HOOK(i,h) _NGI_SET_HOOK(i,h)
#define NGI_CLR_HOOK(i) _NGI_CLR_HOOK(i)
#define NGI_SET_NODE(i,n) _NGI_SET_NODE(i,n)
#define NGI_CLR_NODE(i) _NGI_CLR_NODE(i)
#define NG_FREE_ITEM(item) ng_free_item((item))
#define SAVE_LINE(item) do {} while (0)
#endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
#define NGI_GET_M(i,m) \
do { \
(m) = NGI_M(i); \
_NGI_M(i) = NULL; \
} while (0)
#define NGI_GET_MSG(i,m) \
do { \
(m) = NGI_MSG(i); \
_NGI_MSG(i) = NULL; \
} while (0)
#define NGI_GET_NODE(i,n) /* YOU NOW HAVE THE REFERENCE */ \
do { \
(n) = NGI_NODE(i); \
_NGI_NODE(i) = NULL; \
} while (0)
#define NGI_GET_HOOK(i,h) \
do { \
(h) = NGI_HOOK(i); \
_NGI_HOOK(i) = NULL; \
} while (0)
#define NGI_SET_WRITER(i) ((i)->el_flags &= ~NGQF_QMODE)
#define NGI_SET_READER(i) ((i)->el_flags |= NGQF_QREADER)
#define NGI_QUEUED_READER(i) ((i)->el_flags & NGQF_QREADER)
#define NGI_QUEUED_WRITER(i) (((i)->el_flags & NGQF_QMODE) == NGQF_QWRITER)
/**********************************************************************
* Data macros. Send, manipulate and free.
**********************************************************************/
/*
* Assuming the data is already ok, just set the new address and send
*/
#define NG_FWD_ITEM_HOOK_FLAGS(error, item, hook, flags) \
do { \
(error) = \
ng_address_hook(NULL, (item), (hook), NG_NOFLAGS); \
if (error == 0) { \
SAVE_LINE(item); \
(error) = ng_snd_item((item), (flags)); \
} \
(item) = NULL; \
} while (0)
#define NG_FWD_ITEM_HOOK(error, item, hook) \
NG_FWD_ITEM_HOOK_FLAGS(error, item, hook, NG_NOFLAGS)
/*
* Forward a data packet. Mbuf pointer is updated to new value. We
* presume you dealt with the old one when you update it to the new one
* (or it maybe the old one). We got a packet and possibly had to modify
* the mbuf. You should probably use NGI_GET_M() if you are going to use
* this too.
*/
#define NG_FWD_NEW_DATA_FLAGS(error, item, hook, m, flags) \
do { \
NGI_M(item) = (m); \
(m) = NULL; \
NG_FWD_ITEM_HOOK_FLAGS(error, item, hook, flags); \
} while (0)
#define NG_FWD_NEW_DATA(error, item, hook, m) \
NG_FWD_NEW_DATA_FLAGS(error, item, hook, m, NG_NOFLAGS)
/* Send a previously unpackaged mbuf. XXX: This should be called
* NG_SEND_DATA in future, but this name is kept for compatibility
* reasons.
*/
#define NG_SEND_DATA_FLAGS(error, hook, m, flags) \
do { \
item_p _item; \
if ((_item = ng_package_data((m), flags))) { \
NG_FWD_ITEM_HOOK_FLAGS(error, _item, hook, flags);\
} else { \
(error) = ENOMEM; \
} \
(m) = NULL; \
} while (0)
#define NG_SEND_DATA_ONLY(error, hook, m) \
NG_SEND_DATA_FLAGS(error, hook, m, NG_NOFLAGS)
/* NG_SEND_DATA() compat for meta-data times */
#define NG_SEND_DATA(error, hook, m, x) \
NG_SEND_DATA_FLAGS(error, hook, m, NG_NOFLAGS)
#define NG_FREE_MSG(msg) \
do { \
if ((msg)) { \
free((msg), M_NETGRAPH_MSG); \
(msg) = NULL; \
} \
} while (0)
#define NG_FREE_M(m) \
do { \
if ((m)) { \
m_freem((m)); \
(m) = NULL; \
} \
} while (0)
/*****************************************
* Message macros
*****************************************/
#define NG_SEND_MSG_HOOK(error, here, msg, hook, retaddr) \
do { \
item_p _item; \
if ((_item = ng_package_msg(msg, NG_NOFLAGS)) == NULL) {\
(msg) = NULL; \
(error) = ENOMEM; \
break; \
} \
if (((error) = ng_address_hook((here), (_item), \
(hook), (retaddr))) == 0) { \
SAVE_LINE(_item); \
(error) = ng_snd_item((_item), 0); \
} \
(msg) = NULL; \
} while (0)
#define NG_SEND_MSG_PATH(error, here, msg, path, retaddr) \
do { \
item_p _item; \
if ((_item = ng_package_msg(msg, NG_NOFLAGS)) == NULL) {\
(msg) = NULL; \
(error) = ENOMEM; \
break; \
} \
if (((error) = ng_address_path((here), (_item), \
(path), (retaddr))) == 0) { \
SAVE_LINE(_item); \
(error) = ng_snd_item((_item), 0); \
} \
(msg) = NULL; \
} while (0)
#define NG_SEND_MSG_ID(error, here, msg, ID, retaddr) \
do { \
item_p _item; \
if ((_item = ng_package_msg(msg, NG_NOFLAGS)) == NULL) {\
(msg) = NULL; \
(error) = ENOMEM; \
break; \
} \
if (((error) = ng_address_ID((here), (_item), \
(ID), (retaddr))) == 0) { \
SAVE_LINE(_item); \
(error) = ng_snd_item((_item), 0); \
} \
(msg) = NULL; \
} while (0)
/*
* Redirect the message to the next hop using the given hook.
* ng_retarget_msg() frees the item if there is an error
* and returns an error code. It returns 0 on success.
*/
#define NG_FWD_MSG_HOOK(error, here, item, hook, retaddr) \
do { \
if (((error) = ng_address_hook((here), (item), \
(hook), (retaddr))) == 0) { \
SAVE_LINE(item); \
(error) = ng_snd_item((item), 0); \
} \
(item) = NULL; \
} while (0)
/*
* Send a queue item back to it's originator with a response message.
* Assume original message was removed and freed separatly.
*/
#define NG_RESPOND_MSG(error, here, item, resp) \
do { \
if (resp) { \
ng_ID_t _dest = NGI_RETADDR(item); \
NGI_RETADDR(item) = 0; \
NGI_MSG(item) = resp; \
if ((error = ng_address_ID((here), (item), \
_dest, 0)) == 0) { \
SAVE_LINE(item); \
(error) = ng_snd_item((item), NG_QUEUE);\
} \
} else \
NG_FREE_ITEM(item); \
(item) = NULL; \
} while (0)
/***********************************************************************
******** Structures Definitions and Macros for defining a node *******
***********************************************************************
*
* Here we define the structures needed to actually define a new node
* type.
*/
/*
* Command list -- each node type specifies the command that it knows
* how to convert between ASCII and binary using an array of these.
* The last element in the array must be a terminator with cookie=0.
*/
struct ng_cmdlist {
u_int32_t cookie; /* command typecookie */
int cmd; /* command number */
const char *name; /* command name */
const struct ng_parse_type *mesgType; /* args if !NGF_RESP */
const struct ng_parse_type *respType; /* args if NGF_RESP */
};
/*
* Structure of a node type
* If data is sent to the "rcvdata()" entrypoint then the system
* may decide to defer it until later by queing it with the normal netgraph
* input queuing system. This is decidde by the HK_QUEUE flag being set in
* the flags word of the peer (receiving) hook. The dequeuing mechanism will
* ensure it is not requeued again.
* Note the input queueing system is to allow modules
* to 'release the stack' or to pass data across spl layers.
* The data will be redelivered as soon as the NETISR code runs
* which may be almost immediatly. A node may also do it's own queueing
* for other reasons (e.g. device output queuing).
*/
struct ng_type {
u_int32_t version; /* must equal NG_API_VERSION */
const char *name; /* Unique type name */
modeventhand_t mod_event; /* Module event handler (optional) */
ng_constructor_t *constructor; /* Node constructor */
ng_rcvmsg_t *rcvmsg; /* control messages come here */
ng_close_t *close; /* warn about forthcoming shutdown */
ng_shutdown_t *shutdown; /* reset, and free resources */
ng_newhook_t *newhook; /* first notification of new hook */
ng_findhook_t *findhook; /* only if you have lots of hooks */
ng_connect_t *connect; /* final notification of new hook */
ng_rcvdata_t *rcvdata; /* data comes here */
ng_disconnect_t *disconnect; /* notify on disconnect */
const struct ng_cmdlist *cmdlist; /* commands we can convert */
/* R/W data private to the base netgraph code DON'T TOUCH! */
LIST_ENTRY(ng_type) types; /* linked list of all types */
int refs; /* number of instances */
};
/*
* Use the NETGRAPH_INIT() macro to link a node type into the
* netgraph system. This works for types compiled into the kernel
* as well as KLD modules. The first argument should be the type
* name (eg, echo) and the second a pointer to the type struct.
*
* If a different link time is desired, e.g., a device driver that
* needs to install its netgraph type before probing, use the
* NETGRAPH_INIT_ORDERED() macro instead. Device drivers probably
* want to use SI_SUB_DRIVERS/SI_ORDER_FIRST.
*/
#define NETGRAPH_INIT_ORDERED(typename, typestructp, sub, order) \
static moduledata_t ng_##typename##_mod = { \
"ng_" #typename, \
ng_mod_event, \
(typestructp) \
}; \
DECLARE_MODULE(ng_##typename, ng_##typename##_mod, sub, order); \
MODULE_DEPEND(ng_##typename, netgraph, NG_ABI_VERSION, \
NG_ABI_VERSION, \
NG_ABI_VERSION)
#define NETGRAPH_INIT(tn, tp) \
NETGRAPH_INIT_ORDERED(tn, tp, SI_SUB_PSEUDO, SI_ORDER_MIDDLE)
/* Special malloc() type for netgraph structs and ctrl messages */
/* Only these two types should be visible to nodes */
MALLOC_DECLARE(M_NETGRAPH);
MALLOC_DECLARE(M_NETGRAPH_MSG);
/* declare the base of the netgraph sysclt hierarchy */
/* but only if this file cares about sysctls */
#ifdef SYSCTL_DECL
SYSCTL_DECL(_net_graph);
#endif
/*
* Methods that the nodes can use.
* Many of these methods should usually NOT be used directly but via
* Macros above.
*/
int ng_address_ID(node_p here, item_p item, ng_ID_t ID, ng_ID_t retaddr);
int ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr);
int ng_address_path(node_p here, item_p item, const char *address, ng_ID_t raddr);
int ng_bypass(hook_p hook1, hook_p hook2);
hook_p ng_findhook(node_p node, const char *name);
struct ng_type *ng_findtype(const char *type);
int ng_make_node_common(struct ng_type *typep, node_p *nodep);
int ng_name_node(node_p node, const char *name);
node_p ng_name2noderef(node_p node, const char *name);
int ng_newtype(struct ng_type *tp);
ng_ID_t ng_node2ID(node_p node);
item_p ng_package_data(struct mbuf *m, int flags);
item_p ng_package_msg(struct ng_mesg *msg, int flags);
item_p ng_package_msg_self(node_p here, hook_p hook, struct ng_mesg *msg);
void ng_replace_retaddr(node_p here, item_p item, ng_ID_t retaddr);
int ng_rmhook_self(hook_p hook); /* if a node wants to kill a hook */
int ng_rmnode_self(node_p here); /* if a node wants to suicide */
int ng_rmtype(struct ng_type *tp);
int ng_snd_item(item_p item, int queue);
int ng_send_fn(node_p node, hook_p hook, ng_item_fn *fn, void *arg1,
int arg2);
int ng_send_fn1(node_p node, hook_p hook, ng_item_fn *fn, void *arg1,
int arg2, int flags);
int ng_send_fn2(node_p node, hook_p hook, item_p pitem, ng_item_fn2 *fn,
void *arg1, int arg2, int flags);
int ng_uncallout(struct callout *c, node_p node);
int ng_callout(struct callout *c, node_p node, hook_p hook, int ticks,
ng_item_fn *fn, void * arg1, int arg2);
-#define ng_callout_init(c) callout_init(c, CALLOUT_MPSAFE)
+#define ng_callout_init(c) callout_init(c, 1)
/* Flags for netgraph functions. */
#define NG_NOFLAGS 0x00000000 /* no special options */
#define NG_QUEUE 0x00000001 /* enqueue item, don't dispatch */
#define NG_WAITOK 0x00000002 /* use M_WAITOK, etc. */
/* XXXGL: NG_PROGRESS unused since ng_base.c rev. 1.136. Should be deleted? */
#define NG_PROGRESS 0x00000004 /* return EINPROGRESS if queued */
#define NG_REUSE_ITEM 0x00000008 /* supplied item should be reused */
/*
* prototypes the user should DEFINITELY not use directly
*/
void ng_free_item(item_p item); /* Use NG_FREE_ITEM instead */
int ng_mod_event(module_t mod, int what, void *arg);
/*
* Tag definitions and constants
*/
#define NG_TAG_PRIO 1
struct ng_tag_prio {
struct m_tag tag;
char priority;
char discardability;
};
#define NG_PRIO_CUTOFF 32
#define NG_PRIO_LINKSTATE 64
/* Macros and declarations to keep compatibility with metadata, which
* is obsoleted now. To be deleted.
*/
typedef void *meta_p;
#define _NGI_META(i) NULL
#define NGI_META(i) NULL
#define NG_FREE_META(meta)
#define NGI_GET_META(i,m)
#define ng_copy_meta(meta) NULL
/*
* Mark the current thread when called from the outbound path of the
* network stack, in order to enforce queuing on ng nodes calling into
* the inbound network stack path.
*/
#define NG_OUTBOUND_THREAD_REF() \
curthread->td_ng_outbound++
#define NG_OUTBOUND_THREAD_UNREF() \
do { \
curthread->td_ng_outbound--; \
KASSERT(curthread->td_ng_outbound >= 0, \
("%s: negative td_ng_outbound", __func__)); \
} while (0)
#endif /* _NETGRAPH_NETGRAPH_H_ */
Index: head/sys/netinet/in_pcb.c
===================================================================
--- head/sys/netinet/in_pcb.c (revision 283290)
+++ head/sys/netinet/in_pcb.c (revision 283291)
@@ -1,2618 +1,2618 @@
/*-
* Copyright (c) 1982, 1986, 1991, 1993, 1995
* The Regents of the University of California.
* Copyright (c) 2007-2009 Robert N. M. Watson
* Copyright (c) 2010-2011 Juniper Networks, Inc.
* All rights reserved.
*
* Portions of this software were developed by Robert N. M. Watson under
* contract to Juniper Networks, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)in_pcb.c 8.4 (Berkeley) 5/24/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include "opt_ipsec.h"
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_pcbgroup.h"
#include "opt_rss.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/callout.h>
#include <sys/domain.h>
#include <sys/protosw.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/refcount.h>
#include <sys/jail.h>
#include <sys/kernel.h>
#include <sys/sysctl.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
#include <vm/uma.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_types.h>
#include <net/route.h>
#include <net/rss_config.h>
#include <net/vnet.h>
#if defined(INET) || defined(INET6)
#include <netinet/in.h>
#include <netinet/in_pcb.h>
#include <netinet/ip_var.h>
#include <netinet/tcp_var.h>
#include <netinet/udp.h>
#include <netinet/udp_var.h>
#endif
#ifdef INET
#include <netinet/in_var.h>
#endif
#ifdef INET6
#include <netinet/ip6.h>
#include <netinet6/in6_pcb.h>
#include <netinet6/in6_var.h>
#include <netinet6/ip6_var.h>
#endif /* INET6 */
#ifdef IPSEC
#include <netipsec/ipsec.h>
#include <netipsec/key.h>
#endif /* IPSEC */
#include <security/mac/mac_framework.h>
static struct callout ipport_tick_callout;
/*
* These configure the range of local port addresses assigned to
* "unspecified" outgoing connections/packets/whatever.
*/
VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1; /* 1023 */
VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART; /* 600 */
VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST; /* 10000 */
VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST; /* 65535 */
VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO; /* 49152 */
VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO; /* 65535 */
/*
* Reserved ports accessible only to root. There are significant
* security considerations that must be accounted for when changing these,
* but the security benefits can be great. Please be careful.
*/
VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */
VNET_DEFINE(int, ipport_reservedlow);
/* Variables dealing with random ephemeral port allocation. */
VNET_DEFINE(int, ipport_randomized) = 1; /* user controlled via sysctl */
VNET_DEFINE(int, ipport_randomcps) = 10; /* user controlled via sysctl */
VNET_DEFINE(int, ipport_randomtime) = 45; /* user controlled via sysctl */
VNET_DEFINE(int, ipport_stoprandom); /* toggled by ipport_tick */
VNET_DEFINE(int, ipport_tcpallocs);
static VNET_DEFINE(int, ipport_tcplastcount);
#define V_ipport_tcplastcount VNET(ipport_tcplastcount)
static void in_pcbremlists(struct inpcb *inp);
#ifdef INET
static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
struct in_addr faddr, u_int fport_arg,
struct in_addr laddr, u_int lport_arg,
int lookupflags, struct ifnet *ifp);
#define RANGECHK(var, min, max) \
if ((var) < (min)) { (var) = (min); } \
else if ((var) > (max)) { (var) = (max); }
static int
sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
{
int error;
error = sysctl_handle_int(oidp, arg1, arg2, req);
if (error == 0) {
RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
}
return (error);
}
#undef RANGECHK
static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0,
"IP Ports");
SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
&VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I", "");
SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
&VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I", "");
SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
&VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I", "");
SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
&VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I", "");
SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
&VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I", "");
SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
&VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I", "");
SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
&VNET_NAME(ipport_reservedhigh), 0, "");
SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(ipport_randomcps), 0, "Maximum number of random port "
"allocations before switching to a sequental one");
SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(ipport_randomtime), 0,
"Minimum time to keep sequental port "
"allocation before switching to a random one");
#endif /* INET */
/*
* in_pcb.c: manage the Protocol Control Blocks.
*
* NOTE: It is assumed that most of these functions will be called with
* the pcbinfo lock held, and often, the inpcb lock held, as these utility
* functions often modify hash chains or addresses in pcbs.
*/
/*
* Initialize an inpcbinfo -- we should be able to reduce the number of
* arguments in time.
*/
void
in_pcbinfo_init(struct inpcbinfo *pcbinfo, const char *name,
struct inpcbhead *listhead, int hash_nelements, int porthash_nelements,
char *inpcbzone_name, uma_init inpcbzone_init, uma_fini inpcbzone_fini,
uint32_t inpcbzone_flags, u_int hashfields)
{
INP_INFO_LOCK_INIT(pcbinfo, name);
INP_HASH_LOCK_INIT(pcbinfo, "pcbinfohash"); /* XXXRW: argument? */
#ifdef VIMAGE
pcbinfo->ipi_vnet = curvnet;
#endif
pcbinfo->ipi_listhead = listhead;
LIST_INIT(pcbinfo->ipi_listhead);
pcbinfo->ipi_count = 0;
pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB,
&pcbinfo->ipi_hashmask);
pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
&pcbinfo->ipi_porthashmask);
#ifdef PCBGROUP
in_pcbgroup_init(pcbinfo, hashfields, hash_nelements);
#endif
pcbinfo->ipi_zone = uma_zcreate(inpcbzone_name, sizeof(struct inpcb),
NULL, NULL, inpcbzone_init, inpcbzone_fini, UMA_ALIGN_PTR,
inpcbzone_flags);
uma_zone_set_max(pcbinfo->ipi_zone, maxsockets);
uma_zone_set_warning(pcbinfo->ipi_zone,
"kern.ipc.maxsockets limit reached");
}
/*
* Destroy an inpcbinfo.
*/
void
in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
{
KASSERT(pcbinfo->ipi_count == 0,
("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask);
hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
pcbinfo->ipi_porthashmask);
#ifdef PCBGROUP
in_pcbgroup_destroy(pcbinfo);
#endif
uma_zdestroy(pcbinfo->ipi_zone);
INP_HASH_LOCK_DESTROY(pcbinfo);
INP_INFO_LOCK_DESTROY(pcbinfo);
}
/*
* Allocate a PCB and associate it with the socket.
* On success return with the PCB locked.
*/
int
in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
{
struct inpcb *inp;
int error;
INP_INFO_WLOCK_ASSERT(pcbinfo);
error = 0;
inp = uma_zalloc(pcbinfo->ipi_zone, M_NOWAIT);
if (inp == NULL)
return (ENOBUFS);
bzero(inp, inp_zero_size);
inp->inp_pcbinfo = pcbinfo;
inp->inp_socket = so;
inp->inp_cred = crhold(so->so_cred);
inp->inp_inc.inc_fibnum = so->so_fibnum;
#ifdef MAC
error = mac_inpcb_init(inp, M_NOWAIT);
if (error != 0)
goto out;
mac_inpcb_create(so, inp);
#endif
#ifdef IPSEC
error = ipsec_init_policy(so, &inp->inp_sp);
if (error != 0) {
#ifdef MAC
mac_inpcb_destroy(inp);
#endif
goto out;
}
#endif /*IPSEC*/
#ifdef INET6
if (INP_SOCKAF(so) == AF_INET6) {
inp->inp_vflag |= INP_IPV6PROTO;
if (V_ip6_v6only)
inp->inp_flags |= IN6P_IPV6_V6ONLY;
}
#endif
LIST_INSERT_HEAD(pcbinfo->ipi_listhead, inp, inp_list);
pcbinfo->ipi_count++;
so->so_pcb = (caddr_t)inp;
#ifdef INET6
if (V_ip6_auto_flowlabel)
inp->inp_flags |= IN6P_AUTOFLOWLABEL;
#endif
INP_WLOCK(inp);
inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
refcount_init(&inp->inp_refcount, 1); /* Reference from inpcbinfo */
#if defined(IPSEC) || defined(MAC)
out:
if (error != 0) {
crfree(inp->inp_cred);
uma_zfree(pcbinfo->ipi_zone, inp);
}
#endif
return (error);
}
#ifdef INET
int
in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
{
int anonport, error;
INP_WLOCK_ASSERT(inp);
INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
return (EINVAL);
anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
&inp->inp_lport, cred);
if (error)
return (error);
if (in_pcbinshash(inp) != 0) {
inp->inp_laddr.s_addr = INADDR_ANY;
inp->inp_lport = 0;
return (EAGAIN);
}
if (anonport)
inp->inp_flags |= INP_ANONPORT;
return (0);
}
#endif
/*
* Select a local port (number) to use.
*/
#if defined(INET) || defined(INET6)
int
in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
struct ucred *cred, int lookupflags)
{
struct inpcbinfo *pcbinfo;
struct inpcb *tmpinp;
unsigned short *lastport;
int count, dorandom, error;
u_short aux, first, last, lport;
#ifdef INET
struct in_addr laddr;
#endif
pcbinfo = inp->inp_pcbinfo;
/*
* Because no actual state changes occur here, a global write lock on
* the pcbinfo isn't required.
*/
INP_LOCK_ASSERT(inp);
INP_HASH_LOCK_ASSERT(pcbinfo);
if (inp->inp_flags & INP_HIGHPORT) {
first = V_ipport_hifirstauto; /* sysctl */
last = V_ipport_hilastauto;
lastport = &pcbinfo->ipi_lasthi;
} else if (inp->inp_flags & INP_LOWPORT) {
error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0);
if (error)
return (error);
first = V_ipport_lowfirstauto; /* 1023 */
last = V_ipport_lowlastauto; /* 600 */
lastport = &pcbinfo->ipi_lastlow;
} else {
first = V_ipport_firstauto; /* sysctl */
last = V_ipport_lastauto;
lastport = &pcbinfo->ipi_lastport;
}
/*
* For UDP(-Lite), use random port allocation as long as the user
* allows it. For TCP (and as of yet unknown) connections,
* use random port allocation only if the user allows it AND
* ipport_tick() allows it.
*/
if (V_ipport_randomized &&
(!V_ipport_stoprandom || pcbinfo == &V_udbinfo ||
pcbinfo == &V_ulitecbinfo))
dorandom = 1;
else
dorandom = 0;
/*
* It makes no sense to do random port allocation if
* we have the only port available.
*/
if (first == last)
dorandom = 0;
/* Make sure to not include UDP(-Lite) packets in the count. */
if (pcbinfo != &V_udbinfo || pcbinfo != &V_ulitecbinfo)
V_ipport_tcpallocs++;
/*
* Instead of having two loops further down counting up or down
* make sure that first is always <= last and go with only one
* code path implementing all logic.
*/
if (first > last) {
aux = first;
first = last;
last = aux;
}
#ifdef INET
/* Make the compiler happy. */
laddr.s_addr = 0;
if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
KASSERT(laddrp != NULL, ("%s: laddrp NULL for v4 inp %p",
__func__, inp));
laddr = *laddrp;
}
#endif
tmpinp = NULL; /* Make compiler happy. */
lport = *lportp;
if (dorandom)
*lastport = first + (arc4random() % (last - first));
count = last - first;
do {
if (count-- < 0) /* completely used? */
return (EADDRNOTAVAIL);
++*lastport;
if (*lastport < first || *lastport > last)
*lastport = first;
lport = htons(*lastport);
#ifdef INET6
if ((inp->inp_vflag & INP_IPV6) != 0)
tmpinp = in6_pcblookup_local(pcbinfo,
&inp->in6p_laddr, lport, lookupflags, cred);
#endif
#if defined(INET) && defined(INET6)
else
#endif
#ifdef INET
tmpinp = in_pcblookup_local(pcbinfo, laddr,
lport, lookupflags, cred);
#endif
} while (tmpinp != NULL);
#ifdef INET
if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4)
laddrp->s_addr = laddr.s_addr;
#endif
*lportp = lport;
return (0);
}
/*
* Return cached socket options.
*/
short
inp_so_options(const struct inpcb *inp)
{
short so_options;
so_options = 0;
if ((inp->inp_flags2 & INP_REUSEPORT) != 0)
so_options |= SO_REUSEPORT;
if ((inp->inp_flags2 & INP_REUSEADDR) != 0)
so_options |= SO_REUSEADDR;
return (so_options);
}
#endif /* INET || INET6 */
/*
* Check if a new BINDMULTI socket is allowed to be created.
*
* ni points to the new inp.
* oi points to the exisitng inp.
*
* This checks whether the existing inp also has BINDMULTI and
* whether the credentials match.
*/
int
in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
{
/* Check permissions match */
if ((ni->inp_flags2 & INP_BINDMULTI) &&
(ni->inp_cred->cr_uid !=
oi->inp_cred->cr_uid))
return (0);
/* Check the existing inp has BINDMULTI set */
if ((ni->inp_flags2 & INP_BINDMULTI) &&
((oi->inp_flags2 & INP_BINDMULTI) == 0))
return (0);
/*
* We're okay - either INP_BINDMULTI isn't set on ni, or
* it is and it matches the checks.
*/
return (1);
}
#ifdef INET
/*
* Set up a bind operation on a PCB, performing port allocation
* as required, but do not actually modify the PCB. Callers can
* either complete the bind by setting inp_laddr/inp_lport and
* calling in_pcbinshash(), or they can just use the resulting
* port and address to authorise the sending of a once-off packet.
*
* On error, the values of *laddrp and *lportp are not changed.
*/
int
in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
u_short *lportp, struct ucred *cred)
{
struct socket *so = inp->inp_socket;
struct sockaddr_in *sin;
struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
struct in_addr laddr;
u_short lport = 0;
int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
int error;
/*
* No state changes, so read locks are sufficient here.
*/
INP_LOCK_ASSERT(inp);
INP_HASH_LOCK_ASSERT(pcbinfo);
if (TAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */
return (EADDRNOTAVAIL);
laddr.s_addr = *laddrp;
if (nam != NULL && laddr.s_addr != INADDR_ANY)
return (EINVAL);
if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) == 0)
lookupflags = INPLOOKUP_WILDCARD;
if (nam == NULL) {
if ((error = prison_local_ip4(cred, &laddr)) != 0)
return (error);
} else {
sin = (struct sockaddr_in *)nam;
if (nam->sa_len != sizeof (*sin))
return (EINVAL);
#ifdef notdef
/*
* We should check the family, but old programs
* incorrectly fail to initialize it.
*/
if (sin->sin_family != AF_INET)
return (EAFNOSUPPORT);
#endif
error = prison_local_ip4(cred, &sin->sin_addr);
if (error)
return (error);
if (sin->sin_port != *lportp) {
/* Don't allow the port to change. */
if (*lportp != 0)
return (EINVAL);
lport = sin->sin_port;
}
/* NB: lport is left as 0 if the port isn't being changed. */
if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
/*
* Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
* allow complete duplication of binding if
* SO_REUSEPORT is set, or if SO_REUSEADDR is set
* and a multicast address is bound on both
* new and duplicated sockets.
*/
if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
reuseport = SO_REUSEADDR|SO_REUSEPORT;
} else if (sin->sin_addr.s_addr != INADDR_ANY) {
sin->sin_port = 0; /* yech... */
bzero(&sin->sin_zero, sizeof(sin->sin_zero));
/*
* Is the address a local IP address?
* If INP_BINDANY is set, then the socket may be bound
* to any endpoint address, local or not.
*/
if ((inp->inp_flags & INP_BINDANY) == 0 &&
ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
return (EADDRNOTAVAIL);
}
laddr = sin->sin_addr;
if (lport) {
struct inpcb *t;
struct tcptw *tw;
/* GROSS */
if (ntohs(lport) <= V_ipport_reservedhigh &&
ntohs(lport) >= V_ipport_reservedlow &&
priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT,
0))
return (EACCES);
if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
priv_check_cred(inp->inp_cred,
PRIV_NETINET_REUSEPORT, 0) != 0) {
t = in_pcblookup_local(pcbinfo, sin->sin_addr,
lport, INPLOOKUP_WILDCARD, cred);
/*
* XXX
* This entire block sorely needs a rewrite.
*/
if (t &&
((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
((t->inp_flags & INP_TIMEWAIT) == 0) &&
(so->so_type != SOCK_STREAM ||
ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
(ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
(t->inp_flags2 & INP_REUSEPORT) == 0) &&
(inp->inp_cred->cr_uid !=
t->inp_cred->cr_uid))
return (EADDRINUSE);
/*
* If the socket is a BINDMULTI socket, then
* the credentials need to match and the
* original socket also has to have been bound
* with BINDMULTI.
*/
if (t && (! in_pcbbind_check_bindmulti(inp, t)))
return (EADDRINUSE);
}
t = in_pcblookup_local(pcbinfo, sin->sin_addr,
lport, lookupflags, cred);
if (t && (t->inp_flags & INP_TIMEWAIT)) {
/*
* XXXRW: If an incpb has had its timewait
* state recycled, we treat the address as
* being in use (for now). This is better
* than a panic, but not desirable.
*/
tw = intotw(t);
if (tw == NULL ||
(reuseport & tw->tw_so_options) == 0)
return (EADDRINUSE);
} else if (t &&
((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
(reuseport & inp_so_options(t)) == 0) {
#ifdef INET6
if (ntohl(sin->sin_addr.s_addr) !=
INADDR_ANY ||
ntohl(t->inp_laddr.s_addr) !=
INADDR_ANY ||
(inp->inp_vflag & INP_IPV6PROTO) == 0 ||
(t->inp_vflag & INP_IPV6PROTO) == 0)
#endif
return (EADDRINUSE);
if (t && (! in_pcbbind_check_bindmulti(inp, t)))
return (EADDRINUSE);
}
}
}
if (*lportp != 0)
lport = *lportp;
if (lport == 0) {
error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
if (error != 0)
return (error);
}
*laddrp = laddr.s_addr;
*lportp = lport;
return (0);
}
/*
* Connect from a socket to a specified address.
* Both address and port must be specified in argument sin.
* If don't have a local address for this socket yet,
* then pick one.
*/
int
in_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
struct ucred *cred, struct mbuf *m)
{
u_short lport, fport;
in_addr_t laddr, faddr;
int anonport, error;
INP_WLOCK_ASSERT(inp);
INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
lport = inp->inp_lport;
laddr = inp->inp_laddr.s_addr;
anonport = (lport == 0);
error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport,
NULL, cred);
if (error)
return (error);
/* Do the initial binding of the local address if required. */
if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
inp->inp_lport = lport;
inp->inp_laddr.s_addr = laddr;
if (in_pcbinshash(inp) != 0) {
inp->inp_laddr.s_addr = INADDR_ANY;
inp->inp_lport = 0;
return (EAGAIN);
}
}
/* Commit the remaining changes. */
inp->inp_lport = lport;
inp->inp_laddr.s_addr = laddr;
inp->inp_faddr.s_addr = faddr;
inp->inp_fport = fport;
in_pcbrehash_mbuf(inp, m);
if (anonport)
inp->inp_flags |= INP_ANONPORT;
return (0);
}
int
in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
{
return (in_pcbconnect_mbuf(inp, nam, cred, NULL));
}
/*
* Do proper source address selection on an unbound socket in case
* of connect. Take jails into account as well.
*/
int
in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
struct ucred *cred)
{
struct ifaddr *ifa;
struct sockaddr *sa;
struct sockaddr_in *sin;
struct route sro;
int error;
KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
/*
* Bypass source address selection and use the primary jail IP
* if requested.
*/
if (cred != NULL && !prison_saddrsel_ip4(cred, laddr))
return (0);
error = 0;
bzero(&sro, sizeof(sro));
sin = (struct sockaddr_in *)&sro.ro_dst;
sin->sin_family = AF_INET;
sin->sin_len = sizeof(struct sockaddr_in);
sin->sin_addr.s_addr = faddr->s_addr;
/*
* If route is known our src addr is taken from the i/f,
* else punt.
*
* Find out route to destination.
*/
if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
in_rtalloc_ign(&sro, 0, inp->inp_inc.inc_fibnum);
/*
* If we found a route, use the address corresponding to
* the outgoing interface.
*
* Otherwise assume faddr is reachable on a directly connected
* network and try to find a corresponding interface to take
* the source address from.
*/
if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) {
struct in_ifaddr *ia;
struct ifnet *ifp;
ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
inp->inp_socket->so_fibnum));
if (ia == NULL)
ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
inp->inp_socket->so_fibnum));
if (ia == NULL) {
error = ENETUNREACH;
goto done;
}
if (cred == NULL || !prison_flag(cred, PR_IP4)) {
laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
ifa_free(&ia->ia_ifa);
goto done;
}
ifp = ia->ia_ifp;
ifa_free(&ia->ia_ifa);
ia = NULL;
IF_ADDR_RLOCK(ifp);
TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
sa = ifa->ifa_addr;
if (sa->sa_family != AF_INET)
continue;
sin = (struct sockaddr_in *)sa;
if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
ia = (struct in_ifaddr *)ifa;
break;
}
}
if (ia != NULL) {
laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
IF_ADDR_RUNLOCK(ifp);
goto done;
}
IF_ADDR_RUNLOCK(ifp);
/* 3. As a last resort return the 'default' jail address. */
error = prison_get_ip4(cred, laddr);
goto done;
}
/*
* If the outgoing interface on the route found is not
* a loopback interface, use the address from that interface.
* In case of jails do those three steps:
* 1. check if the interface address belongs to the jail. If so use it.
* 2. check if we have any address on the outgoing interface
* belonging to this jail. If so use it.
* 3. as a last resort return the 'default' jail address.
*/
if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) == 0) {
struct in_ifaddr *ia;
struct ifnet *ifp;
/* If not jailed, use the default returned. */
if (cred == NULL || !prison_flag(cred, PR_IP4)) {
ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
goto done;
}
/* Jailed. */
/* 1. Check if the iface address belongs to the jail. */
sin = (struct sockaddr_in *)sro.ro_rt->rt_ifa->ifa_addr;
if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
goto done;
}
/*
* 2. Check if we have any address on the outgoing interface
* belonging to this jail.
*/
ia = NULL;
ifp = sro.ro_rt->rt_ifp;
IF_ADDR_RLOCK(ifp);
TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
sa = ifa->ifa_addr;
if (sa->sa_family != AF_INET)
continue;
sin = (struct sockaddr_in *)sa;
if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
ia = (struct in_ifaddr *)ifa;
break;
}
}
if (ia != NULL) {
laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
IF_ADDR_RUNLOCK(ifp);
goto done;
}
IF_ADDR_RUNLOCK(ifp);
/* 3. As a last resort return the 'default' jail address. */
error = prison_get_ip4(cred, laddr);
goto done;
}
/*
* The outgoing interface is marked with 'loopback net', so a route
* to ourselves is here.
* Try to find the interface of the destination address and then
* take the address from there. That interface is not necessarily
* a loopback interface.
* In case of jails, check that it is an address of the jail
* and if we cannot find, fall back to the 'default' jail address.
*/
if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
struct sockaddr_in sain;
struct in_ifaddr *ia;
bzero(&sain, sizeof(struct sockaddr_in));
sain.sin_family = AF_INET;
sain.sin_len = sizeof(struct sockaddr_in);
sain.sin_addr.s_addr = faddr->s_addr;
ia = ifatoia(ifa_ifwithdstaddr(sintosa(&sain),
inp->inp_socket->so_fibnum));
if (ia == NULL)
ia = ifatoia(ifa_ifwithnet(sintosa(&sain), 0,
inp->inp_socket->so_fibnum));
if (ia == NULL)
ia = ifatoia(ifa_ifwithaddr(sintosa(&sain)));
if (cred == NULL || !prison_flag(cred, PR_IP4)) {
if (ia == NULL) {
error = ENETUNREACH;
goto done;
}
laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
ifa_free(&ia->ia_ifa);
goto done;
}
/* Jailed. */
if (ia != NULL) {
struct ifnet *ifp;
ifp = ia->ia_ifp;
ifa_free(&ia->ia_ifa);
ia = NULL;
IF_ADDR_RLOCK(ifp);
TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
sa = ifa->ifa_addr;
if (sa->sa_family != AF_INET)
continue;
sin = (struct sockaddr_in *)sa;
if (prison_check_ip4(cred,
&sin->sin_addr) == 0) {
ia = (struct in_ifaddr *)ifa;
break;
}
}
if (ia != NULL) {
laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
IF_ADDR_RUNLOCK(ifp);
goto done;
}
IF_ADDR_RUNLOCK(ifp);
}
/* 3. As a last resort return the 'default' jail address. */
error = prison_get_ip4(cred, laddr);
goto done;
}
done:
if (sro.ro_rt != NULL)
RTFREE(sro.ro_rt);
return (error);
}
/*
* Set up for a connect from a socket to the specified address.
* On entry, *laddrp and *lportp should contain the current local
* address and port for the PCB; these are updated to the values
* that should be placed in inp_laddr and inp_lport to complete
* the connect.
*
* On success, *faddrp and *fportp will be set to the remote address
* and port. These are not updated in the error case.
*
* If the operation fails because the connection already exists,
* *oinpp will be set to the PCB of that connection so that the
* caller can decide to override it. In all other cases, *oinpp
* is set to NULL.
*/
int
in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
struct inpcb **oinpp, struct ucred *cred)
{
struct sockaddr_in *sin = (struct sockaddr_in *)nam;
struct in_ifaddr *ia;
struct inpcb *oinp;
struct in_addr laddr, faddr;
u_short lport, fport;
int error;
/*
* Because a global state change doesn't actually occur here, a read
* lock is sufficient.
*/
INP_LOCK_ASSERT(inp);
INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
if (oinpp != NULL)
*oinpp = NULL;
if (nam->sa_len != sizeof (*sin))
return (EINVAL);
if (sin->sin_family != AF_INET)
return (EAFNOSUPPORT);
if (sin->sin_port == 0)
return (EADDRNOTAVAIL);
laddr.s_addr = *laddrp;
lport = *lportp;
faddr = sin->sin_addr;
fport = sin->sin_port;
if (!TAILQ_EMPTY(&V_in_ifaddrhead)) {
/*
* If the destination address is INADDR_ANY,
* use the primary local address.
* If the supplied address is INADDR_BROADCAST,
* and the primary interface supports broadcast,
* choose the broadcast address for that interface.
*/
if (faddr.s_addr == INADDR_ANY) {
IN_IFADDR_RLOCK();
faddr =
IA_SIN(TAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
IN_IFADDR_RUNLOCK();
if (cred != NULL &&
(error = prison_get_ip4(cred, &faddr)) != 0)
return (error);
} else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
IN_IFADDR_RLOCK();
if (TAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
IFF_BROADCAST)
faddr = satosin(&TAILQ_FIRST(
&V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
IN_IFADDR_RUNLOCK();
}
}
if (laddr.s_addr == INADDR_ANY) {
error = in_pcbladdr(inp, &faddr, &laddr, cred);
/*
* If the destination address is multicast and an outgoing
* interface has been set as a multicast option, prefer the
* address of that interface as our source address.
*/
if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
inp->inp_moptions != NULL) {
struct ip_moptions *imo;
struct ifnet *ifp;
imo = inp->inp_moptions;
if (imo->imo_multicast_ifp != NULL) {
ifp = imo->imo_multicast_ifp;
IN_IFADDR_RLOCK();
TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
if ((ia->ia_ifp == ifp) &&
(cred == NULL ||
prison_check_ip4(cred,
&ia->ia_addr.sin_addr) == 0))
break;
}
if (ia == NULL)
error = EADDRNOTAVAIL;
else {
laddr = ia->ia_addr.sin_addr;
error = 0;
}
IN_IFADDR_RUNLOCK();
}
}
if (error)
return (error);
}
oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, fport,
laddr, lport, 0, NULL);
if (oinp != NULL) {
if (oinpp != NULL)
*oinpp = oinp;
return (EADDRINUSE);
}
if (lport == 0) {
error = in_pcbbind_setup(inp, NULL, &laddr.s_addr, &lport,
cred);
if (error)
return (error);
}
*laddrp = laddr.s_addr;
*lportp = lport;
*faddrp = faddr.s_addr;
*fportp = fport;
return (0);
}
void
in_pcbdisconnect(struct inpcb *inp)
{
INP_WLOCK_ASSERT(inp);
INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
inp->inp_faddr.s_addr = INADDR_ANY;
inp->inp_fport = 0;
in_pcbrehash(inp);
}
#endif /* INET */
/*
* in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
* For most protocols, this will be invoked immediately prior to calling
* in_pcbfree(). However, with TCP the inpcb may significantly outlive the
* socket, in which case in_pcbfree() is deferred.
*/
void
in_pcbdetach(struct inpcb *inp)
{
KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
inp->inp_socket->so_pcb = NULL;
inp->inp_socket = NULL;
}
/*
* in_pcbref() bumps the reference count on an inpcb in order to maintain
* stability of an inpcb pointer despite the inpcb lock being released. This
* is used in TCP when the inpcbinfo lock needs to be acquired or upgraded,
* but where the inpcb lock may already held, or when acquiring a reference
* via a pcbgroup.
*
* in_pcbref() should be used only to provide brief memory stability, and
* must always be followed by a call to INP_WLOCK() and in_pcbrele() to
* garbage collect the inpcb if it has been in_pcbfree()'d from another
* context. Until in_pcbrele() has returned that the inpcb is still valid,
* lock and rele are the *only* safe operations that may be performed on the
* inpcb.
*
* While the inpcb will not be freed, releasing the inpcb lock means that the
* connection's state may change, so the caller should be careful to
* revalidate any cached state on reacquiring the lock. Drop the reference
* using in_pcbrele().
*/
void
in_pcbref(struct inpcb *inp)
{
KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
refcount_acquire(&inp->inp_refcount);
}
/*
* Drop a refcount on an inpcb elevated using in_pcbref(); because a call to
* in_pcbfree() may have been made between in_pcbref() and in_pcbrele(), we
* return a flag indicating whether or not the inpcb remains valid. If it is
* valid, we return with the inpcb lock held.
*
* Notice that, unlike in_pcbref(), the inpcb lock must be held to drop a
* reference on an inpcb. Historically more work was done here (actually, in
* in_pcbfree_internal()) but has been moved to in_pcbfree() to avoid the
* need for the pcbinfo lock in in_pcbrele(). Deferring the free is entirely
* about memory stability (and continued use of the write lock).
*/
int
in_pcbrele_rlocked(struct inpcb *inp)
{
struct inpcbinfo *pcbinfo;
KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
INP_RLOCK_ASSERT(inp);
if (refcount_release(&inp->inp_refcount) == 0) {
/*
* If the inpcb has been freed, let the caller know, even if
* this isn't the last reference.
*/
if (inp->inp_flags2 & INP_FREED) {
INP_RUNLOCK(inp);
return (1);
}
return (0);
}
KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
INP_RUNLOCK(inp);
pcbinfo = inp->inp_pcbinfo;
uma_zfree(pcbinfo->ipi_zone, inp);
return (1);
}
int
in_pcbrele_wlocked(struct inpcb *inp)
{
struct inpcbinfo *pcbinfo;
KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
INP_WLOCK_ASSERT(inp);
if (refcount_release(&inp->inp_refcount) == 0)
return (0);
KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
INP_WUNLOCK(inp);
pcbinfo = inp->inp_pcbinfo;
uma_zfree(pcbinfo->ipi_zone, inp);
return (1);
}
/*
* Temporary wrapper.
*/
int
in_pcbrele(struct inpcb *inp)
{
return (in_pcbrele_wlocked(inp));
}
/*
* Unconditionally schedule an inpcb to be freed by decrementing its
* reference count, which should occur only after the inpcb has been detached
* from its socket. If another thread holds a temporary reference (acquired
* using in_pcbref()) then the free is deferred until that reference is
* released using in_pcbrele(), but the inpcb is still unlocked. Almost all
* work, including removal from global lists, is done in this context, where
* the pcbinfo lock is held.
*/
void
in_pcbfree(struct inpcb *inp)
{
struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
INP_INFO_WLOCK_ASSERT(pcbinfo);
INP_WLOCK_ASSERT(inp);
/* XXXRW: Do as much as possible here. */
#ifdef IPSEC
if (inp->inp_sp != NULL)
ipsec_delete_pcbpolicy(inp);
#endif
inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
in_pcbremlists(inp);
#ifdef INET6
if (inp->inp_vflag & INP_IPV6PROTO) {
ip6_freepcbopts(inp->in6p_outputopts);
if (inp->in6p_moptions != NULL)
ip6_freemoptions(inp->in6p_moptions);
}
#endif
if (inp->inp_options)
(void)m_free(inp->inp_options);
#ifdef INET
if (inp->inp_moptions != NULL)
inp_freemoptions(inp->inp_moptions);
#endif
inp->inp_vflag = 0;
inp->inp_flags2 |= INP_FREED;
crfree(inp->inp_cred);
#ifdef MAC
mac_inpcb_destroy(inp);
#endif
if (!in_pcbrele_wlocked(inp))
INP_WUNLOCK(inp);
}
/*
* in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
* port reservation, and preventing it from being returned by inpcb lookups.
*
* It is used by TCP to mark an inpcb as unused and avoid future packet
* delivery or event notification when a socket remains open but TCP has
* closed. This might occur as a result of a shutdown()-initiated TCP close
* or a RST on the wire, and allows the port binding to be reused while still
* maintaining the invariant that so_pcb always points to a valid inpcb until
* in_pcbdetach().
*
* XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
* in_pcbnotifyall() and in_pcbpurgeif0()?
*/
void
in_pcbdrop(struct inpcb *inp)
{
INP_WLOCK_ASSERT(inp);
/*
* XXXRW: Possibly we should protect the setting of INP_DROPPED with
* the hash lock...?
*/
inp->inp_flags |= INP_DROPPED;
if (inp->inp_flags & INP_INHASHLIST) {
struct inpcbport *phd = inp->inp_phd;
INP_HASH_WLOCK(inp->inp_pcbinfo);
LIST_REMOVE(inp, inp_hash);
LIST_REMOVE(inp, inp_portlist);
if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
LIST_REMOVE(phd, phd_hash);
free(phd, M_PCB);
}
INP_HASH_WUNLOCK(inp->inp_pcbinfo);
inp->inp_flags &= ~INP_INHASHLIST;
#ifdef PCBGROUP
in_pcbgroup_remove(inp);
#endif
}
}
#ifdef INET
/*
* Common routines to return the socket addresses associated with inpcbs.
*/
struct sockaddr *
in_sockaddr(in_port_t port, struct in_addr *addr_p)
{
struct sockaddr_in *sin;
sin = malloc(sizeof *sin, M_SONAME,
M_WAITOK | M_ZERO);
sin->sin_family = AF_INET;
sin->sin_len = sizeof(*sin);
sin->sin_addr = *addr_p;
sin->sin_port = port;
return (struct sockaddr *)sin;
}
int
in_getsockaddr(struct socket *so, struct sockaddr **nam)
{
struct inpcb *inp;
struct in_addr addr;
in_port_t port;
inp = sotoinpcb(so);
KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
INP_RLOCK(inp);
port = inp->inp_lport;
addr = inp->inp_laddr;
INP_RUNLOCK(inp);
*nam = in_sockaddr(port, &addr);
return 0;
}
int
in_getpeeraddr(struct socket *so, struct sockaddr **nam)
{
struct inpcb *inp;
struct in_addr addr;
in_port_t port;
inp = sotoinpcb(so);
KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
INP_RLOCK(inp);
port = inp->inp_fport;
addr = inp->inp_faddr;
INP_RUNLOCK(inp);
*nam = in_sockaddr(port, &addr);
return 0;
}
void
in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
struct inpcb *(*notify)(struct inpcb *, int))
{
struct inpcb *inp, *inp_temp;
INP_INFO_WLOCK(pcbinfo);
LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
INP_WLOCK(inp);
#ifdef INET6
if ((inp->inp_vflag & INP_IPV4) == 0) {
INP_WUNLOCK(inp);
continue;
}
#endif
if (inp->inp_faddr.s_addr != faddr.s_addr ||
inp->inp_socket == NULL) {
INP_WUNLOCK(inp);
continue;
}
if ((*notify)(inp, errno))
INP_WUNLOCK(inp);
}
INP_INFO_WUNLOCK(pcbinfo);
}
void
in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
{
struct inpcb *inp;
struct ip_moptions *imo;
int i, gap;
INP_INFO_RLOCK(pcbinfo);
LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
INP_WLOCK(inp);
imo = inp->inp_moptions;
if ((inp->inp_vflag & INP_IPV4) &&
imo != NULL) {
/*
* Unselect the outgoing interface if it is being
* detached.
*/
if (imo->imo_multicast_ifp == ifp)
imo->imo_multicast_ifp = NULL;
/*
* Drop multicast group membership if we joined
* through the interface being detached.
*/
for (i = 0, gap = 0; i < imo->imo_num_memberships;
i++) {
if (imo->imo_membership[i]->inm_ifp == ifp) {
in_delmulti(imo->imo_membership[i]);
gap++;
} else if (gap != 0)
imo->imo_membership[i - gap] =
imo->imo_membership[i];
}
imo->imo_num_memberships -= gap;
}
INP_WUNLOCK(inp);
}
INP_INFO_RUNLOCK(pcbinfo);
}
/*
* Lookup a PCB based on the local address and port. Caller must hold the
* hash lock. No inpcb locks or references are acquired.
*/
#define INP_LOOKUP_MAPPED_PCB_COST 3
struct inpcb *
in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
u_short lport, int lookupflags, struct ucred *cred)
{
struct inpcb *inp;
#ifdef INET6
int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
#else
int matchwild = 3;
#endif
int wildcard;
KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
("%s: invalid lookup flags %d", __func__, lookupflags));
INP_HASH_LOCK_ASSERT(pcbinfo);
if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
struct inpcbhead *head;
/*
* Look for an unconnected (wildcard foreign addr) PCB that
* matches the local address and port we're looking for.
*/
head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
0, pcbinfo->ipi_hashmask)];
LIST_FOREACH(inp, head, inp_hash) {
#ifdef INET6
/* XXX inp locking */
if ((inp->inp_vflag & INP_IPV4) == 0)
continue;
#endif
if (inp->inp_faddr.s_addr == INADDR_ANY &&
inp->inp_laddr.s_addr == laddr.s_addr &&
inp->inp_lport == lport) {
/*
* Found?
*/
if (cred == NULL ||
prison_equal_ip4(cred->cr_prison,
inp->inp_cred->cr_prison))
return (inp);
}
}
/*
* Not found.
*/
return (NULL);
} else {
struct inpcbporthead *porthash;
struct inpcbport *phd;
struct inpcb *match = NULL;
/*
* Best fit PCB lookup.
*
* First see if this local port is in use by looking on the
* port hash list.
*/
porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
pcbinfo->ipi_porthashmask)];
LIST_FOREACH(phd, porthash, phd_hash) {
if (phd->phd_port == lport)
break;
}
if (phd != NULL) {
/*
* Port is in use by one or more PCBs. Look for best
* fit.
*/
LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
wildcard = 0;
if (cred != NULL &&
!prison_equal_ip4(inp->inp_cred->cr_prison,
cred->cr_prison))
continue;
#ifdef INET6
/* XXX inp locking */
if ((inp->inp_vflag & INP_IPV4) == 0)
continue;
/*
* We never select the PCB that has
* INP_IPV6 flag and is bound to :: if
* we have another PCB which is bound
* to 0.0.0.0. If a PCB has the
* INP_IPV6 flag, then we set its cost
* higher than IPv4 only PCBs.
*
* Note that the case only happens
* when a socket is bound to ::, under
* the condition that the use of the
* mapped address is allowed.
*/
if ((inp->inp_vflag & INP_IPV6) != 0)
wildcard += INP_LOOKUP_MAPPED_PCB_COST;
#endif
if (inp->inp_faddr.s_addr != INADDR_ANY)
wildcard++;
if (inp->inp_laddr.s_addr != INADDR_ANY) {
if (laddr.s_addr == INADDR_ANY)
wildcard++;
else if (inp->inp_laddr.s_addr != laddr.s_addr)
continue;
} else {
if (laddr.s_addr != INADDR_ANY)
wildcard++;
}
if (wildcard < matchwild) {
match = inp;
matchwild = wildcard;
if (matchwild == 0)
break;
}
}
}
return (match);
}
}
#undef INP_LOOKUP_MAPPED_PCB_COST
#ifdef PCBGROUP
/*
* Lookup PCB in hash list, using pcbgroup tables.
*/
static struct inpcb *
in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
u_int lport_arg, int lookupflags, struct ifnet *ifp)
{
struct inpcbhead *head;
struct inpcb *inp, *tmpinp;
u_short fport = fport_arg, lport = lport_arg;
/*
* First look for an exact match.
*/
tmpinp = NULL;
INP_GROUP_LOCK(pcbgroup);
head = &pcbgroup->ipg_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
pcbgroup->ipg_hashmask)];
LIST_FOREACH(inp, head, inp_pcbgrouphash) {
#ifdef INET6
/* XXX inp locking */
if ((inp->inp_vflag & INP_IPV4) == 0)
continue;
#endif
if (inp->inp_faddr.s_addr == faddr.s_addr &&
inp->inp_laddr.s_addr == laddr.s_addr &&
inp->inp_fport == fport &&
inp->inp_lport == lport) {
/*
* XXX We should be able to directly return
* the inp here, without any checks.
* Well unless both bound with SO_REUSEPORT?
*/
if (prison_flag(inp->inp_cred, PR_IP4))
goto found;
if (tmpinp == NULL)
tmpinp = inp;
}
}
if (tmpinp != NULL) {
inp = tmpinp;
goto found;
}
#ifdef RSS
/*
* For incoming connections, we may wish to do a wildcard
* match for an RSS-local socket.
*/
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
struct inpcb *local_wild = NULL, *local_exact = NULL;
#ifdef INET6
struct inpcb *local_wild_mapped = NULL;
#endif
struct inpcb *jail_wild = NULL;
struct inpcbhead *head;
int injail;
/*
* Order of socket selection - we always prefer jails.
* 1. jailed, non-wild.
* 2. jailed, wild.
* 3. non-jailed, non-wild.
* 4. non-jailed, wild.
*/
head = &pcbgroup->ipg_hashbase[INP_PCBHASH(INADDR_ANY,
lport, 0, pcbgroup->ipg_hashmask)];
LIST_FOREACH(inp, head, inp_pcbgrouphash) {
#ifdef INET6
/* XXX inp locking */
if ((inp->inp_vflag & INP_IPV4) == 0)
continue;
#endif
if (inp->inp_faddr.s_addr != INADDR_ANY ||
inp->inp_lport != lport)
continue;
injail = prison_flag(inp->inp_cred, PR_IP4);
if (injail) {
if (prison_check_ip4(inp->inp_cred,
&laddr) != 0)
continue;
} else {
if (local_exact != NULL)
continue;
}
if (inp->inp_laddr.s_addr == laddr.s_addr) {
if (injail)
goto found;
else
local_exact = inp;
} else if (inp->inp_laddr.s_addr == INADDR_ANY) {
#ifdef INET6
/* XXX inp locking, NULL check */
if (inp->inp_vflag & INP_IPV6PROTO)
local_wild_mapped = inp;
else
#endif
if (injail)
jail_wild = inp;
else
local_wild = inp;
}
} /* LIST_FOREACH */
inp = jail_wild;
if (inp == NULL)
inp = local_exact;
if (inp == NULL)
inp = local_wild;
#ifdef INET6
if (inp == NULL)
inp = local_wild_mapped;
#endif
if (inp != NULL)
goto found;
}
#endif
/*
* Then look for a wildcard match, if requested.
*/
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
struct inpcb *local_wild = NULL, *local_exact = NULL;
#ifdef INET6
struct inpcb *local_wild_mapped = NULL;
#endif
struct inpcb *jail_wild = NULL;
struct inpcbhead *head;
int injail;
/*
* Order of socket selection - we always prefer jails.
* 1. jailed, non-wild.
* 2. jailed, wild.
* 3. non-jailed, non-wild.
* 4. non-jailed, wild.
*/
head = &pcbinfo->ipi_wildbase[INP_PCBHASH(INADDR_ANY, lport,
0, pcbinfo->ipi_wildmask)];
LIST_FOREACH(inp, head, inp_pcbgroup_wild) {
#ifdef INET6
/* XXX inp locking */
if ((inp->inp_vflag & INP_IPV4) == 0)
continue;
#endif
if (inp->inp_faddr.s_addr != INADDR_ANY ||
inp->inp_lport != lport)
continue;
injail = prison_flag(inp->inp_cred, PR_IP4);
if (injail) {
if (prison_check_ip4(inp->inp_cred,
&laddr) != 0)
continue;
} else {
if (local_exact != NULL)
continue;
}
if (inp->inp_laddr.s_addr == laddr.s_addr) {
if (injail)
goto found;
else
local_exact = inp;
} else if (inp->inp_laddr.s_addr == INADDR_ANY) {
#ifdef INET6
/* XXX inp locking, NULL check */
if (inp->inp_vflag & INP_IPV6PROTO)
local_wild_mapped = inp;
else
#endif
if (injail)
jail_wild = inp;
else
local_wild = inp;
}
} /* LIST_FOREACH */
inp = jail_wild;
if (inp == NULL)
inp = local_exact;
if (inp == NULL)
inp = local_wild;
#ifdef INET6
if (inp == NULL)
inp = local_wild_mapped;
#endif
if (inp != NULL)
goto found;
} /* if (lookupflags & INPLOOKUP_WILDCARD) */
INP_GROUP_UNLOCK(pcbgroup);
return (NULL);
found:
in_pcbref(inp);
INP_GROUP_UNLOCK(pcbgroup);
if (lookupflags & INPLOOKUP_WLOCKPCB) {
INP_WLOCK(inp);
if (in_pcbrele_wlocked(inp))
return (NULL);
} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
INP_RLOCK(inp);
if (in_pcbrele_rlocked(inp))
return (NULL);
} else
panic("%s: locking bug", __func__);
return (inp);
}
#endif /* PCBGROUP */
/*
* Lookup PCB in hash list, using pcbinfo tables. This variation assumes
* that the caller has locked the hash list, and will not perform any further
* locking or reference operations on either the hash list or the connection.
*/
static struct inpcb *
in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
struct ifnet *ifp)
{
struct inpcbhead *head;
struct inpcb *inp, *tmpinp;
u_short fport = fport_arg, lport = lport_arg;
KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
("%s: invalid lookup flags %d", __func__, lookupflags));
INP_HASH_LOCK_ASSERT(pcbinfo);
/*
* First look for an exact match.
*/
tmpinp = NULL;
head = &pcbinfo->ipi_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
pcbinfo->ipi_hashmask)];
LIST_FOREACH(inp, head, inp_hash) {
#ifdef INET6
/* XXX inp locking */
if ((inp->inp_vflag & INP_IPV4) == 0)
continue;
#endif
if (inp->inp_faddr.s_addr == faddr.s_addr &&
inp->inp_laddr.s_addr == laddr.s_addr &&
inp->inp_fport == fport &&
inp->inp_lport == lport) {
/*
* XXX We should be able to directly return
* the inp here, without any checks.
* Well unless both bound with SO_REUSEPORT?
*/
if (prison_flag(inp->inp_cred, PR_IP4))
return (inp);
if (tmpinp == NULL)
tmpinp = inp;
}
}
if (tmpinp != NULL)
return (tmpinp);
/*
* Then look for a wildcard match, if requested.
*/
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
struct inpcb *local_wild = NULL, *local_exact = NULL;
#ifdef INET6
struct inpcb *local_wild_mapped = NULL;
#endif
struct inpcb *jail_wild = NULL;
int injail;
/*
* Order of socket selection - we always prefer jails.
* 1. jailed, non-wild.
* 2. jailed, wild.
* 3. non-jailed, non-wild.
* 4. non-jailed, wild.
*/
head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
0, pcbinfo->ipi_hashmask)];
LIST_FOREACH(inp, head, inp_hash) {
#ifdef INET6
/* XXX inp locking */
if ((inp->inp_vflag & INP_IPV4) == 0)
continue;
#endif
if (inp->inp_faddr.s_addr != INADDR_ANY ||
inp->inp_lport != lport)
continue;
injail = prison_flag(inp->inp_cred, PR_IP4);
if (injail) {
if (prison_check_ip4(inp->inp_cred,
&laddr) != 0)
continue;
} else {
if (local_exact != NULL)
continue;
}
if (inp->inp_laddr.s_addr == laddr.s_addr) {
if (injail)
return (inp);
else
local_exact = inp;
} else if (inp->inp_laddr.s_addr == INADDR_ANY) {
#ifdef INET6
/* XXX inp locking, NULL check */
if (inp->inp_vflag & INP_IPV6PROTO)
local_wild_mapped = inp;
else
#endif
if (injail)
jail_wild = inp;
else
local_wild = inp;
}
} /* LIST_FOREACH */
if (jail_wild != NULL)
return (jail_wild);
if (local_exact != NULL)
return (local_exact);
if (local_wild != NULL)
return (local_wild);
#ifdef INET6
if (local_wild_mapped != NULL)
return (local_wild_mapped);
#endif
} /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
return (NULL);
}
/*
* Lookup PCB in hash list, using pcbinfo tables. This variation locks the
* hash list lock, and will return the inpcb locked (i.e., requires
* INPLOOKUP_LOCKPCB).
*/
static struct inpcb *
in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
struct ifnet *ifp)
{
struct inpcb *inp;
INP_HASH_RLOCK(pcbinfo);
inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
if (inp != NULL) {
in_pcbref(inp);
INP_HASH_RUNLOCK(pcbinfo);
if (lookupflags & INPLOOKUP_WLOCKPCB) {
INP_WLOCK(inp);
if (in_pcbrele_wlocked(inp))
return (NULL);
} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
INP_RLOCK(inp);
if (in_pcbrele_rlocked(inp))
return (NULL);
} else
panic("%s: locking bug", __func__);
} else
INP_HASH_RUNLOCK(pcbinfo);
return (inp);
}
/*
* Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
* from which a pre-calculated hash value may be extracted.
*
* Possibly more of this logic should be in in_pcbgroup.c.
*/
struct inpcb *
in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp)
{
#if defined(PCBGROUP) && !defined(RSS)
struct inpcbgroup *pcbgroup;
#endif
KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
("%s: invalid lookup flags %d", __func__, lookupflags));
KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
("%s: LOCKPCB not set", __func__));
/*
* When not using RSS, use connection groups in preference to the
* reservation table when looking up 4-tuples. When using RSS, just
* use the reservation table, due to the cost of the Toeplitz hash
* in software.
*
* XXXRW: This policy belongs in the pcbgroup code, as in principle
* we could be doing RSS with a non-Toeplitz hash that is affordable
* in software.
*/
#if defined(PCBGROUP) && !defined(RSS)
if (in_pcbgroup_enabled(pcbinfo)) {
pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
fport);
return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
laddr, lport, lookupflags, ifp));
}
#endif
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
lookupflags, ifp));
}
struct inpcb *
in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
struct ifnet *ifp, struct mbuf *m)
{
#ifdef PCBGROUP
struct inpcbgroup *pcbgroup;
#endif
KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
("%s: invalid lookup flags %d", __func__, lookupflags));
KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
("%s: LOCKPCB not set", __func__));
#ifdef PCBGROUP
/*
* If we can use a hardware-generated hash to look up the connection
* group, use that connection group to find the inpcb. Otherwise
* fall back on a software hash -- or the reservation table if we're
* using RSS.
*
* XXXRW: As above, that policy belongs in the pcbgroup code.
*/
if (in_pcbgroup_enabled(pcbinfo) &&
!(M_HASHTYPE_TEST(m, M_HASHTYPE_NONE))) {
pcbgroup = in_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
m->m_pkthdr.flowid);
if (pcbgroup != NULL)
return (in_pcblookup_group(pcbinfo, pcbgroup, faddr,
fport, laddr, lport, lookupflags, ifp));
#ifndef RSS
pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
fport);
return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
laddr, lport, lookupflags, ifp));
#endif
}
#endif
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
lookupflags, ifp));
}
#endif /* INET */
/*
* Insert PCB onto various hash lists.
*/
static int
in_pcbinshash_internal(struct inpcb *inp, int do_pcbgroup_update)
{
struct inpcbhead *pcbhash;
struct inpcbporthead *pcbporthash;
struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
struct inpcbport *phd;
u_int32_t hashkey_faddr;
INP_WLOCK_ASSERT(inp);
INP_HASH_WLOCK_ASSERT(pcbinfo);
KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
("in_pcbinshash: INP_INHASHLIST"));
#ifdef INET6
if (inp->inp_vflag & INP_IPV6)
hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
else
#endif
hashkey_faddr = inp->inp_faddr.s_addr;
pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
pcbporthash = &pcbinfo->ipi_porthashbase[
INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
/*
* Go through port list and look for a head for this lport.
*/
LIST_FOREACH(phd, pcbporthash, phd_hash) {
if (phd->phd_port == inp->inp_lport)
break;
}
/*
* If none exists, malloc one and tack it on.
*/
if (phd == NULL) {
phd = malloc(sizeof(struct inpcbport), M_PCB, M_NOWAIT);
if (phd == NULL) {
return (ENOBUFS); /* XXX */
}
phd->phd_port = inp->inp_lport;
LIST_INIT(&phd->phd_pcblist);
LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
}
inp->inp_phd = phd;
LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
inp->inp_flags |= INP_INHASHLIST;
#ifdef PCBGROUP
if (do_pcbgroup_update)
in_pcbgroup_update(inp);
#endif
return (0);
}
/*
* For now, there are two public interfaces to insert an inpcb into the hash
* lists -- one that does update pcbgroups, and one that doesn't. The latter
* is used only in the TCP syncache, where in_pcbinshash is called before the
* full 4-tuple is set for the inpcb, and we don't want to install in the
* pcbgroup until later.
*
* XXXRW: This seems like a misfeature. in_pcbinshash should always update
* connection groups, and partially initialised inpcbs should not be exposed
* to either reservation hash tables or pcbgroups.
*/
int
in_pcbinshash(struct inpcb *inp)
{
return (in_pcbinshash_internal(inp, 1));
}
int
in_pcbinshash_nopcbgroup(struct inpcb *inp)
{
return (in_pcbinshash_internal(inp, 0));
}
/*
* Move PCB to the proper hash bucket when { faddr, fport } have been
* changed. NOTE: This does not handle the case of the lport changing (the
* hashed port list would have to be updated as well), so the lport must
* not change after in_pcbinshash() has been called.
*/
void
in_pcbrehash_mbuf(struct inpcb *inp, struct mbuf *m)
{
struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
struct inpcbhead *head;
u_int32_t hashkey_faddr;
INP_WLOCK_ASSERT(inp);
INP_HASH_WLOCK_ASSERT(pcbinfo);
KASSERT(inp->inp_flags & INP_INHASHLIST,
("in_pcbrehash: !INP_INHASHLIST"));
#ifdef INET6
if (inp->inp_vflag & INP_IPV6)
hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
else
#endif
hashkey_faddr = inp->inp_faddr.s_addr;
head = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
LIST_REMOVE(inp, inp_hash);
LIST_INSERT_HEAD(head, inp, inp_hash);
#ifdef PCBGROUP
if (m != NULL)
in_pcbgroup_update_mbuf(inp, m);
else
in_pcbgroup_update(inp);
#endif
}
void
in_pcbrehash(struct inpcb *inp)
{
in_pcbrehash_mbuf(inp, NULL);
}
/*
* Remove PCB from various lists.
*/
static void
in_pcbremlists(struct inpcb *inp)
{
struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
INP_INFO_WLOCK_ASSERT(pcbinfo);
INP_WLOCK_ASSERT(inp);
inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
if (inp->inp_flags & INP_INHASHLIST) {
struct inpcbport *phd = inp->inp_phd;
INP_HASH_WLOCK(pcbinfo);
LIST_REMOVE(inp, inp_hash);
LIST_REMOVE(inp, inp_portlist);
if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
LIST_REMOVE(phd, phd_hash);
free(phd, M_PCB);
}
INP_HASH_WUNLOCK(pcbinfo);
inp->inp_flags &= ~INP_INHASHLIST;
}
LIST_REMOVE(inp, inp_list);
pcbinfo->ipi_count--;
#ifdef PCBGROUP
in_pcbgroup_remove(inp);
#endif
}
/*
* A set label operation has occurred at the socket layer, propagate the
* label change into the in_pcb for the socket.
*/
void
in_pcbsosetlabel(struct socket *so)
{
#ifdef MAC
struct inpcb *inp;
inp = sotoinpcb(so);
KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
INP_WLOCK(inp);
SOCK_LOCK(so);
mac_inpcb_sosetlabel(so, inp);
SOCK_UNLOCK(so);
INP_WUNLOCK(inp);
#endif
}
/*
* ipport_tick runs once per second, determining if random port allocation
* should be continued. If more than ipport_randomcps ports have been
* allocated in the last second, then we return to sequential port
* allocation. We return to random allocation only once we drop below
* ipport_randomcps for at least ipport_randomtime seconds.
*/
static void
ipport_tick(void *xtp)
{
VNET_ITERATOR_DECL(vnet_iter);
VNET_LIST_RLOCK_NOSLEEP();
VNET_FOREACH(vnet_iter) {
CURVNET_SET(vnet_iter); /* XXX appease INVARIANTS here */
if (V_ipport_tcpallocs <=
V_ipport_tcplastcount + V_ipport_randomcps) {
if (V_ipport_stoprandom > 0)
V_ipport_stoprandom--;
} else
V_ipport_stoprandom = V_ipport_randomtime;
V_ipport_tcplastcount = V_ipport_tcpallocs;
CURVNET_RESTORE();
}
VNET_LIST_RUNLOCK_NOSLEEP();
callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
}
static void
ip_fini(void *xtp)
{
callout_stop(&ipport_tick_callout);
}
/*
* The ipport_callout should start running at about the time we attach the
* inet or inet6 domains.
*/
static void
ipport_tick_init(const void *unused __unused)
{
/* Start ipport_tick. */
- callout_init(&ipport_tick_callout, CALLOUT_MPSAFE);
+ callout_init(&ipport_tick_callout, 1);
callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
EVENTHANDLER_REGISTER(shutdown_pre_sync, ip_fini, NULL,
SHUTDOWN_PRI_DEFAULT);
}
SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
ipport_tick_init, NULL);
void
inp_wlock(struct inpcb *inp)
{
INP_WLOCK(inp);
}
void
inp_wunlock(struct inpcb *inp)
{
INP_WUNLOCK(inp);
}
void
inp_rlock(struct inpcb *inp)
{
INP_RLOCK(inp);
}
void
inp_runlock(struct inpcb *inp)
{
INP_RUNLOCK(inp);
}
#ifdef INVARIANTS
void
inp_lock_assert(struct inpcb *inp)
{
INP_WLOCK_ASSERT(inp);
}
void
inp_unlock_assert(struct inpcb *inp)
{
INP_UNLOCK_ASSERT(inp);
}
#endif
void
inp_apply_all(void (*func)(struct inpcb *, void *), void *arg)
{
struct inpcb *inp;
INP_INFO_RLOCK(&V_tcbinfo);
LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) {
INP_WLOCK(inp);
func(inp, arg);
INP_WUNLOCK(inp);
}
INP_INFO_RUNLOCK(&V_tcbinfo);
}
struct socket *
inp_inpcbtosocket(struct inpcb *inp)
{
INP_WLOCK_ASSERT(inp);
return (inp->inp_socket);
}
struct tcpcb *
inp_inpcbtotcpcb(struct inpcb *inp)
{
INP_WLOCK_ASSERT(inp);
return ((struct tcpcb *)inp->inp_ppcb);
}
int
inp_ip_tos_get(const struct inpcb *inp)
{
return (inp->inp_ip_tos);
}
void
inp_ip_tos_set(struct inpcb *inp, int val)
{
inp->inp_ip_tos = val;
}
void
inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
uint32_t *faddr, uint16_t *fp)
{
INP_LOCK_ASSERT(inp);
*laddr = inp->inp_laddr.s_addr;
*faddr = inp->inp_faddr.s_addr;
*lp = inp->inp_lport;
*fp = inp->inp_fport;
}
struct inpcb *
so_sotoinpcb(struct socket *so)
{
return (sotoinpcb(so));
}
struct tcpcb *
so_sototcpcb(struct socket *so)
{
return (sototcpcb(so));
}
#ifdef DDB
static void
db_print_indent(int indent)
{
int i;
for (i = 0; i < indent; i++)
db_printf(" ");
}
static void
db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
{
char faddr_str[48], laddr_str[48];
db_print_indent(indent);
db_printf("%s at %p\n", name, inc);
indent += 2;
#ifdef INET6
if (inc->inc_flags & INC_ISIPV6) {
/* IPv6. */
ip6_sprintf(laddr_str, &inc->inc6_laddr);
ip6_sprintf(faddr_str, &inc->inc6_faddr);
} else
#endif
{
/* IPv4. */
inet_ntoa_r(inc->inc_laddr, laddr_str);
inet_ntoa_r(inc->inc_faddr, faddr_str);
}
db_print_indent(indent);
db_printf("inc_laddr %s inc_lport %u\n", laddr_str,
ntohs(inc->inc_lport));
db_print_indent(indent);
db_printf("inc_faddr %s inc_fport %u\n", faddr_str,
ntohs(inc->inc_fport));
}
static void
db_print_inpflags(int inp_flags)
{
int comma;
comma = 0;
if (inp_flags & INP_RECVOPTS) {
db_printf("%sINP_RECVOPTS", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_RECVRETOPTS) {
db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_RECVDSTADDR) {
db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_HDRINCL) {
db_printf("%sINP_HDRINCL", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_HIGHPORT) {
db_printf("%sINP_HIGHPORT", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_LOWPORT) {
db_printf("%sINP_LOWPORT", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_ANONPORT) {
db_printf("%sINP_ANONPORT", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_RECVIF) {
db_printf("%sINP_RECVIF", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_MTUDISC) {
db_printf("%sINP_MTUDISC", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_RECVTTL) {
db_printf("%sINP_RECVTTL", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_DONTFRAG) {
db_printf("%sINP_DONTFRAG", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_RECVTOS) {
db_printf("%sINP_RECVTOS", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_IPV6_V6ONLY) {
db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_PKTINFO) {
db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_HOPLIMIT) {
db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_HOPOPTS) {
db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_DSTOPTS) {
db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_RTHDR) {
db_printf("%sIN6P_RTHDR", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_RTHDRDSTOPTS) {
db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_TCLASS) {
db_printf("%sIN6P_TCLASS", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_AUTOFLOWLABEL) {
db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_TIMEWAIT) {
db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_ONESBCAST) {
db_printf("%sINP_ONESBCAST", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_DROPPED) {
db_printf("%sINP_DROPPED", comma ? ", " : "");
comma = 1;
}
if (inp_flags & INP_SOCKREF) {
db_printf("%sINP_SOCKREF", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_RFC2292) {
db_printf("%sIN6P_RFC2292", comma ? ", " : "");
comma = 1;
}
if (inp_flags & IN6P_MTU) {
db_printf("IN6P_MTU%s", comma ? ", " : "");
comma = 1;
}
}
static void
db_print_inpvflag(u_char inp_vflag)
{
int comma;
comma = 0;
if (inp_vflag & INP_IPV4) {
db_printf("%sINP_IPV4", comma ? ", " : "");
comma = 1;
}
if (inp_vflag & INP_IPV6) {
db_printf("%sINP_IPV6", comma ? ", " : "");
comma = 1;
}
if (inp_vflag & INP_IPV6PROTO) {
db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
comma = 1;
}
}
static void
db_print_inpcb(struct inpcb *inp, const char *name, int indent)
{
db_print_indent(indent);
db_printf("%s at %p\n", name, inp);
indent += 2;
db_print_indent(indent);
db_printf("inp_flow: 0x%x\n", inp->inp_flow);
db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
db_print_indent(indent);
db_printf("inp_ppcb: %p inp_pcbinfo: %p inp_socket: %p\n",
inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
db_print_indent(indent);
db_printf("inp_label: %p inp_flags: 0x%x (",
inp->inp_label, inp->inp_flags);
db_print_inpflags(inp->inp_flags);
db_printf(")\n");
db_print_indent(indent);
db_printf("inp_sp: %p inp_vflag: 0x%x (", inp->inp_sp,
inp->inp_vflag);
db_print_inpvflag(inp->inp_vflag);
db_printf(")\n");
db_print_indent(indent);
db_printf("inp_ip_ttl: %d inp_ip_p: %d inp_ip_minttl: %d\n",
inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
db_print_indent(indent);
#ifdef INET6
if (inp->inp_vflag & INP_IPV6) {
db_printf("in6p_options: %p in6p_outputopts: %p "
"in6p_moptions: %p\n", inp->in6p_options,
inp->in6p_outputopts, inp->in6p_moptions);
db_printf("in6p_icmp6filt: %p in6p_cksum %d "
"in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
inp->in6p_hops);
} else
#endif
{
db_printf("inp_ip_tos: %d inp_ip_options: %p "
"inp_ip_moptions: %p\n", inp->inp_ip_tos,
inp->inp_options, inp->inp_moptions);
}
db_print_indent(indent);
db_printf("inp_phd: %p inp_gencnt: %ju\n", inp->inp_phd,
(uintmax_t)inp->inp_gencnt);
}
DB_SHOW_COMMAND(inpcb, db_show_inpcb)
{
struct inpcb *inp;
if (!have_addr) {
db_printf("usage: show inpcb <addr>\n");
return;
}
inp = (struct inpcb *)addr;
db_print_inpcb(inp, "inpcb", 0);
}
#endif /* DDB */
Index: head/sys/netinet/ip_mroute.c
===================================================================
--- head/sys/netinet/ip_mroute.c (revision 283290)
+++ head/sys/netinet/ip_mroute.c (revision 283291)
@@ -1,2948 +1,2948 @@
/*-
* Copyright (c) 1989 Stephen Deering
* Copyright (c) 1992, 1993
* The Regents of the University of California. All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* Stephen Deering of Stanford University.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ip_mroute.c 8.2 (Berkeley) 11/15/93
*/
/*
* IP multicast forwarding procedures
*
* Written by David Waitzman, BBN Labs, August 1988.
* Modified by Steve Deering, Stanford, February 1989.
* Modified by Mark J. Steiglitz, Stanford, May, 1991
* Modified by Van Jacobson, LBL, January 1993
* Modified by Ajit Thyagarajan, PARC, August 1993
* Modified by Bill Fenner, PARC, April 1995
* Modified by Ahmed Helmy, SGI, June 1996
* Modified by George Edmond Eddy (Rusty), ISI, February 1998
* Modified by Pavlin Radoslavov, USC/ISI, May 1998, August 1999, October 2000
* Modified by Hitoshi Asaeda, WIDE, August 2000
* Modified by Pavlin Radoslavov, ICSI, October 2002
*
* MROUTING Revision: 3.5
* and PIM-SMv2 and PIM-DM support, advanced API support,
* bandwidth metering and signaling
*/
/*
* TODO: Prefix functions with ipmf_.
* TODO: Maintain a refcount on if_allmulti() in ifnet or in the protocol
* domain attachment (if_afdata) so we can track consumers of that service.
* TODO: Deprecate routing socket path for SIOCGETSGCNT and SIOCGETVIFCNT,
* move it to socket options.
* TODO: Cleanup LSRR removal further.
* TODO: Push RSVP stubs into raw_ip.c.
* TODO: Use bitstring.h for vif set.
* TODO: Fix mrt6_ioctl dangling ref when dynamically loaded.
* TODO: Sync ip6_mroute.c with this file.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_mrouting.h"
#define _PIM_VT 1
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/stddef.h>
#include <sys/lock.h>
#include <sys/ktr.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/priv.h>
#include <sys/protosw.h>
#include <sys/signalvar.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sockio.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/systm.h>
#include <sys/time.h>
#include <sys/counter.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/netisr.h>
#include <net/route.h>
#include <net/vnet.h>
#include <netinet/in.h>
#include <netinet/igmp.h>
#include <netinet/in_systm.h>
#include <netinet/in_var.h>
#include <netinet/ip.h>
#include <netinet/ip_encap.h>
#include <netinet/ip_mroute.h>
#include <netinet/ip_var.h>
#include <netinet/ip_options.h>
#include <netinet/pim.h>
#include <netinet/pim_var.h>
#include <netinet/udp.h>
#include <machine/in_cksum.h>
#ifndef KTR_IPMF
#define KTR_IPMF KTR_INET
#endif
#define VIFI_INVALID ((vifi_t) -1)
static VNET_DEFINE(uint32_t, last_tv_sec); /* last time we processed this */
#define V_last_tv_sec VNET(last_tv_sec)
static MALLOC_DEFINE(M_MRTABLE, "mroutetbl", "multicast forwarding cache");
/*
* Locking. We use two locks: one for the virtual interface table and
* one for the forwarding table. These locks may be nested in which case
* the VIF lock must always be taken first. Note that each lock is used
* to cover not only the specific data structure but also related data
* structures.
*/
static struct mtx mrouter_mtx;
#define MROUTER_LOCK() mtx_lock(&mrouter_mtx)
#define MROUTER_UNLOCK() mtx_unlock(&mrouter_mtx)
#define MROUTER_LOCK_ASSERT() mtx_assert(&mrouter_mtx, MA_OWNED)
#define MROUTER_LOCK_INIT() \
mtx_init(&mrouter_mtx, "IPv4 multicast forwarding", NULL, MTX_DEF)
#define MROUTER_LOCK_DESTROY() mtx_destroy(&mrouter_mtx)
static int ip_mrouter_cnt; /* # of vnets with active mrouters */
static int ip_mrouter_unloading; /* Allow no more V_ip_mrouter sockets */
static VNET_PCPUSTAT_DEFINE(struct mrtstat, mrtstat);
VNET_PCPUSTAT_SYSINIT(mrtstat);
VNET_PCPUSTAT_SYSUNINIT(mrtstat);
SYSCTL_VNET_PCPUSTAT(_net_inet_ip, OID_AUTO, mrtstat, struct mrtstat,
mrtstat, "IPv4 Multicast Forwarding Statistics (struct mrtstat, "
"netinet/ip_mroute.h)");
static VNET_DEFINE(u_long, mfchash);
#define V_mfchash VNET(mfchash)
#define MFCHASH(a, g) \
((((a).s_addr >> 20) ^ ((a).s_addr >> 10) ^ (a).s_addr ^ \
((g).s_addr >> 20) ^ ((g).s_addr >> 10) ^ (g).s_addr) & V_mfchash)
#define MFCHASHSIZE 256
static u_long mfchashsize; /* Hash size */
static VNET_DEFINE(u_char *, nexpire); /* 0..mfchashsize-1 */
#define V_nexpire VNET(nexpire)
static VNET_DEFINE(LIST_HEAD(mfchashhdr, mfc)*, mfchashtbl);
#define V_mfchashtbl VNET(mfchashtbl)
static struct mtx mfc_mtx;
#define MFC_LOCK() mtx_lock(&mfc_mtx)
#define MFC_UNLOCK() mtx_unlock(&mfc_mtx)
#define MFC_LOCK_ASSERT() mtx_assert(&mfc_mtx, MA_OWNED)
#define MFC_LOCK_INIT() \
mtx_init(&mfc_mtx, "IPv4 multicast forwarding cache", NULL, MTX_DEF)
#define MFC_LOCK_DESTROY() mtx_destroy(&mfc_mtx)
static VNET_DEFINE(vifi_t, numvifs);
#define V_numvifs VNET(numvifs)
static VNET_DEFINE(struct vif, viftable[MAXVIFS]);
#define V_viftable VNET(viftable)
SYSCTL_OPAQUE(_net_inet_ip, OID_AUTO, viftable, CTLFLAG_VNET | CTLFLAG_RD,
&VNET_NAME(viftable), sizeof(V_viftable), "S,vif[MAXVIFS]",
"IPv4 Multicast Interfaces (struct vif[MAXVIFS], netinet/ip_mroute.h)");
static struct mtx vif_mtx;
#define VIF_LOCK() mtx_lock(&vif_mtx)
#define VIF_UNLOCK() mtx_unlock(&vif_mtx)
#define VIF_LOCK_ASSERT() mtx_assert(&vif_mtx, MA_OWNED)
#define VIF_LOCK_INIT() \
mtx_init(&vif_mtx, "IPv4 multicast interfaces", NULL, MTX_DEF)
#define VIF_LOCK_DESTROY() mtx_destroy(&vif_mtx)
static eventhandler_tag if_detach_event_tag = NULL;
static VNET_DEFINE(struct callout, expire_upcalls_ch);
#define V_expire_upcalls_ch VNET(expire_upcalls_ch)
#define EXPIRE_TIMEOUT (hz / 4) /* 4x / second */
#define UPCALL_EXPIRE 6 /* number of timeouts */
/*
* Bandwidth meter variables and constants
*/
static MALLOC_DEFINE(M_BWMETER, "bwmeter", "multicast upcall bw meters");
/*
* Pending timeouts are stored in a hash table, the key being the
* expiration time. Periodically, the entries are analysed and processed.
*/
#define BW_METER_BUCKETS 1024
static VNET_DEFINE(struct bw_meter*, bw_meter_timers[BW_METER_BUCKETS]);
#define V_bw_meter_timers VNET(bw_meter_timers)
static VNET_DEFINE(struct callout, bw_meter_ch);
#define V_bw_meter_ch VNET(bw_meter_ch)
#define BW_METER_PERIOD (hz) /* periodical handling of bw meters */
/*
* Pending upcalls are stored in a vector which is flushed when
* full, or periodically
*/
static VNET_DEFINE(struct bw_upcall, bw_upcalls[BW_UPCALLS_MAX]);
#define V_bw_upcalls VNET(bw_upcalls)
static VNET_DEFINE(u_int, bw_upcalls_n); /* # of pending upcalls */
#define V_bw_upcalls_n VNET(bw_upcalls_n)
static VNET_DEFINE(struct callout, bw_upcalls_ch);
#define V_bw_upcalls_ch VNET(bw_upcalls_ch)
#define BW_UPCALLS_PERIOD (hz) /* periodical flush of bw upcalls */
static VNET_PCPUSTAT_DEFINE(struct pimstat, pimstat);
VNET_PCPUSTAT_SYSINIT(pimstat);
VNET_PCPUSTAT_SYSUNINIT(pimstat);
SYSCTL_NODE(_net_inet, IPPROTO_PIM, pim, CTLFLAG_RW, 0, "PIM");
SYSCTL_VNET_PCPUSTAT(_net_inet_pim, PIMCTL_STATS, stats, struct pimstat,
pimstat, "PIM Statistics (struct pimstat, netinet/pim_var.h)");
static u_long pim_squelch_wholepkt = 0;
SYSCTL_ULONG(_net_inet_pim, OID_AUTO, squelch_wholepkt, CTLFLAG_RW,
&pim_squelch_wholepkt, 0,
"Disable IGMP_WHOLEPKT notifications if rendezvous point is unspecified");
extern struct domain inetdomain;
static const struct protosw in_pim_protosw = {
.pr_type = SOCK_RAW,
.pr_domain = &inetdomain,
.pr_protocol = IPPROTO_PIM,
.pr_flags = PR_ATOMIC|PR_ADDR|PR_LASTHDR,
.pr_input = pim_input,
.pr_output = rip_output,
.pr_ctloutput = rip_ctloutput,
.pr_usrreqs = &rip_usrreqs
};
static const struct encaptab *pim_encap_cookie;
static int pim_encapcheck(const struct mbuf *, int, int, void *);
/*
* Note: the PIM Register encapsulation adds the following in front of a
* data packet:
*
* struct pim_encap_hdr {
* struct ip ip;
* struct pim_encap_pimhdr pim;
* }
*
*/
struct pim_encap_pimhdr {
struct pim pim;
uint32_t flags;
};
#define PIM_ENCAP_TTL 64
static struct ip pim_encap_iphdr = {
#if BYTE_ORDER == LITTLE_ENDIAN
sizeof(struct ip) >> 2,
IPVERSION,
#else
IPVERSION,
sizeof(struct ip) >> 2,
#endif
0, /* tos */
sizeof(struct ip), /* total length */
0, /* id */
0, /* frag offset */
PIM_ENCAP_TTL,
IPPROTO_PIM,
0, /* checksum */
};
static struct pim_encap_pimhdr pim_encap_pimhdr = {
{
PIM_MAKE_VT(PIM_VERSION, PIM_REGISTER), /* PIM vers and message type */
0, /* reserved */
0, /* checksum */
},
0 /* flags */
};
static VNET_DEFINE(vifi_t, reg_vif_num) = VIFI_INVALID;
#define V_reg_vif_num VNET(reg_vif_num)
static VNET_DEFINE(struct ifnet, multicast_register_if);
#define V_multicast_register_if VNET(multicast_register_if)
/*
* Private variables.
*/
static u_long X_ip_mcast_src(int);
static int X_ip_mforward(struct ip *, struct ifnet *, struct mbuf *,
struct ip_moptions *);
static int X_ip_mrouter_done(void);
static int X_ip_mrouter_get(struct socket *, struct sockopt *);
static int X_ip_mrouter_set(struct socket *, struct sockopt *);
static int X_legal_vif_num(int);
static int X_mrt_ioctl(u_long, caddr_t, int);
static int add_bw_upcall(struct bw_upcall *);
static int add_mfc(struct mfcctl2 *);
static int add_vif(struct vifctl *);
static void bw_meter_prepare_upcall(struct bw_meter *, struct timeval *);
static void bw_meter_process(void);
static void bw_meter_receive_packet(struct bw_meter *, int,
struct timeval *);
static void bw_upcalls_send(void);
static int del_bw_upcall(struct bw_upcall *);
static int del_mfc(struct mfcctl2 *);
static int del_vif(vifi_t);
static int del_vif_locked(vifi_t);
static void expire_bw_meter_process(void *);
static void expire_bw_upcalls_send(void *);
static void expire_mfc(struct mfc *);
static void expire_upcalls(void *);
static void free_bw_list(struct bw_meter *);
static int get_sg_cnt(struct sioc_sg_req *);
static int get_vif_cnt(struct sioc_vif_req *);
static void if_detached_event(void *, struct ifnet *);
static int ip_mdq(struct mbuf *, struct ifnet *, struct mfc *, vifi_t);
static int ip_mrouter_init(struct socket *, int);
static __inline struct mfc *
mfc_find(struct in_addr *, struct in_addr *);
static void phyint_send(struct ip *, struct vif *, struct mbuf *);
static struct mbuf *
pim_register_prepare(struct ip *, struct mbuf *);
static int pim_register_send(struct ip *, struct vif *,
struct mbuf *, struct mfc *);
static int pim_register_send_rp(struct ip *, struct vif *,
struct mbuf *, struct mfc *);
static int pim_register_send_upcall(struct ip *, struct vif *,
struct mbuf *, struct mfc *);
static void schedule_bw_meter(struct bw_meter *, struct timeval *);
static void send_packet(struct vif *, struct mbuf *);
static int set_api_config(uint32_t *);
static int set_assert(int);
static int socket_send(struct socket *, struct mbuf *,
struct sockaddr_in *);
static void unschedule_bw_meter(struct bw_meter *);
/*
* Kernel multicast forwarding API capabilities and setup.
* If more API capabilities are added to the kernel, they should be
* recorded in `mrt_api_support'.
*/
#define MRT_API_VERSION 0x0305
static const int mrt_api_version = MRT_API_VERSION;
static const uint32_t mrt_api_support = (MRT_MFC_FLAGS_DISABLE_WRONGVIF |
MRT_MFC_FLAGS_BORDER_VIF |
MRT_MFC_RP |
MRT_MFC_BW_UPCALL);
static VNET_DEFINE(uint32_t, mrt_api_config);
#define V_mrt_api_config VNET(mrt_api_config)
static VNET_DEFINE(int, pim_assert_enabled);
#define V_pim_assert_enabled VNET(pim_assert_enabled)
static struct timeval pim_assert_interval = { 3, 0 }; /* Rate limit */
/*
* Find a route for a given origin IP address and multicast group address.
* Statistics must be updated by the caller.
*/
static __inline struct mfc *
mfc_find(struct in_addr *o, struct in_addr *g)
{
struct mfc *rt;
MFC_LOCK_ASSERT();
LIST_FOREACH(rt, &V_mfchashtbl[MFCHASH(*o, *g)], mfc_hash) {
if (in_hosteq(rt->mfc_origin, *o) &&
in_hosteq(rt->mfc_mcastgrp, *g) &&
TAILQ_EMPTY(&rt->mfc_stall))
break;
}
return (rt);
}
/*
* Handle MRT setsockopt commands to modify the multicast forwarding tables.
*/
static int
X_ip_mrouter_set(struct socket *so, struct sockopt *sopt)
{
int error, optval;
vifi_t vifi;
struct vifctl vifc;
struct mfcctl2 mfc;
struct bw_upcall bw_upcall;
uint32_t i;
if (so != V_ip_mrouter && sopt->sopt_name != MRT_INIT)
return EPERM;
error = 0;
switch (sopt->sopt_name) {
case MRT_INIT:
error = sooptcopyin(sopt, &optval, sizeof optval, sizeof optval);
if (error)
break;
error = ip_mrouter_init(so, optval);
break;
case MRT_DONE:
error = ip_mrouter_done();
break;
case MRT_ADD_VIF:
error = sooptcopyin(sopt, &vifc, sizeof vifc, sizeof vifc);
if (error)
break;
error = add_vif(&vifc);
break;
case MRT_DEL_VIF:
error = sooptcopyin(sopt, &vifi, sizeof vifi, sizeof vifi);
if (error)
break;
error = del_vif(vifi);
break;
case MRT_ADD_MFC:
case MRT_DEL_MFC:
/*
* select data size depending on API version.
*/
if (sopt->sopt_name == MRT_ADD_MFC &&
V_mrt_api_config & MRT_API_FLAGS_ALL) {
error = sooptcopyin(sopt, &mfc, sizeof(struct mfcctl2),
sizeof(struct mfcctl2));
} else {
error = sooptcopyin(sopt, &mfc, sizeof(struct mfcctl),
sizeof(struct mfcctl));
bzero((caddr_t)&mfc + sizeof(struct mfcctl),
sizeof(mfc) - sizeof(struct mfcctl));
}
if (error)
break;
if (sopt->sopt_name == MRT_ADD_MFC)
error = add_mfc(&mfc);
else
error = del_mfc(&mfc);
break;
case MRT_ASSERT:
error = sooptcopyin(sopt, &optval, sizeof optval, sizeof optval);
if (error)
break;
set_assert(optval);
break;
case MRT_API_CONFIG:
error = sooptcopyin(sopt, &i, sizeof i, sizeof i);
if (!error)
error = set_api_config(&i);
if (!error)
error = sooptcopyout(sopt, &i, sizeof i);
break;
case MRT_ADD_BW_UPCALL:
case MRT_DEL_BW_UPCALL:
error = sooptcopyin(sopt, &bw_upcall, sizeof bw_upcall,
sizeof bw_upcall);
if (error)
break;
if (sopt->sopt_name == MRT_ADD_BW_UPCALL)
error = add_bw_upcall(&bw_upcall);
else
error = del_bw_upcall(&bw_upcall);
break;
default:
error = EOPNOTSUPP;
break;
}
return error;
}
/*
* Handle MRT getsockopt commands
*/
static int
X_ip_mrouter_get(struct socket *so, struct sockopt *sopt)
{
int error;
switch (sopt->sopt_name) {
case MRT_VERSION:
error = sooptcopyout(sopt, &mrt_api_version, sizeof mrt_api_version);
break;
case MRT_ASSERT:
error = sooptcopyout(sopt, &V_pim_assert_enabled,
sizeof V_pim_assert_enabled);
break;
case MRT_API_SUPPORT:
error = sooptcopyout(sopt, &mrt_api_support, sizeof mrt_api_support);
break;
case MRT_API_CONFIG:
error = sooptcopyout(sopt, &V_mrt_api_config, sizeof V_mrt_api_config);
break;
default:
error = EOPNOTSUPP;
break;
}
return error;
}
/*
* Handle ioctl commands to obtain information from the cache
*/
static int
X_mrt_ioctl(u_long cmd, caddr_t data, int fibnum __unused)
{
int error = 0;
/*
* Currently the only function calling this ioctl routine is rtioctl().
* Typically, only root can create the raw socket in order to execute
* this ioctl method, however the request might be coming from a prison
*/
error = priv_check(curthread, PRIV_NETINET_MROUTE);
if (error)
return (error);
switch (cmd) {
case (SIOCGETVIFCNT):
error = get_vif_cnt((struct sioc_vif_req *)data);
break;
case (SIOCGETSGCNT):
error = get_sg_cnt((struct sioc_sg_req *)data);
break;
default:
error = EINVAL;
break;
}
return error;
}
/*
* returns the packet, byte, rpf-failure count for the source group provided
*/
static int
get_sg_cnt(struct sioc_sg_req *req)
{
struct mfc *rt;
MFC_LOCK();
rt = mfc_find(&req->src, &req->grp);
if (rt == NULL) {
MFC_UNLOCK();
req->pktcnt = req->bytecnt = req->wrong_if = 0xffffffff;
return EADDRNOTAVAIL;
}
req->pktcnt = rt->mfc_pkt_cnt;
req->bytecnt = rt->mfc_byte_cnt;
req->wrong_if = rt->mfc_wrong_if;
MFC_UNLOCK();
return 0;
}
/*
* returns the input and output packet and byte counts on the vif provided
*/
static int
get_vif_cnt(struct sioc_vif_req *req)
{
vifi_t vifi = req->vifi;
VIF_LOCK();
if (vifi >= V_numvifs) {
VIF_UNLOCK();
return EINVAL;
}
req->icount = V_viftable[vifi].v_pkt_in;
req->ocount = V_viftable[vifi].v_pkt_out;
req->ibytes = V_viftable[vifi].v_bytes_in;
req->obytes = V_viftable[vifi].v_bytes_out;
VIF_UNLOCK();
return 0;
}
static void
if_detached_event(void *arg __unused, struct ifnet *ifp)
{
vifi_t vifi;
u_long i;
MROUTER_LOCK();
if (V_ip_mrouter == NULL) {
MROUTER_UNLOCK();
return;
}
VIF_LOCK();
MFC_LOCK();
/*
* Tear down multicast forwarder state associated with this ifnet.
* 1. Walk the vif list, matching vifs against this ifnet.
* 2. Walk the multicast forwarding cache (mfc) looking for
* inner matches with this vif's index.
* 3. Expire any matching multicast forwarding cache entries.
* 4. Free vif state. This should disable ALLMULTI on the interface.
*/
for (vifi = 0; vifi < V_numvifs; vifi++) {
if (V_viftable[vifi].v_ifp != ifp)
continue;
for (i = 0; i < mfchashsize; i++) {
struct mfc *rt, *nrt;
LIST_FOREACH_SAFE(rt, &V_mfchashtbl[i], mfc_hash, nrt) {
if (rt->mfc_parent == vifi) {
expire_mfc(rt);
}
}
}
del_vif_locked(vifi);
}
MFC_UNLOCK();
VIF_UNLOCK();
MROUTER_UNLOCK();
}
/*
* Enable multicast forwarding.
*/
static int
ip_mrouter_init(struct socket *so, int version)
{
CTR3(KTR_IPMF, "%s: so_type %d, pr_protocol %d", __func__,
so->so_type, so->so_proto->pr_protocol);
if (so->so_type != SOCK_RAW || so->so_proto->pr_protocol != IPPROTO_IGMP)
return EOPNOTSUPP;
if (version != 1)
return ENOPROTOOPT;
MROUTER_LOCK();
if (ip_mrouter_unloading) {
MROUTER_UNLOCK();
return ENOPROTOOPT;
}
if (V_ip_mrouter != NULL) {
MROUTER_UNLOCK();
return EADDRINUSE;
}
V_mfchashtbl = hashinit_flags(mfchashsize, M_MRTABLE, &V_mfchash,
HASH_NOWAIT);
callout_reset(&V_expire_upcalls_ch, EXPIRE_TIMEOUT, expire_upcalls,
curvnet);
callout_reset(&V_bw_upcalls_ch, BW_UPCALLS_PERIOD, expire_bw_upcalls_send,
curvnet);
callout_reset(&V_bw_meter_ch, BW_METER_PERIOD, expire_bw_meter_process,
curvnet);
V_ip_mrouter = so;
ip_mrouter_cnt++;
MROUTER_UNLOCK();
CTR1(KTR_IPMF, "%s: done", __func__);
return 0;
}
/*
* Disable multicast forwarding.
*/
static int
X_ip_mrouter_done(void)
{
struct ifnet *ifp;
u_long i;
vifi_t vifi;
MROUTER_LOCK();
if (V_ip_mrouter == NULL) {
MROUTER_UNLOCK();
return EINVAL;
}
/*
* Detach/disable hooks to the reset of the system.
*/
V_ip_mrouter = NULL;
ip_mrouter_cnt--;
V_mrt_api_config = 0;
VIF_LOCK();
/*
* For each phyint in use, disable promiscuous reception of all IP
* multicasts.
*/
for (vifi = 0; vifi < V_numvifs; vifi++) {
if (!in_nullhost(V_viftable[vifi].v_lcl_addr) &&
!(V_viftable[vifi].v_flags & (VIFF_TUNNEL | VIFF_REGISTER))) {
ifp = V_viftable[vifi].v_ifp;
if_allmulti(ifp, 0);
}
}
bzero((caddr_t)V_viftable, sizeof(V_viftable));
V_numvifs = 0;
V_pim_assert_enabled = 0;
VIF_UNLOCK();
callout_stop(&V_expire_upcalls_ch);
callout_stop(&V_bw_upcalls_ch);
callout_stop(&V_bw_meter_ch);
MFC_LOCK();
/*
* Free all multicast forwarding cache entries.
* Do not use hashdestroy(), as we must perform other cleanup.
*/
for (i = 0; i < mfchashsize; i++) {
struct mfc *rt, *nrt;
LIST_FOREACH_SAFE(rt, &V_mfchashtbl[i], mfc_hash, nrt) {
expire_mfc(rt);
}
}
free(V_mfchashtbl, M_MRTABLE);
V_mfchashtbl = NULL;
bzero(V_nexpire, sizeof(V_nexpire[0]) * mfchashsize);
V_bw_upcalls_n = 0;
bzero(V_bw_meter_timers, sizeof(V_bw_meter_timers));
MFC_UNLOCK();
V_reg_vif_num = VIFI_INVALID;
MROUTER_UNLOCK();
CTR1(KTR_IPMF, "%s: done", __func__);
return 0;
}
/*
* Set PIM assert processing global
*/
static int
set_assert(int i)
{
if ((i != 1) && (i != 0))
return EINVAL;
V_pim_assert_enabled = i;
return 0;
}
/*
* Configure API capabilities
*/
int
set_api_config(uint32_t *apival)
{
u_long i;
/*
* We can set the API capabilities only if it is the first operation
* after MRT_INIT. I.e.:
* - there are no vifs installed
* - pim_assert is not enabled
* - the MFC table is empty
*/
if (V_numvifs > 0) {
*apival = 0;
return EPERM;
}
if (V_pim_assert_enabled) {
*apival = 0;
return EPERM;
}
MFC_LOCK();
for (i = 0; i < mfchashsize; i++) {
if (LIST_FIRST(&V_mfchashtbl[i]) != NULL) {
MFC_UNLOCK();
*apival = 0;
return EPERM;
}
}
MFC_UNLOCK();
V_mrt_api_config = *apival & mrt_api_support;
*apival = V_mrt_api_config;
return 0;
}
/*
* Add a vif to the vif table
*/
static int
add_vif(struct vifctl *vifcp)
{
struct vif *vifp = V_viftable + vifcp->vifc_vifi;
struct sockaddr_in sin = {sizeof sin, AF_INET};
struct ifaddr *ifa;
struct ifnet *ifp;
int error;
VIF_LOCK();
if (vifcp->vifc_vifi >= MAXVIFS) {
VIF_UNLOCK();
return EINVAL;
}
/* rate limiting is no longer supported by this code */
if (vifcp->vifc_rate_limit != 0) {
log(LOG_ERR, "rate limiting is no longer supported\n");
VIF_UNLOCK();
return EINVAL;
}
if (!in_nullhost(vifp->v_lcl_addr)) {
VIF_UNLOCK();
return EADDRINUSE;
}
if (in_nullhost(vifcp->vifc_lcl_addr)) {
VIF_UNLOCK();
return EADDRNOTAVAIL;
}
/* Find the interface with an address in AF_INET family */
if (vifcp->vifc_flags & VIFF_REGISTER) {
/*
* XXX: Because VIFF_REGISTER does not really need a valid
* local interface (e.g. it could be 127.0.0.2), we don't
* check its address.
*/
ifp = NULL;
} else {
sin.sin_addr = vifcp->vifc_lcl_addr;
ifa = ifa_ifwithaddr((struct sockaddr *)&sin);
if (ifa == NULL) {
VIF_UNLOCK();
return EADDRNOTAVAIL;
}
ifp = ifa->ifa_ifp;
ifa_free(ifa);
}
if ((vifcp->vifc_flags & VIFF_TUNNEL) != 0) {
CTR1(KTR_IPMF, "%s: tunnels are no longer supported", __func__);
VIF_UNLOCK();
return EOPNOTSUPP;
} else if (vifcp->vifc_flags & VIFF_REGISTER) {
ifp = &V_multicast_register_if;
CTR2(KTR_IPMF, "%s: add register vif for ifp %p", __func__, ifp);
if (V_reg_vif_num == VIFI_INVALID) {
if_initname(&V_multicast_register_if, "register_vif", 0);
V_multicast_register_if.if_flags = IFF_LOOPBACK;
V_reg_vif_num = vifcp->vifc_vifi;
}
} else { /* Make sure the interface supports multicast */
if ((ifp->if_flags & IFF_MULTICAST) == 0) {
VIF_UNLOCK();
return EOPNOTSUPP;
}
/* Enable promiscuous reception of all IP multicasts from the if */
error = if_allmulti(ifp, 1);
if (error) {
VIF_UNLOCK();
return error;
}
}
vifp->v_flags = vifcp->vifc_flags;
vifp->v_threshold = vifcp->vifc_threshold;
vifp->v_lcl_addr = vifcp->vifc_lcl_addr;
vifp->v_rmt_addr = vifcp->vifc_rmt_addr;
vifp->v_ifp = ifp;
/* initialize per vif pkt counters */
vifp->v_pkt_in = 0;
vifp->v_pkt_out = 0;
vifp->v_bytes_in = 0;
vifp->v_bytes_out = 0;
/* Adjust numvifs up if the vifi is higher than numvifs */
if (V_numvifs <= vifcp->vifc_vifi)
V_numvifs = vifcp->vifc_vifi + 1;
VIF_UNLOCK();
CTR4(KTR_IPMF, "%s: add vif %d laddr %s thresh %x", __func__,
(int)vifcp->vifc_vifi, inet_ntoa(vifcp->vifc_lcl_addr),
(int)vifcp->vifc_threshold);
return 0;
}
/*
* Delete a vif from the vif table
*/
static int
del_vif_locked(vifi_t vifi)
{
struct vif *vifp;
VIF_LOCK_ASSERT();
if (vifi >= V_numvifs) {
return EINVAL;
}
vifp = &V_viftable[vifi];
if (in_nullhost(vifp->v_lcl_addr)) {
return EADDRNOTAVAIL;
}
if (!(vifp->v_flags & (VIFF_TUNNEL | VIFF_REGISTER)))
if_allmulti(vifp->v_ifp, 0);
if (vifp->v_flags & VIFF_REGISTER)
V_reg_vif_num = VIFI_INVALID;
bzero((caddr_t)vifp, sizeof (*vifp));
CTR2(KTR_IPMF, "%s: delete vif %d", __func__, (int)vifi);
/* Adjust numvifs down */
for (vifi = V_numvifs; vifi > 0; vifi--)
if (!in_nullhost(V_viftable[vifi-1].v_lcl_addr))
break;
V_numvifs = vifi;
return 0;
}
static int
del_vif(vifi_t vifi)
{
int cc;
VIF_LOCK();
cc = del_vif_locked(vifi);
VIF_UNLOCK();
return cc;
}
/*
* update an mfc entry without resetting counters and S,G addresses.
*/
static void
update_mfc_params(struct mfc *rt, struct mfcctl2 *mfccp)
{
int i;
rt->mfc_parent = mfccp->mfcc_parent;
for (i = 0; i < V_numvifs; i++) {
rt->mfc_ttls[i] = mfccp->mfcc_ttls[i];
rt->mfc_flags[i] = mfccp->mfcc_flags[i] & V_mrt_api_config &
MRT_MFC_FLAGS_ALL;
}
/* set the RP address */
if (V_mrt_api_config & MRT_MFC_RP)
rt->mfc_rp = mfccp->mfcc_rp;
else
rt->mfc_rp.s_addr = INADDR_ANY;
}
/*
* fully initialize an mfc entry from the parameter.
*/
static void
init_mfc_params(struct mfc *rt, struct mfcctl2 *mfccp)
{
rt->mfc_origin = mfccp->mfcc_origin;
rt->mfc_mcastgrp = mfccp->mfcc_mcastgrp;
update_mfc_params(rt, mfccp);
/* initialize pkt counters per src-grp */
rt->mfc_pkt_cnt = 0;
rt->mfc_byte_cnt = 0;
rt->mfc_wrong_if = 0;
timevalclear(&rt->mfc_last_assert);
}
static void
expire_mfc(struct mfc *rt)
{
struct rtdetq *rte, *nrte;
MFC_LOCK_ASSERT();
free_bw_list(rt->mfc_bw_meter);
TAILQ_FOREACH_SAFE(rte, &rt->mfc_stall, rte_link, nrte) {
m_freem(rte->m);
TAILQ_REMOVE(&rt->mfc_stall, rte, rte_link);
free(rte, M_MRTABLE);
}
LIST_REMOVE(rt, mfc_hash);
free(rt, M_MRTABLE);
}
/*
* Add an mfc entry
*/
static int
add_mfc(struct mfcctl2 *mfccp)
{
struct mfc *rt;
struct rtdetq *rte, *nrte;
u_long hash = 0;
u_short nstl;
VIF_LOCK();
MFC_LOCK();
rt = mfc_find(&mfccp->mfcc_origin, &mfccp->mfcc_mcastgrp);
/* If an entry already exists, just update the fields */
if (rt) {
CTR4(KTR_IPMF, "%s: update mfc orig %s group %lx parent %x",
__func__, inet_ntoa(mfccp->mfcc_origin),
(u_long)ntohl(mfccp->mfcc_mcastgrp.s_addr),
mfccp->mfcc_parent);
update_mfc_params(rt, mfccp);
MFC_UNLOCK();
VIF_UNLOCK();
return (0);
}
/*
* Find the entry for which the upcall was made and update
*/
nstl = 0;
hash = MFCHASH(mfccp->mfcc_origin, mfccp->mfcc_mcastgrp);
LIST_FOREACH(rt, &V_mfchashtbl[hash], mfc_hash) {
if (in_hosteq(rt->mfc_origin, mfccp->mfcc_origin) &&
in_hosteq(rt->mfc_mcastgrp, mfccp->mfcc_mcastgrp) &&
!TAILQ_EMPTY(&rt->mfc_stall)) {
CTR5(KTR_IPMF,
"%s: add mfc orig %s group %lx parent %x qh %p",
__func__, inet_ntoa(mfccp->mfcc_origin),
(u_long)ntohl(mfccp->mfcc_mcastgrp.s_addr),
mfccp->mfcc_parent,
TAILQ_FIRST(&rt->mfc_stall));
if (nstl++)
CTR1(KTR_IPMF, "%s: multiple matches", __func__);
init_mfc_params(rt, mfccp);
rt->mfc_expire = 0; /* Don't clean this guy up */
V_nexpire[hash]--;
/* Free queued packets, but attempt to forward them first. */
TAILQ_FOREACH_SAFE(rte, &rt->mfc_stall, rte_link, nrte) {
if (rte->ifp != NULL)
ip_mdq(rte->m, rte->ifp, rt, -1);
m_freem(rte->m);
TAILQ_REMOVE(&rt->mfc_stall, rte, rte_link);
rt->mfc_nstall--;
free(rte, M_MRTABLE);
}
}
}
/*
* It is possible that an entry is being inserted without an upcall
*/
if (nstl == 0) {
CTR1(KTR_IPMF, "%s: adding mfc w/o upcall", __func__);
LIST_FOREACH(rt, &V_mfchashtbl[hash], mfc_hash) {
if (in_hosteq(rt->mfc_origin, mfccp->mfcc_origin) &&
in_hosteq(rt->mfc_mcastgrp, mfccp->mfcc_mcastgrp)) {
init_mfc_params(rt, mfccp);
if (rt->mfc_expire)
V_nexpire[hash]--;
rt->mfc_expire = 0;
break; /* XXX */
}
}
if (rt == NULL) { /* no upcall, so make a new entry */
rt = (struct mfc *)malloc(sizeof(*rt), M_MRTABLE, M_NOWAIT);
if (rt == NULL) {
MFC_UNLOCK();
VIF_UNLOCK();
return (ENOBUFS);
}
init_mfc_params(rt, mfccp);
TAILQ_INIT(&rt->mfc_stall);
rt->mfc_nstall = 0;
rt->mfc_expire = 0;
rt->mfc_bw_meter = NULL;
/* insert new entry at head of hash chain */
LIST_INSERT_HEAD(&V_mfchashtbl[hash], rt, mfc_hash);
}
}
MFC_UNLOCK();
VIF_UNLOCK();
return (0);
}
/*
* Delete an mfc entry
*/
static int
del_mfc(struct mfcctl2 *mfccp)
{
struct in_addr origin;
struct in_addr mcastgrp;
struct mfc *rt;
origin = mfccp->mfcc_origin;
mcastgrp = mfccp->mfcc_mcastgrp;
CTR3(KTR_IPMF, "%s: delete mfc orig %s group %lx", __func__,
inet_ntoa(origin), (u_long)ntohl(mcastgrp.s_addr));
MFC_LOCK();
rt = mfc_find(&origin, &mcastgrp);
if (rt == NULL) {
MFC_UNLOCK();
return EADDRNOTAVAIL;
}
/*
* free the bw_meter entries
*/
free_bw_list(rt->mfc_bw_meter);
rt->mfc_bw_meter = NULL;
LIST_REMOVE(rt, mfc_hash);
free(rt, M_MRTABLE);
MFC_UNLOCK();
return (0);
}
/*
* Send a message to the routing daemon on the multicast routing socket.
*/
static int
socket_send(struct socket *s, struct mbuf *mm, struct sockaddr_in *src)
{
if (s) {
SOCKBUF_LOCK(&s->so_rcv);
if (sbappendaddr_locked(&s->so_rcv, (struct sockaddr *)src, mm,
NULL) != 0) {
sorwakeup_locked(s);
return 0;
}
SOCKBUF_UNLOCK(&s->so_rcv);
}
m_freem(mm);
return -1;
}
/*
* IP multicast forwarding function. This function assumes that the packet
* pointed to by "ip" has arrived on (or is about to be sent to) the interface
* pointed to by "ifp", and the packet is to be relayed to other networks
* that have members of the packet's destination IP multicast group.
*
* The packet is returned unscathed to the caller, unless it is
* erroneous, in which case a non-zero return value tells the caller to
* discard it.
*/
#define TUNNEL_LEN 12 /* # bytes of IP option for tunnel encapsulation */
static int
X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
struct ip_moptions *imo)
{
struct mfc *rt;
int error;
vifi_t vifi;
CTR3(KTR_IPMF, "ip_mforward: delete mfc orig %s group %lx ifp %p",
inet_ntoa(ip->ip_src), (u_long)ntohl(ip->ip_dst.s_addr), ifp);
if (ip->ip_hl < (sizeof(struct ip) + TUNNEL_LEN) >> 2 ||
((u_char *)(ip + 1))[1] != IPOPT_LSRR ) {
/*
* Packet arrived via a physical interface or
* an encapsulated tunnel or a register_vif.
*/
} else {
/*
* Packet arrived through a source-route tunnel.
* Source-route tunnels are no longer supported.
*/
return (1);
}
VIF_LOCK();
MFC_LOCK();
if (imo && ((vifi = imo->imo_multicast_vif) < V_numvifs)) {
if (ip->ip_ttl < MAXTTL)
ip->ip_ttl++; /* compensate for -1 in *_send routines */
error = ip_mdq(m, ifp, NULL, vifi);
MFC_UNLOCK();
VIF_UNLOCK();
return error;
}
/*
* Don't forward a packet with time-to-live of zero or one,
* or a packet destined to a local-only group.
*/
if (ip->ip_ttl <= 1 || IN_LOCAL_GROUP(ntohl(ip->ip_dst.s_addr))) {
MFC_UNLOCK();
VIF_UNLOCK();
return 0;
}
/*
* Determine forwarding vifs from the forwarding cache table
*/
MRTSTAT_INC(mrts_mfc_lookups);
rt = mfc_find(&ip->ip_src, &ip->ip_dst);
/* Entry exists, so forward if necessary */
if (rt != NULL) {
error = ip_mdq(m, ifp, rt, -1);
MFC_UNLOCK();
VIF_UNLOCK();
return error;
} else {
/*
* If we don't have a route for packet's origin,
* Make a copy of the packet & send message to routing daemon
*/
struct mbuf *mb0;
struct rtdetq *rte;
u_long hash;
int hlen = ip->ip_hl << 2;
MRTSTAT_INC(mrts_mfc_misses);
MRTSTAT_INC(mrts_no_route);
CTR2(KTR_IPMF, "ip_mforward: no mfc for (%s,%lx)",
inet_ntoa(ip->ip_src), (u_long)ntohl(ip->ip_dst.s_addr));
/*
* Allocate mbufs early so that we don't do extra work if we are
* just going to fail anyway. Make sure to pullup the header so
* that other people can't step on it.
*/
rte = (struct rtdetq *)malloc((sizeof *rte), M_MRTABLE,
M_NOWAIT|M_ZERO);
if (rte == NULL) {
MFC_UNLOCK();
VIF_UNLOCK();
return ENOBUFS;
}
mb0 = m_copypacket(m, M_NOWAIT);
if (mb0 && (!M_WRITABLE(mb0) || mb0->m_len < hlen))
mb0 = m_pullup(mb0, hlen);
if (mb0 == NULL) {
free(rte, M_MRTABLE);
MFC_UNLOCK();
VIF_UNLOCK();
return ENOBUFS;
}
/* is there an upcall waiting for this flow ? */
hash = MFCHASH(ip->ip_src, ip->ip_dst);
LIST_FOREACH(rt, &V_mfchashtbl[hash], mfc_hash) {
if (in_hosteq(ip->ip_src, rt->mfc_origin) &&
in_hosteq(ip->ip_dst, rt->mfc_mcastgrp) &&
!TAILQ_EMPTY(&rt->mfc_stall))
break;
}
if (rt == NULL) {
int i;
struct igmpmsg *im;
struct sockaddr_in k_igmpsrc = { sizeof k_igmpsrc, AF_INET };
struct mbuf *mm;
/*
* Locate the vifi for the incoming interface for this packet.
* If none found, drop packet.
*/
for (vifi = 0; vifi < V_numvifs &&
V_viftable[vifi].v_ifp != ifp; vifi++)
;
if (vifi >= V_numvifs) /* vif not found, drop packet */
goto non_fatal;
/* no upcall, so make a new entry */
rt = (struct mfc *)malloc(sizeof(*rt), M_MRTABLE, M_NOWAIT);
if (rt == NULL)
goto fail;
/* Make a copy of the header to send to the user level process */
mm = m_copy(mb0, 0, hlen);
if (mm == NULL)
goto fail1;
/*
* Send message to routing daemon to install
* a route into the kernel table
*/
im = mtod(mm, struct igmpmsg *);
im->im_msgtype = IGMPMSG_NOCACHE;
im->im_mbz = 0;
im->im_vif = vifi;
MRTSTAT_INC(mrts_upcalls);
k_igmpsrc.sin_addr = ip->ip_src;
if (socket_send(V_ip_mrouter, mm, &k_igmpsrc) < 0) {
CTR0(KTR_IPMF, "ip_mforward: socket queue full");
MRTSTAT_INC(mrts_upq_sockfull);
fail1:
free(rt, M_MRTABLE);
fail:
free(rte, M_MRTABLE);
m_freem(mb0);
MFC_UNLOCK();
VIF_UNLOCK();
return ENOBUFS;
}
/* insert new entry at head of hash chain */
rt->mfc_origin.s_addr = ip->ip_src.s_addr;
rt->mfc_mcastgrp.s_addr = ip->ip_dst.s_addr;
rt->mfc_expire = UPCALL_EXPIRE;
V_nexpire[hash]++;
for (i = 0; i < V_numvifs; i++) {
rt->mfc_ttls[i] = 0;
rt->mfc_flags[i] = 0;
}
rt->mfc_parent = -1;
/* clear the RP address */
rt->mfc_rp.s_addr = INADDR_ANY;
rt->mfc_bw_meter = NULL;
/* initialize pkt counters per src-grp */
rt->mfc_pkt_cnt = 0;
rt->mfc_byte_cnt = 0;
rt->mfc_wrong_if = 0;
timevalclear(&rt->mfc_last_assert);
TAILQ_INIT(&rt->mfc_stall);
rt->mfc_nstall = 0;
/* link into table */
LIST_INSERT_HEAD(&V_mfchashtbl[hash], rt, mfc_hash);
TAILQ_INSERT_HEAD(&rt->mfc_stall, rte, rte_link);
rt->mfc_nstall++;
} else {
/* determine if queue has overflowed */
if (rt->mfc_nstall > MAX_UPQ) {
MRTSTAT_INC(mrts_upq_ovflw);
non_fatal:
free(rte, M_MRTABLE);
m_freem(mb0);
MFC_UNLOCK();
VIF_UNLOCK();
return (0);
}
TAILQ_INSERT_TAIL(&rt->mfc_stall, rte, rte_link);
rt->mfc_nstall++;
}
rte->m = mb0;
rte->ifp = ifp;
MFC_UNLOCK();
VIF_UNLOCK();
return 0;
}
}
/*
* Clean up the cache entry if upcall is not serviced
*/
static void
expire_upcalls(void *arg)
{
u_long i;
CURVNET_SET((struct vnet *) arg);
MFC_LOCK();
for (i = 0; i < mfchashsize; i++) {
struct mfc *rt, *nrt;
if (V_nexpire[i] == 0)
continue;
LIST_FOREACH_SAFE(rt, &V_mfchashtbl[i], mfc_hash, nrt) {
if (TAILQ_EMPTY(&rt->mfc_stall))
continue;
if (rt->mfc_expire == 0 || --rt->mfc_expire > 0)
continue;
/*
* free the bw_meter entries
*/
while (rt->mfc_bw_meter != NULL) {
struct bw_meter *x = rt->mfc_bw_meter;
rt->mfc_bw_meter = x->bm_mfc_next;
free(x, M_BWMETER);
}
MRTSTAT_INC(mrts_cache_cleanups);
CTR3(KTR_IPMF, "%s: expire (%lx, %lx)", __func__,
(u_long)ntohl(rt->mfc_origin.s_addr),
(u_long)ntohl(rt->mfc_mcastgrp.s_addr));
expire_mfc(rt);
}
}
MFC_UNLOCK();
callout_reset(&V_expire_upcalls_ch, EXPIRE_TIMEOUT, expire_upcalls,
curvnet);
CURVNET_RESTORE();
}
/*
* Packet forwarding routine once entry in the cache is made
*/
static int
ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif)
{
struct ip *ip = mtod(m, struct ip *);
vifi_t vifi;
int plen = ntohs(ip->ip_len);
VIF_LOCK_ASSERT();
/*
* If xmt_vif is not -1, send on only the requested vif.
*
* (since vifi_t is u_short, -1 becomes MAXUSHORT, which > numvifs.)
*/
if (xmt_vif < V_numvifs) {
if (V_viftable[xmt_vif].v_flags & VIFF_REGISTER)
pim_register_send(ip, V_viftable + xmt_vif, m, rt);
else
phyint_send(ip, V_viftable + xmt_vif, m);
return 1;
}
/*
* Don't forward if it didn't arrive from the parent vif for its origin.
*/
vifi = rt->mfc_parent;
if ((vifi >= V_numvifs) || (V_viftable[vifi].v_ifp != ifp)) {
CTR4(KTR_IPMF, "%s: rx on wrong ifp %p (vifi %d, v_ifp %p)",
__func__, ifp, (int)vifi, V_viftable[vifi].v_ifp);
MRTSTAT_INC(mrts_wrong_if);
++rt->mfc_wrong_if;
/*
* If we are doing PIM assert processing, send a message
* to the routing daemon.
*
* XXX: A PIM-SM router needs the WRONGVIF detection so it
* can complete the SPT switch, regardless of the type
* of the iif (broadcast media, GRE tunnel, etc).
*/
if (V_pim_assert_enabled && (vifi < V_numvifs) &&
V_viftable[vifi].v_ifp) {
if (ifp == &V_multicast_register_if)
PIMSTAT_INC(pims_rcv_registers_wrongiif);
/* Get vifi for the incoming packet */
for (vifi = 0; vifi < V_numvifs && V_viftable[vifi].v_ifp != ifp;
vifi++)
;
if (vifi >= V_numvifs)
return 0; /* The iif is not found: ignore the packet. */
if (rt->mfc_flags[vifi] & MRT_MFC_FLAGS_DISABLE_WRONGVIF)
return 0; /* WRONGVIF disabled: ignore the packet */
if (ratecheck(&rt->mfc_last_assert, &pim_assert_interval)) {
struct sockaddr_in k_igmpsrc = { sizeof k_igmpsrc, AF_INET };
struct igmpmsg *im;
int hlen = ip->ip_hl << 2;
struct mbuf *mm = m_copy(m, 0, hlen);
if (mm && (!M_WRITABLE(mm) || mm->m_len < hlen))
mm = m_pullup(mm, hlen);
if (mm == NULL)
return ENOBUFS;
im = mtod(mm, struct igmpmsg *);
im->im_msgtype = IGMPMSG_WRONGVIF;
im->im_mbz = 0;
im->im_vif = vifi;
MRTSTAT_INC(mrts_upcalls);
k_igmpsrc.sin_addr = im->im_src;
if (socket_send(V_ip_mrouter, mm, &k_igmpsrc) < 0) {
CTR1(KTR_IPMF, "%s: socket queue full", __func__);
MRTSTAT_INC(mrts_upq_sockfull);
return ENOBUFS;
}
}
}
return 0;
}
/* If I sourced this packet, it counts as output, else it was input. */
if (in_hosteq(ip->ip_src, V_viftable[vifi].v_lcl_addr)) {
V_viftable[vifi].v_pkt_out++;
V_viftable[vifi].v_bytes_out += plen;
} else {
V_viftable[vifi].v_pkt_in++;
V_viftable[vifi].v_bytes_in += plen;
}
rt->mfc_pkt_cnt++;
rt->mfc_byte_cnt += plen;
/*
* For each vif, decide if a copy of the packet should be forwarded.
* Forward if:
* - the ttl exceeds the vif's threshold
* - there are group members downstream on interface
*/
for (vifi = 0; vifi < V_numvifs; vifi++)
if ((rt->mfc_ttls[vifi] > 0) && (ip->ip_ttl > rt->mfc_ttls[vifi])) {
V_viftable[vifi].v_pkt_out++;
V_viftable[vifi].v_bytes_out += plen;
if (V_viftable[vifi].v_flags & VIFF_REGISTER)
pim_register_send(ip, V_viftable + vifi, m, rt);
else
phyint_send(ip, V_viftable + vifi, m);
}
/*
* Perform upcall-related bw measuring.
*/
if (rt->mfc_bw_meter != NULL) {
struct bw_meter *x;
struct timeval now;
microtime(&now);
MFC_LOCK_ASSERT();
for (x = rt->mfc_bw_meter; x != NULL; x = x->bm_mfc_next)
bw_meter_receive_packet(x, plen, &now);
}
return 0;
}
/*
* Check if a vif number is legal/ok. This is used by in_mcast.c.
*/
static int
X_legal_vif_num(int vif)
{
int ret;
ret = 0;
if (vif < 0)
return (ret);
VIF_LOCK();
if (vif < V_numvifs)
ret = 1;
VIF_UNLOCK();
return (ret);
}
/*
* Return the local address used by this vif
*/
static u_long
X_ip_mcast_src(int vifi)
{
in_addr_t addr;
addr = INADDR_ANY;
if (vifi < 0)
return (addr);
VIF_LOCK();
if (vifi < V_numvifs)
addr = V_viftable[vifi].v_lcl_addr.s_addr;
VIF_UNLOCK();
return (addr);
}
static void
phyint_send(struct ip *ip, struct vif *vifp, struct mbuf *m)
{
struct mbuf *mb_copy;
int hlen = ip->ip_hl << 2;
VIF_LOCK_ASSERT();
/*
* Make a new reference to the packet; make sure that
* the IP header is actually copied, not just referenced,
* so that ip_output() only scribbles on the copy.
*/
mb_copy = m_copypacket(m, M_NOWAIT);
if (mb_copy && (!M_WRITABLE(mb_copy) || mb_copy->m_len < hlen))
mb_copy = m_pullup(mb_copy, hlen);
if (mb_copy == NULL)
return;
send_packet(vifp, mb_copy);
}
static void
send_packet(struct vif *vifp, struct mbuf *m)
{
struct ip_moptions imo;
struct in_multi *imm[2];
int error;
VIF_LOCK_ASSERT();
imo.imo_multicast_ifp = vifp->v_ifp;
imo.imo_multicast_ttl = mtod(m, struct ip *)->ip_ttl - 1;
imo.imo_multicast_loop = 1;
imo.imo_multicast_vif = -1;
imo.imo_num_memberships = 0;
imo.imo_max_memberships = 2;
imo.imo_membership = &imm[0];
/*
* Re-entrancy should not be a problem here, because
* the packets that we send out and are looped back at us
* should get rejected because they appear to come from
* the loopback interface, thus preventing looping.
*/
error = ip_output(m, NULL, NULL, IP_FORWARDING, &imo, NULL);
CTR3(KTR_IPMF, "%s: vif %td err %d", __func__,
(ptrdiff_t)(vifp - V_viftable), error);
}
/*
* Stubs for old RSVP socket shim implementation.
*/
static int
X_ip_rsvp_vif(struct socket *so __unused, struct sockopt *sopt __unused)
{
return (EOPNOTSUPP);
}
static void
X_ip_rsvp_force_done(struct socket *so __unused)
{
}
static int
X_rsvp_input(struct mbuf **mp, int *offp, int proto)
{
struct mbuf *m;
m = *mp;
*mp = NULL;
if (!V_rsvp_on)
m_freem(m);
return (IPPROTO_DONE);
}
/*
* Code for bandwidth monitors
*/
/*
* Define common interface for timeval-related methods
*/
#define BW_TIMEVALCMP(tvp, uvp, cmp) timevalcmp((tvp), (uvp), cmp)
#define BW_TIMEVALDECR(vvp, uvp) timevalsub((vvp), (uvp))
#define BW_TIMEVALADD(vvp, uvp) timevaladd((vvp), (uvp))
static uint32_t
compute_bw_meter_flags(struct bw_upcall *req)
{
uint32_t flags = 0;
if (req->bu_flags & BW_UPCALL_UNIT_PACKETS)
flags |= BW_METER_UNIT_PACKETS;
if (req->bu_flags & BW_UPCALL_UNIT_BYTES)
flags |= BW_METER_UNIT_BYTES;
if (req->bu_flags & BW_UPCALL_GEQ)
flags |= BW_METER_GEQ;
if (req->bu_flags & BW_UPCALL_LEQ)
flags |= BW_METER_LEQ;
return flags;
}
/*
* Add a bw_meter entry
*/
static int
add_bw_upcall(struct bw_upcall *req)
{
struct mfc *mfc;
struct timeval delta = { BW_UPCALL_THRESHOLD_INTERVAL_MIN_SEC,
BW_UPCALL_THRESHOLD_INTERVAL_MIN_USEC };
struct timeval now;
struct bw_meter *x;
uint32_t flags;
if (!(V_mrt_api_config & MRT_MFC_BW_UPCALL))
return EOPNOTSUPP;
/* Test if the flags are valid */
if (!(req->bu_flags & (BW_UPCALL_UNIT_PACKETS | BW_UPCALL_UNIT_BYTES)))
return EINVAL;
if (!(req->bu_flags & (BW_UPCALL_GEQ | BW_UPCALL_LEQ)))
return EINVAL;
if ((req->bu_flags & (BW_UPCALL_GEQ | BW_UPCALL_LEQ))
== (BW_UPCALL_GEQ | BW_UPCALL_LEQ))
return EINVAL;
/* Test if the threshold time interval is valid */
if (BW_TIMEVALCMP(&req->bu_threshold.b_time, &delta, <))
return EINVAL;
flags = compute_bw_meter_flags(req);
/*
* Find if we have already same bw_meter entry
*/
MFC_LOCK();
mfc = mfc_find(&req->bu_src, &req->bu_dst);
if (mfc == NULL) {
MFC_UNLOCK();
return EADDRNOTAVAIL;
}
for (x = mfc->mfc_bw_meter; x != NULL; x = x->bm_mfc_next) {
if ((BW_TIMEVALCMP(&x->bm_threshold.b_time,
&req->bu_threshold.b_time, ==)) &&
(x->bm_threshold.b_packets == req->bu_threshold.b_packets) &&
(x->bm_threshold.b_bytes == req->bu_threshold.b_bytes) &&
(x->bm_flags & BW_METER_USER_FLAGS) == flags) {
MFC_UNLOCK();
return 0; /* XXX Already installed */
}
}
/* Allocate the new bw_meter entry */
x = (struct bw_meter *)malloc(sizeof(*x), M_BWMETER, M_NOWAIT);
if (x == NULL) {
MFC_UNLOCK();
return ENOBUFS;
}
/* Set the new bw_meter entry */
x->bm_threshold.b_time = req->bu_threshold.b_time;
microtime(&now);
x->bm_start_time = now;
x->bm_threshold.b_packets = req->bu_threshold.b_packets;
x->bm_threshold.b_bytes = req->bu_threshold.b_bytes;
x->bm_measured.b_packets = 0;
x->bm_measured.b_bytes = 0;
x->bm_flags = flags;
x->bm_time_next = NULL;
x->bm_time_hash = BW_METER_BUCKETS;
/* Add the new bw_meter entry to the front of entries for this MFC */
x->bm_mfc = mfc;
x->bm_mfc_next = mfc->mfc_bw_meter;
mfc->mfc_bw_meter = x;
schedule_bw_meter(x, &now);
MFC_UNLOCK();
return 0;
}
static void
free_bw_list(struct bw_meter *list)
{
while (list != NULL) {
struct bw_meter *x = list;
list = list->bm_mfc_next;
unschedule_bw_meter(x);
free(x, M_BWMETER);
}
}
/*
* Delete one or multiple bw_meter entries
*/
static int
del_bw_upcall(struct bw_upcall *req)
{
struct mfc *mfc;
struct bw_meter *x;
if (!(V_mrt_api_config & MRT_MFC_BW_UPCALL))
return EOPNOTSUPP;
MFC_LOCK();
/* Find the corresponding MFC entry */
mfc = mfc_find(&req->bu_src, &req->bu_dst);
if (mfc == NULL) {
MFC_UNLOCK();
return EADDRNOTAVAIL;
} else if (req->bu_flags & BW_UPCALL_DELETE_ALL) {
/*
* Delete all bw_meter entries for this mfc
*/
struct bw_meter *list;
list = mfc->mfc_bw_meter;
mfc->mfc_bw_meter = NULL;
free_bw_list(list);
MFC_UNLOCK();
return 0;
} else { /* Delete a single bw_meter entry */
struct bw_meter *prev;
uint32_t flags = 0;
flags = compute_bw_meter_flags(req);
/* Find the bw_meter entry to delete */
for (prev = NULL, x = mfc->mfc_bw_meter; x != NULL;
prev = x, x = x->bm_mfc_next) {
if ((BW_TIMEVALCMP(&x->bm_threshold.b_time,
&req->bu_threshold.b_time, ==)) &&
(x->bm_threshold.b_packets == req->bu_threshold.b_packets) &&
(x->bm_threshold.b_bytes == req->bu_threshold.b_bytes) &&
(x->bm_flags & BW_METER_USER_FLAGS) == flags)
break;
}
if (x != NULL) { /* Delete entry from the list for this MFC */
if (prev != NULL)
prev->bm_mfc_next = x->bm_mfc_next; /* remove from middle*/
else
x->bm_mfc->mfc_bw_meter = x->bm_mfc_next;/* new head of list */
unschedule_bw_meter(x);
MFC_UNLOCK();
/* Free the bw_meter entry */
free(x, M_BWMETER);
return 0;
} else {
MFC_UNLOCK();
return EINVAL;
}
}
/* NOTREACHED */
}
/*
* Perform bandwidth measurement processing that may result in an upcall
*/
static void
bw_meter_receive_packet(struct bw_meter *x, int plen, struct timeval *nowp)
{
struct timeval delta;
MFC_LOCK_ASSERT();
delta = *nowp;
BW_TIMEVALDECR(&delta, &x->bm_start_time);
if (x->bm_flags & BW_METER_GEQ) {
/*
* Processing for ">=" type of bw_meter entry
*/
if (BW_TIMEVALCMP(&delta, &x->bm_threshold.b_time, >)) {
/* Reset the bw_meter entry */
x->bm_start_time = *nowp;
x->bm_measured.b_packets = 0;
x->bm_measured.b_bytes = 0;
x->bm_flags &= ~BW_METER_UPCALL_DELIVERED;
}
/* Record that a packet is received */
x->bm_measured.b_packets++;
x->bm_measured.b_bytes += plen;
/*
* Test if we should deliver an upcall
*/
if (!(x->bm_flags & BW_METER_UPCALL_DELIVERED)) {
if (((x->bm_flags & BW_METER_UNIT_PACKETS) &&
(x->bm_measured.b_packets >= x->bm_threshold.b_packets)) ||
((x->bm_flags & BW_METER_UNIT_BYTES) &&
(x->bm_measured.b_bytes >= x->bm_threshold.b_bytes))) {
/* Prepare an upcall for delivery */
bw_meter_prepare_upcall(x, nowp);
x->bm_flags |= BW_METER_UPCALL_DELIVERED;
}
}
} else if (x->bm_flags & BW_METER_LEQ) {
/*
* Processing for "<=" type of bw_meter entry
*/
if (BW_TIMEVALCMP(&delta, &x->bm_threshold.b_time, >)) {
/*
* We are behind time with the multicast forwarding table
* scanning for "<=" type of bw_meter entries, so test now
* if we should deliver an upcall.
*/
if (((x->bm_flags & BW_METER_UNIT_PACKETS) &&
(x->bm_measured.b_packets <= x->bm_threshold.b_packets)) ||
((x->bm_flags & BW_METER_UNIT_BYTES) &&
(x->bm_measured.b_bytes <= x->bm_threshold.b_bytes))) {
/* Prepare an upcall for delivery */
bw_meter_prepare_upcall(x, nowp);
}
/* Reschedule the bw_meter entry */
unschedule_bw_meter(x);
schedule_bw_meter(x, nowp);
}
/* Record that a packet is received */
x->bm_measured.b_packets++;
x->bm_measured.b_bytes += plen;
/*
* Test if we should restart the measuring interval
*/
if ((x->bm_flags & BW_METER_UNIT_PACKETS &&
x->bm_measured.b_packets <= x->bm_threshold.b_packets) ||
(x->bm_flags & BW_METER_UNIT_BYTES &&
x->bm_measured.b_bytes <= x->bm_threshold.b_bytes)) {
/* Don't restart the measuring interval */
} else {
/* Do restart the measuring interval */
/*
* XXX: note that we don't unschedule and schedule, because this
* might be too much overhead per packet. Instead, when we process
* all entries for a given timer hash bin, we check whether it is
* really a timeout. If not, we reschedule at that time.
*/
x->bm_start_time = *nowp;
x->bm_measured.b_packets = 0;
x->bm_measured.b_bytes = 0;
x->bm_flags &= ~BW_METER_UPCALL_DELIVERED;
}
}
}
/*
* Prepare a bandwidth-related upcall
*/
static void
bw_meter_prepare_upcall(struct bw_meter *x, struct timeval *nowp)
{
struct timeval delta;
struct bw_upcall *u;
MFC_LOCK_ASSERT();
/*
* Compute the measured time interval
*/
delta = *nowp;
BW_TIMEVALDECR(&delta, &x->bm_start_time);
/*
* If there are too many pending upcalls, deliver them now
*/
if (V_bw_upcalls_n >= BW_UPCALLS_MAX)
bw_upcalls_send();
/*
* Set the bw_upcall entry
*/
u = &V_bw_upcalls[V_bw_upcalls_n++];
u->bu_src = x->bm_mfc->mfc_origin;
u->bu_dst = x->bm_mfc->mfc_mcastgrp;
u->bu_threshold.b_time = x->bm_threshold.b_time;
u->bu_threshold.b_packets = x->bm_threshold.b_packets;
u->bu_threshold.b_bytes = x->bm_threshold.b_bytes;
u->bu_measured.b_time = delta;
u->bu_measured.b_packets = x->bm_measured.b_packets;
u->bu_measured.b_bytes = x->bm_measured.b_bytes;
u->bu_flags = 0;
if (x->bm_flags & BW_METER_UNIT_PACKETS)
u->bu_flags |= BW_UPCALL_UNIT_PACKETS;
if (x->bm_flags & BW_METER_UNIT_BYTES)
u->bu_flags |= BW_UPCALL_UNIT_BYTES;
if (x->bm_flags & BW_METER_GEQ)
u->bu_flags |= BW_UPCALL_GEQ;
if (x->bm_flags & BW_METER_LEQ)
u->bu_flags |= BW_UPCALL_LEQ;
}
/*
* Send the pending bandwidth-related upcalls
*/
static void
bw_upcalls_send(void)
{
struct mbuf *m;
int len = V_bw_upcalls_n * sizeof(V_bw_upcalls[0]);
struct sockaddr_in k_igmpsrc = { sizeof k_igmpsrc, AF_INET };
static struct igmpmsg igmpmsg = { 0, /* unused1 */
0, /* unused2 */
IGMPMSG_BW_UPCALL,/* im_msgtype */
0, /* im_mbz */
0, /* im_vif */
0, /* unused3 */
{ 0 }, /* im_src */
{ 0 } }; /* im_dst */
MFC_LOCK_ASSERT();
if (V_bw_upcalls_n == 0)
return; /* No pending upcalls */
V_bw_upcalls_n = 0;
/*
* Allocate a new mbuf, initialize it with the header and
* the payload for the pending calls.
*/
m = m_gethdr(M_NOWAIT, MT_DATA);
if (m == NULL) {
log(LOG_WARNING, "bw_upcalls_send: cannot allocate mbuf\n");
return;
}
m_copyback(m, 0, sizeof(struct igmpmsg), (caddr_t)&igmpmsg);
m_copyback(m, sizeof(struct igmpmsg), len, (caddr_t)&V_bw_upcalls[0]);
/*
* Send the upcalls
* XXX do we need to set the address in k_igmpsrc ?
*/
MRTSTAT_INC(mrts_upcalls);
if (socket_send(V_ip_mrouter, m, &k_igmpsrc) < 0) {
log(LOG_WARNING, "bw_upcalls_send: ip_mrouter socket queue full\n");
MRTSTAT_INC(mrts_upq_sockfull);
}
}
/*
* Compute the timeout hash value for the bw_meter entries
*/
#define BW_METER_TIMEHASH(bw_meter, hash) \
do { \
struct timeval next_timeval = (bw_meter)->bm_start_time; \
\
BW_TIMEVALADD(&next_timeval, &(bw_meter)->bm_threshold.b_time); \
(hash) = next_timeval.tv_sec; \
if (next_timeval.tv_usec) \
(hash)++; /* XXX: make sure we don't timeout early */ \
(hash) %= BW_METER_BUCKETS; \
} while (0)
/*
* Schedule a timer to process periodically bw_meter entry of type "<="
* by linking the entry in the proper hash bucket.
*/
static void
schedule_bw_meter(struct bw_meter *x, struct timeval *nowp)
{
int time_hash;
MFC_LOCK_ASSERT();
if (!(x->bm_flags & BW_METER_LEQ))
return; /* XXX: we schedule timers only for "<=" entries */
/*
* Reset the bw_meter entry
*/
x->bm_start_time = *nowp;
x->bm_measured.b_packets = 0;
x->bm_measured.b_bytes = 0;
x->bm_flags &= ~BW_METER_UPCALL_DELIVERED;
/*
* Compute the timeout hash value and insert the entry
*/
BW_METER_TIMEHASH(x, time_hash);
x->bm_time_next = V_bw_meter_timers[time_hash];
V_bw_meter_timers[time_hash] = x;
x->bm_time_hash = time_hash;
}
/*
* Unschedule the periodic timer that processes bw_meter entry of type "<="
* by removing the entry from the proper hash bucket.
*/
static void
unschedule_bw_meter(struct bw_meter *x)
{
int time_hash;
struct bw_meter *prev, *tmp;
MFC_LOCK_ASSERT();
if (!(x->bm_flags & BW_METER_LEQ))
return; /* XXX: we schedule timers only for "<=" entries */
/*
* Compute the timeout hash value and delete the entry
*/
time_hash = x->bm_time_hash;
if (time_hash >= BW_METER_BUCKETS)
return; /* Entry was not scheduled */
for (prev = NULL, tmp = V_bw_meter_timers[time_hash];
tmp != NULL; prev = tmp, tmp = tmp->bm_time_next)
if (tmp == x)
break;
if (tmp == NULL)
panic("unschedule_bw_meter: bw_meter entry not found");
if (prev != NULL)
prev->bm_time_next = x->bm_time_next;
else
V_bw_meter_timers[time_hash] = x->bm_time_next;
x->bm_time_next = NULL;
x->bm_time_hash = BW_METER_BUCKETS;
}
/*
* Process all "<=" type of bw_meter that should be processed now,
* and for each entry prepare an upcall if necessary. Each processed
* entry is rescheduled again for the (periodic) processing.
*
* This is run periodically (once per second normally). On each round,
* all the potentially matching entries are in the hash slot that we are
* looking at.
*/
static void
bw_meter_process()
{
uint32_t loops;
int i;
struct timeval now, process_endtime;
microtime(&now);
if (V_last_tv_sec == now.tv_sec)
return; /* nothing to do */
loops = now.tv_sec - V_last_tv_sec;
V_last_tv_sec = now.tv_sec;
if (loops > BW_METER_BUCKETS)
loops = BW_METER_BUCKETS;
MFC_LOCK();
/*
* Process all bins of bw_meter entries from the one after the last
* processed to the current one. On entry, i points to the last bucket
* visited, so we need to increment i at the beginning of the loop.
*/
for (i = (now.tv_sec - loops) % BW_METER_BUCKETS; loops > 0; loops--) {
struct bw_meter *x, *tmp_list;
if (++i >= BW_METER_BUCKETS)
i = 0;
/* Disconnect the list of bw_meter entries from the bin */
tmp_list = V_bw_meter_timers[i];
V_bw_meter_timers[i] = NULL;
/* Process the list of bw_meter entries */
while (tmp_list != NULL) {
x = tmp_list;
tmp_list = tmp_list->bm_time_next;
/* Test if the time interval is over */
process_endtime = x->bm_start_time;
BW_TIMEVALADD(&process_endtime, &x->bm_threshold.b_time);
if (BW_TIMEVALCMP(&process_endtime, &now, >)) {
/* Not yet: reschedule, but don't reset */
int time_hash;
BW_METER_TIMEHASH(x, time_hash);
if (time_hash == i && process_endtime.tv_sec == now.tv_sec) {
/*
* XXX: somehow the bin processing is a bit ahead of time.
* Put the entry in the next bin.
*/
if (++time_hash >= BW_METER_BUCKETS)
time_hash = 0;
}
x->bm_time_next = V_bw_meter_timers[time_hash];
V_bw_meter_timers[time_hash] = x;
x->bm_time_hash = time_hash;
continue;
}
/*
* Test if we should deliver an upcall
*/
if (((x->bm_flags & BW_METER_UNIT_PACKETS) &&
(x->bm_measured.b_packets <= x->bm_threshold.b_packets)) ||
((x->bm_flags & BW_METER_UNIT_BYTES) &&
(x->bm_measured.b_bytes <= x->bm_threshold.b_bytes))) {
/* Prepare an upcall for delivery */
bw_meter_prepare_upcall(x, &now);
}
/*
* Reschedule for next processing
*/
schedule_bw_meter(x, &now);
}
}
/* Send all upcalls that are pending delivery */
bw_upcalls_send();
MFC_UNLOCK();
}
/*
* A periodic function for sending all upcalls that are pending delivery
*/
static void
expire_bw_upcalls_send(void *arg)
{
CURVNET_SET((struct vnet *) arg);
MFC_LOCK();
bw_upcalls_send();
MFC_UNLOCK();
callout_reset(&V_bw_upcalls_ch, BW_UPCALLS_PERIOD, expire_bw_upcalls_send,
curvnet);
CURVNET_RESTORE();
}
/*
* A periodic function for periodic scanning of the multicast forwarding
* table for processing all "<=" bw_meter entries.
*/
static void
expire_bw_meter_process(void *arg)
{
CURVNET_SET((struct vnet *) arg);
if (V_mrt_api_config & MRT_MFC_BW_UPCALL)
bw_meter_process();
callout_reset(&V_bw_meter_ch, BW_METER_PERIOD, expire_bw_meter_process,
curvnet);
CURVNET_RESTORE();
}
/*
* End of bandwidth monitoring code
*/
/*
* Send the packet up to the user daemon, or eventually do kernel encapsulation
*
*/
static int
pim_register_send(struct ip *ip, struct vif *vifp, struct mbuf *m,
struct mfc *rt)
{
struct mbuf *mb_copy, *mm;
/*
* Do not send IGMP_WHOLEPKT notifications to userland, if the
* rendezvous point was unspecified, and we were told not to.
*/
if (pim_squelch_wholepkt != 0 && (V_mrt_api_config & MRT_MFC_RP) &&
in_nullhost(rt->mfc_rp))
return 0;
mb_copy = pim_register_prepare(ip, m);
if (mb_copy == NULL)
return ENOBUFS;
/*
* Send all the fragments. Note that the mbuf for each fragment
* is freed by the sending machinery.
*/
for (mm = mb_copy; mm; mm = mb_copy) {
mb_copy = mm->m_nextpkt;
mm->m_nextpkt = 0;
mm = m_pullup(mm, sizeof(struct ip));
if (mm != NULL) {
ip = mtod(mm, struct ip *);
if ((V_mrt_api_config & MRT_MFC_RP) && !in_nullhost(rt->mfc_rp)) {
pim_register_send_rp(ip, vifp, mm, rt);
} else {
pim_register_send_upcall(ip, vifp, mm, rt);
}
}
}
return 0;
}
/*
* Return a copy of the data packet that is ready for PIM Register
* encapsulation.
* XXX: Note that in the returned copy the IP header is a valid one.
*/
static struct mbuf *
pim_register_prepare(struct ip *ip, struct mbuf *m)
{
struct mbuf *mb_copy = NULL;
int mtu;
/* Take care of delayed checksums */
if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
in_delayed_cksum(m);
m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
}
/*
* Copy the old packet & pullup its IP header into the
* new mbuf so we can modify it.
*/
mb_copy = m_copypacket(m, M_NOWAIT);
if (mb_copy == NULL)
return NULL;
mb_copy = m_pullup(mb_copy, ip->ip_hl << 2);
if (mb_copy == NULL)
return NULL;
/* take care of the TTL */
ip = mtod(mb_copy, struct ip *);
--ip->ip_ttl;
/* Compute the MTU after the PIM Register encapsulation */
mtu = 0xffff - sizeof(pim_encap_iphdr) - sizeof(pim_encap_pimhdr);
if (ntohs(ip->ip_len) <= mtu) {
/* Turn the IP header into a valid one */
ip->ip_sum = 0;
ip->ip_sum = in_cksum(mb_copy, ip->ip_hl << 2);
} else {
/* Fragment the packet */
mb_copy->m_pkthdr.csum_flags |= CSUM_IP;
if (ip_fragment(ip, &mb_copy, mtu, 0) != 0) {
m_freem(mb_copy);
return NULL;
}
}
return mb_copy;
}
/*
* Send an upcall with the data packet to the user-level process.
*/
static int
pim_register_send_upcall(struct ip *ip, struct vif *vifp,
struct mbuf *mb_copy, struct mfc *rt)
{
struct mbuf *mb_first;
int len = ntohs(ip->ip_len);
struct igmpmsg *im;
struct sockaddr_in k_igmpsrc = { sizeof k_igmpsrc, AF_INET };
VIF_LOCK_ASSERT();
/*
* Add a new mbuf with an upcall header
*/
mb_first = m_gethdr(M_NOWAIT, MT_DATA);
if (mb_first == NULL) {
m_freem(mb_copy);
return ENOBUFS;
}
mb_first->m_data += max_linkhdr;
mb_first->m_pkthdr.len = len + sizeof(struct igmpmsg);
mb_first->m_len = sizeof(struct igmpmsg);
mb_first->m_next = mb_copy;
/* Send message to routing daemon */
im = mtod(mb_first, struct igmpmsg *);
im->im_msgtype = IGMPMSG_WHOLEPKT;
im->im_mbz = 0;
im->im_vif = vifp - V_viftable;
im->im_src = ip->ip_src;
im->im_dst = ip->ip_dst;
k_igmpsrc.sin_addr = ip->ip_src;
MRTSTAT_INC(mrts_upcalls);
if (socket_send(V_ip_mrouter, mb_first, &k_igmpsrc) < 0) {
CTR1(KTR_IPMF, "%s: socket queue full", __func__);
MRTSTAT_INC(mrts_upq_sockfull);
return ENOBUFS;
}
/* Keep statistics */
PIMSTAT_INC(pims_snd_registers_msgs);
PIMSTAT_ADD(pims_snd_registers_bytes, len);
return 0;
}
/*
* Encapsulate the data packet in PIM Register message and send it to the RP.
*/
static int
pim_register_send_rp(struct ip *ip, struct vif *vifp, struct mbuf *mb_copy,
struct mfc *rt)
{
struct mbuf *mb_first;
struct ip *ip_outer;
struct pim_encap_pimhdr *pimhdr;
int len = ntohs(ip->ip_len);
vifi_t vifi = rt->mfc_parent;
VIF_LOCK_ASSERT();
if ((vifi >= V_numvifs) || in_nullhost(V_viftable[vifi].v_lcl_addr)) {
m_freem(mb_copy);
return EADDRNOTAVAIL; /* The iif vif is invalid */
}
/*
* Add a new mbuf with the encapsulating header
*/
mb_first = m_gethdr(M_NOWAIT, MT_DATA);
if (mb_first == NULL) {
m_freem(mb_copy);
return ENOBUFS;
}
mb_first->m_data += max_linkhdr;
mb_first->m_len = sizeof(pim_encap_iphdr) + sizeof(pim_encap_pimhdr);
mb_first->m_next = mb_copy;
mb_first->m_pkthdr.len = len + mb_first->m_len;
/*
* Fill in the encapsulating IP and PIM header
*/
ip_outer = mtod(mb_first, struct ip *);
*ip_outer = pim_encap_iphdr;
ip_outer->ip_len = htons(len + sizeof(pim_encap_iphdr) +
sizeof(pim_encap_pimhdr));
ip_outer->ip_src = V_viftable[vifi].v_lcl_addr;
ip_outer->ip_dst = rt->mfc_rp;
/*
* Copy the inner header TOS to the outer header, and take care of the
* IP_DF bit.
*/
ip_outer->ip_tos = ip->ip_tos;
if (ip->ip_off & htons(IP_DF))
ip_outer->ip_off |= htons(IP_DF);
ip_fillid(ip_outer);
pimhdr = (struct pim_encap_pimhdr *)((caddr_t)ip_outer
+ sizeof(pim_encap_iphdr));
*pimhdr = pim_encap_pimhdr;
/* If the iif crosses a border, set the Border-bit */
if (rt->mfc_flags[vifi] & MRT_MFC_FLAGS_BORDER_VIF & V_mrt_api_config)
pimhdr->flags |= htonl(PIM_BORDER_REGISTER);
mb_first->m_data += sizeof(pim_encap_iphdr);
pimhdr->pim.pim_cksum = in_cksum(mb_first, sizeof(pim_encap_pimhdr));
mb_first->m_data -= sizeof(pim_encap_iphdr);
send_packet(vifp, mb_first);
/* Keep statistics */
PIMSTAT_INC(pims_snd_registers_msgs);
PIMSTAT_ADD(pims_snd_registers_bytes, len);
return 0;
}
/*
* pim_encapcheck() is called by the encap4_input() path at runtime to
* determine if a packet is for PIM; allowing PIM to be dynamically loaded
* into the kernel.
*/
static int
pim_encapcheck(const struct mbuf *m, int off, int proto, void *arg)
{
#ifdef DIAGNOSTIC
KASSERT(proto == IPPROTO_PIM, ("not for IPPROTO_PIM"));
#endif
if (proto != IPPROTO_PIM)
return 0; /* not for us; reject the datagram. */
return 64; /* claim the datagram. */
}
/*
* PIM-SMv2 and PIM-DM messages processing.
* Receives and verifies the PIM control messages, and passes them
* up to the listening socket, using rip_input().
* The only message with special processing is the PIM_REGISTER message
* (used by PIM-SM): the PIM header is stripped off, and the inner packet
* is passed to if_simloop().
*/
int
pim_input(struct mbuf **mp, int *offp, int proto)
{
struct mbuf *m = *mp;
struct ip *ip = mtod(m, struct ip *);
struct pim *pim;
int iphlen = *offp;
int minlen;
int datalen = ntohs(ip->ip_len) - iphlen;
int ip_tos;
*mp = NULL;
/* Keep statistics */
PIMSTAT_INC(pims_rcv_total_msgs);
PIMSTAT_ADD(pims_rcv_total_bytes, datalen);
/*
* Validate lengths
*/
if (datalen < PIM_MINLEN) {
PIMSTAT_INC(pims_rcv_tooshort);
CTR3(KTR_IPMF, "%s: short packet (%d) from %s",
__func__, datalen, inet_ntoa(ip->ip_src));
m_freem(m);
return (IPPROTO_DONE);
}
/*
* If the packet is at least as big as a REGISTER, go agead
* and grab the PIM REGISTER header size, to avoid another
* possible m_pullup() later.
*
* PIM_MINLEN == pimhdr + u_int32_t == 4 + 4 = 8
* PIM_REG_MINLEN == pimhdr + reghdr + encap_iphdr == 4 + 4 + 20 = 28
*/
minlen = iphlen + (datalen >= PIM_REG_MINLEN ? PIM_REG_MINLEN : PIM_MINLEN);
/*
* Get the IP and PIM headers in contiguous memory, and
* possibly the PIM REGISTER header.
*/
if (m->m_len < minlen && (m = m_pullup(m, minlen)) == 0) {
CTR1(KTR_IPMF, "%s: m_pullup() failed", __func__);
return (IPPROTO_DONE);
}
/* m_pullup() may have given us a new mbuf so reset ip. */
ip = mtod(m, struct ip *);
ip_tos = ip->ip_tos;
/* adjust mbuf to point to the PIM header */
m->m_data += iphlen;
m->m_len -= iphlen;
pim = mtod(m, struct pim *);
/*
* Validate checksum. If PIM REGISTER, exclude the data packet.
*
* XXX: some older PIMv2 implementations don't make this distinction,
* so for compatibility reason perform the checksum over part of the
* message, and if error, then over the whole message.
*/
if (PIM_VT_T(pim->pim_vt) == PIM_REGISTER && in_cksum(m, PIM_MINLEN) == 0) {
/* do nothing, checksum okay */
} else if (in_cksum(m, datalen)) {
PIMSTAT_INC(pims_rcv_badsum);
CTR1(KTR_IPMF, "%s: invalid checksum", __func__);
m_freem(m);
return (IPPROTO_DONE);
}
/* PIM version check */
if (PIM_VT_V(pim->pim_vt) < PIM_VERSION) {
PIMSTAT_INC(pims_rcv_badversion);
CTR3(KTR_IPMF, "%s: bad version %d expect %d", __func__,
(int)PIM_VT_V(pim->pim_vt), PIM_VERSION);
m_freem(m);
return (IPPROTO_DONE);
}
/* restore mbuf back to the outer IP */
m->m_data -= iphlen;
m->m_len += iphlen;
if (PIM_VT_T(pim->pim_vt) == PIM_REGISTER) {
/*
* Since this is a REGISTER, we'll make a copy of the register
* headers ip + pim + u_int32 + encap_ip, to be passed up to the
* routing daemon.
*/
struct sockaddr_in dst = { sizeof(dst), AF_INET };
struct mbuf *mcp;
struct ip *encap_ip;
u_int32_t *reghdr;
struct ifnet *vifp;
VIF_LOCK();
if ((V_reg_vif_num >= V_numvifs) || (V_reg_vif_num == VIFI_INVALID)) {
VIF_UNLOCK();
CTR2(KTR_IPMF, "%s: register vif not set: %d", __func__,
(int)V_reg_vif_num);
m_freem(m);
return (IPPROTO_DONE);
}
/* XXX need refcnt? */
vifp = V_viftable[V_reg_vif_num].v_ifp;
VIF_UNLOCK();
/*
* Validate length
*/
if (datalen < PIM_REG_MINLEN) {
PIMSTAT_INC(pims_rcv_tooshort);
PIMSTAT_INC(pims_rcv_badregisters);
CTR1(KTR_IPMF, "%s: register packet size too small", __func__);
m_freem(m);
return (IPPROTO_DONE);
}
reghdr = (u_int32_t *)(pim + 1);
encap_ip = (struct ip *)(reghdr + 1);
CTR3(KTR_IPMF, "%s: register: encap ip src %s len %d",
__func__, inet_ntoa(encap_ip->ip_src), ntohs(encap_ip->ip_len));
/* verify the version number of the inner packet */
if (encap_ip->ip_v != IPVERSION) {
PIMSTAT_INC(pims_rcv_badregisters);
CTR1(KTR_IPMF, "%s: bad encap ip version", __func__);
m_freem(m);
return (IPPROTO_DONE);
}
/* verify the inner packet is destined to a mcast group */
if (!IN_MULTICAST(ntohl(encap_ip->ip_dst.s_addr))) {
PIMSTAT_INC(pims_rcv_badregisters);
CTR2(KTR_IPMF, "%s: bad encap ip dest %s", __func__,
inet_ntoa(encap_ip->ip_dst));
m_freem(m);
return (IPPROTO_DONE);
}
/* If a NULL_REGISTER, pass it to the daemon */
if ((ntohl(*reghdr) & PIM_NULL_REGISTER))
goto pim_input_to_daemon;
/*
* Copy the TOS from the outer IP header to the inner IP header.
*/
if (encap_ip->ip_tos != ip_tos) {
/* Outer TOS -> inner TOS */
encap_ip->ip_tos = ip_tos;
/* Recompute the inner header checksum. Sigh... */
/* adjust mbuf to point to the inner IP header */
m->m_data += (iphlen + PIM_MINLEN);
m->m_len -= (iphlen + PIM_MINLEN);
encap_ip->ip_sum = 0;
encap_ip->ip_sum = in_cksum(m, encap_ip->ip_hl << 2);
/* restore mbuf to point back to the outer IP header */
m->m_data -= (iphlen + PIM_MINLEN);
m->m_len += (iphlen + PIM_MINLEN);
}
/*
* Decapsulate the inner IP packet and loopback to forward it
* as a normal multicast packet. Also, make a copy of the
* outer_iphdr + pimhdr + reghdr + encap_iphdr
* to pass to the daemon later, so it can take the appropriate
* actions (e.g., send back PIM_REGISTER_STOP).
* XXX: here m->m_data points to the outer IP header.
*/
mcp = m_copy(m, 0, iphlen + PIM_REG_MINLEN);
if (mcp == NULL) {
CTR1(KTR_IPMF, "%s: m_copy() failed", __func__);
m_freem(m);
return (IPPROTO_DONE);
}
/* Keep statistics */
/* XXX: registers_bytes include only the encap. mcast pkt */
PIMSTAT_INC(pims_rcv_registers_msgs);
PIMSTAT_ADD(pims_rcv_registers_bytes, ntohs(encap_ip->ip_len));
/*
* forward the inner ip packet; point m_data at the inner ip.
*/
m_adj(m, iphlen + PIM_MINLEN);
CTR4(KTR_IPMF,
"%s: forward decap'd REGISTER: src %lx dst %lx vif %d",
__func__,
(u_long)ntohl(encap_ip->ip_src.s_addr),
(u_long)ntohl(encap_ip->ip_dst.s_addr),
(int)V_reg_vif_num);
/* NB: vifp was collected above; can it change on us? */
if_simloop(vifp, m, dst.sin_family, 0);
/* prepare the register head to send to the mrouting daemon */
m = mcp;
}
pim_input_to_daemon:
/*
* Pass the PIM message up to the daemon; if it is a Register message,
* pass the 'head' only up to the daemon. This includes the
* outer IP header, PIM header, PIM-Register header and the
* inner IP header.
* XXX: the outer IP header pkt size of a Register is not adjust to
* reflect the fact that the inner multicast data is truncated.
*/
*mp = m;
rip_input(mp, offp, proto);
return (IPPROTO_DONE);
}
static int
sysctl_mfctable(SYSCTL_HANDLER_ARGS)
{
struct mfc *rt;
int error, i;
if (req->newptr)
return (EPERM);
if (V_mfchashtbl == NULL) /* XXX unlocked */
return (0);
error = sysctl_wire_old_buffer(req, 0);
if (error)
return (error);
MFC_LOCK();
for (i = 0; i < mfchashsize; i++) {
LIST_FOREACH(rt, &V_mfchashtbl[i], mfc_hash) {
error = SYSCTL_OUT(req, rt, sizeof(struct mfc));
if (error)
goto out_locked;
}
}
out_locked:
MFC_UNLOCK();
return (error);
}
static SYSCTL_NODE(_net_inet_ip, OID_AUTO, mfctable, CTLFLAG_RD,
sysctl_mfctable, "IPv4 Multicast Forwarding Table "
"(struct *mfc[mfchashsize], netinet/ip_mroute.h)");
static void
vnet_mroute_init(const void *unused __unused)
{
MALLOC(V_nexpire, u_char *, mfchashsize, M_MRTABLE, M_WAITOK|M_ZERO);
bzero(V_bw_meter_timers, sizeof(V_bw_meter_timers));
- callout_init(&V_expire_upcalls_ch, CALLOUT_MPSAFE);
- callout_init(&V_bw_upcalls_ch, CALLOUT_MPSAFE);
- callout_init(&V_bw_meter_ch, CALLOUT_MPSAFE);
+ callout_init(&V_expire_upcalls_ch, 1);
+ callout_init(&V_bw_upcalls_ch, 1);
+ callout_init(&V_bw_meter_ch, 1);
}
VNET_SYSINIT(vnet_mroute_init, SI_SUB_PSEUDO, SI_ORDER_ANY, vnet_mroute_init,
NULL);
static void
vnet_mroute_uninit(const void *unused __unused)
{
FREE(V_nexpire, M_MRTABLE);
V_nexpire = NULL;
}
VNET_SYSUNINIT(vnet_mroute_uninit, SI_SUB_PSEUDO, SI_ORDER_MIDDLE,
vnet_mroute_uninit, NULL);
static int
ip_mroute_modevent(module_t mod, int type, void *unused)
{
switch (type) {
case MOD_LOAD:
MROUTER_LOCK_INIT();
if_detach_event_tag = EVENTHANDLER_REGISTER(ifnet_departure_event,
if_detached_event, NULL, EVENTHANDLER_PRI_ANY);
if (if_detach_event_tag == NULL) {
printf("ip_mroute: unable to register "
"ifnet_departure_event handler\n");
MROUTER_LOCK_DESTROY();
return (EINVAL);
}
MFC_LOCK_INIT();
VIF_LOCK_INIT();
mfchashsize = MFCHASHSIZE;
if (TUNABLE_ULONG_FETCH("net.inet.ip.mfchashsize", &mfchashsize) &&
!powerof2(mfchashsize)) {
printf("WARNING: %s not a power of 2; using default\n",
"net.inet.ip.mfchashsize");
mfchashsize = MFCHASHSIZE;
}
pim_squelch_wholepkt = 0;
TUNABLE_ULONG_FETCH("net.inet.pim.squelch_wholepkt",
&pim_squelch_wholepkt);
pim_encap_cookie = encap_attach_func(AF_INET, IPPROTO_PIM,
pim_encapcheck, &in_pim_protosw, NULL);
if (pim_encap_cookie == NULL) {
printf("ip_mroute: unable to attach pim encap\n");
VIF_LOCK_DESTROY();
MFC_LOCK_DESTROY();
MROUTER_LOCK_DESTROY();
return (EINVAL);
}
ip_mcast_src = X_ip_mcast_src;
ip_mforward = X_ip_mforward;
ip_mrouter_done = X_ip_mrouter_done;
ip_mrouter_get = X_ip_mrouter_get;
ip_mrouter_set = X_ip_mrouter_set;
ip_rsvp_force_done = X_ip_rsvp_force_done;
ip_rsvp_vif = X_ip_rsvp_vif;
legal_vif_num = X_legal_vif_num;
mrt_ioctl = X_mrt_ioctl;
rsvp_input_p = X_rsvp_input;
break;
case MOD_UNLOAD:
/*
* Typically module unload happens after the user-level
* process has shutdown the kernel services (the check
* below insures someone can't just yank the module out
* from under a running process). But if the module is
* just loaded and then unloaded w/o starting up a user
* process we still need to cleanup.
*/
MROUTER_LOCK();
if (ip_mrouter_cnt != 0) {
MROUTER_UNLOCK();
return (EINVAL);
}
ip_mrouter_unloading = 1;
MROUTER_UNLOCK();
EVENTHANDLER_DEREGISTER(ifnet_departure_event, if_detach_event_tag);
if (pim_encap_cookie) {
encap_detach(pim_encap_cookie);
pim_encap_cookie = NULL;
}
ip_mcast_src = NULL;
ip_mforward = NULL;
ip_mrouter_done = NULL;
ip_mrouter_get = NULL;
ip_mrouter_set = NULL;
ip_rsvp_force_done = NULL;
ip_rsvp_vif = NULL;
legal_vif_num = NULL;
mrt_ioctl = NULL;
rsvp_input_p = NULL;
VIF_LOCK_DESTROY();
MFC_LOCK_DESTROY();
MROUTER_LOCK_DESTROY();
break;
default:
return EOPNOTSUPP;
}
return 0;
}
static moduledata_t ip_mroutemod = {
"ip_mroute",
ip_mroute_modevent,
0
};
DECLARE_MODULE(ip_mroute, ip_mroutemod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE);
Index: head/sys/netinet/tcp_hostcache.c
===================================================================
--- head/sys/netinet/tcp_hostcache.c (revision 283290)
+++ head/sys/netinet/tcp_hostcache.c (revision 283291)
@@ -1,723 +1,723 @@
/*-
* Copyright (c) 2002 Andre Oppermann, Internet Business Solutions AG
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* The tcp_hostcache moves the tcp-specific cached metrics from the routing
* table to a dedicated structure indexed by the remote IP address. It keeps
* information on the measured TCP parameters of past TCP sessions to allow
* better initial start values to be used with later connections to/from the
* same source. Depending on the network parameters (delay, bandwidth, max
* MTU, congestion window) between local and remote sites, this can lead to
* significant speed-ups for new TCP connections after the first one.
*
* Due to the tcp_hostcache, all TCP-specific metrics information in the
* routing table have been removed. The inpcb no longer keeps a pointer to
* the routing entry, and protocol-initiated route cloning has been removed
* as well. With these changes, the routing table has gone back to being
* more lightwight and only carries information related to packet forwarding.
*
* tcp_hostcache is designed for multiple concurrent access in SMP
* environments and high contention. All bucket rows have their own lock and
* thus multiple lookups and modifies can be done at the same time as long as
* they are in different bucket rows. If a request for insertion of a new
* record can't be satisfied, it simply returns an empty structure. Nobody
* and nothing outside of tcp_hostcache.c will ever point directly to any
* entry in the tcp_hostcache. All communication is done in an
* object-oriented way and only functions of tcp_hostcache will manipulate
* hostcache entries. Otherwise, we are unable to achieve good behaviour in
* concurrent access situations. Since tcp_hostcache is only caching
* information, there are no fatal consequences if we either can't satisfy
* any particular request or have to drop/overwrite an existing entry because
* of bucket limit memory constrains.
*/
/*
* Many thanks to jlemon for basic structure of tcp_syncache which is being
* followed here.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet6.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/malloc.h>
#include <sys/sbuf.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sysctl.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/route.h>
#include <net/vnet.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/in_var.h>
#include <netinet/in_pcb.h>
#include <netinet/ip_var.h>
#ifdef INET6
#include <netinet/ip6.h>
#include <netinet6/ip6_var.h>
#endif
#include <netinet/tcp.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp_hostcache.h>
#ifdef INET6
#include <netinet6/tcp6_var.h>
#endif
#include <vm/uma.h>
/* Arbitrary values */
#define TCP_HOSTCACHE_HASHSIZE 512
#define TCP_HOSTCACHE_BUCKETLIMIT 30
#define TCP_HOSTCACHE_EXPIRE 60*60 /* one hour */
#define TCP_HOSTCACHE_PRUNE 5*60 /* every 5 minutes */
static VNET_DEFINE(struct tcp_hostcache, tcp_hostcache);
#define V_tcp_hostcache VNET(tcp_hostcache)
static VNET_DEFINE(struct callout, tcp_hc_callout);
#define V_tcp_hc_callout VNET(tcp_hc_callout)
static struct hc_metrics *tcp_hc_lookup(struct in_conninfo *);
static struct hc_metrics *tcp_hc_insert(struct in_conninfo *);
static int sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS);
static int sysctl_tcp_hc_purgenow(SYSCTL_HANDLER_ARGS);
static void tcp_hc_purge_internal(int);
static void tcp_hc_purge(void *);
static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hostcache, CTLFLAG_RW, 0,
"TCP Host cache");
SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, cachelimit, CTLFLAG_VNET | CTLFLAG_RDTUN,
&VNET_NAME(tcp_hostcache.cache_limit), 0,
"Overall entry limit for hostcache");
SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
&VNET_NAME(tcp_hostcache.hashsize), 0,
"Size of TCP hostcache hashtable");
SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, bucketlimit,
CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_hostcache.bucket_limit), 0,
"Per-bucket hash limit for hostcache");
SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, count, CTLFLAG_VNET | CTLFLAG_RD,
&VNET_NAME(tcp_hostcache.cache_count), 0,
"Current number of entries in hostcache");
SYSCTL_INT(_net_inet_tcp_hostcache, OID_AUTO, expire, CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(tcp_hostcache.expire), 0,
"Expire time of TCP hostcache entries");
SYSCTL_INT(_net_inet_tcp_hostcache, OID_AUTO, prune, CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(tcp_hostcache.prune), 0,
"Time between purge runs");
SYSCTL_INT(_net_inet_tcp_hostcache, OID_AUTO, purge, CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(tcp_hostcache.purgeall), 0,
"Expire all entires on next purge run");
SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, list,
CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP, 0, 0,
sysctl_tcp_hc_list, "A", "List of all hostcache entries");
SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, purgenow,
CTLTYPE_INT | CTLFLAG_RW, NULL, 0,
sysctl_tcp_hc_purgenow, "I", "Immediately purge all entries");
static MALLOC_DEFINE(M_HOSTCACHE, "hostcache", "TCP hostcache");
#define HOSTCACHE_HASH(ip) \
(((ip)->s_addr ^ ((ip)->s_addr >> 7) ^ ((ip)->s_addr >> 17)) & \
V_tcp_hostcache.hashmask)
/* XXX: What is the recommended hash to get good entropy for IPv6 addresses? */
#define HOSTCACHE_HASH6(ip6) \
(((ip6)->s6_addr32[0] ^ \
(ip6)->s6_addr32[1] ^ \
(ip6)->s6_addr32[2] ^ \
(ip6)->s6_addr32[3]) & \
V_tcp_hostcache.hashmask)
#define THC_LOCK(lp) mtx_lock(lp)
#define THC_UNLOCK(lp) mtx_unlock(lp)
void
tcp_hc_init(void)
{
u_int cache_limit;
int i;
/*
* Initialize hostcache structures.
*/
V_tcp_hostcache.cache_count = 0;
V_tcp_hostcache.hashsize = TCP_HOSTCACHE_HASHSIZE;
V_tcp_hostcache.bucket_limit = TCP_HOSTCACHE_BUCKETLIMIT;
V_tcp_hostcache.expire = TCP_HOSTCACHE_EXPIRE;
V_tcp_hostcache.prune = TCP_HOSTCACHE_PRUNE;
TUNABLE_INT_FETCH("net.inet.tcp.hostcache.hashsize",
&V_tcp_hostcache.hashsize);
if (!powerof2(V_tcp_hostcache.hashsize)) {
printf("WARNING: hostcache hash size is not a power of 2.\n");
V_tcp_hostcache.hashsize = TCP_HOSTCACHE_HASHSIZE; /* default */
}
V_tcp_hostcache.hashmask = V_tcp_hostcache.hashsize - 1;
TUNABLE_INT_FETCH("net.inet.tcp.hostcache.bucketlimit",
&V_tcp_hostcache.bucket_limit);
cache_limit = V_tcp_hostcache.hashsize * V_tcp_hostcache.bucket_limit;
V_tcp_hostcache.cache_limit = cache_limit;
TUNABLE_INT_FETCH("net.inet.tcp.hostcache.cachelimit",
&V_tcp_hostcache.cache_limit);
if (V_tcp_hostcache.cache_limit > cache_limit)
V_tcp_hostcache.cache_limit = cache_limit;
/*
* Allocate the hash table.
*/
V_tcp_hostcache.hashbase = (struct hc_head *)
malloc(V_tcp_hostcache.hashsize * sizeof(struct hc_head),
M_HOSTCACHE, M_WAITOK | M_ZERO);
/*
* Initialize the hash buckets.
*/
for (i = 0; i < V_tcp_hostcache.hashsize; i++) {
TAILQ_INIT(&V_tcp_hostcache.hashbase[i].hch_bucket);
V_tcp_hostcache.hashbase[i].hch_length = 0;
mtx_init(&V_tcp_hostcache.hashbase[i].hch_mtx, "tcp_hc_entry",
NULL, MTX_DEF);
}
/*
* Allocate the hostcache entries.
*/
V_tcp_hostcache.zone =
uma_zcreate("hostcache", sizeof(struct hc_metrics),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
uma_zone_set_max(V_tcp_hostcache.zone, V_tcp_hostcache.cache_limit);
/*
* Set up periodic cache cleanup.
*/
- callout_init(&V_tcp_hc_callout, CALLOUT_MPSAFE);
+ callout_init(&V_tcp_hc_callout, 1);
callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz,
tcp_hc_purge, curvnet);
}
#ifdef VIMAGE
void
tcp_hc_destroy(void)
{
int i;
callout_drain(&V_tcp_hc_callout);
/* Purge all hc entries. */
tcp_hc_purge_internal(1);
/* Free the uma zone and the allocated hash table. */
uma_zdestroy(V_tcp_hostcache.zone);
for (i = 0; i < V_tcp_hostcache.hashsize; i++)
mtx_destroy(&V_tcp_hostcache.hashbase[i].hch_mtx);
free(V_tcp_hostcache.hashbase, M_HOSTCACHE);
}
#endif
/*
* Internal function: look up an entry in the hostcache or return NULL.
*
* If an entry has been returned, the caller becomes responsible for
* unlocking the bucket row after he is done reading/modifying the entry.
*/
static struct hc_metrics *
tcp_hc_lookup(struct in_conninfo *inc)
{
int hash;
struct hc_head *hc_head;
struct hc_metrics *hc_entry;
KASSERT(inc != NULL, ("tcp_hc_lookup with NULL in_conninfo pointer"));
/*
* Hash the foreign ip address.
*/
if (inc->inc_flags & INC_ISIPV6)
hash = HOSTCACHE_HASH6(&inc->inc6_faddr);
else
hash = HOSTCACHE_HASH(&inc->inc_faddr);
hc_head = &V_tcp_hostcache.hashbase[hash];
/*
* Acquire lock for this bucket row; we release the lock if we don't
* find an entry, otherwise the caller has to unlock after he is
* done.
*/
THC_LOCK(&hc_head->hch_mtx);
/*
* Iterate through entries in bucket row looking for a match.
*/
TAILQ_FOREACH(hc_entry, &hc_head->hch_bucket, rmx_q) {
if (inc->inc_flags & INC_ISIPV6) {
/* XXX: check ip6_zoneid */
if (memcmp(&inc->inc6_faddr, &hc_entry->ip6,
sizeof(inc->inc6_faddr)) == 0)
return hc_entry;
} else {
if (memcmp(&inc->inc_faddr, &hc_entry->ip4,
sizeof(inc->inc_faddr)) == 0)
return hc_entry;
}
}
/*
* We were unsuccessful and didn't find anything.
*/
THC_UNLOCK(&hc_head->hch_mtx);
return NULL;
}
/*
* Internal function: insert an entry into the hostcache or return NULL if
* unable to allocate a new one.
*
* If an entry has been returned, the caller becomes responsible for
* unlocking the bucket row after he is done reading/modifying the entry.
*/
static struct hc_metrics *
tcp_hc_insert(struct in_conninfo *inc)
{
int hash;
struct hc_head *hc_head;
struct hc_metrics *hc_entry;
KASSERT(inc != NULL, ("tcp_hc_insert with NULL in_conninfo pointer"));
/*
* Hash the foreign ip address.
*/
if (inc->inc_flags & INC_ISIPV6)
hash = HOSTCACHE_HASH6(&inc->inc6_faddr);
else
hash = HOSTCACHE_HASH(&inc->inc_faddr);
hc_head = &V_tcp_hostcache.hashbase[hash];
/*
* Acquire lock for this bucket row; we release the lock if we don't
* find an entry, otherwise the caller has to unlock after he is
* done.
*/
THC_LOCK(&hc_head->hch_mtx);
/*
* If the bucket limit is reached, reuse the least-used element.
*/
if (hc_head->hch_length >= V_tcp_hostcache.bucket_limit ||
V_tcp_hostcache.cache_count >= V_tcp_hostcache.cache_limit) {
hc_entry = TAILQ_LAST(&hc_head->hch_bucket, hc_qhead);
/*
* At first we were dropping the last element, just to
* reacquire it in the next two lines again, which isn't very
* efficient. Instead just reuse the least used element.
* We may drop something that is still "in-use" but we can be
* "lossy".
* Just give up if this bucket row is empty and we don't have
* anything to replace.
*/
if (hc_entry == NULL) {
THC_UNLOCK(&hc_head->hch_mtx);
return NULL;
}
TAILQ_REMOVE(&hc_head->hch_bucket, hc_entry, rmx_q);
V_tcp_hostcache.hashbase[hash].hch_length--;
V_tcp_hostcache.cache_count--;
TCPSTAT_INC(tcps_hc_bucketoverflow);
#if 0
uma_zfree(V_tcp_hostcache.zone, hc_entry);
#endif
} else {
/*
* Allocate a new entry, or balk if not possible.
*/
hc_entry = uma_zalloc(V_tcp_hostcache.zone, M_NOWAIT);
if (hc_entry == NULL) {
THC_UNLOCK(&hc_head->hch_mtx);
return NULL;
}
}
/*
* Initialize basic information of hostcache entry.
*/
bzero(hc_entry, sizeof(*hc_entry));
if (inc->inc_flags & INC_ISIPV6) {
hc_entry->ip6 = inc->inc6_faddr;
hc_entry->ip6_zoneid = inc->inc6_zoneid;
} else
hc_entry->ip4 = inc->inc_faddr;
hc_entry->rmx_head = hc_head;
hc_entry->rmx_expire = V_tcp_hostcache.expire;
/*
* Put it upfront.
*/
TAILQ_INSERT_HEAD(&hc_head->hch_bucket, hc_entry, rmx_q);
V_tcp_hostcache.hashbase[hash].hch_length++;
V_tcp_hostcache.cache_count++;
TCPSTAT_INC(tcps_hc_added);
return hc_entry;
}
/*
* External function: look up an entry in the hostcache and fill out the
* supplied TCP metrics structure. Fills in NULL when no entry was found or
* a value is not set.
*/
void
tcp_hc_get(struct in_conninfo *inc, struct hc_metrics_lite *hc_metrics_lite)
{
struct hc_metrics *hc_entry;
/*
* Find the right bucket.
*/
hc_entry = tcp_hc_lookup(inc);
/*
* If we don't have an existing object.
*/
if (hc_entry == NULL) {
bzero(hc_metrics_lite, sizeof(*hc_metrics_lite));
return;
}
hc_entry->rmx_hits++;
hc_entry->rmx_expire = V_tcp_hostcache.expire; /* start over again */
hc_metrics_lite->rmx_mtu = hc_entry->rmx_mtu;
hc_metrics_lite->rmx_ssthresh = hc_entry->rmx_ssthresh;
hc_metrics_lite->rmx_rtt = hc_entry->rmx_rtt;
hc_metrics_lite->rmx_rttvar = hc_entry->rmx_rttvar;
hc_metrics_lite->rmx_bandwidth = hc_entry->rmx_bandwidth;
hc_metrics_lite->rmx_cwnd = hc_entry->rmx_cwnd;
hc_metrics_lite->rmx_sendpipe = hc_entry->rmx_sendpipe;
hc_metrics_lite->rmx_recvpipe = hc_entry->rmx_recvpipe;
/*
* Unlock bucket row.
*/
THC_UNLOCK(&hc_entry->rmx_head->hch_mtx);
}
/*
* External function: look up an entry in the hostcache and return the
* discovered path MTU. Returns NULL if no entry is found or value is not
* set.
*/
u_long
tcp_hc_getmtu(struct in_conninfo *inc)
{
struct hc_metrics *hc_entry;
u_long mtu;
hc_entry = tcp_hc_lookup(inc);
if (hc_entry == NULL) {
return 0;
}
hc_entry->rmx_hits++;
hc_entry->rmx_expire = V_tcp_hostcache.expire; /* start over again */
mtu = hc_entry->rmx_mtu;
THC_UNLOCK(&hc_entry->rmx_head->hch_mtx);
return mtu;
}
/*
* External function: update the MTU value of an entry in the hostcache.
* Creates a new entry if none was found.
*/
void
tcp_hc_updatemtu(struct in_conninfo *inc, u_long mtu)
{
struct hc_metrics *hc_entry;
/*
* Find the right bucket.
*/
hc_entry = tcp_hc_lookup(inc);
/*
* If we don't have an existing object, try to insert a new one.
*/
if (hc_entry == NULL) {
hc_entry = tcp_hc_insert(inc);
if (hc_entry == NULL)
return;
}
hc_entry->rmx_updates++;
hc_entry->rmx_expire = V_tcp_hostcache.expire; /* start over again */
hc_entry->rmx_mtu = mtu;
/*
* Put it upfront so we find it faster next time.
*/
TAILQ_REMOVE(&hc_entry->rmx_head->hch_bucket, hc_entry, rmx_q);
TAILQ_INSERT_HEAD(&hc_entry->rmx_head->hch_bucket, hc_entry, rmx_q);
/*
* Unlock bucket row.
*/
THC_UNLOCK(&hc_entry->rmx_head->hch_mtx);
}
/*
* External function: update the TCP metrics of an entry in the hostcache.
* Creates a new entry if none was found.
*/
void
tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
{
struct hc_metrics *hc_entry;
hc_entry = tcp_hc_lookup(inc);
if (hc_entry == NULL) {
hc_entry = tcp_hc_insert(inc);
if (hc_entry == NULL)
return;
}
hc_entry->rmx_updates++;
hc_entry->rmx_expire = V_tcp_hostcache.expire; /* start over again */
if (hcml->rmx_rtt != 0) {
if (hc_entry->rmx_rtt == 0)
hc_entry->rmx_rtt = hcml->rmx_rtt;
else
hc_entry->rmx_rtt =
(hc_entry->rmx_rtt + hcml->rmx_rtt) / 2;
TCPSTAT_INC(tcps_cachedrtt);
}
if (hcml->rmx_rttvar != 0) {
if (hc_entry->rmx_rttvar == 0)
hc_entry->rmx_rttvar = hcml->rmx_rttvar;
else
hc_entry->rmx_rttvar =
(hc_entry->rmx_rttvar + hcml->rmx_rttvar) / 2;
TCPSTAT_INC(tcps_cachedrttvar);
}
if (hcml->rmx_ssthresh != 0) {
if (hc_entry->rmx_ssthresh == 0)
hc_entry->rmx_ssthresh = hcml->rmx_ssthresh;
else
hc_entry->rmx_ssthresh =
(hc_entry->rmx_ssthresh + hcml->rmx_ssthresh) / 2;
TCPSTAT_INC(tcps_cachedssthresh);
}
if (hcml->rmx_bandwidth != 0) {
if (hc_entry->rmx_bandwidth == 0)
hc_entry->rmx_bandwidth = hcml->rmx_bandwidth;
else
hc_entry->rmx_bandwidth =
(hc_entry->rmx_bandwidth + hcml->rmx_bandwidth) / 2;
/* TCPSTAT_INC(tcps_cachedbandwidth); */
}
if (hcml->rmx_cwnd != 0) {
if (hc_entry->rmx_cwnd == 0)
hc_entry->rmx_cwnd = hcml->rmx_cwnd;
else
hc_entry->rmx_cwnd =
(hc_entry->rmx_cwnd + hcml->rmx_cwnd) / 2;
/* TCPSTAT_INC(tcps_cachedcwnd); */
}
if (hcml->rmx_sendpipe != 0) {
if (hc_entry->rmx_sendpipe == 0)
hc_entry->rmx_sendpipe = hcml->rmx_sendpipe;
else
hc_entry->rmx_sendpipe =
(hc_entry->rmx_sendpipe + hcml->rmx_sendpipe) /2;
/* TCPSTAT_INC(tcps_cachedsendpipe); */
}
if (hcml->rmx_recvpipe != 0) {
if (hc_entry->rmx_recvpipe == 0)
hc_entry->rmx_recvpipe = hcml->rmx_recvpipe;
else
hc_entry->rmx_recvpipe =
(hc_entry->rmx_recvpipe + hcml->rmx_recvpipe) /2;
/* TCPSTAT_INC(tcps_cachedrecvpipe); */
}
TAILQ_REMOVE(&hc_entry->rmx_head->hch_bucket, hc_entry, rmx_q);
TAILQ_INSERT_HEAD(&hc_entry->rmx_head->hch_bucket, hc_entry, rmx_q);
THC_UNLOCK(&hc_entry->rmx_head->hch_mtx);
}
/*
* Sysctl function: prints the list and values of all hostcache entries in
* unsorted order.
*/
static int
sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS)
{
const int linesize = 128;
struct sbuf sb;
int i, error;
struct hc_metrics *hc_entry;
#ifdef INET6
char ip6buf[INET6_ADDRSTRLEN];
#endif
sbuf_new(&sb, NULL, linesize * (V_tcp_hostcache.cache_count + 1),
SBUF_INCLUDENUL);
sbuf_printf(&sb,
"\nIP address MTU SSTRESH RTT RTTVAR BANDWIDTH "
" CWND SENDPIPE RECVPIPE HITS UPD EXP\n");
#define msec(u) (((u) + 500) / 1000)
for (i = 0; i < V_tcp_hostcache.hashsize; i++) {
THC_LOCK(&V_tcp_hostcache.hashbase[i].hch_mtx);
TAILQ_FOREACH(hc_entry, &V_tcp_hostcache.hashbase[i].hch_bucket,
rmx_q) {
sbuf_printf(&sb,
"%-15s %5lu %8lu %6lums %6lums %9lu %8lu %8lu %8lu "
"%4lu %4lu %4i\n",
hc_entry->ip4.s_addr ? inet_ntoa(hc_entry->ip4) :
#ifdef INET6
ip6_sprintf(ip6buf, &hc_entry->ip6),
#else
"IPv6?",
#endif
hc_entry->rmx_mtu,
hc_entry->rmx_ssthresh,
msec(hc_entry->rmx_rtt *
(RTM_RTTUNIT / (hz * TCP_RTT_SCALE))),
msec(hc_entry->rmx_rttvar *
(RTM_RTTUNIT / (hz * TCP_RTTVAR_SCALE))),
hc_entry->rmx_bandwidth * 8,
hc_entry->rmx_cwnd,
hc_entry->rmx_sendpipe,
hc_entry->rmx_recvpipe,
hc_entry->rmx_hits,
hc_entry->rmx_updates,
hc_entry->rmx_expire);
}
THC_UNLOCK(&V_tcp_hostcache.hashbase[i].hch_mtx);
}
#undef msec
error = sbuf_finish(&sb);
if (error == 0)
error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
sbuf_delete(&sb);
return(error);
}
/*
* Caller has to make sure the curvnet is set properly.
*/
static void
tcp_hc_purge_internal(int all)
{
struct hc_metrics *hc_entry, *hc_next;
int i;
for (i = 0; i < V_tcp_hostcache.hashsize; i++) {
THC_LOCK(&V_tcp_hostcache.hashbase[i].hch_mtx);
TAILQ_FOREACH_SAFE(hc_entry,
&V_tcp_hostcache.hashbase[i].hch_bucket, rmx_q, hc_next) {
if (all || hc_entry->rmx_expire <= 0) {
TAILQ_REMOVE(&V_tcp_hostcache.hashbase[i].hch_bucket,
hc_entry, rmx_q);
uma_zfree(V_tcp_hostcache.zone, hc_entry);
V_tcp_hostcache.hashbase[i].hch_length--;
V_tcp_hostcache.cache_count--;
} else
hc_entry->rmx_expire -= V_tcp_hostcache.prune;
}
THC_UNLOCK(&V_tcp_hostcache.hashbase[i].hch_mtx);
}
}
/*
* Expire and purge (old|all) entries in the tcp_hostcache. Runs
* periodically from the callout.
*/
static void
tcp_hc_purge(void *arg)
{
CURVNET_SET((struct vnet *) arg);
int all = 0;
if (V_tcp_hostcache.purgeall) {
all = 1;
V_tcp_hostcache.purgeall = 0;
}
tcp_hc_purge_internal(all);
callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz,
tcp_hc_purge, arg);
CURVNET_RESTORE();
}
/*
* Expire and purge all entries in hostcache immediately.
*/
static int
sysctl_tcp_hc_purgenow(SYSCTL_HANDLER_ARGS)
{
int error, val;
val = 0;
error = sysctl_handle_int(oidp, &val, 0, req);
if (error || !req->newptr)
return (error);
tcp_hc_purge_internal(1);
callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz,
tcp_hc_purge, curvnet);
return (0);
}
Index: head/sys/netinet/tcp_subr.c
===================================================================
--- head/sys/netinet/tcp_subr.c (revision 283290)
+++ head/sys/netinet/tcp_subr.c (revision 283291)
@@ -1,2519 +1,2519 @@
/*-
* Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)tcp_subr.c 8.2 (Berkeley) 5/24/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_compat.h"
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
#include "opt_tcpdebug.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/callout.h>
#include <sys/hhook.h>
#include <sys/kernel.h>
#include <sys/khelp.h>
#include <sys/sysctl.h>
#include <sys/jail.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#ifdef INET6
#include <sys/domain.h>
#endif
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/sdt.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/protosw.h>
#include <sys/random.h>
#include <vm/uma.h>
#include <net/route.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/vnet.h>
#include <netinet/cc.h>
#include <netinet/in.h>
#include <netinet/in_kdtrace.h>
#include <netinet/in_pcb.h>
#include <netinet/in_systm.h>
#include <netinet/in_var.h>
#include <netinet/ip.h>
#include <netinet/ip_icmp.h>
#include <netinet/ip_var.h>
#ifdef INET6
#include <netinet/ip6.h>
#include <netinet6/in6_pcb.h>
#include <netinet6/ip6_var.h>
#include <netinet6/scope6_var.h>
#include <netinet6/nd6.h>
#endif
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp_syncache.h>
#ifdef INET6
#include <netinet6/tcp6_var.h>
#endif
#include <netinet/tcpip.h>
#ifdef TCPDEBUG
#include <netinet/tcp_debug.h>
#endif
#ifdef INET6
#include <netinet6/ip6protosw.h>
#endif
#ifdef TCP_OFFLOAD
#include <netinet/tcp_offload.h>
#endif
#ifdef IPSEC
#include <netipsec/ipsec.h>
#include <netipsec/xform.h>
#ifdef INET6
#include <netipsec/ipsec6.h>
#endif
#include <netipsec/key.h>
#include <sys/syslog.h>
#endif /*IPSEC*/
#include <machine/in_cksum.h>
#include <sys/md5.h>
#include <security/mac/mac_framework.h>
VNET_DEFINE(int, tcp_mssdflt) = TCP_MSS;
#ifdef INET6
VNET_DEFINE(int, tcp_v6mssdflt) = TCP6_MSS;
#endif
static int
sysctl_net_inet_tcp_mss_check(SYSCTL_HANDLER_ARGS)
{
int error, new;
new = V_tcp_mssdflt;
error = sysctl_handle_int(oidp, &new, 0, req);
if (error == 0 && req->newptr) {
if (new < TCP_MINMSS)
error = EINVAL;
else
V_tcp_mssdflt = new;
}
return (error);
}
SYSCTL_PROC(_net_inet_tcp, TCPCTL_MSSDFLT, mssdflt,
CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW, &VNET_NAME(tcp_mssdflt), 0,
&sysctl_net_inet_tcp_mss_check, "I",
"Default TCP Maximum Segment Size");
#ifdef INET6
static int
sysctl_net_inet_tcp_mss_v6_check(SYSCTL_HANDLER_ARGS)
{
int error, new;
new = V_tcp_v6mssdflt;
error = sysctl_handle_int(oidp, &new, 0, req);
if (error == 0 && req->newptr) {
if (new < TCP_MINMSS)
error = EINVAL;
else
V_tcp_v6mssdflt = new;
}
return (error);
}
SYSCTL_PROC(_net_inet_tcp, TCPCTL_V6MSSDFLT, v6mssdflt,
CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW, &VNET_NAME(tcp_v6mssdflt), 0,
&sysctl_net_inet_tcp_mss_v6_check, "I",
"Default TCP Maximum Segment Size for IPv6");
#endif /* INET6 */
/*
* Minimum MSS we accept and use. This prevents DoS attacks where
* we are forced to a ridiculous low MSS like 20 and send hundreds
* of packets instead of one. The effect scales with the available
* bandwidth and quickly saturates the CPU and network interface
* with packet generation and sending. Set to zero to disable MINMSS
* checking. This setting prevents us from sending too small packets.
*/
VNET_DEFINE(int, tcp_minmss) = TCP_MINMSS;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, minmss, CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(tcp_minmss), 0,
"Minimum TCP Maximum Segment Size");
VNET_DEFINE(int, tcp_do_rfc1323) = 1;
SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1323, rfc1323, CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(tcp_do_rfc1323), 0,
"Enable rfc1323 (high performance TCP) extensions");
static int tcp_log_debug = 0;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, log_debug, CTLFLAG_RW,
&tcp_log_debug, 0, "Log errors caused by incoming TCP segments");
static int tcp_tcbhashsize;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcbhashsize, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
&tcp_tcbhashsize, 0, "Size of TCP control-block hashtable");
static int do_tcpdrain = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, do_tcpdrain, CTLFLAG_RW, &do_tcpdrain, 0,
"Enable tcp_drain routine for extra help when low on mbufs");
SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_VNET | CTLFLAG_RD,
&VNET_NAME(tcbinfo.ipi_count), 0, "Number of active PCBs");
static VNET_DEFINE(int, icmp_may_rst) = 1;
#define V_icmp_may_rst VNET(icmp_may_rst)
SYSCTL_INT(_net_inet_tcp, OID_AUTO, icmp_may_rst, CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(icmp_may_rst), 0,
"Certain ICMP unreachable messages may abort connections in SYN_SENT");
static VNET_DEFINE(int, tcp_isn_reseed_interval) = 0;
#define V_tcp_isn_reseed_interval VNET(tcp_isn_reseed_interval)
SYSCTL_INT(_net_inet_tcp, OID_AUTO, isn_reseed_interval, CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(tcp_isn_reseed_interval), 0,
"Seconds between reseeding of ISN secret");
static int tcp_soreceive_stream;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, soreceive_stream, CTLFLAG_RDTUN,
&tcp_soreceive_stream, 0, "Using soreceive_stream for TCP sockets");
#ifdef TCP_SIGNATURE
static int tcp_sig_checksigs = 1;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, signature_verify_input, CTLFLAG_RW,
&tcp_sig_checksigs, 0, "Verify RFC2385 digests on inbound traffic");
#endif
VNET_DEFINE(uma_zone_t, sack_hole_zone);
#define V_sack_hole_zone VNET(sack_hole_zone)
VNET_DEFINE(struct hhook_head *, tcp_hhh[HHOOK_TCP_LAST+1]);
static struct inpcb *tcp_notify(struct inpcb *, int);
static struct inpcb *tcp_mtudisc_notify(struct inpcb *, int);
static char * tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th,
void *ip4hdr, const void *ip6hdr);
static void tcp_timer_discard(struct tcpcb *, uint32_t);
/*
* Target size of TCP PCB hash tables. Must be a power of two.
*
* Note that this can be overridden by the kernel environment
* variable net.inet.tcp.tcbhashsize
*/
#ifndef TCBHASHSIZE
#define TCBHASHSIZE 0
#endif
/*
* XXX
* Callouts should be moved into struct tcp directly. They are currently
* separate because the tcpcb structure is exported to userland for sysctl
* parsing purposes, which do not know about callouts.
*/
struct tcpcb_mem {
struct tcpcb tcb;
struct tcp_timer tt;
struct cc_var ccv;
struct osd osd;
};
static VNET_DEFINE(uma_zone_t, tcpcb_zone);
#define V_tcpcb_zone VNET(tcpcb_zone)
MALLOC_DEFINE(M_TCPLOG, "tcplog", "TCP address and flags print buffers");
static struct mtx isn_mtx;
#define ISN_LOCK_INIT() mtx_init(&isn_mtx, "isn_mtx", NULL, MTX_DEF)
#define ISN_LOCK() mtx_lock(&isn_mtx)
#define ISN_UNLOCK() mtx_unlock(&isn_mtx)
/*
* TCP initialization.
*/
static void
tcp_zone_change(void *tag)
{
uma_zone_set_max(V_tcbinfo.ipi_zone, maxsockets);
uma_zone_set_max(V_tcpcb_zone, maxsockets);
tcp_tw_zone_change();
}
static int
tcp_inpcb_init(void *mem, int size, int flags)
{
struct inpcb *inp = mem;
INP_LOCK_INIT(inp, "inp", "tcpinp");
return (0);
}
/*
* Take a value and get the next power of 2 that doesn't overflow.
* Used to size the tcp_inpcb hash buckets.
*/
static int
maketcp_hashsize(int size)
{
int hashsize;
/*
* auto tune.
* get the next power of 2 higher than maxsockets.
*/
hashsize = 1 << fls(size);
/* catch overflow, and just go one power of 2 smaller */
if (hashsize < size) {
hashsize = 1 << (fls(size) - 1);
}
return (hashsize);
}
void
tcp_init(void)
{
const char *tcbhash_tuneable;
int hashsize;
tcbhash_tuneable = "net.inet.tcp.tcbhashsize";
if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN,
&V_tcp_hhh[HHOOK_TCP_EST_IN], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
printf("%s: WARNING: unable to register helper hook\n", __func__);
if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT,
&V_tcp_hhh[HHOOK_TCP_EST_OUT], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
printf("%s: WARNING: unable to register helper hook\n", __func__);
hashsize = TCBHASHSIZE;
TUNABLE_INT_FETCH(tcbhash_tuneable, &hashsize);
if (hashsize == 0) {
/*
* Auto tune the hash size based on maxsockets.
* A perfect hash would have a 1:1 mapping
* (hashsize = maxsockets) however it's been
* suggested that O(2) average is better.
*/
hashsize = maketcp_hashsize(maxsockets / 4);
/*
* Our historical default is 512,
* do not autotune lower than this.
*/
if (hashsize < 512)
hashsize = 512;
if (bootverbose)
printf("%s: %s auto tuned to %d\n", __func__,
tcbhash_tuneable, hashsize);
}
/*
* We require a hashsize to be a power of two.
* Previously if it was not a power of two we would just reset it
* back to 512, which could be a nasty surprise if you did not notice
* the error message.
* Instead what we do is clip it to the closest power of two lower
* than the specified hash value.
*/
if (!powerof2(hashsize)) {
int oldhashsize = hashsize;
hashsize = maketcp_hashsize(hashsize);
/* prevent absurdly low value */
if (hashsize < 16)
hashsize = 16;
printf("%s: WARNING: TCB hash size not a power of 2, "
"clipped from %d to %d.\n", __func__, oldhashsize,
hashsize);
}
in_pcbinfo_init(&V_tcbinfo, "tcp", &V_tcb, hashsize, hashsize,
"tcp_inpcb", tcp_inpcb_init, NULL, UMA_ZONE_NOFREE,
IPI_HASHFIELDS_4TUPLE);
/*
* These have to be type stable for the benefit of the timers.
*/
V_tcpcb_zone = uma_zcreate("tcpcb", sizeof(struct tcpcb_mem),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
uma_zone_set_max(V_tcpcb_zone, maxsockets);
uma_zone_set_warning(V_tcpcb_zone, "kern.ipc.maxsockets limit reached");
tcp_tw_init();
syncache_init();
tcp_hc_init();
TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack);
V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
/* Skip initialization of globals for non-default instances. */
if (!IS_DEFAULT_VNET(curvnet))
return;
/* XXX virtualize those bellow? */
tcp_delacktime = TCPTV_DELACK;
tcp_keepinit = TCPTV_KEEP_INIT;
tcp_keepidle = TCPTV_KEEP_IDLE;
tcp_keepintvl = TCPTV_KEEPINTVL;
tcp_maxpersistidle = TCPTV_KEEP_IDLE;
tcp_msl = TCPTV_MSL;
tcp_rexmit_min = TCPTV_MIN;
if (tcp_rexmit_min < 1)
tcp_rexmit_min = 1;
tcp_rexmit_slop = TCPTV_CPU_VAR;
tcp_finwait2_timeout = TCPTV_FINWAIT2_TIMEOUT;
tcp_tcbhashsize = hashsize;
if (tcp_soreceive_stream) {
#ifdef INET
tcp_usrreqs.pru_soreceive = soreceive_stream;
#endif
#ifdef INET6
tcp6_usrreqs.pru_soreceive = soreceive_stream;
#endif /* INET6 */
}
#ifdef INET6
#define TCP_MINPROTOHDR (sizeof(struct ip6_hdr) + sizeof(struct tcphdr))
#else /* INET6 */
#define TCP_MINPROTOHDR (sizeof(struct tcpiphdr))
#endif /* INET6 */
if (max_protohdr < TCP_MINPROTOHDR)
max_protohdr = TCP_MINPROTOHDR;
if (max_linkhdr + TCP_MINPROTOHDR > MHLEN)
panic("tcp_init");
#undef TCP_MINPROTOHDR
ISN_LOCK_INIT();
EVENTHANDLER_REGISTER(shutdown_pre_sync, tcp_fini, NULL,
SHUTDOWN_PRI_DEFAULT);
EVENTHANDLER_REGISTER(maxsockets_change, tcp_zone_change, NULL,
EVENTHANDLER_PRI_ANY);
}
#ifdef VIMAGE
void
tcp_destroy(void)
{
int error;
tcp_hc_destroy();
syncache_destroy();
tcp_tw_destroy();
in_pcbinfo_destroy(&V_tcbinfo);
uma_zdestroy(V_sack_hole_zone);
uma_zdestroy(V_tcpcb_zone);
error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_IN]);
if (error != 0) {
printf("%s: WARNING: unable to deregister helper hook "
"type=%d, id=%d: error %d returned\n", __func__,
HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN, error);
}
error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_OUT]);
if (error != 0) {
printf("%s: WARNING: unable to deregister helper hook "
"type=%d, id=%d: error %d returned\n", __func__,
HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT, error);
}
}
#endif
void
tcp_fini(void *xtp)
{
}
/*
* Fill in the IP and TCP headers for an outgoing packet, given the tcpcb.
* tcp_template used to store this data in mbufs, but we now recopy it out
* of the tcpcb each time to conserve mbufs.
*/
void
tcpip_fillheaders(struct inpcb *inp, void *ip_ptr, void *tcp_ptr)
{
struct tcphdr *th = (struct tcphdr *)tcp_ptr;
INP_WLOCK_ASSERT(inp);
#ifdef INET6
if ((inp->inp_vflag & INP_IPV6) != 0) {
struct ip6_hdr *ip6;
ip6 = (struct ip6_hdr *)ip_ptr;
ip6->ip6_flow = (ip6->ip6_flow & ~IPV6_FLOWINFO_MASK) |
(inp->inp_flow & IPV6_FLOWINFO_MASK);
ip6->ip6_vfc = (ip6->ip6_vfc & ~IPV6_VERSION_MASK) |
(IPV6_VERSION & IPV6_VERSION_MASK);
ip6->ip6_nxt = IPPROTO_TCP;
ip6->ip6_plen = htons(sizeof(struct tcphdr));
ip6->ip6_src = inp->in6p_laddr;
ip6->ip6_dst = inp->in6p_faddr;
}
#endif /* INET6 */
#if defined(INET6) && defined(INET)
else
#endif
#ifdef INET
{
struct ip *ip;
ip = (struct ip *)ip_ptr;
ip->ip_v = IPVERSION;
ip->ip_hl = 5;
ip->ip_tos = inp->inp_ip_tos;
ip->ip_len = 0;
ip->ip_id = 0;
ip->ip_off = 0;
ip->ip_ttl = inp->inp_ip_ttl;
ip->ip_sum = 0;
ip->ip_p = IPPROTO_TCP;
ip->ip_src = inp->inp_laddr;
ip->ip_dst = inp->inp_faddr;
}
#endif /* INET */
th->th_sport = inp->inp_lport;
th->th_dport = inp->inp_fport;
th->th_seq = 0;
th->th_ack = 0;
th->th_x2 = 0;
th->th_off = 5;
th->th_flags = 0;
th->th_win = 0;
th->th_urp = 0;
th->th_sum = 0; /* in_pseudo() is called later for ipv4 */
}
/*
* Create template to be used to send tcp packets on a connection.
* Allocates an mbuf and fills in a skeletal tcp/ip header. The only
* use for this function is in keepalives, which use tcp_respond.
*/
struct tcptemp *
tcpip_maketemplate(struct inpcb *inp)
{
struct tcptemp *t;
t = malloc(sizeof(*t), M_TEMP, M_NOWAIT);
if (t == NULL)
return (NULL);
tcpip_fillheaders(inp, (void *)&t->tt_ipgen, (void *)&t->tt_t);
return (t);
}
/*
* Send a single message to the TCP at address specified by
* the given TCP/IP header. If m == NULL, then we make a copy
* of the tcpiphdr at th and send directly to the addressed host.
* This is used to force keep alive messages out using the TCP
* template for a connection. If flags are given then we send
* a message back to the TCP which originated the segment th,
* and discard the mbuf containing it and any other attached mbufs.
*
* In any case the ack and sequence number of the transmitted
* segment are as specified by the parameters.
*
* NOTE: If m != NULL, then th must point to *inside* the mbuf.
*/
void
tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m,
tcp_seq ack, tcp_seq seq, int flags)
{
int tlen;
int win = 0;
struct ip *ip;
struct tcphdr *nth;
#ifdef INET6
struct ip6_hdr *ip6;
int isipv6;
#endif /* INET6 */
int ipflags = 0;
struct inpcb *inp;
KASSERT(tp != NULL || m != NULL, ("tcp_respond: tp and m both NULL"));
#ifdef INET6
isipv6 = ((struct ip *)ipgen)->ip_v == (IPV6_VERSION >> 4);
ip6 = ipgen;
#endif /* INET6 */
ip = ipgen;
if (tp != NULL) {
inp = tp->t_inpcb;
KASSERT(inp != NULL, ("tcp control block w/o inpcb"));
INP_WLOCK_ASSERT(inp);
} else
inp = NULL;
if (tp != NULL) {
if (!(flags & TH_RST)) {
win = sbspace(&inp->inp_socket->so_rcv);
if (win > (long)TCP_MAXWIN << tp->rcv_scale)
win = (long)TCP_MAXWIN << tp->rcv_scale;
}
}
if (m == NULL) {
m = m_gethdr(M_NOWAIT, MT_DATA);
if (m == NULL)
return;
tlen = 0;
m->m_data += max_linkhdr;
#ifdef INET6
if (isipv6) {
bcopy((caddr_t)ip6, mtod(m, caddr_t),
sizeof(struct ip6_hdr));
ip6 = mtod(m, struct ip6_hdr *);
nth = (struct tcphdr *)(ip6 + 1);
} else
#endif /* INET6 */
{
bcopy((caddr_t)ip, mtod(m, caddr_t), sizeof(struct ip));
ip = mtod(m, struct ip *);
nth = (struct tcphdr *)(ip + 1);
}
bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
flags = TH_ACK;
} else {
/*
* reuse the mbuf.
* XXX MRT We inherrit the FIB, which is lucky.
*/
m_freem(m->m_next);
m->m_next = NULL;
m->m_data = (caddr_t)ipgen;
/* m_len is set later */
tlen = 0;
#define xchg(a,b,type) { type t; t=a; a=b; b=t; }
#ifdef INET6
if (isipv6) {
xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
nth = (struct tcphdr *)(ip6 + 1);
} else
#endif /* INET6 */
{
xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
nth = (struct tcphdr *)(ip + 1);
}
if (th != nth) {
/*
* this is usually a case when an extension header
* exists between the IPv6 header and the
* TCP header.
*/
nth->th_sport = th->th_sport;
nth->th_dport = th->th_dport;
}
xchg(nth->th_dport, nth->th_sport, uint16_t);
#undef xchg
}
#ifdef INET6
if (isipv6) {
ip6->ip6_flow = 0;
ip6->ip6_vfc = IPV6_VERSION;
ip6->ip6_nxt = IPPROTO_TCP;
tlen += sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
ip6->ip6_plen = htons(tlen - sizeof(*ip6));
}
#endif
#if defined(INET) && defined(INET6)
else
#endif
#ifdef INET
{
tlen += sizeof (struct tcpiphdr);
ip->ip_len = htons(tlen);
ip->ip_ttl = V_ip_defttl;
if (V_path_mtu_discovery)
ip->ip_off |= htons(IP_DF);
}
#endif
m->m_len = tlen;
m->m_pkthdr.len = tlen;
m->m_pkthdr.rcvif = NULL;
#ifdef MAC
if (inp != NULL) {
/*
* Packet is associated with a socket, so allow the
* label of the response to reflect the socket label.
*/
INP_WLOCK_ASSERT(inp);
mac_inpcb_create_mbuf(inp, m);
} else {
/*
* Packet is not associated with a socket, so possibly
* update the label in place.
*/
mac_netinet_tcp_reply(m);
}
#endif
nth->th_seq = htonl(seq);
nth->th_ack = htonl(ack);
nth->th_x2 = 0;
nth->th_off = sizeof (struct tcphdr) >> 2;
nth->th_flags = flags;
if (tp != NULL)
nth->th_win = htons((u_short) (win >> tp->rcv_scale));
else
nth->th_win = htons((u_short)win);
nth->th_urp = 0;
m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
#ifdef INET6
if (isipv6) {
m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
nth->th_sum = in6_cksum_pseudo(ip6,
tlen - sizeof(struct ip6_hdr), IPPROTO_TCP, 0);
ip6->ip6_hlim = in6_selecthlim(tp != NULL ? tp->t_inpcb :
NULL, NULL);
}
#endif /* INET6 */
#if defined(INET6) && defined(INET)
else
#endif
#ifdef INET
{
m->m_pkthdr.csum_flags = CSUM_TCP;
nth->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
htons((u_short)(tlen - sizeof(struct ip) + ip->ip_p)));
}
#endif /* INET */
#ifdef TCPDEBUG
if (tp == NULL || (inp->inp_socket->so_options & SO_DEBUG))
tcp_trace(TA_OUTPUT, 0, tp, mtod(m, void *), th, 0);
#endif
if (flags & TH_RST)
TCP_PROBE5(accept__refused, NULL, NULL, mtod(m, const char *),
tp, nth);
TCP_PROBE5(send, NULL, tp, mtod(m, const char *), tp, nth);
#ifdef INET6
if (isipv6)
(void) ip6_output(m, NULL, NULL, ipflags, NULL, NULL, inp);
#endif /* INET6 */
#if defined(INET) && defined(INET6)
else
#endif
#ifdef INET
(void) ip_output(m, NULL, NULL, ipflags, NULL, inp);
#endif
}
/*
* Create a new TCP control block, making an
* empty reassembly queue and hooking it to the argument
* protocol control block. The `inp' parameter must have
* come from the zone allocator set up in tcp_init().
*/
struct tcpcb *
tcp_newtcpcb(struct inpcb *inp)
{
struct tcpcb_mem *tm;
struct tcpcb *tp;
#ifdef INET6
int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
#endif /* INET6 */
tm = uma_zalloc(V_tcpcb_zone, M_NOWAIT | M_ZERO);
if (tm == NULL)
return (NULL);
tp = &tm->tcb;
/* Initialise cc_var struct for this tcpcb. */
tp->ccv = &tm->ccv;
tp->ccv->type = IPPROTO_TCP;
tp->ccv->ccvc.tcp = tp;
/*
* Use the current system default CC algorithm.
*/
CC_LIST_RLOCK();
KASSERT(!STAILQ_EMPTY(&cc_list), ("cc_list is empty!"));
CC_ALGO(tp) = CC_DEFAULT();
CC_LIST_RUNLOCK();
if (CC_ALGO(tp)->cb_init != NULL)
if (CC_ALGO(tp)->cb_init(tp->ccv) > 0) {
uma_zfree(V_tcpcb_zone, tm);
return (NULL);
}
tp->osd = &tm->osd;
if (khelp_init_osd(HELPER_CLASS_TCP, tp->osd)) {
uma_zfree(V_tcpcb_zone, tm);
return (NULL);
}
#ifdef VIMAGE
tp->t_vnet = inp->inp_vnet;
#endif
tp->t_timers = &tm->tt;
/* LIST_INIT(&tp->t_segq); */ /* XXX covered by M_ZERO */
tp->t_maxseg = tp->t_maxopd =
#ifdef INET6
isipv6 ? V_tcp_v6mssdflt :
#endif /* INET6 */
V_tcp_mssdflt;
/* Set up our timeouts. */
- callout_init(&tp->t_timers->tt_rexmt, CALLOUT_MPSAFE);
- callout_init(&tp->t_timers->tt_persist, CALLOUT_MPSAFE);
- callout_init(&tp->t_timers->tt_keep, CALLOUT_MPSAFE);
- callout_init(&tp->t_timers->tt_2msl, CALLOUT_MPSAFE);
- callout_init(&tp->t_timers->tt_delack, CALLOUT_MPSAFE);
+ callout_init(&tp->t_timers->tt_rexmt, 1);
+ callout_init(&tp->t_timers->tt_persist, 1);
+ callout_init(&tp->t_timers->tt_keep, 1);
+ callout_init(&tp->t_timers->tt_2msl, 1);
+ callout_init(&tp->t_timers->tt_delack, 1);
if (V_tcp_do_rfc1323)
tp->t_flags = (TF_REQ_SCALE|TF_REQ_TSTMP);
if (V_tcp_do_sack)
tp->t_flags |= TF_SACK_PERMIT;
TAILQ_INIT(&tp->snd_holes);
/*
* The tcpcb will hold a reference on its inpcb until tcp_discardcb()
* is called.
*/
in_pcbref(inp); /* Reference for tcpcb */
tp->t_inpcb = inp;
/*
* Init srtt to TCPTV_SRTTBASE (0), so we can tell that we have no
* rtt estimate. Set rttvar so that srtt + 4 * rttvar gives
* reasonable initial retransmit time.
*/
tp->t_srtt = TCPTV_SRTTBASE;
tp->t_rttvar = ((TCPTV_RTOBASE - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
tp->t_rttmin = tcp_rexmit_min;
tp->t_rxtcur = TCPTV_RTOBASE;
tp->snd_cwnd = TCP_MAXWIN << TCP_MAX_WINSHIFT;
tp->snd_ssthresh = TCP_MAXWIN << TCP_MAX_WINSHIFT;
tp->t_rcvtime = ticks;
/*
* IPv4 TTL initialization is necessary for an IPv6 socket as well,
* because the socket may be bound to an IPv6 wildcard address,
* which may match an IPv4-mapped IPv6 address.
*/
inp->inp_ip_ttl = V_ip_defttl;
inp->inp_ppcb = tp;
return (tp); /* XXX */
}
/*
* Switch the congestion control algorithm back to NewReno for any active
* control blocks using an algorithm which is about to go away.
* This ensures the CC framework can allow the unload to proceed without leaving
* any dangling pointers which would trigger a panic.
* Returning non-zero would inform the CC framework that something went wrong
* and it would be unsafe to allow the unload to proceed. However, there is no
* way for this to occur with this implementation so we always return zero.
*/
int
tcp_ccalgounload(struct cc_algo *unload_algo)
{
struct cc_algo *tmpalgo;
struct inpcb *inp;
struct tcpcb *tp;
VNET_ITERATOR_DECL(vnet_iter);
/*
* Check all active control blocks across all network stacks and change
* any that are using "unload_algo" back to NewReno. If "unload_algo"
* requires cleanup code to be run, call it.
*/
VNET_LIST_RLOCK();
VNET_FOREACH(vnet_iter) {
CURVNET_SET(vnet_iter);
INP_INFO_RLOCK(&V_tcbinfo);
/*
* New connections already part way through being initialised
* with the CC algo we're removing will not race with this code
* because the INP_INFO_WLOCK is held during initialisation. We
* therefore don't enter the loop below until the connection
* list has stabilised.
*/
LIST_FOREACH(inp, &V_tcb, inp_list) {
INP_WLOCK(inp);
/* Important to skip tcptw structs. */
if (!(inp->inp_flags & INP_TIMEWAIT) &&
(tp = intotcpcb(inp)) != NULL) {
/*
* By holding INP_WLOCK here, we are assured
* that the connection is not currently
* executing inside the CC module's functions
* i.e. it is safe to make the switch back to
* NewReno.
*/
if (CC_ALGO(tp) == unload_algo) {
tmpalgo = CC_ALGO(tp);
/* NewReno does not require any init. */
CC_ALGO(tp) = &newreno_cc_algo;
if (tmpalgo->cb_destroy != NULL)
tmpalgo->cb_destroy(tp->ccv);
}
}
INP_WUNLOCK(inp);
}
INP_INFO_RUNLOCK(&V_tcbinfo);
CURVNET_RESTORE();
}
VNET_LIST_RUNLOCK();
return (0);
}
/*
* Drop a TCP connection, reporting
* the specified error. If connection is synchronized,
* then send a RST to peer.
*/
struct tcpcb *
tcp_drop(struct tcpcb *tp, int errno)
{
struct socket *so = tp->t_inpcb->inp_socket;
INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
INP_WLOCK_ASSERT(tp->t_inpcb);
if (TCPS_HAVERCVDSYN(tp->t_state)) {
tcp_state_change(tp, TCPS_CLOSED);
(void) tcp_output(tp);
TCPSTAT_INC(tcps_drops);
} else
TCPSTAT_INC(tcps_conndrops);
if (errno == ETIMEDOUT && tp->t_softerror)
errno = tp->t_softerror;
so->so_error = errno;
return (tcp_close(tp));
}
void
tcp_discardcb(struct tcpcb *tp)
{
struct inpcb *inp = tp->t_inpcb;
struct socket *so = inp->inp_socket;
#ifdef INET6
int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
#endif /* INET6 */
int released;
INP_WLOCK_ASSERT(inp);
/*
* Make sure that all of our timers are stopped before we delete the
* PCB.
*
* If stopping a timer fails, we schedule a discard function in same
* callout, and the last discard function called will take care of
* deleting the tcpcb.
*/
tcp_timer_stop(tp, TT_REXMT);
tcp_timer_stop(tp, TT_PERSIST);
tcp_timer_stop(tp, TT_KEEP);
tcp_timer_stop(tp, TT_2MSL);
tcp_timer_stop(tp, TT_DELACK);
/*
* If we got enough samples through the srtt filter,
* save the rtt and rttvar in the routing entry.
* 'Enough' is arbitrarily defined as 4 rtt samples.
* 4 samples is enough for the srtt filter to converge
* to within enough % of the correct value; fewer samples
* and we could save a bogus rtt. The danger is not high
* as tcp quickly recovers from everything.
* XXX: Works very well but needs some more statistics!
*/
if (tp->t_rttupdated >= 4) {
struct hc_metrics_lite metrics;
u_long ssthresh;
bzero(&metrics, sizeof(metrics));
/*
* Update the ssthresh always when the conditions below
* are satisfied. This gives us better new start value
* for the congestion avoidance for new connections.
* ssthresh is only set if packet loss occured on a session.
*
* XXXRW: 'so' may be NULL here, and/or socket buffer may be
* being torn down. Ideally this code would not use 'so'.
*/
ssthresh = tp->snd_ssthresh;
if (ssthresh != 0 && ssthresh < so->so_snd.sb_hiwat / 2) {
/*
* convert the limit from user data bytes to
* packets then to packet data bytes.
*/
ssthresh = (ssthresh + tp->t_maxseg / 2) / tp->t_maxseg;
if (ssthresh < 2)
ssthresh = 2;
ssthresh *= (u_long)(tp->t_maxseg +
#ifdef INET6
(isipv6 ? sizeof (struct ip6_hdr) +
sizeof (struct tcphdr) :
#endif
sizeof (struct tcpiphdr)
#ifdef INET6
)
#endif
);
} else
ssthresh = 0;
metrics.rmx_ssthresh = ssthresh;
metrics.rmx_rtt = tp->t_srtt;
metrics.rmx_rttvar = tp->t_rttvar;
metrics.rmx_cwnd = tp->snd_cwnd;
metrics.rmx_sendpipe = 0;
metrics.rmx_recvpipe = 0;
tcp_hc_update(&inp->inp_inc, &metrics);
}
/* free the reassembly queue, if any */
tcp_reass_flush(tp);
#ifdef TCP_OFFLOAD
/* Disconnect offload device, if any. */
if (tp->t_flags & TF_TOE)
tcp_offload_detach(tp);
#endif
tcp_free_sackholes(tp);
/* Allow the CC algorithm to clean up after itself. */
if (CC_ALGO(tp)->cb_destroy != NULL)
CC_ALGO(tp)->cb_destroy(tp->ccv);
khelp_destroy_osd(tp->osd);
CC_ALGO(tp) = NULL;
inp->inp_ppcb = NULL;
if ((tp->t_timers->tt_flags & TT_MASK) == 0) {
/* We own the last reference on tcpcb, let's free it. */
tp->t_inpcb = NULL;
uma_zfree(V_tcpcb_zone, tp);
released = in_pcbrele_wlocked(inp);
KASSERT(!released, ("%s: inp %p should not have been released "
"here", __func__, inp));
}
}
void
tcp_timer_2msl_discard(void *xtp)
{
tcp_timer_discard((struct tcpcb *)xtp, TT_2MSL);
}
void
tcp_timer_keep_discard(void *xtp)
{
tcp_timer_discard((struct tcpcb *)xtp, TT_KEEP);
}
void
tcp_timer_persist_discard(void *xtp)
{
tcp_timer_discard((struct tcpcb *)xtp, TT_PERSIST);
}
void
tcp_timer_rexmt_discard(void *xtp)
{
tcp_timer_discard((struct tcpcb *)xtp, TT_REXMT);
}
void
tcp_timer_delack_discard(void *xtp)
{
tcp_timer_discard((struct tcpcb *)xtp, TT_DELACK);
}
void
tcp_timer_discard(struct tcpcb *tp, uint32_t timer_type)
{
struct inpcb *inp;
CURVNET_SET(tp->t_vnet);
INP_INFO_WLOCK(&V_tcbinfo);
inp = tp->t_inpcb;
KASSERT(inp != NULL, ("%s: tp %p tp->t_inpcb == NULL",
__func__, tp));
INP_WLOCK(inp);
KASSERT((tp->t_timers->tt_flags & TT_STOPPED) != 0,
("%s: tcpcb has to be stopped here", __func__));
KASSERT((tp->t_timers->tt_flags & timer_type) != 0,
("%s: discard callout should be running", __func__));
tp->t_timers->tt_flags &= ~timer_type;
if ((tp->t_timers->tt_flags & TT_MASK) == 0) {
/* We own the last reference on this tcpcb, let's free it. */
tp->t_inpcb = NULL;
uma_zfree(V_tcpcb_zone, tp);
if (in_pcbrele_wlocked(inp)) {
INP_INFO_WUNLOCK(&V_tcbinfo);
CURVNET_RESTORE();
return;
}
}
INP_WUNLOCK(inp);
INP_INFO_WUNLOCK(&V_tcbinfo);
CURVNET_RESTORE();
}
/*
* Attempt to close a TCP control block, marking it as dropped, and freeing
* the socket if we hold the only reference.
*/
struct tcpcb *
tcp_close(struct tcpcb *tp)
{
struct inpcb *inp = tp->t_inpcb;
struct socket *so;
INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
INP_WLOCK_ASSERT(inp);
#ifdef TCP_OFFLOAD
if (tp->t_state == TCPS_LISTEN)
tcp_offload_listen_stop(tp);
#endif
in_pcbdrop(inp);
TCPSTAT_INC(tcps_closed);
KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
so = inp->inp_socket;
soisdisconnected(so);
if (inp->inp_flags & INP_SOCKREF) {
KASSERT(so->so_state & SS_PROTOREF,
("tcp_close: !SS_PROTOREF"));
inp->inp_flags &= ~INP_SOCKREF;
INP_WUNLOCK(inp);
ACCEPT_LOCK();
SOCK_LOCK(so);
so->so_state &= ~SS_PROTOREF;
sofree(so);
return (NULL);
}
return (tp);
}
void
tcp_drain(void)
{
VNET_ITERATOR_DECL(vnet_iter);
if (!do_tcpdrain)
return;
VNET_LIST_RLOCK_NOSLEEP();
VNET_FOREACH(vnet_iter) {
CURVNET_SET(vnet_iter);
struct inpcb *inpb;
struct tcpcb *tcpb;
/*
* Walk the tcpbs, if existing, and flush the reassembly queue,
* if there is one...
* XXX: The "Net/3" implementation doesn't imply that the TCP
* reassembly queue should be flushed, but in a situation
* where we're really low on mbufs, this is potentially
* useful.
*/
INP_INFO_RLOCK(&V_tcbinfo);
LIST_FOREACH(inpb, V_tcbinfo.ipi_listhead, inp_list) {
if (inpb->inp_flags & INP_TIMEWAIT)
continue;
INP_WLOCK(inpb);
if ((tcpb = intotcpcb(inpb)) != NULL) {
tcp_reass_flush(tcpb);
tcp_clean_sackreport(tcpb);
}
INP_WUNLOCK(inpb);
}
INP_INFO_RUNLOCK(&V_tcbinfo);
CURVNET_RESTORE();
}
VNET_LIST_RUNLOCK_NOSLEEP();
}
/*
* Notify a tcp user of an asynchronous error;
* store error as soft error, but wake up user
* (for now, won't do anything until can select for soft error).
*
* Do not wake up user since there currently is no mechanism for
* reporting soft errors (yet - a kqueue filter may be added).
*/
static struct inpcb *
tcp_notify(struct inpcb *inp, int error)
{
struct tcpcb *tp;
INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
INP_WLOCK_ASSERT(inp);
if ((inp->inp_flags & INP_TIMEWAIT) ||
(inp->inp_flags & INP_DROPPED))
return (inp);
tp = intotcpcb(inp);
KASSERT(tp != NULL, ("tcp_notify: tp == NULL"));
/*
* Ignore some errors if we are hooked up.
* If connection hasn't completed, has retransmitted several times,
* and receives a second error, give up now. This is better
* than waiting a long time to establish a connection that
* can never complete.
*/
if (tp->t_state == TCPS_ESTABLISHED &&
(error == EHOSTUNREACH || error == ENETUNREACH ||
error == EHOSTDOWN)) {
return (inp);
} else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 &&
tp->t_softerror) {
tp = tcp_drop(tp, error);
if (tp != NULL)
return (inp);
else
return (NULL);
} else {
tp->t_softerror = error;
return (inp);
}
#if 0
wakeup( &so->so_timeo);
sorwakeup(so);
sowwakeup(so);
#endif
}
static int
tcp_pcblist(SYSCTL_HANDLER_ARGS)
{
int error, i, m, n, pcb_count;
struct inpcb *inp, **inp_list;
inp_gen_t gencnt;
struct xinpgen xig;
/*
* The process of preparing the TCB list is too time-consuming and
* resource-intensive to repeat twice on every request.
*/
if (req->oldptr == NULL) {
n = V_tcbinfo.ipi_count + syncache_pcbcount();
n += imax(n / 8, 10);
req->oldidx = 2 * (sizeof xig) + n * sizeof(struct xtcpcb);
return (0);
}
if (req->newptr != NULL)
return (EPERM);
/*
* OK, now we're committed to doing something.
*/
INP_INFO_RLOCK(&V_tcbinfo);
gencnt = V_tcbinfo.ipi_gencnt;
n = V_tcbinfo.ipi_count;
INP_INFO_RUNLOCK(&V_tcbinfo);
m = syncache_pcbcount();
error = sysctl_wire_old_buffer(req, 2 * (sizeof xig)
+ (n + m) * sizeof(struct xtcpcb));
if (error != 0)
return (error);
xig.xig_len = sizeof xig;
xig.xig_count = n + m;
xig.xig_gen = gencnt;
xig.xig_sogen = so_gencnt;
error = SYSCTL_OUT(req, &xig, sizeof xig);
if (error)
return (error);
error = syncache_pcblist(req, m, &pcb_count);
if (error)
return (error);
inp_list = malloc(n * sizeof *inp_list, M_TEMP, M_WAITOK);
if (inp_list == NULL)
return (ENOMEM);
INP_INFO_RLOCK(&V_tcbinfo);
for (inp = LIST_FIRST(V_tcbinfo.ipi_listhead), i = 0;
inp != NULL && i < n; inp = LIST_NEXT(inp, inp_list)) {
INP_WLOCK(inp);
if (inp->inp_gencnt <= gencnt) {
/*
* XXX: This use of cr_cansee(), introduced with
* TCP state changes, is not quite right, but for
* now, better than nothing.
*/
if (inp->inp_flags & INP_TIMEWAIT) {
if (intotw(inp) != NULL)
error = cr_cansee(req->td->td_ucred,
intotw(inp)->tw_cred);
else
error = EINVAL; /* Skip this inp. */
} else
error = cr_canseeinpcb(req->td->td_ucred, inp);
if (error == 0) {
in_pcbref(inp);
inp_list[i++] = inp;
}
}
INP_WUNLOCK(inp);
}
INP_INFO_RUNLOCK(&V_tcbinfo);
n = i;
error = 0;
for (i = 0; i < n; i++) {
inp = inp_list[i];
INP_RLOCK(inp);
if (inp->inp_gencnt <= gencnt) {
struct xtcpcb xt;
void *inp_ppcb;
bzero(&xt, sizeof(xt));
xt.xt_len = sizeof xt;
/* XXX should avoid extra copy */
bcopy(inp, &xt.xt_inp, sizeof *inp);
inp_ppcb = inp->inp_ppcb;
if (inp_ppcb == NULL)
bzero((char *) &xt.xt_tp, sizeof xt.xt_tp);
else if (inp->inp_flags & INP_TIMEWAIT) {
bzero((char *) &xt.xt_tp, sizeof xt.xt_tp);
xt.xt_tp.t_state = TCPS_TIME_WAIT;
} else {
bcopy(inp_ppcb, &xt.xt_tp, sizeof xt.xt_tp);
if (xt.xt_tp.t_timers)
tcp_timer_to_xtimer(&xt.xt_tp, xt.xt_tp.t_timers, &xt.xt_timer);
}
if (inp->inp_socket != NULL)
sotoxsocket(inp->inp_socket, &xt.xt_socket);
else {
bzero(&xt.xt_socket, sizeof xt.xt_socket);
xt.xt_socket.xso_protocol = IPPROTO_TCP;
}
xt.xt_inp.inp_gencnt = inp->inp_gencnt;
INP_RUNLOCK(inp);
error = SYSCTL_OUT(req, &xt, sizeof xt);
} else
INP_RUNLOCK(inp);
}
INP_INFO_WLOCK(&V_tcbinfo);
for (i = 0; i < n; i++) {
inp = inp_list[i];
INP_RLOCK(inp);
if (!in_pcbrele_rlocked(inp))
INP_RUNLOCK(inp);
}
INP_INFO_WUNLOCK(&V_tcbinfo);
if (!error) {
/*
* Give the user an updated idea of our state.
* If the generation differs from what we told
* her before, she knows that something happened
* while we were processing this request, and it
* might be necessary to retry.
*/
INP_INFO_RLOCK(&V_tcbinfo);
xig.xig_gen = V_tcbinfo.ipi_gencnt;
xig.xig_sogen = so_gencnt;
xig.xig_count = V_tcbinfo.ipi_count + pcb_count;
INP_INFO_RUNLOCK(&V_tcbinfo);
error = SYSCTL_OUT(req, &xig, sizeof xig);
}
free(inp_list, M_TEMP);
return (error);
}
SYSCTL_PROC(_net_inet_tcp, TCPCTL_PCBLIST, pcblist,
CTLTYPE_OPAQUE | CTLFLAG_RD, NULL, 0,
tcp_pcblist, "S,xtcpcb", "List of active TCP connections");
#ifdef INET
static int
tcp_getcred(SYSCTL_HANDLER_ARGS)
{
struct xucred xuc;
struct sockaddr_in addrs[2];
struct inpcb *inp;
int error;
error = priv_check(req->td, PRIV_NETINET_GETCRED);
if (error)
return (error);
error = SYSCTL_IN(req, addrs, sizeof(addrs));
if (error)
return (error);
inp = in_pcblookup(&V_tcbinfo, addrs[1].sin_addr, addrs[1].sin_port,
addrs[0].sin_addr, addrs[0].sin_port, INPLOOKUP_RLOCKPCB, NULL);
if (inp != NULL) {
if (inp->inp_socket == NULL)
error = ENOENT;
if (error == 0)
error = cr_canseeinpcb(req->td->td_ucred, inp);
if (error == 0)
cru2x(inp->inp_cred, &xuc);
INP_RUNLOCK(inp);
} else
error = ENOENT;
if (error == 0)
error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
return (error);
}
SYSCTL_PROC(_net_inet_tcp, OID_AUTO, getcred,
CTLTYPE_OPAQUE|CTLFLAG_RW|CTLFLAG_PRISON, 0, 0,
tcp_getcred, "S,xucred", "Get the xucred of a TCP connection");
#endif /* INET */
#ifdef INET6
static int
tcp6_getcred(SYSCTL_HANDLER_ARGS)
{
struct xucred xuc;
struct sockaddr_in6 addrs[2];
struct inpcb *inp;
int error;
#ifdef INET
int mapped = 0;
#endif
error = priv_check(req->td, PRIV_NETINET_GETCRED);
if (error)
return (error);
error = SYSCTL_IN(req, addrs, sizeof(addrs));
if (error)
return (error);
if ((error = sa6_embedscope(&addrs[0], V_ip6_use_defzone)) != 0 ||
(error = sa6_embedscope(&addrs[1], V_ip6_use_defzone)) != 0) {
return (error);
}
if (IN6_IS_ADDR_V4MAPPED(&addrs[0].sin6_addr)) {
#ifdef INET
if (IN6_IS_ADDR_V4MAPPED(&addrs[1].sin6_addr))
mapped = 1;
else
#endif
return (EINVAL);
}
#ifdef INET
if (mapped == 1)
inp = in_pcblookup(&V_tcbinfo,
*(struct in_addr *)&addrs[1].sin6_addr.s6_addr[12],
addrs[1].sin6_port,
*(struct in_addr *)&addrs[0].sin6_addr.s6_addr[12],
addrs[0].sin6_port, INPLOOKUP_RLOCKPCB, NULL);
else
#endif
inp = in6_pcblookup(&V_tcbinfo,
&addrs[1].sin6_addr, addrs[1].sin6_port,
&addrs[0].sin6_addr, addrs[0].sin6_port,
INPLOOKUP_RLOCKPCB, NULL);
if (inp != NULL) {
if (inp->inp_socket == NULL)
error = ENOENT;
if (error == 0)
error = cr_canseeinpcb(req->td->td_ucred, inp);
if (error == 0)
cru2x(inp->inp_cred, &xuc);
INP_RUNLOCK(inp);
} else
error = ENOENT;
if (error == 0)
error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
return (error);
}
SYSCTL_PROC(_net_inet6_tcp6, OID_AUTO, getcred,
CTLTYPE_OPAQUE|CTLFLAG_RW|CTLFLAG_PRISON, 0, 0,
tcp6_getcred, "S,xucred", "Get the xucred of a TCP6 connection");
#endif /* INET6 */
#ifdef INET
void
tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip)
{
struct ip *ip = vip;
struct tcphdr *th;
struct in_addr faddr;
struct inpcb *inp;
struct tcpcb *tp;
struct inpcb *(*notify)(struct inpcb *, int) = tcp_notify;
struct icmp *icp;
struct in_conninfo inc;
tcp_seq icmp_tcp_seq;
int mtu;
faddr = ((struct sockaddr_in *)sa)->sin_addr;
if (sa->sa_family != AF_INET || faddr.s_addr == INADDR_ANY)
return;
if (cmd == PRC_MSGSIZE)
notify = tcp_mtudisc_notify;
else if (V_icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB ||
cmd == PRC_UNREACH_PORT || cmd == PRC_TIMXCEED_INTRANS) && ip)
notify = tcp_drop_syn_sent;
/*
* Redirects don't need to be handled up here.
*/
else if (PRC_IS_REDIRECT(cmd))
return;
/*
* Hostdead is ugly because it goes linearly through all PCBs.
* XXX: We never get this from ICMP, otherwise it makes an
* excellent DoS attack on machines with many connections.
*/
else if (cmd == PRC_HOSTDEAD)
ip = NULL;
else if ((unsigned)cmd >= PRC_NCMDS || inetctlerrmap[cmd] == 0)
return;
if (ip != NULL) {
icp = (struct icmp *)((caddr_t)ip
- offsetof(struct icmp, icmp_ip));
th = (struct tcphdr *)((caddr_t)ip
+ (ip->ip_hl << 2));
INP_INFO_WLOCK(&V_tcbinfo);
inp = in_pcblookup(&V_tcbinfo, faddr, th->th_dport,
ip->ip_src, th->th_sport, INPLOOKUP_WLOCKPCB, NULL);
if (inp != NULL) {
if (!(inp->inp_flags & INP_TIMEWAIT) &&
!(inp->inp_flags & INP_DROPPED) &&
!(inp->inp_socket == NULL)) {
icmp_tcp_seq = htonl(th->th_seq);
tp = intotcpcb(inp);
if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) &&
SEQ_LT(icmp_tcp_seq, tp->snd_max)) {
if (cmd == PRC_MSGSIZE) {
/*
* MTU discovery:
* If we got a needfrag set the MTU
* in the route to the suggested new
* value (if given) and then notify.
*/
bzero(&inc, sizeof(inc));
inc.inc_faddr = faddr;
inc.inc_fibnum =
inp->inp_inc.inc_fibnum;
mtu = ntohs(icp->icmp_nextmtu);
/*
* If no alternative MTU was
* proposed, try the next smaller
* one.
*/
if (!mtu)
mtu = ip_next_mtu(
ntohs(ip->ip_len), 1);
if (mtu < V_tcp_minmss
+ sizeof(struct tcpiphdr))
mtu = V_tcp_minmss
+ sizeof(struct tcpiphdr);
/*
* Only cache the MTU if it
* is smaller than the interface
* or route MTU. tcp_mtudisc()
* will do right thing by itself.
*/
if (mtu <= tcp_maxmtu(&inc, NULL))
tcp_hc_updatemtu(&inc, mtu);
tcp_mtudisc(inp, mtu);
} else
inp = (*notify)(inp,
inetctlerrmap[cmd]);
}
}
if (inp != NULL)
INP_WUNLOCK(inp);
} else {
bzero(&inc, sizeof(inc));
inc.inc_fport = th->th_dport;
inc.inc_lport = th->th_sport;
inc.inc_faddr = faddr;
inc.inc_laddr = ip->ip_src;
syncache_unreach(&inc, th);
}
INP_INFO_WUNLOCK(&V_tcbinfo);
} else
in_pcbnotifyall(&V_tcbinfo, faddr, inetctlerrmap[cmd], notify);
}
#endif /* INET */
#ifdef INET6
void
tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d)
{
struct tcphdr th;
struct inpcb *(*notify)(struct inpcb *, int) = tcp_notify;
struct ip6_hdr *ip6;
struct mbuf *m;
struct ip6ctlparam *ip6cp = NULL;
const struct sockaddr_in6 *sa6_src = NULL;
int off;
struct tcp_portonly {
u_int16_t th_sport;
u_int16_t th_dport;
} *thp;
if (sa->sa_family != AF_INET6 ||
sa->sa_len != sizeof(struct sockaddr_in6))
return;
if (cmd == PRC_MSGSIZE)
notify = tcp_mtudisc_notify;
else if (!PRC_IS_REDIRECT(cmd) &&
((unsigned)cmd >= PRC_NCMDS || inet6ctlerrmap[cmd] == 0))
return;
/* if the parameter is from icmp6, decode it. */
if (d != NULL) {
ip6cp = (struct ip6ctlparam *)d;
m = ip6cp->ip6c_m;
ip6 = ip6cp->ip6c_ip6;
off = ip6cp->ip6c_off;
sa6_src = ip6cp->ip6c_src;
} else {
m = NULL;
ip6 = NULL;
off = 0; /* fool gcc */
sa6_src = &sa6_any;
}
if (ip6 != NULL) {
struct in_conninfo inc;
/*
* XXX: We assume that when IPV6 is non NULL,
* M and OFF are valid.
*/
/* check if we can safely examine src and dst ports */
if (m->m_pkthdr.len < off + sizeof(*thp))
return;
bzero(&th, sizeof(th));
m_copydata(m, off, sizeof(*thp), (caddr_t)&th);
in6_pcbnotify(&V_tcbinfo, sa, th.th_dport,
(struct sockaddr *)ip6cp->ip6c_src,
th.th_sport, cmd, NULL, notify);
bzero(&inc, sizeof(inc));
inc.inc_fport = th.th_dport;
inc.inc_lport = th.th_sport;
inc.inc6_faddr = ((struct sockaddr_in6 *)sa)->sin6_addr;
inc.inc6_laddr = ip6cp->ip6c_src->sin6_addr;
inc.inc_flags |= INC_ISIPV6;
INP_INFO_WLOCK(&V_tcbinfo);
syncache_unreach(&inc, &th);
INP_INFO_WUNLOCK(&V_tcbinfo);
} else
in6_pcbnotify(&V_tcbinfo, sa, 0, (const struct sockaddr *)sa6_src,
0, cmd, NULL, notify);
}
#endif /* INET6 */
/*
* Following is where TCP initial sequence number generation occurs.
*
* There are two places where we must use initial sequence numbers:
* 1. In SYN-ACK packets.
* 2. In SYN packets.
*
* All ISNs for SYN-ACK packets are generated by the syncache. See
* tcp_syncache.c for details.
*
* The ISNs in SYN packets must be monotonic; TIME_WAIT recycling
* depends on this property. In addition, these ISNs should be
* unguessable so as to prevent connection hijacking. To satisfy
* the requirements of this situation, the algorithm outlined in
* RFC 1948 is used, with only small modifications.
*
* Implementation details:
*
* Time is based off the system timer, and is corrected so that it
* increases by one megabyte per second. This allows for proper
* recycling on high speed LANs while still leaving over an hour
* before rollover.
*
* As reading the *exact* system time is too expensive to be done
* whenever setting up a TCP connection, we increment the time
* offset in two ways. First, a small random positive increment
* is added to isn_offset for each connection that is set up.
* Second, the function tcp_isn_tick fires once per clock tick
* and increments isn_offset as necessary so that sequence numbers
* are incremented at approximately ISN_BYTES_PER_SECOND. The
* random positive increments serve only to ensure that the same
* exact sequence number is never sent out twice (as could otherwise
* happen when a port is recycled in less than the system tick
* interval.)
*
* net.inet.tcp.isn_reseed_interval controls the number of seconds
* between seeding of isn_secret. This is normally set to zero,
* as reseeding should not be necessary.
*
* Locking of the global variables isn_secret, isn_last_reseed, isn_offset,
* isn_offset_old, and isn_ctx is performed using the TCP pcbinfo lock. In
* general, this means holding an exclusive (write) lock.
*/
#define ISN_BYTES_PER_SECOND 1048576
#define ISN_STATIC_INCREMENT 4096
#define ISN_RANDOM_INCREMENT (4096 - 1)
static VNET_DEFINE(u_char, isn_secret[32]);
static VNET_DEFINE(int, isn_last);
static VNET_DEFINE(int, isn_last_reseed);
static VNET_DEFINE(u_int32_t, isn_offset);
static VNET_DEFINE(u_int32_t, isn_offset_old);
#define V_isn_secret VNET(isn_secret)
#define V_isn_last VNET(isn_last)
#define V_isn_last_reseed VNET(isn_last_reseed)
#define V_isn_offset VNET(isn_offset)
#define V_isn_offset_old VNET(isn_offset_old)
tcp_seq
tcp_new_isn(struct tcpcb *tp)
{
MD5_CTX isn_ctx;
u_int32_t md5_buffer[4];
tcp_seq new_isn;
u_int32_t projected_offset;
INP_WLOCK_ASSERT(tp->t_inpcb);
ISN_LOCK();
/* Seed if this is the first use, reseed if requested. */
if ((V_isn_last_reseed == 0) || ((V_tcp_isn_reseed_interval > 0) &&
(((u_int)V_isn_last_reseed + (u_int)V_tcp_isn_reseed_interval*hz)
< (u_int)ticks))) {
read_random(&V_isn_secret, sizeof(V_isn_secret));
V_isn_last_reseed = ticks;
}
/* Compute the md5 hash and return the ISN. */
MD5Init(&isn_ctx);
MD5Update(&isn_ctx, (u_char *) &tp->t_inpcb->inp_fport, sizeof(u_short));
MD5Update(&isn_ctx, (u_char *) &tp->t_inpcb->inp_lport, sizeof(u_short));
#ifdef INET6
if ((tp->t_inpcb->inp_vflag & INP_IPV6) != 0) {
MD5Update(&isn_ctx, (u_char *) &tp->t_inpcb->in6p_faddr,
sizeof(struct in6_addr));
MD5Update(&isn_ctx, (u_char *) &tp->t_inpcb->in6p_laddr,
sizeof(struct in6_addr));
} else
#endif
{
MD5Update(&isn_ctx, (u_char *) &tp->t_inpcb->inp_faddr,
sizeof(struct in_addr));
MD5Update(&isn_ctx, (u_char *) &tp->t_inpcb->inp_laddr,
sizeof(struct in_addr));
}
MD5Update(&isn_ctx, (u_char *) &V_isn_secret, sizeof(V_isn_secret));
MD5Final((u_char *) &md5_buffer, &isn_ctx);
new_isn = (tcp_seq) md5_buffer[0];
V_isn_offset += ISN_STATIC_INCREMENT +
(arc4random() & ISN_RANDOM_INCREMENT);
if (ticks != V_isn_last) {
projected_offset = V_isn_offset_old +
ISN_BYTES_PER_SECOND / hz * (ticks - V_isn_last);
if (SEQ_GT(projected_offset, V_isn_offset))
V_isn_offset = projected_offset;
V_isn_offset_old = V_isn_offset;
V_isn_last = ticks;
}
new_isn += V_isn_offset;
ISN_UNLOCK();
return (new_isn);
}
/*
* When a specific ICMP unreachable message is received and the
* connection state is SYN-SENT, drop the connection. This behavior
* is controlled by the icmp_may_rst sysctl.
*/
struct inpcb *
tcp_drop_syn_sent(struct inpcb *inp, int errno)
{
struct tcpcb *tp;
INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
INP_WLOCK_ASSERT(inp);
if ((inp->inp_flags & INP_TIMEWAIT) ||
(inp->inp_flags & INP_DROPPED))
return (inp);
tp = intotcpcb(inp);
if (tp->t_state != TCPS_SYN_SENT)
return (inp);
tp = tcp_drop(tp, errno);
if (tp != NULL)
return (inp);
else
return (NULL);
}
/*
* When `need fragmentation' ICMP is received, update our idea of the MSS
* based on the new value. Also nudge TCP to send something, since we
* know the packet we just sent was dropped.
* This duplicates some code in the tcp_mss() function in tcp_input.c.
*/
static struct inpcb *
tcp_mtudisc_notify(struct inpcb *inp, int error)
{
return (tcp_mtudisc(inp, -1));
}
struct inpcb *
tcp_mtudisc(struct inpcb *inp, int mtuoffer)
{
struct tcpcb *tp;
struct socket *so;
INP_WLOCK_ASSERT(inp);
if ((inp->inp_flags & INP_TIMEWAIT) ||
(inp->inp_flags & INP_DROPPED))
return (inp);
tp = intotcpcb(inp);
KASSERT(tp != NULL, ("tcp_mtudisc: tp == NULL"));
tcp_mss_update(tp, -1, mtuoffer, NULL, NULL);
so = inp->inp_socket;
SOCKBUF_LOCK(&so->so_snd);
/* If the mss is larger than the socket buffer, decrease the mss. */
if (so->so_snd.sb_hiwat < tp->t_maxseg)
tp->t_maxseg = so->so_snd.sb_hiwat;
SOCKBUF_UNLOCK(&so->so_snd);
TCPSTAT_INC(tcps_mturesent);
tp->t_rtttime = 0;
tp->snd_nxt = tp->snd_una;
tcp_free_sackholes(tp);
tp->snd_recover = tp->snd_max;
if (tp->t_flags & TF_SACK_PERMIT)
EXIT_FASTRECOVERY(tp->t_flags);
tcp_output(tp);
return (inp);
}
#ifdef INET
/*
* Look-up the routing entry to the peer of this inpcb. If no route
* is found and it cannot be allocated, then return 0. This routine
* is called by TCP routines that access the rmx structure and by
* tcp_mss_update to get the peer/interface MTU.
*/
u_long
tcp_maxmtu(struct in_conninfo *inc, struct tcp_ifcap *cap)
{
struct route sro;
struct sockaddr_in *dst;
struct ifnet *ifp;
u_long maxmtu = 0;
KASSERT(inc != NULL, ("tcp_maxmtu with NULL in_conninfo pointer"));
bzero(&sro, sizeof(sro));
if (inc->inc_faddr.s_addr != INADDR_ANY) {
dst = (struct sockaddr_in *)&sro.ro_dst;
dst->sin_family = AF_INET;
dst->sin_len = sizeof(*dst);
dst->sin_addr = inc->inc_faddr;
in_rtalloc_ign(&sro, 0, inc->inc_fibnum);
}
if (sro.ro_rt != NULL) {
ifp = sro.ro_rt->rt_ifp;
if (sro.ro_rt->rt_mtu == 0)
maxmtu = ifp->if_mtu;
else
maxmtu = min(sro.ro_rt->rt_mtu, ifp->if_mtu);
/* Report additional interface capabilities. */
if (cap != NULL) {
if (ifp->if_capenable & IFCAP_TSO4 &&
ifp->if_hwassist & CSUM_TSO) {
cap->ifcap |= CSUM_TSO;
cap->tsomax = ifp->if_hw_tsomax;
cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
}
}
RTFREE(sro.ro_rt);
}
return (maxmtu);
}
#endif /* INET */
#ifdef INET6
u_long
tcp_maxmtu6(struct in_conninfo *inc, struct tcp_ifcap *cap)
{
struct route_in6 sro6;
struct ifnet *ifp;
u_long maxmtu = 0;
KASSERT(inc != NULL, ("tcp_maxmtu6 with NULL in_conninfo pointer"));
bzero(&sro6, sizeof(sro6));
if (!IN6_IS_ADDR_UNSPECIFIED(&inc->inc6_faddr)) {
sro6.ro_dst.sin6_family = AF_INET6;
sro6.ro_dst.sin6_len = sizeof(struct sockaddr_in6);
sro6.ro_dst.sin6_addr = inc->inc6_faddr;
in6_rtalloc_ign(&sro6, 0, inc->inc_fibnum);
}
if (sro6.ro_rt != NULL) {
ifp = sro6.ro_rt->rt_ifp;
if (sro6.ro_rt->rt_mtu == 0)
maxmtu = IN6_LINKMTU(sro6.ro_rt->rt_ifp);
else
maxmtu = min(sro6.ro_rt->rt_mtu,
IN6_LINKMTU(sro6.ro_rt->rt_ifp));
/* Report additional interface capabilities. */
if (cap != NULL) {
if (ifp->if_capenable & IFCAP_TSO6 &&
ifp->if_hwassist & CSUM_TSO) {
cap->ifcap |= CSUM_TSO;
cap->tsomax = ifp->if_hw_tsomax;
cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
}
}
RTFREE(sro6.ro_rt);
}
return (maxmtu);
}
#endif /* INET6 */
#ifdef IPSEC
/* compute ESP/AH header size for TCP, including outer IP header. */
size_t
ipsec_hdrsiz_tcp(struct tcpcb *tp)
{
struct inpcb *inp;
struct mbuf *m;
size_t hdrsiz;
struct ip *ip;
#ifdef INET6
struct ip6_hdr *ip6;
#endif
struct tcphdr *th;
if ((tp == NULL) || ((inp = tp->t_inpcb) == NULL))
return (0);
m = m_gethdr(M_NOWAIT, MT_DATA);
if (!m)
return (0);
#ifdef INET6
if ((inp->inp_vflag & INP_IPV6) != 0) {
ip6 = mtod(m, struct ip6_hdr *);
th = (struct tcphdr *)(ip6 + 1);
m->m_pkthdr.len = m->m_len =
sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
tcpip_fillheaders(inp, ip6, th);
hdrsiz = ipsec_hdrsiz(m, IPSEC_DIR_OUTBOUND, inp);
} else
#endif /* INET6 */
{
ip = mtod(m, struct ip *);
th = (struct tcphdr *)(ip + 1);
m->m_pkthdr.len = m->m_len = sizeof(struct tcpiphdr);
tcpip_fillheaders(inp, ip, th);
hdrsiz = ipsec_hdrsiz(m, IPSEC_DIR_OUTBOUND, inp);
}
m_free(m);
return (hdrsiz);
}
#endif /* IPSEC */
#ifdef TCP_SIGNATURE
/*
* Callback function invoked by m_apply() to digest TCP segment data
* contained within an mbuf chain.
*/
static int
tcp_signature_apply(void *fstate, void *data, u_int len)
{
MD5Update(fstate, (u_char *)data, len);
return (0);
}
/*
* XXX The key is retrieved from the system's PF_KEY SADB, by keying a
* search with the destination IP address, and a 'magic SPI' to be
* determined by the application. This is hardcoded elsewhere to 1179
*/
struct secasvar *
tcp_get_sav(struct mbuf *m, u_int direction)
{
union sockaddr_union dst;
struct secasvar *sav;
struct ip *ip;
#ifdef INET6
struct ip6_hdr *ip6;
char ip6buf[INET6_ADDRSTRLEN];
#endif
/* Extract the destination from the IP header in the mbuf. */
bzero(&dst, sizeof(union sockaddr_union));
ip = mtod(m, struct ip *);
#ifdef INET6
ip6 = NULL; /* Make the compiler happy. */
#endif
switch (ip->ip_v) {
#ifdef INET
case IPVERSION:
dst.sa.sa_len = sizeof(struct sockaddr_in);
dst.sa.sa_family = AF_INET;
dst.sin.sin_addr = (direction == IPSEC_DIR_INBOUND) ?
ip->ip_src : ip->ip_dst;
break;
#endif
#ifdef INET6
case (IPV6_VERSION >> 4):
ip6 = mtod(m, struct ip6_hdr *);
dst.sa.sa_len = sizeof(struct sockaddr_in6);
dst.sa.sa_family = AF_INET6;
dst.sin6.sin6_addr = (direction == IPSEC_DIR_INBOUND) ?
ip6->ip6_src : ip6->ip6_dst;
break;
#endif
default:
return (NULL);
/* NOTREACHED */
break;
}
/* Look up an SADB entry which matches the address of the peer. */
sav = KEY_ALLOCSA(&dst, IPPROTO_TCP, htonl(TCP_SIG_SPI));
if (sav == NULL) {
ipseclog((LOG_ERR, "%s: SADB lookup failed for %s\n", __func__,
(ip->ip_v == IPVERSION) ? inet_ntoa(dst.sin.sin_addr) :
#ifdef INET6
(ip->ip_v == (IPV6_VERSION >> 4)) ?
ip6_sprintf(ip6buf, &dst.sin6.sin6_addr) :
#endif
"(unsupported)"));
}
return (sav);
}
/*
* Compute TCP-MD5 hash of a TCP segment. (RFC2385)
*
* Parameters:
* m pointer to head of mbuf chain
* len length of TCP segment data, excluding options
* optlen length of TCP segment options
* buf pointer to storage for computed MD5 digest
* sav pointer to security assosiation
*
* We do this over ip, tcphdr, segment data, and the key in the SADB.
* When called from tcp_input(), we can be sure that th_sum has been
* zeroed out and verified already.
*
* Releases reference to SADB key before return.
*
* Return 0 if successful, otherwise return -1.
*
*/
int
tcp_signature_do_compute(struct mbuf *m, int len, int optlen,
u_char *buf, struct secasvar *sav)
{
#ifdef INET
struct ippseudo ippseudo;
#endif
MD5_CTX ctx;
int doff;
struct ip *ip;
#ifdef INET
struct ipovly *ipovly;
#endif
struct tcphdr *th;
#ifdef INET6
struct ip6_hdr *ip6;
struct in6_addr in6;
uint32_t plen;
uint16_t nhdr;
#endif
u_short savecsum;
KASSERT(m != NULL, ("NULL mbuf chain"));
KASSERT(buf != NULL, ("NULL signature pointer"));
/* Extract the destination from the IP header in the mbuf. */
ip = mtod(m, struct ip *);
#ifdef INET6
ip6 = NULL; /* Make the compiler happy. */
#endif
MD5Init(&ctx);
/*
* Step 1: Update MD5 hash with IP(v6) pseudo-header.
*
* XXX The ippseudo header MUST be digested in network byte order,
* or else we'll fail the regression test. Assume all fields we've
* been doing arithmetic on have been in host byte order.
* XXX One cannot depend on ipovly->ih_len here. When called from
* tcp_output(), the underlying ip_len member has not yet been set.
*/
switch (ip->ip_v) {
#ifdef INET
case IPVERSION:
ipovly = (struct ipovly *)ip;
ippseudo.ippseudo_src = ipovly->ih_src;
ippseudo.ippseudo_dst = ipovly->ih_dst;
ippseudo.ippseudo_pad = 0;
ippseudo.ippseudo_p = IPPROTO_TCP;
ippseudo.ippseudo_len = htons(len + sizeof(struct tcphdr) +
optlen);
MD5Update(&ctx, (char *)&ippseudo, sizeof(struct ippseudo));
th = (struct tcphdr *)((u_char *)ip + sizeof(struct ip));
doff = sizeof(struct ip) + sizeof(struct tcphdr) + optlen;
break;
#endif
#ifdef INET6
/*
* RFC 2385, 2.0 Proposal
* For IPv6, the pseudo-header is as described in RFC 2460, namely the
* 128-bit source IPv6 address, 128-bit destination IPv6 address, zero-
* extended next header value (to form 32 bits), and 32-bit segment
* length.
* Note: Upper-Layer Packet Length comes before Next Header.
*/
case (IPV6_VERSION >> 4):
in6 = ip6->ip6_src;
in6_clearscope(&in6);
MD5Update(&ctx, (char *)&in6, sizeof(struct in6_addr));
in6 = ip6->ip6_dst;
in6_clearscope(&in6);
MD5Update(&ctx, (char *)&in6, sizeof(struct in6_addr));
plen = htonl(len + sizeof(struct tcphdr) + optlen);
MD5Update(&ctx, (char *)&plen, sizeof(uint32_t));
nhdr = 0;
MD5Update(&ctx, (char *)&nhdr, sizeof(uint8_t));
MD5Update(&ctx, (char *)&nhdr, sizeof(uint8_t));
MD5Update(&ctx, (char *)&nhdr, sizeof(uint8_t));
nhdr = IPPROTO_TCP;
MD5Update(&ctx, (char *)&nhdr, sizeof(uint8_t));
th = (struct tcphdr *)((u_char *)ip6 + sizeof(struct ip6_hdr));
doff = sizeof(struct ip6_hdr) + sizeof(struct tcphdr) + optlen;
break;
#endif
default:
KEY_FREESAV(&sav);
return (-1);
/* NOTREACHED */
break;
}
/*
* Step 2: Update MD5 hash with TCP header, excluding options.
* The TCP checksum must be set to zero.
*/
savecsum = th->th_sum;
th->th_sum = 0;
MD5Update(&ctx, (char *)th, sizeof(struct tcphdr));
th->th_sum = savecsum;
/*
* Step 3: Update MD5 hash with TCP segment data.
* Use m_apply() to avoid an early m_pullup().
*/
if (len > 0)
m_apply(m, doff, len, tcp_signature_apply, &ctx);
/*
* Step 4: Update MD5 hash with shared secret.
*/
MD5Update(&ctx, sav->key_auth->key_data, _KEYLEN(sav->key_auth));
MD5Final(buf, &ctx);
key_sa_recordxfer(sav, m);
KEY_FREESAV(&sav);
return (0);
}
/*
* Compute TCP-MD5 hash of a TCP segment. (RFC2385)
*
* Return 0 if successful, otherwise return -1.
*/
int
tcp_signature_compute(struct mbuf *m, int _unused, int len, int optlen,
u_char *buf, u_int direction)
{
struct secasvar *sav;
if ((sav = tcp_get_sav(m, direction)) == NULL)
return (-1);
return (tcp_signature_do_compute(m, len, optlen, buf, sav));
}
/*
* Verify the TCP-MD5 hash of a TCP segment. (RFC2385)
*
* Parameters:
* m pointer to head of mbuf chain
* len length of TCP segment data, excluding options
* optlen length of TCP segment options
* buf pointer to storage for computed MD5 digest
* direction direction of flow (IPSEC_DIR_INBOUND or OUTBOUND)
*
* Return 1 if successful, otherwise return 0.
*/
int
tcp_signature_verify(struct mbuf *m, int off0, int tlen, int optlen,
struct tcpopt *to, struct tcphdr *th, u_int tcpbflag)
{
char tmpdigest[TCP_SIGLEN];
if (tcp_sig_checksigs == 0)
return (1);
if ((tcpbflag & TF_SIGNATURE) == 0) {
if ((to->to_flags & TOF_SIGNATURE) != 0) {
/*
* If this socket is not expecting signature but
* the segment contains signature just fail.
*/
TCPSTAT_INC(tcps_sig_err_sigopt);
TCPSTAT_INC(tcps_sig_rcvbadsig);
return (0);
}
/* Signature is not expected, and not present in segment. */
return (1);
}
/*
* If this socket is expecting signature but the segment does not
* contain any just fail.
*/
if ((to->to_flags & TOF_SIGNATURE) == 0) {
TCPSTAT_INC(tcps_sig_err_nosigopt);
TCPSTAT_INC(tcps_sig_rcvbadsig);
return (0);
}
if (tcp_signature_compute(m, off0, tlen, optlen, &tmpdigest[0],
IPSEC_DIR_INBOUND) == -1) {
TCPSTAT_INC(tcps_sig_err_buildsig);
TCPSTAT_INC(tcps_sig_rcvbadsig);
return (0);
}
if (bcmp(to->to_signature, &tmpdigest[0], TCP_SIGLEN) != 0) {
TCPSTAT_INC(tcps_sig_rcvbadsig);
return (0);
}
TCPSTAT_INC(tcps_sig_rcvgoodsig);
return (1);
}
#endif /* TCP_SIGNATURE */
static int
sysctl_drop(SYSCTL_HANDLER_ARGS)
{
/* addrs[0] is a foreign socket, addrs[1] is a local one. */
struct sockaddr_storage addrs[2];
struct inpcb *inp;
struct tcpcb *tp;
struct tcptw *tw;
struct sockaddr_in *fin, *lin;
#ifdef INET6
struct sockaddr_in6 *fin6, *lin6;
#endif
int error;
inp = NULL;
fin = lin = NULL;
#ifdef INET6
fin6 = lin6 = NULL;
#endif
error = 0;
if (req->oldptr != NULL || req->oldlen != 0)
return (EINVAL);
if (req->newptr == NULL)
return (EPERM);
if (req->newlen < sizeof(addrs))
return (ENOMEM);
error = SYSCTL_IN(req, &addrs, sizeof(addrs));
if (error)
return (error);
switch (addrs[0].ss_family) {
#ifdef INET6
case AF_INET6:
fin6 = (struct sockaddr_in6 *)&addrs[0];
lin6 = (struct sockaddr_in6 *)&addrs[1];
if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
lin6->sin6_len != sizeof(struct sockaddr_in6))
return (EINVAL);
if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
return (EINVAL);
in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
fin = (struct sockaddr_in *)&addrs[0];
lin = (struct sockaddr_in *)&addrs[1];
break;
}
error = sa6_embedscope(fin6, V_ip6_use_defzone);
if (error)
return (error);
error = sa6_embedscope(lin6, V_ip6_use_defzone);
if (error)
return (error);
break;
#endif
#ifdef INET
case AF_INET:
fin = (struct sockaddr_in *)&addrs[0];
lin = (struct sockaddr_in *)&addrs[1];
if (fin->sin_len != sizeof(struct sockaddr_in) ||
lin->sin_len != sizeof(struct sockaddr_in))
return (EINVAL);
break;
#endif
default:
return (EINVAL);
}
INP_INFO_WLOCK(&V_tcbinfo);
switch (addrs[0].ss_family) {
#ifdef INET6
case AF_INET6:
inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
INPLOOKUP_WLOCKPCB, NULL);
break;
#endif
#ifdef INET
case AF_INET:
inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
break;
#endif
}
if (inp != NULL) {
if (inp->inp_flags & INP_TIMEWAIT) {
/*
* XXXRW: There currently exists a state where an
* inpcb is present, but its timewait state has been
* discarded. For now, don't allow dropping of this
* type of inpcb.
*/
tw = intotw(inp);
if (tw != NULL)
tcp_twclose(tw, 0);
else
INP_WUNLOCK(inp);
} else if (!(inp->inp_flags & INP_DROPPED) &&
!(inp->inp_socket->so_options & SO_ACCEPTCONN)) {
tp = intotcpcb(inp);
tp = tcp_drop(tp, ECONNABORTED);
if (tp != NULL)
INP_WUNLOCK(inp);
} else
INP_WUNLOCK(inp);
} else
error = ESRCH;
INP_INFO_WUNLOCK(&V_tcbinfo);
return (error);
}
SYSCTL_PROC(_net_inet_tcp, TCPCTL_DROP, drop,
CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP, NULL,
0, sysctl_drop, "", "Drop TCP connection");
/*
* Generate a standardized TCP log line for use throughout the
* tcp subsystem. Memory allocation is done with M_NOWAIT to
* allow use in the interrupt context.
*
* NB: The caller MUST free(s, M_TCPLOG) the returned string.
* NB: The function may return NULL if memory allocation failed.
*
* Due to header inclusion and ordering limitations the struct ip
* and ip6_hdr pointers have to be passed as void pointers.
*/
char *
tcp_log_vain(struct in_conninfo *inc, struct tcphdr *th, void *ip4hdr,
const void *ip6hdr)
{
/* Is logging enabled? */
if (tcp_log_in_vain == 0)
return (NULL);
return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
}
char *
tcp_log_addrs(struct in_conninfo *inc, struct tcphdr *th, void *ip4hdr,
const void *ip6hdr)
{
/* Is logging enabled? */
if (tcp_log_debug == 0)
return (NULL);
return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
}
static char *
tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th, void *ip4hdr,
const void *ip6hdr)
{
char *s, *sp;
size_t size;
struct ip *ip;
#ifdef INET6
const struct ip6_hdr *ip6;
ip6 = (const struct ip6_hdr *)ip6hdr;
#endif /* INET6 */
ip = (struct ip *)ip4hdr;
/*
* The log line looks like this:
* "TCP: [1.2.3.4]:50332 to [1.2.3.4]:80 tcpflags 0x2<SYN>"
*/
size = sizeof("TCP: []:12345 to []:12345 tcpflags 0x2<>") +
sizeof(PRINT_TH_FLAGS) + 1 +
#ifdef INET6
2 * INET6_ADDRSTRLEN;
#else
2 * INET_ADDRSTRLEN;
#endif /* INET6 */
s = malloc(size, M_TCPLOG, M_ZERO|M_NOWAIT);
if (s == NULL)
return (NULL);
strcat(s, "TCP: [");
sp = s + strlen(s);
if (inc && ((inc->inc_flags & INC_ISIPV6) == 0)) {
inet_ntoa_r(inc->inc_faddr, sp);
sp = s + strlen(s);
sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
sp = s + strlen(s);
inet_ntoa_r(inc->inc_laddr, sp);
sp = s + strlen(s);
sprintf(sp, "]:%i", ntohs(inc->inc_lport));
#ifdef INET6
} else if (inc) {
ip6_sprintf(sp, &inc->inc6_faddr);
sp = s + strlen(s);
sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
sp = s + strlen(s);
ip6_sprintf(sp, &inc->inc6_laddr);
sp = s + strlen(s);
sprintf(sp, "]:%i", ntohs(inc->inc_lport));
} else if (ip6 && th) {
ip6_sprintf(sp, &ip6->ip6_src);
sp = s + strlen(s);
sprintf(sp, "]:%i to [", ntohs(th->th_sport));
sp = s + strlen(s);
ip6_sprintf(sp, &ip6->ip6_dst);
sp = s + strlen(s);
sprintf(sp, "]:%i", ntohs(th->th_dport));
#endif /* INET6 */
#ifdef INET
} else if (ip && th) {
inet_ntoa_r(ip->ip_src, sp);
sp = s + strlen(s);
sprintf(sp, "]:%i to [", ntohs(th->th_sport));
sp = s + strlen(s);
inet_ntoa_r(ip->ip_dst, sp);
sp = s + strlen(s);
sprintf(sp, "]:%i", ntohs(th->th_dport));
#endif /* INET */
} else {
free(s, M_TCPLOG);
return (NULL);
}
sp = s + strlen(s);
if (th)
sprintf(sp, " tcpflags 0x%b", th->th_flags, PRINT_TH_FLAGS);
if (*(s + size - 1) != '\0')
panic("%s: string too long", __func__);
return (s);
}
/*
* A subroutine which makes it easy to track TCP state changes with DTrace.
* This function shouldn't be called for t_state initializations that don't
* correspond to actual TCP state transitions.
*/
void
tcp_state_change(struct tcpcb *tp, int newstate)
{
#if defined(KDTRACE_HOOKS)
int pstate = tp->t_state;
#endif
tp->t_state = newstate;
TCP_PROBE6(state__change, NULL, tp, NULL, tp, NULL, pstate);
}
Index: head/sys/netinet6/in6_rmx.c
===================================================================
--- head/sys/netinet6/in6_rmx.c (revision 283290)
+++ head/sys/netinet6/in6_rmx.c (revision 283291)
@@ -1,322 +1,322 @@
/*-
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the project nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $KAME: in6_rmx.c,v 1.11 2001/07/26 06:53:16 jinmei Exp $
*/
/*-
* Copyright 1994, 1995 Massachusetts Institute of Technology
*
* Permission to use, copy, modify, and distribute this software and
* its documentation for any purpose and without fee is hereby
* granted, provided that both the above copyright notice and this
* permission notice appear in all copies, that both the above
* copyright notice and this permission notice appear in all
* supporting documentation, and that the name of M.I.T. not be used
* in advertising or publicity pertaining to distribution of the
* software without specific, written prior permission. M.I.T. makes
* no representations about the suitability of this software for any
* purpose. It is provided "as is" without express or implied
* warranty.
*
* THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS
* ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
* SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/queue.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/mbuf.h>
#include <sys/rwlock.h>
#include <sys/syslog.h>
#include <sys/callout.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/ip_var.h>
#include <netinet/in_var.h>
#include <netinet/ip6.h>
#include <netinet6/ip6_var.h>
#include <netinet/icmp6.h>
#include <netinet6/nd6.h>
#include <netinet/tcp.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
extern int in6_inithead(void **head, int off);
#ifdef VIMAGE
extern int in6_detachhead(void **head, int off);
#endif
/*
* Do what we need to do when inserting a route.
*/
static struct radix_node *
in6_addroute(void *v_arg, void *n_arg, struct radix_node_head *head,
struct radix_node *treenodes)
{
struct rtentry *rt = (struct rtentry *)treenodes;
struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)rt_key(rt);
struct radix_node *ret;
RADIX_NODE_HEAD_WLOCK_ASSERT(head);
if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr))
rt->rt_flags |= RTF_MULTICAST;
/*
* A little bit of help for both IPv6 output and input:
* For local addresses, we make sure that RTF_LOCAL is set,
* with the thought that this might one day be used to speed up
* ip_input().
*
* We also mark routes to multicast addresses as such, because
* it's easy to do and might be useful (but this is much more
* dubious since it's so easy to inspect the address). (This
* is done above.)
*
* XXX
* should elaborate the code.
*/
if (rt->rt_flags & RTF_HOST) {
if (IN6_ARE_ADDR_EQUAL(&satosin6(rt->rt_ifa->ifa_addr)
->sin6_addr,
&sin6->sin6_addr)) {
rt->rt_flags |= RTF_LOCAL;
}
}
if (rt->rt_ifp != NULL) {
/*
* Check route MTU:
* inherit interface MTU if not set or
* check if MTU is too large.
*/
if (rt->rt_mtu == 0) {
rt->rt_mtu = IN6_LINKMTU(rt->rt_ifp);
} else if (rt->rt_mtu > IN6_LINKMTU(rt->rt_ifp))
rt->rt_mtu = IN6_LINKMTU(rt->rt_ifp);
}
ret = rn_addroute(v_arg, n_arg, head, treenodes);
if (ret == NULL) {
struct rtentry *rt2;
/*
* We are trying to add a net route, but can't.
* The following case should be allowed, so we'll make a
* special check for this:
* Two IPv6 addresses with the same prefix is assigned
* to a single interrface.
* # ifconfig if0 inet6 3ffe:0501::1 prefix 64 alias (*1)
* # ifconfig if0 inet6 3ffe:0501::2 prefix 64 alias (*2)
* In this case, (*1) and (*2) want to add the same
* net route entry, 3ffe:0501:: -> if0.
* This case should not raise an error.
*/
rt2 = in6_rtalloc1((struct sockaddr *)sin6, 0, RTF_RNH_LOCKED,
rt->rt_fibnum);
if (rt2) {
if (((rt2->rt_flags & (RTF_HOST|RTF_GATEWAY)) == 0)
&& rt2->rt_gateway
&& rt2->rt_gateway->sa_family == AF_LINK
&& rt2->rt_ifp == rt->rt_ifp) {
ret = rt2->rt_nodes;
}
RTFREE_LOCKED(rt2);
}
}
return (ret);
}
/*
* Age old PMTUs.
*/
struct mtuex_arg {
struct radix_node_head *rnh;
time_t nextstop;
};
static VNET_DEFINE(struct callout, rtq_mtutimer);
#define V_rtq_mtutimer VNET(rtq_mtutimer)
static int
in6_mtuexpire(struct radix_node *rn, void *rock)
{
struct rtentry *rt = (struct rtentry *)rn;
struct mtuex_arg *ap = rock;
/* sanity */
if (!rt)
panic("rt == NULL in in6_mtuexpire");
if (rt->rt_expire && !(rt->rt_flags & RTF_PROBEMTU)) {
if (rt->rt_expire <= time_uptime) {
rt->rt_flags |= RTF_PROBEMTU;
} else {
ap->nextstop = lmin(ap->nextstop, rt->rt_expire);
}
}
return 0;
}
#define MTUTIMO_DEFAULT (60*1)
static void
in6_mtutimo_one(struct radix_node_head *rnh)
{
struct mtuex_arg arg;
arg.rnh = rnh;
arg.nextstop = time_uptime + MTUTIMO_DEFAULT;
RADIX_NODE_HEAD_LOCK(rnh);
rnh->rnh_walktree(rnh, in6_mtuexpire, &arg);
RADIX_NODE_HEAD_UNLOCK(rnh);
}
static void
in6_mtutimo(void *rock)
{
CURVNET_SET_QUIET((struct vnet *) rock);
struct radix_node_head *rnh;
struct timeval atv;
u_int fibnum;
for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
rnh = rt_tables_get_rnh(fibnum, AF_INET6);
if (rnh != NULL)
in6_mtutimo_one(rnh);
}
atv.tv_sec = MTUTIMO_DEFAULT;
atv.tv_usec = 0;
callout_reset(&V_rtq_mtutimer, tvtohz(&atv), in6_mtutimo, rock);
CURVNET_RESTORE();
}
/*
* Initialize our routing tree.
*/
static VNET_DEFINE(int, _in6_rt_was_here);
#define V__in6_rt_was_here VNET(_in6_rt_was_here)
int
in6_inithead(void **head, int off)
{
struct radix_node_head *rnh;
if (!rn_inithead(head, offsetof(struct sockaddr_in6, sin6_addr) << 3))
return (0);
rnh = *head;
RADIX_NODE_HEAD_LOCK_INIT(rnh);
rnh->rnh_addaddr = in6_addroute;
if (V__in6_rt_was_here == 0) {
- callout_init(&V_rtq_mtutimer, CALLOUT_MPSAFE);
+ callout_init(&V_rtq_mtutimer, 1);
in6_mtutimo(curvnet); /* kick off timeout first time */
V__in6_rt_was_here = 1;
}
return (1);
}
#ifdef VIMAGE
int
in6_detachhead(void **head, int off)
{
callout_drain(&V_rtq_mtutimer);
return (1);
}
#endif
/*
* Extended API for IPv6 FIB support.
*/
void
in6_rtredirect(struct sockaddr *dst, struct sockaddr *gw, struct sockaddr *nm,
int flags, struct sockaddr *src, u_int fibnum)
{
rtredirect_fib(dst, gw, nm, flags, src, fibnum);
}
int
in6_rtrequest(int req, struct sockaddr *dst, struct sockaddr *gw,
struct sockaddr *mask, int flags, struct rtentry **ret_nrt, u_int fibnum)
{
return (rtrequest_fib(req, dst, gw, mask, flags, ret_nrt, fibnum));
}
void
in6_rtalloc(struct route_in6 *ro, u_int fibnum)
{
rtalloc_ign_fib((struct route *)ro, 0ul, fibnum);
}
void
in6_rtalloc_ign(struct route_in6 *ro, u_long ignflags, u_int fibnum)
{
rtalloc_ign_fib((struct route *)ro, ignflags, fibnum);
}
struct rtentry *
in6_rtalloc1(struct sockaddr *dst, int report, u_long ignflags, u_int fibnum)
{
return (rtalloc1_fib(dst, report, ignflags, fibnum));
}
Index: head/sys/netipsec/key.c
===================================================================
--- head/sys/netipsec/key.c (revision 283290)
+++ head/sys/netipsec/key.c (revision 283291)
@@ -1,7805 +1,7805 @@
/* $FreeBSD$ */
/* $KAME: key.c,v 1.191 2001/06/27 10:46:49 sakane Exp $ */
/*-
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the project nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* This code is referd to RFC 2367
*/
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/mbuf.h>
#include <sys/domain.h>
#include <sys/protosw.h>
#include <sys/malloc.h>
#include <sys/rmlock.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sysctl.h>
#include <sys/errno.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/refcount.h>
#include <sys/syslog.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/vnet.h>
#include <net/raw_cb.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/in_var.h>
#ifdef INET6
#include <netinet/ip6.h>
#include <netinet6/in6_var.h>
#include <netinet6/ip6_var.h>
#endif /* INET6 */
#if defined(INET) || defined(INET6)
#include <netinet/in_pcb.h>
#endif
#ifdef INET6
#include <netinet6/in6_pcb.h>
#endif /* INET6 */
#include <net/pfkeyv2.h>
#include <netipsec/keydb.h>
#include <netipsec/key.h>
#include <netipsec/keysock.h>
#include <netipsec/key_debug.h>
#include <netipsec/ipsec.h>
#ifdef INET6
#include <netipsec/ipsec6.h>
#endif
#include <netipsec/xform.h>
#include <machine/stdarg.h>
/* randomness */
#include <sys/random.h>
#define FULLMASK 0xff
#define _BITS(bytes) ((bytes) << 3)
/*
* Note on SA reference counting:
* - SAs that are not in DEAD state will have (total external reference + 1)
* following value in reference count field. they cannot be freed and are
* referenced from SA header.
* - SAs that are in DEAD state will have (total external reference)
* in reference count field. they are ready to be freed. reference from
* SA header will be removed in key_delsav(), when the reference count
* field hits 0 (= no external reference other than from SA header.
*/
VNET_DEFINE(u_int32_t, key_debug_level) = 0;
static VNET_DEFINE(u_int, key_spi_trycnt) = 1000;
static VNET_DEFINE(u_int32_t, key_spi_minval) = 0x100;
static VNET_DEFINE(u_int32_t, key_spi_maxval) = 0x0fffffff; /* XXX */
static VNET_DEFINE(u_int32_t, policy_id) = 0;
/*interval to initialize randseed,1(m)*/
static VNET_DEFINE(u_int, key_int_random) = 60;
/* interval to expire acquiring, 30(s)*/
static VNET_DEFINE(u_int, key_larval_lifetime) = 30;
/* counter for blocking SADB_ACQUIRE.*/
static VNET_DEFINE(int, key_blockacq_count) = 10;
/* lifetime for blocking SADB_ACQUIRE.*/
static VNET_DEFINE(int, key_blockacq_lifetime) = 20;
/* preferred old sa rather than new sa.*/
static VNET_DEFINE(int, key_preferred_oldsa) = 1;
#define V_key_spi_trycnt VNET(key_spi_trycnt)
#define V_key_spi_minval VNET(key_spi_minval)
#define V_key_spi_maxval VNET(key_spi_maxval)
#define V_policy_id VNET(policy_id)
#define V_key_int_random VNET(key_int_random)
#define V_key_larval_lifetime VNET(key_larval_lifetime)
#define V_key_blockacq_count VNET(key_blockacq_count)
#define V_key_blockacq_lifetime VNET(key_blockacq_lifetime)
#define V_key_preferred_oldsa VNET(key_preferred_oldsa)
static VNET_DEFINE(u_int32_t, acq_seq) = 0;
#define V_acq_seq VNET(acq_seq)
/* SPD */
static VNET_DEFINE(TAILQ_HEAD(_sptree, secpolicy), sptree[IPSEC_DIR_MAX]);
static struct rmlock sptree_lock;
#define V_sptree VNET(sptree)
#define SPTREE_LOCK_INIT() rm_init(&sptree_lock, "sptree")
#define SPTREE_LOCK_DESTROY() rm_destroy(&sptree_lock)
#define SPTREE_RLOCK_TRACKER struct rm_priotracker sptree_tracker
#define SPTREE_RLOCK() rm_rlock(&sptree_lock, &sptree_tracker)
#define SPTREE_RUNLOCK() rm_runlock(&sptree_lock, &sptree_tracker)
#define SPTREE_RLOCK_ASSERT() rm_assert(&sptree_lock, RA_RLOCKED)
#define SPTREE_WLOCK() rm_wlock(&sptree_lock)
#define SPTREE_WUNLOCK() rm_wunlock(&sptree_lock)
#define SPTREE_WLOCK_ASSERT() rm_assert(&sptree_lock, RA_WLOCKED)
#define SPTREE_UNLOCK_ASSERT() rm_assert(&sptree_lock, RA_UNLOCKED)
static VNET_DEFINE(LIST_HEAD(_sahtree, secashead), sahtree); /* SAD */
#define V_sahtree VNET(sahtree)
static struct mtx sahtree_lock;
#define SAHTREE_LOCK_INIT() \
mtx_init(&sahtree_lock, "sahtree", \
"fast ipsec security association database", MTX_DEF)
#define SAHTREE_LOCK_DESTROY() mtx_destroy(&sahtree_lock)
#define SAHTREE_LOCK() mtx_lock(&sahtree_lock)
#define SAHTREE_UNLOCK() mtx_unlock(&sahtree_lock)
#define SAHTREE_LOCK_ASSERT() mtx_assert(&sahtree_lock, MA_OWNED)
/* registed list */
static VNET_DEFINE(LIST_HEAD(_regtree, secreg), regtree[SADB_SATYPE_MAX + 1]);
#define V_regtree VNET(regtree)
static struct mtx regtree_lock;
#define REGTREE_LOCK_INIT() \
mtx_init(&regtree_lock, "regtree", "fast ipsec regtree", MTX_DEF)
#define REGTREE_LOCK_DESTROY() mtx_destroy(&regtree_lock)
#define REGTREE_LOCK() mtx_lock(&regtree_lock)
#define REGTREE_UNLOCK() mtx_unlock(&regtree_lock)
#define REGTREE_LOCK_ASSERT() mtx_assert(&regtree_lock, MA_OWNED)
static VNET_DEFINE(LIST_HEAD(_acqtree, secacq), acqtree); /* acquiring list */
#define V_acqtree VNET(acqtree)
static struct mtx acq_lock;
#define ACQ_LOCK_INIT() \
mtx_init(&acq_lock, "acqtree", "fast ipsec acquire list", MTX_DEF)
#define ACQ_LOCK_DESTROY() mtx_destroy(&acq_lock)
#define ACQ_LOCK() mtx_lock(&acq_lock)
#define ACQ_UNLOCK() mtx_unlock(&acq_lock)
#define ACQ_LOCK_ASSERT() mtx_assert(&acq_lock, MA_OWNED)
/* SP acquiring list */
static VNET_DEFINE(LIST_HEAD(_spacqtree, secspacq), spacqtree);
#define V_spacqtree VNET(spacqtree)
static struct mtx spacq_lock;
#define SPACQ_LOCK_INIT() \
mtx_init(&spacq_lock, "spacqtree", \
"fast ipsec security policy acquire list", MTX_DEF)
#define SPACQ_LOCK_DESTROY() mtx_destroy(&spacq_lock)
#define SPACQ_LOCK() mtx_lock(&spacq_lock)
#define SPACQ_UNLOCK() mtx_unlock(&spacq_lock)
#define SPACQ_LOCK_ASSERT() mtx_assert(&spacq_lock, MA_OWNED)
/* search order for SAs */
static const u_int saorder_state_valid_prefer_old[] = {
SADB_SASTATE_DYING, SADB_SASTATE_MATURE,
};
static const u_int saorder_state_valid_prefer_new[] = {
SADB_SASTATE_MATURE, SADB_SASTATE_DYING,
};
static const u_int saorder_state_alive[] = {
/* except DEAD */
SADB_SASTATE_MATURE, SADB_SASTATE_DYING, SADB_SASTATE_LARVAL
};
static const u_int saorder_state_any[] = {
SADB_SASTATE_MATURE, SADB_SASTATE_DYING,
SADB_SASTATE_LARVAL, SADB_SASTATE_DEAD
};
static const int minsize[] = {
sizeof(struct sadb_msg), /* SADB_EXT_RESERVED */
sizeof(struct sadb_sa), /* SADB_EXT_SA */
sizeof(struct sadb_lifetime), /* SADB_EXT_LIFETIME_CURRENT */
sizeof(struct sadb_lifetime), /* SADB_EXT_LIFETIME_HARD */
sizeof(struct sadb_lifetime), /* SADB_EXT_LIFETIME_SOFT */
sizeof(struct sadb_address), /* SADB_EXT_ADDRESS_SRC */
sizeof(struct sadb_address), /* SADB_EXT_ADDRESS_DST */
sizeof(struct sadb_address), /* SADB_EXT_ADDRESS_PROXY */
sizeof(struct sadb_key), /* SADB_EXT_KEY_AUTH */
sizeof(struct sadb_key), /* SADB_EXT_KEY_ENCRYPT */
sizeof(struct sadb_ident), /* SADB_EXT_IDENTITY_SRC */
sizeof(struct sadb_ident), /* SADB_EXT_IDENTITY_DST */
sizeof(struct sadb_sens), /* SADB_EXT_SENSITIVITY */
sizeof(struct sadb_prop), /* SADB_EXT_PROPOSAL */
sizeof(struct sadb_supported), /* SADB_EXT_SUPPORTED_AUTH */
sizeof(struct sadb_supported), /* SADB_EXT_SUPPORTED_ENCRYPT */
sizeof(struct sadb_spirange), /* SADB_EXT_SPIRANGE */
0, /* SADB_X_EXT_KMPRIVATE */
sizeof(struct sadb_x_policy), /* SADB_X_EXT_POLICY */
sizeof(struct sadb_x_sa2), /* SADB_X_SA2 */
sizeof(struct sadb_x_nat_t_type),/* SADB_X_EXT_NAT_T_TYPE */
sizeof(struct sadb_x_nat_t_port),/* SADB_X_EXT_NAT_T_SPORT */
sizeof(struct sadb_x_nat_t_port),/* SADB_X_EXT_NAT_T_DPORT */
sizeof(struct sadb_address), /* SADB_X_EXT_NAT_T_OAI */
sizeof(struct sadb_address), /* SADB_X_EXT_NAT_T_OAR */
sizeof(struct sadb_x_nat_t_frag),/* SADB_X_EXT_NAT_T_FRAG */
};
static const int maxsize[] = {
sizeof(struct sadb_msg), /* SADB_EXT_RESERVED */
sizeof(struct sadb_sa), /* SADB_EXT_SA */
sizeof(struct sadb_lifetime), /* SADB_EXT_LIFETIME_CURRENT */
sizeof(struct sadb_lifetime), /* SADB_EXT_LIFETIME_HARD */
sizeof(struct sadb_lifetime), /* SADB_EXT_LIFETIME_SOFT */
0, /* SADB_EXT_ADDRESS_SRC */
0, /* SADB_EXT_ADDRESS_DST */
0, /* SADB_EXT_ADDRESS_PROXY */
0, /* SADB_EXT_KEY_AUTH */
0, /* SADB_EXT_KEY_ENCRYPT */
0, /* SADB_EXT_IDENTITY_SRC */
0, /* SADB_EXT_IDENTITY_DST */
0, /* SADB_EXT_SENSITIVITY */
0, /* SADB_EXT_PROPOSAL */
0, /* SADB_EXT_SUPPORTED_AUTH */
0, /* SADB_EXT_SUPPORTED_ENCRYPT */
sizeof(struct sadb_spirange), /* SADB_EXT_SPIRANGE */
0, /* SADB_X_EXT_KMPRIVATE */
0, /* SADB_X_EXT_POLICY */
sizeof(struct sadb_x_sa2), /* SADB_X_SA2 */
sizeof(struct sadb_x_nat_t_type),/* SADB_X_EXT_NAT_T_TYPE */
sizeof(struct sadb_x_nat_t_port),/* SADB_X_EXT_NAT_T_SPORT */
sizeof(struct sadb_x_nat_t_port),/* SADB_X_EXT_NAT_T_DPORT */
0, /* SADB_X_EXT_NAT_T_OAI */
0, /* SADB_X_EXT_NAT_T_OAR */
sizeof(struct sadb_x_nat_t_frag),/* SADB_X_EXT_NAT_T_FRAG */
};
static VNET_DEFINE(int, ipsec_esp_keymin) = 256;
static VNET_DEFINE(int, ipsec_esp_auth) = 0;
static VNET_DEFINE(int, ipsec_ah_keymin) = 128;
#define V_ipsec_esp_keymin VNET(ipsec_esp_keymin)
#define V_ipsec_esp_auth VNET(ipsec_esp_auth)
#define V_ipsec_ah_keymin VNET(ipsec_ah_keymin)
#ifdef SYSCTL_DECL
SYSCTL_DECL(_net_key);
#endif
SYSCTL_INT(_net_key, KEYCTL_DEBUG_LEVEL, debug,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_debug_level), 0, "");
/* max count of trial for the decision of spi value */
SYSCTL_INT(_net_key, KEYCTL_SPI_TRY, spi_trycnt,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_spi_trycnt), 0, "");
/* minimum spi value to allocate automatically. */
SYSCTL_INT(_net_key, KEYCTL_SPI_MIN_VALUE, spi_minval,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_spi_minval), 0, "");
/* maximun spi value to allocate automatically. */
SYSCTL_INT(_net_key, KEYCTL_SPI_MAX_VALUE, spi_maxval,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_spi_maxval), 0, "");
/* interval to initialize randseed */
SYSCTL_INT(_net_key, KEYCTL_RANDOM_INT, int_random,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_int_random), 0, "");
/* lifetime for larval SA */
SYSCTL_INT(_net_key, KEYCTL_LARVAL_LIFETIME, larval_lifetime,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_larval_lifetime), 0, "");
/* counter for blocking to send SADB_ACQUIRE to IKEd */
SYSCTL_INT(_net_key, KEYCTL_BLOCKACQ_COUNT, blockacq_count,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_blockacq_count), 0, "");
/* lifetime for blocking to send SADB_ACQUIRE to IKEd */
SYSCTL_INT(_net_key, KEYCTL_BLOCKACQ_LIFETIME, blockacq_lifetime,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_blockacq_lifetime), 0, "");
/* ESP auth */
SYSCTL_INT(_net_key, KEYCTL_ESP_AUTH, esp_auth,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ipsec_esp_auth), 0, "");
/* minimum ESP key length */
SYSCTL_INT(_net_key, KEYCTL_ESP_KEYMIN, esp_keymin,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ipsec_esp_keymin), 0, "");
/* minimum AH key length */
SYSCTL_INT(_net_key, KEYCTL_AH_KEYMIN, ah_keymin,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ipsec_ah_keymin), 0, "");
/* perfered old SA rather than new SA */
SYSCTL_INT(_net_key, KEYCTL_PREFERED_OLDSA, preferred_oldsa,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(key_preferred_oldsa), 0, "");
#define __LIST_CHAINED(elm) \
(!((elm)->chain.le_next == NULL && (elm)->chain.le_prev == NULL))
#define LIST_INSERT_TAIL(head, elm, type, field) \
do {\
struct type *curelm = LIST_FIRST(head); \
if (curelm == NULL) {\
LIST_INSERT_HEAD(head, elm, field); \
} else { \
while (LIST_NEXT(curelm, field)) \
curelm = LIST_NEXT(curelm, field);\
LIST_INSERT_AFTER(curelm, elm, field);\
}\
} while (0)
#define KEY_CHKSASTATE(head, sav, name) \
do { \
if ((head) != (sav)) { \
ipseclog((LOG_DEBUG, "%s: state mismatched (TREE=%d SA=%d)\n", \
(name), (head), (sav))); \
continue; \
} \
} while (0)
#define KEY_CHKSPDIR(head, sp, name) \
do { \
if ((head) != (sp)) { \
ipseclog((LOG_DEBUG, "%s: direction mismatched (TREE=%d SP=%d), " \
"anyway continue.\n", \
(name), (head), (sp))); \
} \
} while (0)
MALLOC_DEFINE(M_IPSEC_SA, "secasvar", "ipsec security association");
MALLOC_DEFINE(M_IPSEC_SAH, "sahead", "ipsec sa head");
MALLOC_DEFINE(M_IPSEC_SP, "ipsecpolicy", "ipsec security policy");
MALLOC_DEFINE(M_IPSEC_SR, "ipsecrequest", "ipsec security request");
MALLOC_DEFINE(M_IPSEC_MISC, "ipsec-misc", "ipsec miscellaneous");
MALLOC_DEFINE(M_IPSEC_SAQ, "ipsec-saq", "ipsec sa acquire");
MALLOC_DEFINE(M_IPSEC_SAR, "ipsec-reg", "ipsec sa acquire");
/*
* set parameters into secpolicyindex buffer.
* Must allocate secpolicyindex buffer passed to this function.
*/
#define KEY_SETSECSPIDX(_dir, s, d, ps, pd, ulp, idx) \
do { \
bzero((idx), sizeof(struct secpolicyindex)); \
(idx)->dir = (_dir); \
(idx)->prefs = (ps); \
(idx)->prefd = (pd); \
(idx)->ul_proto = (ulp); \
bcopy((s), &(idx)->src, ((const struct sockaddr *)(s))->sa_len); \
bcopy((d), &(idx)->dst, ((const struct sockaddr *)(d))->sa_len); \
} while (0)
/*
* set parameters into secasindex buffer.
* Must allocate secasindex buffer before calling this function.
*/
#define KEY_SETSECASIDX(p, m, r, s, d, idx) \
do { \
bzero((idx), sizeof(struct secasindex)); \
(idx)->proto = (p); \
(idx)->mode = (m); \
(idx)->reqid = (r); \
bcopy((s), &(idx)->src, ((const struct sockaddr *)(s))->sa_len); \
bcopy((d), &(idx)->dst, ((const struct sockaddr *)(d))->sa_len); \
} while (0)
/* key statistics */
struct _keystat {
u_long getspi_count; /* the avarage of count to try to get new SPI */
} keystat;
struct sadb_msghdr {
struct sadb_msg *msg;
struct sadb_ext *ext[SADB_EXT_MAX + 1];
int extoff[SADB_EXT_MAX + 1];
int extlen[SADB_EXT_MAX + 1];
};
#ifndef IPSEC_DEBUG2
static struct callout key_timer;
#endif
static struct secasvar *key_allocsa_policy(const struct secasindex *);
static void key_freesp_so(struct secpolicy **);
static struct secasvar *key_do_allocsa_policy(struct secashead *, u_int);
static void key_unlink(struct secpolicy *);
static struct secpolicy *key_getsp(struct secpolicyindex *);
static struct secpolicy *key_getspbyid(u_int32_t);
static u_int32_t key_newreqid(void);
static struct mbuf *key_gather_mbuf(struct mbuf *,
const struct sadb_msghdr *, int, int, ...);
static int key_spdadd(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static u_int32_t key_getnewspid(void);
static int key_spddelete(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_spddelete2(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_spdget(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_spdflush(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_spddump(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static struct mbuf *key_setdumpsp(struct secpolicy *,
u_int8_t, u_int32_t, u_int32_t);
static u_int key_getspreqmsglen(struct secpolicy *);
static int key_spdexpire(struct secpolicy *);
static struct secashead *key_newsah(struct secasindex *);
static void key_delsah(struct secashead *);
static struct secasvar *key_newsav(struct mbuf *,
const struct sadb_msghdr *, struct secashead *, int *,
const char*, int);
#define KEY_NEWSAV(m, sadb, sah, e) \
key_newsav(m, sadb, sah, e, __FILE__, __LINE__)
static void key_delsav(struct secasvar *);
static struct secashead *key_getsah(struct secasindex *);
static struct secasvar *key_checkspidup(struct secasindex *, u_int32_t);
static struct secasvar *key_getsavbyspi(struct secashead *, u_int32_t);
static int key_setsaval(struct secasvar *, struct mbuf *,
const struct sadb_msghdr *);
static int key_mature(struct secasvar *);
static struct mbuf *key_setdumpsa(struct secasvar *, u_int8_t,
u_int8_t, u_int32_t, u_int32_t);
static struct mbuf *key_setsadbmsg(u_int8_t, u_int16_t, u_int8_t,
u_int32_t, pid_t, u_int16_t);
static struct mbuf *key_setsadbsa(struct secasvar *);
static struct mbuf *key_setsadbaddr(u_int16_t,
const struct sockaddr *, u_int8_t, u_int16_t);
#ifdef IPSEC_NAT_T
static struct mbuf *key_setsadbxport(u_int16_t, u_int16_t);
static struct mbuf *key_setsadbxtype(u_int16_t);
#endif
static void key_porttosaddr(struct sockaddr *, u_int16_t);
#define KEY_PORTTOSADDR(saddr, port) \
key_porttosaddr((struct sockaddr *)(saddr), (port))
static struct mbuf *key_setsadbxsa2(u_int8_t, u_int32_t, u_int32_t);
static struct mbuf *key_setsadbxpolicy(u_int16_t, u_int8_t,
u_int32_t);
static struct seckey *key_dup_keymsg(const struct sadb_key *, u_int,
struct malloc_type *);
static struct seclifetime *key_dup_lifemsg(const struct sadb_lifetime *src,
struct malloc_type *type);
#ifdef INET6
static int key_ismyaddr6(struct sockaddr_in6 *);
#endif
/* flags for key_cmpsaidx() */
#define CMP_HEAD 1 /* protocol, addresses. */
#define CMP_MODE_REQID 2 /* additionally HEAD, reqid, mode. */
#define CMP_REQID 3 /* additionally HEAD, reaid. */
#define CMP_EXACTLY 4 /* all elements. */
static int key_cmpsaidx(const struct secasindex *,
const struct secasindex *, int);
static int key_cmpspidx_exactly(struct secpolicyindex *,
struct secpolicyindex *);
static int key_cmpspidx_withmask(struct secpolicyindex *,
struct secpolicyindex *);
static int key_sockaddrcmp(const struct sockaddr *,
const struct sockaddr *, int);
static int key_bbcmp(const void *, const void *, u_int);
static u_int16_t key_satype2proto(u_int8_t);
static u_int8_t key_proto2satype(u_int16_t);
static int key_getspi(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static u_int32_t key_do_getnewspi(struct sadb_spirange *,
struct secasindex *);
static int key_update(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
#ifdef IPSEC_DOSEQCHECK
static struct secasvar *key_getsavbyseq(struct secashead *, u_int32_t);
#endif
static int key_add(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_setident(struct secashead *, struct mbuf *,
const struct sadb_msghdr *);
static struct mbuf *key_getmsgbuf_x1(struct mbuf *,
const struct sadb_msghdr *);
static int key_delete(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_delete_all(struct socket *, struct mbuf *,
const struct sadb_msghdr *, u_int16_t);
static int key_get(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static void key_getcomb_setlifetime(struct sadb_comb *);
static struct mbuf *key_getcomb_esp(void);
static struct mbuf *key_getcomb_ah(void);
static struct mbuf *key_getcomb_ipcomp(void);
static struct mbuf *key_getprop(const struct secasindex *);
static int key_acquire(const struct secasindex *, struct secpolicy *);
static struct secacq *key_newacq(const struct secasindex *);
static struct secacq *key_getacq(const struct secasindex *);
static struct secacq *key_getacqbyseq(u_int32_t);
static struct secspacq *key_newspacq(struct secpolicyindex *);
static struct secspacq *key_getspacq(struct secpolicyindex *);
static int key_acquire2(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_register(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_expire(struct secasvar *, int);
static int key_flush(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_dump(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_promisc(struct socket *, struct mbuf *,
const struct sadb_msghdr *);
static int key_senderror(struct socket *, struct mbuf *, int);
static int key_validate_ext(const struct sadb_ext *, int);
static int key_align(struct mbuf *, struct sadb_msghdr *);
static struct mbuf *key_setlifetime(struct seclifetime *src,
u_int16_t exttype);
static struct mbuf *key_setkey(struct seckey *src, u_int16_t exttype);
#if 0
static const char *key_getfqdn(void);
static const char *key_getuserfqdn(void);
#endif
static void key_sa_chgstate(struct secasvar *, u_int8_t);
static __inline void
sa_initref(struct secasvar *sav)
{
refcount_init(&sav->refcnt, 1);
}
static __inline void
sa_addref(struct secasvar *sav)
{
refcount_acquire(&sav->refcnt);
IPSEC_ASSERT(sav->refcnt != 0, ("SA refcnt overflow"));
}
static __inline int
sa_delref(struct secasvar *sav)
{
IPSEC_ASSERT(sav->refcnt > 0, ("SA refcnt underflow"));
return (refcount_release(&sav->refcnt));
}
#define SP_ADDREF(p) refcount_acquire(&(p)->refcnt)
#define SP_DELREF(p) refcount_release(&(p)->refcnt)
/*
* Update the refcnt while holding the SPTREE lock.
*/
void
key_addref(struct secpolicy *sp)
{
SP_ADDREF(sp);
}
/*
* Return 0 when there are known to be no SP's for the specified
* direction. Otherwise return 1. This is used by IPsec code
* to optimize performance.
*/
int
key_havesp(u_int dir)
{
return (dir == IPSEC_DIR_INBOUND || dir == IPSEC_DIR_OUTBOUND ?
TAILQ_FIRST(&V_sptree[dir]) != NULL : 1);
}
/* %%% IPsec policy management */
/*
* allocating a SP for OUTBOUND or INBOUND packet.
* Must call key_freesp() later.
* OUT: NULL: not found
* others: found and return the pointer.
*/
struct secpolicy *
key_allocsp(struct secpolicyindex *spidx, u_int dir, const char* where,
int tag)
{
SPTREE_RLOCK_TRACKER;
struct secpolicy *sp;
IPSEC_ASSERT(spidx != NULL, ("null spidx"));
IPSEC_ASSERT(dir == IPSEC_DIR_INBOUND || dir == IPSEC_DIR_OUTBOUND,
("invalid direction %u", dir));
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s from %s:%u\n", __func__, where, tag));
/* get a SP entry */
KEYDEBUG(KEYDEBUG_IPSEC_DATA,
printf("*** objects\n");
kdebug_secpolicyindex(spidx));
SPTREE_RLOCK();
TAILQ_FOREACH(sp, &V_sptree[dir], chain) {
KEYDEBUG(KEYDEBUG_IPSEC_DATA,
printf("*** in SPD\n");
kdebug_secpolicyindex(&sp->spidx));
if (key_cmpspidx_withmask(&sp->spidx, spidx))
goto found;
}
sp = NULL;
found:
if (sp) {
/* sanity check */
KEY_CHKSPDIR(sp->spidx.dir, dir, __func__);
/* found a SPD entry */
sp->lastused = time_second;
SP_ADDREF(sp);
}
SPTREE_RUNLOCK();
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s return SP:%p (ID=%u) refcnt %u\n", __func__,
sp, sp ? sp->id : 0, sp ? sp->refcnt : 0));
return sp;
}
/*
* allocating a SP for OUTBOUND or INBOUND packet.
* Must call key_freesp() later.
* OUT: NULL: not found
* others: found and return the pointer.
*/
struct secpolicy *
key_allocsp2(u_int32_t spi, union sockaddr_union *dst, u_int8_t proto,
u_int dir, const char* where, int tag)
{
SPTREE_RLOCK_TRACKER;
struct secpolicy *sp;
IPSEC_ASSERT(dst != NULL, ("null dst"));
IPSEC_ASSERT(dir == IPSEC_DIR_INBOUND || dir == IPSEC_DIR_OUTBOUND,
("invalid direction %u", dir));
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s from %s:%u\n", __func__, where, tag));
/* get a SP entry */
KEYDEBUG(KEYDEBUG_IPSEC_DATA,
printf("*** objects\n");
printf("spi %u proto %u dir %u\n", spi, proto, dir);
kdebug_sockaddr(&dst->sa));
SPTREE_RLOCK();
TAILQ_FOREACH(sp, &V_sptree[dir], chain) {
KEYDEBUG(KEYDEBUG_IPSEC_DATA,
printf("*** in SPD\n");
kdebug_secpolicyindex(&sp->spidx));
/* compare simple values, then dst address */
if (sp->spidx.ul_proto != proto)
continue;
/* NB: spi's must exist and match */
if (!sp->req || !sp->req->sav || sp->req->sav->spi != spi)
continue;
if (key_sockaddrcmp(&sp->spidx.dst.sa, &dst->sa, 1) == 0)
goto found;
}
sp = NULL;
found:
if (sp) {
/* sanity check */
KEY_CHKSPDIR(sp->spidx.dir, dir, __func__);
/* found a SPD entry */
sp->lastused = time_second;
SP_ADDREF(sp);
}
SPTREE_RUNLOCK();
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s return SP:%p (ID=%u) refcnt %u\n", __func__,
sp, sp ? sp->id : 0, sp ? sp->refcnt : 0));
return sp;
}
#if 0
/*
* return a policy that matches this particular inbound packet.
* XXX slow
*/
struct secpolicy *
key_gettunnel(const struct sockaddr *osrc,
const struct sockaddr *odst,
const struct sockaddr *isrc,
const struct sockaddr *idst,
const char* where, int tag)
{
struct secpolicy *sp;
const int dir = IPSEC_DIR_INBOUND;
struct ipsecrequest *r1, *r2, *p;
struct secpolicyindex spidx;
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s from %s:%u\n", __func__, where, tag));
if (isrc->sa_family != idst->sa_family) {
ipseclog((LOG_ERR, "%s: protocol family mismatched %d != %d\n.",
__func__, isrc->sa_family, idst->sa_family));
sp = NULL;
goto done;
}
SPTREE_LOCK();
LIST_FOREACH(sp, &V_sptree[dir], chain) {
if (sp->state == IPSEC_SPSTATE_DEAD)
continue;
r1 = r2 = NULL;
for (p = sp->req; p; p = p->next) {
if (p->saidx.mode != IPSEC_MODE_TUNNEL)
continue;
r1 = r2;
r2 = p;
if (!r1) {
/* here we look at address matches only */
spidx = sp->spidx;
if (isrc->sa_len > sizeof(spidx.src) ||
idst->sa_len > sizeof(spidx.dst))
continue;
bcopy(isrc, &spidx.src, isrc->sa_len);
bcopy(idst, &spidx.dst, idst->sa_len);
if (!key_cmpspidx_withmask(&sp->spidx, &spidx))
continue;
} else {
if (key_sockaddrcmp(&r1->saidx.src.sa, isrc, 0) ||
key_sockaddrcmp(&r1->saidx.dst.sa, idst, 0))
continue;
}
if (key_sockaddrcmp(&r2->saidx.src.sa, osrc, 0) ||
key_sockaddrcmp(&r2->saidx.dst.sa, odst, 0))
continue;
goto found;
}
}
sp = NULL;
found:
if (sp) {
sp->lastused = time_second;
SP_ADDREF(sp);
}
SPTREE_UNLOCK();
done:
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s return SP:%p (ID=%u) refcnt %u\n", __func__,
sp, sp ? sp->id : 0, sp ? sp->refcnt : 0));
return sp;
}
#endif
/*
* allocating an SA entry for an *OUTBOUND* packet.
* checking each request entries in SP, and acquire an SA if need.
* OUT: 0: there are valid requests.
* ENOENT: policy may be valid, but SA with REQUIRE is on acquiring.
*/
int
key_checkrequest(struct ipsecrequest *isr, const struct secasindex *saidx)
{
u_int level;
int error;
struct secasvar *sav;
IPSEC_ASSERT(isr != NULL, ("null isr"));
IPSEC_ASSERT(saidx != NULL, ("null saidx"));
IPSEC_ASSERT(saidx->mode == IPSEC_MODE_TRANSPORT ||
saidx->mode == IPSEC_MODE_TUNNEL,
("unexpected policy %u", saidx->mode));
/*
* XXX guard against protocol callbacks from the crypto
* thread as they reference ipsecrequest.sav which we
* temporarily null out below. Need to rethink how we
* handle bundled SA's in the callback thread.
*/
IPSECREQUEST_LOCK_ASSERT(isr);
/* get current level */
level = ipsec_get_reqlevel(isr);
/*
* We check new SA in the IPsec request because a different
* SA may be involved each time this request is checked, either
* because new SAs are being configured, or this request is
* associated with an unconnected datagram socket, or this request
* is associated with a system default policy.
*
* key_allocsa_policy should allocate the oldest SA available.
* See key_do_allocsa_policy(), and draft-jenkins-ipsec-rekeying-03.txt.
*/
sav = key_allocsa_policy(saidx);
if (sav != isr->sav) {
/* SA need to be updated. */
if (!IPSECREQUEST_UPGRADE(isr)) {
/* Kick everyone off. */
IPSECREQUEST_UNLOCK(isr);
IPSECREQUEST_WLOCK(isr);
}
if (isr->sav != NULL)
KEY_FREESAV(&isr->sav);
isr->sav = sav;
IPSECREQUEST_DOWNGRADE(isr);
} else if (sav != NULL)
KEY_FREESAV(&sav);
/* When there is SA. */
if (isr->sav != NULL) {
if (isr->sav->state != SADB_SASTATE_MATURE &&
isr->sav->state != SADB_SASTATE_DYING)
return EINVAL;
return 0;
}
/* there is no SA */
error = key_acquire(saidx, isr->sp);
if (error != 0) {
/* XXX What should I do ? */
ipseclog((LOG_DEBUG, "%s: error %d returned from key_acquire\n",
__func__, error));
return error;
}
if (level != IPSEC_LEVEL_REQUIRE) {
/* XXX sigh, the interface to this routine is botched */
IPSEC_ASSERT(isr->sav == NULL, ("unexpected SA"));
return 0;
} else {
return ENOENT;
}
}
/*
* allocating a SA for policy entry from SAD.
* NOTE: searching SAD of aliving state.
* OUT: NULL: not found.
* others: found and return the pointer.
*/
static struct secasvar *
key_allocsa_policy(const struct secasindex *saidx)
{
#define N(a) _ARRAYLEN(a)
struct secashead *sah;
struct secasvar *sav;
u_int stateidx, arraysize;
const u_int *state_valid;
state_valid = NULL; /* silence gcc */
arraysize = 0; /* silence gcc */
SAHTREE_LOCK();
LIST_FOREACH(sah, &V_sahtree, chain) {
if (sah->state == SADB_SASTATE_DEAD)
continue;
if (key_cmpsaidx(&sah->saidx, saidx, CMP_MODE_REQID)) {
if (V_key_preferred_oldsa) {
state_valid = saorder_state_valid_prefer_old;
arraysize = N(saorder_state_valid_prefer_old);
} else {
state_valid = saorder_state_valid_prefer_new;
arraysize = N(saorder_state_valid_prefer_new);
}
break;
}
}
SAHTREE_UNLOCK();
if (sah == NULL)
return NULL;
/* search valid state */
for (stateidx = 0; stateidx < arraysize; stateidx++) {
sav = key_do_allocsa_policy(sah, state_valid[stateidx]);
if (sav != NULL)
return sav;
}
return NULL;
#undef N
}
/*
* searching SAD with direction, protocol, mode and state.
* called by key_allocsa_policy().
* OUT:
* NULL : not found
* others : found, pointer to a SA.
*/
static struct secasvar *
key_do_allocsa_policy(struct secashead *sah, u_int state)
{
struct secasvar *sav, *nextsav, *candidate, *d;
/* initilize */
candidate = NULL;
SAHTREE_LOCK();
for (sav = LIST_FIRST(&sah->savtree[state]);
sav != NULL;
sav = nextsav) {
nextsav = LIST_NEXT(sav, chain);
/* sanity check */
KEY_CHKSASTATE(sav->state, state, __func__);
/* initialize */
if (candidate == NULL) {
candidate = sav;
continue;
}
/* Which SA is the better ? */
IPSEC_ASSERT(candidate->lft_c != NULL,
("null candidate lifetime"));
IPSEC_ASSERT(sav->lft_c != NULL, ("null sav lifetime"));
/* What the best method is to compare ? */
if (V_key_preferred_oldsa) {
if (candidate->lft_c->addtime >
sav->lft_c->addtime) {
candidate = sav;
}
continue;
/*NOTREACHED*/
}
/* preferred new sa rather than old sa */
if (candidate->lft_c->addtime <
sav->lft_c->addtime) {
d = candidate;
candidate = sav;
} else
d = sav;
/*
* prepared to delete the SA when there is more
* suitable candidate and the lifetime of the SA is not
* permanent.
*/
if (d->lft_h->addtime != 0) {
struct mbuf *m, *result;
u_int8_t satype;
key_sa_chgstate(d, SADB_SASTATE_DEAD);
IPSEC_ASSERT(d->refcnt > 0, ("bogus ref count"));
satype = key_proto2satype(d->sah->saidx.proto);
if (satype == 0)
goto msgfail;
m = key_setsadbmsg(SADB_DELETE, 0,
satype, 0, 0, d->refcnt - 1);
if (!m)
goto msgfail;
result = m;
/* set sadb_address for saidx's. */
m = key_setsadbaddr(SADB_EXT_ADDRESS_SRC,
&d->sah->saidx.src.sa,
d->sah->saidx.src.sa.sa_len << 3,
IPSEC_ULPROTO_ANY);
if (!m)
goto msgfail;
m_cat(result, m);
/* set sadb_address for saidx's. */
m = key_setsadbaddr(SADB_EXT_ADDRESS_DST,
&d->sah->saidx.dst.sa,
d->sah->saidx.dst.sa.sa_len << 3,
IPSEC_ULPROTO_ANY);
if (!m)
goto msgfail;
m_cat(result, m);
/* create SA extension */
m = key_setsadbsa(d);
if (!m)
goto msgfail;
m_cat(result, m);
if (result->m_len < sizeof(struct sadb_msg)) {
result = m_pullup(result,
sizeof(struct sadb_msg));
if (result == NULL)
goto msgfail;
}
result->m_pkthdr.len = 0;
for (m = result; m; m = m->m_next)
result->m_pkthdr.len += m->m_len;
mtod(result, struct sadb_msg *)->sadb_msg_len =
PFKEY_UNIT64(result->m_pkthdr.len);
if (key_sendup_mbuf(NULL, result,
KEY_SENDUP_REGISTERED))
goto msgfail;
msgfail:
KEY_FREESAV(&d);
}
}
if (candidate) {
sa_addref(candidate);
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s cause refcnt++:%d SA:%p\n",
__func__, candidate->refcnt, candidate));
}
SAHTREE_UNLOCK();
return candidate;
}
/*
* allocating a usable SA entry for a *INBOUND* packet.
* Must call key_freesav() later.
* OUT: positive: pointer to a usable sav (i.e. MATURE or DYING state).
* NULL: not found, or error occured.
*
* In the comparison, no source address is used--for RFC2401 conformance.
* To quote, from section 4.1:
* A security association is uniquely identified by a triple consisting
* of a Security Parameter Index (SPI), an IP Destination Address, and a
* security protocol (AH or ESP) identifier.
* Note that, however, we do need to keep source address in IPsec SA.
* IKE specification and PF_KEY specification do assume that we
* keep source address in IPsec SA. We see a tricky situation here.
*/
struct secasvar *
key_allocsa(union sockaddr_union *dst, u_int proto, u_int32_t spi,
const char* where, int tag)
{
struct secashead *sah;
struct secasvar *sav;
u_int stateidx, arraysize, state;
const u_int *saorder_state_valid;
#ifdef IPSEC_NAT_T
int natt_chkport;
#endif
IPSEC_ASSERT(dst != NULL, ("null dst address"));
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s from %s:%u\n", __func__, where, tag));
#ifdef IPSEC_NAT_T
natt_chkport = (dst->sa.sa_family == AF_INET &&
dst->sa.sa_len == sizeof(struct sockaddr_in) &&
dst->sin.sin_port != 0);
#endif
/*
* searching SAD.
* XXX: to be checked internal IP header somewhere. Also when
* IPsec tunnel packet is received. But ESP tunnel mode is
* encrypted so we can't check internal IP header.
*/
SAHTREE_LOCK();
if (V_key_preferred_oldsa) {
saorder_state_valid = saorder_state_valid_prefer_old;
arraysize = _ARRAYLEN(saorder_state_valid_prefer_old);
} else {
saorder_state_valid = saorder_state_valid_prefer_new;
arraysize = _ARRAYLEN(saorder_state_valid_prefer_new);
}
LIST_FOREACH(sah, &V_sahtree, chain) {
int checkport;
/* search valid state */
for (stateidx = 0; stateidx < arraysize; stateidx++) {
state = saorder_state_valid[stateidx];
LIST_FOREACH(sav, &sah->savtree[state], chain) {
/* sanity check */
KEY_CHKSASTATE(sav->state, state, __func__);
/* do not return entries w/ unusable state */
if (sav->state != SADB_SASTATE_MATURE &&
sav->state != SADB_SASTATE_DYING)
continue;
if (proto != sav->sah->saidx.proto)
continue;
if (spi != sav->spi)
continue;
checkport = 0;
#ifdef IPSEC_NAT_T
/*
* Really only check ports when this is a NAT-T
* SA. Otherwise other lookups providing ports
* might suffer.
*/
if (sav->natt_type && natt_chkport)
checkport = 1;
#endif
#if 0 /* don't check src */
/* check src address */
if (key_sockaddrcmp(&src->sa,
&sav->sah->saidx.src.sa, checkport) != 0)
continue;
#endif
/* check dst address */
if (key_sockaddrcmp(&dst->sa,
&sav->sah->saidx.dst.sa, checkport) != 0)
continue;
sa_addref(sav);
goto done;
}
}
}
sav = NULL;
done:
SAHTREE_UNLOCK();
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s return SA:%p; refcnt %u\n", __func__,
sav, sav ? sav->refcnt : 0));
return sav;
}
/*
* Must be called after calling key_allocsp().
* For both the packet without socket and key_freeso().
*/
void
_key_freesp(struct secpolicy **spp, const char* where, int tag)
{
struct ipsecrequest *isr, *nextisr;
struct secpolicy *sp = *spp;
IPSEC_ASSERT(sp != NULL, ("null sp"));
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s SP:%p (ID=%u) from %s:%u; refcnt now %u\n",
__func__, sp, sp->id, where, tag, sp->refcnt));
if (SP_DELREF(sp) == 0)
return;
*spp = NULL;
for (isr = sp->req; isr != NULL; isr = nextisr) {
if (isr->sav != NULL) {
KEY_FREESAV(&isr->sav);
isr->sav = NULL;
}
nextisr = isr->next;
ipsec_delisr(isr);
}
free(sp, M_IPSEC_SP);
}
static void
key_unlink(struct secpolicy *sp)
{
IPSEC_ASSERT(sp != NULL, ("null sp"));
IPSEC_ASSERT(sp->spidx.dir == IPSEC_DIR_INBOUND ||
sp->spidx.dir == IPSEC_DIR_OUTBOUND,
("invalid direction %u", sp->spidx.dir));
SPTREE_UNLOCK_ASSERT();
SPTREE_WLOCK();
if (sp->state == IPSEC_SPSTATE_DEAD) {
SPTREE_WUNLOCK();
return;
}
sp->state = IPSEC_SPSTATE_DEAD;
TAILQ_REMOVE(&V_sptree[sp->spidx.dir], sp, chain);
SPTREE_WUNLOCK();
KEY_FREESP(&sp);
}
/*
* Must be called after calling key_allocsp().
* For the packet with socket.
*/
void
key_freeso(struct socket *so)
{
IPSEC_ASSERT(so != NULL, ("null so"));
switch (so->so_proto->pr_domain->dom_family) {
#if defined(INET) || defined(INET6)
#ifdef INET
case PF_INET:
#endif
#ifdef INET6
case PF_INET6:
#endif
{
struct inpcb *pcb = sotoinpcb(so);
/* Does it have a PCB ? */
if (pcb == NULL)
return;
key_freesp_so(&pcb->inp_sp->sp_in);
key_freesp_so(&pcb->inp_sp->sp_out);
}
break;
#endif /* INET || INET6 */
default:
ipseclog((LOG_DEBUG, "%s: unknown address family=%d.\n",
__func__, so->so_proto->pr_domain->dom_family));
return;
}
}
static void
key_freesp_so(struct secpolicy **sp)
{
IPSEC_ASSERT(sp != NULL && *sp != NULL, ("null sp"));
if ((*sp)->policy == IPSEC_POLICY_ENTRUST ||
(*sp)->policy == IPSEC_POLICY_BYPASS)
return;
IPSEC_ASSERT((*sp)->policy == IPSEC_POLICY_IPSEC,
("invalid policy %u", (*sp)->policy));
KEY_FREESP(sp);
}
void
key_addrefsa(struct secasvar *sav, const char* where, int tag)
{
IPSEC_ASSERT(sav != NULL, ("null sav"));
IPSEC_ASSERT(sav->refcnt > 0, ("refcount must exist"));
sa_addref(sav);
}
/*
* Must be called after calling key_allocsa().
* This function is called by key_freesp() to free some SA allocated
* for a policy.
*/
void
key_freesav(struct secasvar **psav, const char* where, int tag)
{
struct secasvar *sav = *psav;
IPSEC_ASSERT(sav != NULL, ("null sav"));
if (sa_delref(sav)) {
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s SA:%p (SPI %u) from %s:%u; refcnt now %u\n",
__func__, sav, ntohl(sav->spi), where, tag, sav->refcnt));
*psav = NULL;
key_delsav(sav);
} else {
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s SA:%p (SPI %u) from %s:%u; refcnt now %u\n",
__func__, sav, ntohl(sav->spi), where, tag, sav->refcnt));
}
}
/* %%% SPD management */
/*
* search SPD
* OUT: NULL : not found
* others : found, pointer to a SP.
*/
static struct secpolicy *
key_getsp(struct secpolicyindex *spidx)
{
SPTREE_RLOCK_TRACKER;
struct secpolicy *sp;
IPSEC_ASSERT(spidx != NULL, ("null spidx"));
SPTREE_RLOCK();
TAILQ_FOREACH(sp, &V_sptree[spidx->dir], chain) {
if (key_cmpspidx_exactly(spidx, &sp->spidx)) {
SP_ADDREF(sp);
break;
}
}
SPTREE_RUNLOCK();
return sp;
}
/*
* get SP by index.
* OUT: NULL : not found
* others : found, pointer to a SP.
*/
static struct secpolicy *
key_getspbyid(u_int32_t id)
{
SPTREE_RLOCK_TRACKER;
struct secpolicy *sp;
SPTREE_RLOCK();
TAILQ_FOREACH(sp, &V_sptree[IPSEC_DIR_INBOUND], chain) {
if (sp->id == id) {
SP_ADDREF(sp);
goto done;
}
}
TAILQ_FOREACH(sp, &V_sptree[IPSEC_DIR_OUTBOUND], chain) {
if (sp->id == id) {
SP_ADDREF(sp);
goto done;
}
}
done:
SPTREE_RUNLOCK();
return sp;
}
struct secpolicy *
key_newsp(const char* where, int tag)
{
struct secpolicy *newsp = NULL;
newsp = (struct secpolicy *)
malloc(sizeof(struct secpolicy), M_IPSEC_SP, M_NOWAIT|M_ZERO);
if (newsp)
refcount_init(&newsp->refcnt, 1);
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s from %s:%u return SP:%p\n", __func__,
where, tag, newsp));
return newsp;
}
/*
* create secpolicy structure from sadb_x_policy structure.
* NOTE: `state', `secpolicyindex' in secpolicy structure are not set,
* so must be set properly later.
*/
struct secpolicy *
key_msg2sp(struct sadb_x_policy *xpl0, size_t len, int *error)
{
struct secpolicy *newsp;
IPSEC_ASSERT(xpl0 != NULL, ("null xpl0"));
IPSEC_ASSERT(len >= sizeof(*xpl0), ("policy too short: %zu", len));
if (len != PFKEY_EXTLEN(xpl0)) {
ipseclog((LOG_DEBUG, "%s: Invalid msg length.\n", __func__));
*error = EINVAL;
return NULL;
}
if ((newsp = KEY_NEWSP()) == NULL) {
*error = ENOBUFS;
return NULL;
}
newsp->spidx.dir = xpl0->sadb_x_policy_dir;
newsp->policy = xpl0->sadb_x_policy_type;
/* check policy */
switch (xpl0->sadb_x_policy_type) {
case IPSEC_POLICY_DISCARD:
case IPSEC_POLICY_NONE:
case IPSEC_POLICY_ENTRUST:
case IPSEC_POLICY_BYPASS:
newsp->req = NULL;
break;
case IPSEC_POLICY_IPSEC:
{
int tlen;
struct sadb_x_ipsecrequest *xisr;
struct ipsecrequest **p_isr = &newsp->req;
/* validity check */
if (PFKEY_EXTLEN(xpl0) < sizeof(*xpl0)) {
ipseclog((LOG_DEBUG, "%s: Invalid msg length.\n",
__func__));
KEY_FREESP(&newsp);
*error = EINVAL;
return NULL;
}
tlen = PFKEY_EXTLEN(xpl0) - sizeof(*xpl0);
xisr = (struct sadb_x_ipsecrequest *)(xpl0 + 1);
while (tlen > 0) {
/* length check */
if (xisr->sadb_x_ipsecrequest_len < sizeof(*xisr)) {
ipseclog((LOG_DEBUG, "%s: invalid ipsecrequest "
"length.\n", __func__));
KEY_FREESP(&newsp);
*error = EINVAL;
return NULL;
}
/* allocate request buffer */
/* NB: data structure is zero'd */
*p_isr = ipsec_newisr();
if ((*p_isr) == NULL) {
ipseclog((LOG_DEBUG,
"%s: No more memory.\n", __func__));
KEY_FREESP(&newsp);
*error = ENOBUFS;
return NULL;
}
/* set values */
switch (xisr->sadb_x_ipsecrequest_proto) {
case IPPROTO_ESP:
case IPPROTO_AH:
case IPPROTO_IPCOMP:
break;
default:
ipseclog((LOG_DEBUG,
"%s: invalid proto type=%u\n", __func__,
xisr->sadb_x_ipsecrequest_proto));
KEY_FREESP(&newsp);
*error = EPROTONOSUPPORT;
return NULL;
}
(*p_isr)->saidx.proto = xisr->sadb_x_ipsecrequest_proto;
switch (xisr->sadb_x_ipsecrequest_mode) {
case IPSEC_MODE_TRANSPORT:
case IPSEC_MODE_TUNNEL:
break;
case IPSEC_MODE_ANY:
default:
ipseclog((LOG_DEBUG,
"%s: invalid mode=%u\n", __func__,
xisr->sadb_x_ipsecrequest_mode));
KEY_FREESP(&newsp);
*error = EINVAL;
return NULL;
}
(*p_isr)->saidx.mode = xisr->sadb_x_ipsecrequest_mode;
switch (xisr->sadb_x_ipsecrequest_level) {
case IPSEC_LEVEL_DEFAULT:
case IPSEC_LEVEL_USE:
case IPSEC_LEVEL_REQUIRE:
break;
case IPSEC_LEVEL_UNIQUE:
/* validity check */
/*
* If range violation of reqid, kernel will
* update it, don't refuse it.
*/
if (xisr->sadb_x_ipsecrequest_reqid
> IPSEC_MANUAL_REQID_MAX) {
ipseclog((LOG_DEBUG,
"%s: reqid=%d range "
"violation, updated by kernel.\n",
__func__,
xisr->sadb_x_ipsecrequest_reqid));
xisr->sadb_x_ipsecrequest_reqid = 0;
}
/* allocate new reqid id if reqid is zero. */
if (xisr->sadb_x_ipsecrequest_reqid == 0) {
u_int32_t reqid;
if ((reqid = key_newreqid()) == 0) {
KEY_FREESP(&newsp);
*error = ENOBUFS;
return NULL;
}
(*p_isr)->saidx.reqid = reqid;
xisr->sadb_x_ipsecrequest_reqid = reqid;
} else {
/* set it for manual keying. */
(*p_isr)->saidx.reqid =
xisr->sadb_x_ipsecrequest_reqid;
}
break;
default:
ipseclog((LOG_DEBUG, "%s: invalid level=%u\n",
__func__,
xisr->sadb_x_ipsecrequest_level));
KEY_FREESP(&newsp);
*error = EINVAL;
return NULL;
}
(*p_isr)->level = xisr->sadb_x_ipsecrequest_level;
/* set IP addresses if there */
if (xisr->sadb_x_ipsecrequest_len > sizeof(*xisr)) {
struct sockaddr *paddr;
paddr = (struct sockaddr *)(xisr + 1);
/* validity check */
if (paddr->sa_len
> sizeof((*p_isr)->saidx.src)) {
ipseclog((LOG_DEBUG, "%s: invalid "
"request address length.\n",
__func__));
KEY_FREESP(&newsp);
*error = EINVAL;
return NULL;
}
bcopy(paddr, &(*p_isr)->saidx.src,
paddr->sa_len);
paddr = (struct sockaddr *)((caddr_t)paddr
+ paddr->sa_len);
/* validity check */
if (paddr->sa_len
> sizeof((*p_isr)->saidx.dst)) {
ipseclog((LOG_DEBUG, "%s: invalid "
"request address length.\n",
__func__));
KEY_FREESP(&newsp);
*error = EINVAL;
return NULL;
}
bcopy(paddr, &(*p_isr)->saidx.dst,
paddr->sa_len);
}
(*p_isr)->sp = newsp;
/* initialization for the next. */
p_isr = &(*p_isr)->next;
tlen -= xisr->sadb_x_ipsecrequest_len;
/* validity check */
if (tlen < 0) {
ipseclog((LOG_DEBUG, "%s: becoming tlen < 0.\n",
__func__));
KEY_FREESP(&newsp);
*error = EINVAL;
return NULL;
}
xisr = (struct sadb_x_ipsecrequest *)((caddr_t)xisr
+ xisr->sadb_x_ipsecrequest_len);
}
}
break;
default:
ipseclog((LOG_DEBUG, "%s: invalid policy type.\n", __func__));
KEY_FREESP(&newsp);
*error = EINVAL;
return NULL;
}
*error = 0;
return newsp;
}
static u_int32_t
key_newreqid()
{
static u_int32_t auto_reqid = IPSEC_MANUAL_REQID_MAX + 1;
auto_reqid = (auto_reqid == ~0
? IPSEC_MANUAL_REQID_MAX + 1 : auto_reqid + 1);
/* XXX should be unique check */
return auto_reqid;
}
/*
* copy secpolicy struct to sadb_x_policy structure indicated.
*/
struct mbuf *
key_sp2msg(struct secpolicy *sp)
{
struct sadb_x_policy *xpl;
int tlen;
caddr_t p;
struct mbuf *m;
IPSEC_ASSERT(sp != NULL, ("null policy"));
tlen = key_getspreqmsglen(sp);
m = m_get2(tlen, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return (NULL);
m_align(m, tlen);
m->m_len = tlen;
xpl = mtod(m, struct sadb_x_policy *);
bzero(xpl, tlen);
xpl->sadb_x_policy_len = PFKEY_UNIT64(tlen);
xpl->sadb_x_policy_exttype = SADB_X_EXT_POLICY;
xpl->sadb_x_policy_type = sp->policy;
xpl->sadb_x_policy_dir = sp->spidx.dir;
xpl->sadb_x_policy_id = sp->id;
p = (caddr_t)xpl + sizeof(*xpl);
/* if is the policy for ipsec ? */
if (sp->policy == IPSEC_POLICY_IPSEC) {
struct sadb_x_ipsecrequest *xisr;
struct ipsecrequest *isr;
for (isr = sp->req; isr != NULL; isr = isr->next) {
xisr = (struct sadb_x_ipsecrequest *)p;
xisr->sadb_x_ipsecrequest_proto = isr->saidx.proto;
xisr->sadb_x_ipsecrequest_mode = isr->saidx.mode;
xisr->sadb_x_ipsecrequest_level = isr->level;
xisr->sadb_x_ipsecrequest_reqid = isr->saidx.reqid;
p += sizeof(*xisr);
bcopy(&isr->saidx.src, p, isr->saidx.src.sa.sa_len);
p += isr->saidx.src.sa.sa_len;
bcopy(&isr->saidx.dst, p, isr->saidx.dst.sa.sa_len);
p += isr->saidx.src.sa.sa_len;
xisr->sadb_x_ipsecrequest_len =
PFKEY_ALIGN8(sizeof(*xisr)
+ isr->saidx.src.sa.sa_len
+ isr->saidx.dst.sa.sa_len);
}
}
return m;
}
/* m will not be freed nor modified */
static struct mbuf *
key_gather_mbuf(struct mbuf *m, const struct sadb_msghdr *mhp,
int ndeep, int nitem, ...)
{
va_list ap;
int idx;
int i;
struct mbuf *result = NULL, *n;
int len;
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
va_start(ap, nitem);
for (i = 0; i < nitem; i++) {
idx = va_arg(ap, int);
if (idx < 0 || idx > SADB_EXT_MAX)
goto fail;
/* don't attempt to pull empty extension */
if (idx == SADB_EXT_RESERVED && mhp->msg == NULL)
continue;
if (idx != SADB_EXT_RESERVED &&
(mhp->ext[idx] == NULL || mhp->extlen[idx] == 0))
continue;
if (idx == SADB_EXT_RESERVED) {
len = PFKEY_ALIGN8(sizeof(struct sadb_msg));
IPSEC_ASSERT(len <= MHLEN, ("header too big %u", len));
MGETHDR(n, M_NOWAIT, MT_DATA);
if (!n)
goto fail;
n->m_len = len;
n->m_next = NULL;
m_copydata(m, 0, sizeof(struct sadb_msg),
mtod(n, caddr_t));
} else if (i < ndeep) {
len = mhp->extlen[idx];
n = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (n == NULL)
goto fail;
m_align(n, len);
n->m_len = len;
m_copydata(m, mhp->extoff[idx], mhp->extlen[idx],
mtod(n, caddr_t));
} else {
n = m_copym(m, mhp->extoff[idx], mhp->extlen[idx],
M_NOWAIT);
}
if (n == NULL)
goto fail;
if (result)
m_cat(result, n);
else
result = n;
}
va_end(ap);
if ((result->m_flags & M_PKTHDR) != 0) {
result->m_pkthdr.len = 0;
for (n = result; n; n = n->m_next)
result->m_pkthdr.len += n->m_len;
}
return result;
fail:
m_freem(result);
va_end(ap);
return NULL;
}
/*
* SADB_X_SPDADD, SADB_X_SPDSETIDX or SADB_X_SPDUPDATE processing
* add an entry to SP database, when received
* <base, address(SD), (lifetime(H),) policy>
* from the user(?).
* Adding to SP database,
* and send
* <base, address(SD), (lifetime(H),) policy>
* to the socket which was send.
*
* SPDADD set a unique policy entry.
* SPDSETIDX like SPDADD without a part of policy requests.
* SPDUPDATE replace a unique policy entry.
*
* m will always be freed.
*/
static int
key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct sadb_address *src0, *dst0;
struct sadb_x_policy *xpl0, *xpl;
struct sadb_lifetime *lft = NULL;
struct secpolicyindex spidx;
struct secpolicy *newsp;
int error;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
if (mhp->ext[SADB_EXT_ADDRESS_SRC] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_DST] == NULL ||
mhp->ext[SADB_X_EXT_POLICY] == NULL) {
ipseclog((LOG_DEBUG, "key_spdadd: invalid message is passed.\n"));
return key_senderror(so, m, EINVAL);
}
if (mhp->extlen[SADB_EXT_ADDRESS_SRC] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_EXT_ADDRESS_DST] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_X_EXT_POLICY] < sizeof(struct sadb_x_policy)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_EXT_LIFETIME_HARD] != NULL) {
if (mhp->extlen[SADB_EXT_LIFETIME_HARD]
< sizeof(struct sadb_lifetime)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
lft = (struct sadb_lifetime *)mhp->ext[SADB_EXT_LIFETIME_HARD];
}
src0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_SRC];
dst0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_DST];
xpl0 = (struct sadb_x_policy *)mhp->ext[SADB_X_EXT_POLICY];
/*
* Note: do not parse SADB_X_EXT_NAT_T_* here:
* we are processing traffic endpoints.
*/
/* make secindex */
/* XXX boundary check against sa_len */
KEY_SETSECSPIDX(xpl0->sadb_x_policy_dir,
src0 + 1,
dst0 + 1,
src0->sadb_address_prefixlen,
dst0->sadb_address_prefixlen,
src0->sadb_address_proto,
&spidx);
/* checking the direciton. */
switch (xpl0->sadb_x_policy_dir) {
case IPSEC_DIR_INBOUND:
case IPSEC_DIR_OUTBOUND:
break;
default:
ipseclog((LOG_DEBUG, "%s: Invalid SP direction.\n", __func__));
mhp->msg->sadb_msg_errno = EINVAL;
return 0;
}
/* check policy */
/* key_spdadd() accepts DISCARD, NONE and IPSEC. */
if (xpl0->sadb_x_policy_type == IPSEC_POLICY_ENTRUST
|| xpl0->sadb_x_policy_type == IPSEC_POLICY_BYPASS) {
ipseclog((LOG_DEBUG, "%s: Invalid policy type.\n", __func__));
return key_senderror(so, m, EINVAL);
}
/* policy requests are mandatory when action is ipsec. */
if (mhp->msg->sadb_msg_type != SADB_X_SPDSETIDX
&& xpl0->sadb_x_policy_type == IPSEC_POLICY_IPSEC
&& mhp->extlen[SADB_X_EXT_POLICY] <= sizeof(*xpl0)) {
ipseclog((LOG_DEBUG, "%s: some policy requests part required\n",
__func__));
return key_senderror(so, m, EINVAL);
}
/*
* checking there is SP already or not.
* SPDUPDATE doesn't depend on whether there is a SP or not.
* If the type is either SPDADD or SPDSETIDX AND a SP is found,
* then error.
*/
newsp = key_getsp(&spidx);
if (mhp->msg->sadb_msg_type == SADB_X_SPDUPDATE) {
if (newsp) {
key_unlink(newsp);
KEY_FREESP(&newsp);
}
} else {
if (newsp != NULL) {
KEY_FREESP(&newsp);
ipseclog((LOG_DEBUG, "%s: a SP entry exists already.\n",
__func__));
return key_senderror(so, m, EEXIST);
}
}
/* XXX: there is race between key_getsp and key_msg2sp. */
/* allocation new SP entry */
if ((newsp = key_msg2sp(xpl0, PFKEY_EXTLEN(xpl0), &error)) == NULL) {
return key_senderror(so, m, error);
}
if ((newsp->id = key_getnewspid()) == 0) {
KEY_FREESP(&newsp);
return key_senderror(so, m, ENOBUFS);
}
/* XXX boundary check against sa_len */
KEY_SETSECSPIDX(xpl0->sadb_x_policy_dir,
src0 + 1,
dst0 + 1,
src0->sadb_address_prefixlen,
dst0->sadb_address_prefixlen,
src0->sadb_address_proto,
&newsp->spidx);
/* sanity check on addr pair */
if (((struct sockaddr *)(src0 + 1))->sa_family !=
((struct sockaddr *)(dst0+ 1))->sa_family) {
KEY_FREESP(&newsp);
return key_senderror(so, m, EINVAL);
}
if (((struct sockaddr *)(src0 + 1))->sa_len !=
((struct sockaddr *)(dst0+ 1))->sa_len) {
KEY_FREESP(&newsp);
return key_senderror(so, m, EINVAL);
}
#if 1
if (newsp->req && newsp->req->saidx.src.sa.sa_family &&
newsp->req->saidx.dst.sa.sa_family) {
if (newsp->req->saidx.src.sa.sa_family !=
newsp->req->saidx.dst.sa.sa_family) {
KEY_FREESP(&newsp);
return key_senderror(so, m, EINVAL);
}
}
#endif
newsp->created = time_second;
newsp->lastused = newsp->created;
newsp->lifetime = lft ? lft->sadb_lifetime_addtime : 0;
newsp->validtime = lft ? lft->sadb_lifetime_usetime : 0;
SPTREE_WLOCK();
TAILQ_INSERT_TAIL(&V_sptree[newsp->spidx.dir], newsp, chain);
newsp->state = IPSEC_SPSTATE_ALIVE;
SPTREE_WUNLOCK();
/* delete the entry in spacqtree */
if (mhp->msg->sadb_msg_type == SADB_X_SPDUPDATE) {
struct secspacq *spacq = key_getspacq(&spidx);
if (spacq != NULL) {
/* reset counter in order to deletion by timehandler. */
spacq->created = time_second;
spacq->count = 0;
SPACQ_UNLOCK();
}
}
{
struct mbuf *n, *mpolicy;
struct sadb_msg *newmsg;
int off;
/*
* Note: do not send SADB_X_EXT_NAT_T_* here:
* we are sending traffic endpoints.
*/
/* create new sadb_msg to reply. */
if (lft) {
n = key_gather_mbuf(m, mhp, 2, 5, SADB_EXT_RESERVED,
SADB_X_EXT_POLICY, SADB_EXT_LIFETIME_HARD,
SADB_EXT_ADDRESS_SRC, SADB_EXT_ADDRESS_DST);
} else {
n = key_gather_mbuf(m, mhp, 2, 4, SADB_EXT_RESERVED,
SADB_X_EXT_POLICY,
SADB_EXT_ADDRESS_SRC, SADB_EXT_ADDRESS_DST);
}
if (!n)
return key_senderror(so, m, ENOBUFS);
if (n->m_len < sizeof(*newmsg)) {
n = m_pullup(n, sizeof(*newmsg));
if (!n)
return key_senderror(so, m, ENOBUFS);
}
newmsg = mtod(n, struct sadb_msg *);
newmsg->sadb_msg_errno = 0;
newmsg->sadb_msg_len = PFKEY_UNIT64(n->m_pkthdr.len);
off = 0;
mpolicy = m_pulldown(n, PFKEY_ALIGN8(sizeof(struct sadb_msg)),
sizeof(*xpl), &off);
if (mpolicy == NULL) {
/* n is already freed */
return key_senderror(so, m, ENOBUFS);
}
xpl = (struct sadb_x_policy *)(mtod(mpolicy, caddr_t) + off);
if (xpl->sadb_x_policy_exttype != SADB_X_EXT_POLICY) {
m_freem(n);
return key_senderror(so, m, EINVAL);
}
xpl->sadb_x_policy_id = newsp->id;
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ALL);
}
}
/*
* get new policy id.
* OUT:
* 0: failure.
* others: success.
*/
static u_int32_t
key_getnewspid()
{
u_int32_t newid = 0;
int count = V_key_spi_trycnt; /* XXX */
struct secpolicy *sp;
/* when requesting to allocate spi ranged */
while (count--) {
newid = (V_policy_id = (V_policy_id == ~0 ? 1 : V_policy_id + 1));
if ((sp = key_getspbyid(newid)) == NULL)
break;
KEY_FREESP(&sp);
}
if (count == 0 || newid == 0) {
ipseclog((LOG_DEBUG, "%s: to allocate policy id is failed.\n",
__func__));
return 0;
}
return newid;
}
/*
* SADB_SPDDELETE processing
* receive
* <base, address(SD), policy(*)>
* from the user(?), and set SADB_SASTATE_DEAD,
* and send,
* <base, address(SD), policy(*)>
* to the ikmpd.
* policy(*) including direction of policy.
*
* m will always be freed.
*/
static int
key_spddelete(struct socket *so, struct mbuf *m,
const struct sadb_msghdr *mhp)
{
struct sadb_address *src0, *dst0;
struct sadb_x_policy *xpl0;
struct secpolicyindex spidx;
struct secpolicy *sp;
IPSEC_ASSERT(so != NULL, ("null so"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
if (mhp->ext[SADB_EXT_ADDRESS_SRC] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_DST] == NULL ||
mhp->ext[SADB_X_EXT_POLICY] == NULL) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->extlen[SADB_EXT_ADDRESS_SRC] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_EXT_ADDRESS_DST] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_X_EXT_POLICY] < sizeof(struct sadb_x_policy)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
src0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_SRC];
dst0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_DST];
xpl0 = (struct sadb_x_policy *)mhp->ext[SADB_X_EXT_POLICY];
/*
* Note: do not parse SADB_X_EXT_NAT_T_* here:
* we are processing traffic endpoints.
*/
/* make secindex */
/* XXX boundary check against sa_len */
KEY_SETSECSPIDX(xpl0->sadb_x_policy_dir,
src0 + 1,
dst0 + 1,
src0->sadb_address_prefixlen,
dst0->sadb_address_prefixlen,
src0->sadb_address_proto,
&spidx);
/* checking the direciton. */
switch (xpl0->sadb_x_policy_dir) {
case IPSEC_DIR_INBOUND:
case IPSEC_DIR_OUTBOUND:
break;
default:
ipseclog((LOG_DEBUG, "%s: Invalid SP direction.\n", __func__));
return key_senderror(so, m, EINVAL);
}
/* Is there SP in SPD ? */
if ((sp = key_getsp(&spidx)) == NULL) {
ipseclog((LOG_DEBUG, "%s: no SP found.\n", __func__));
return key_senderror(so, m, EINVAL);
}
/* save policy id to buffer to be returned. */
xpl0->sadb_x_policy_id = sp->id;
key_unlink(sp);
KEY_FREESP(&sp);
{
struct mbuf *n;
struct sadb_msg *newmsg;
/*
* Note: do not send SADB_X_EXT_NAT_T_* here:
* we are sending traffic endpoints.
*/
/* create new sadb_msg to reply. */
n = key_gather_mbuf(m, mhp, 1, 4, SADB_EXT_RESERVED,
SADB_X_EXT_POLICY, SADB_EXT_ADDRESS_SRC, SADB_EXT_ADDRESS_DST);
if (!n)
return key_senderror(so, m, ENOBUFS);
newmsg = mtod(n, struct sadb_msg *);
newmsg->sadb_msg_errno = 0;
newmsg->sadb_msg_len = PFKEY_UNIT64(n->m_pkthdr.len);
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ALL);
}
}
/*
* SADB_SPDDELETE2 processing
* receive
* <base, policy(*)>
* from the user(?), and set SADB_SASTATE_DEAD,
* and send,
* <base, policy(*)>
* to the ikmpd.
* policy(*) including direction of policy.
*
* m will always be freed.
*/
static int
key_spddelete2(struct socket *so, struct mbuf *m,
const struct sadb_msghdr *mhp)
{
u_int32_t id;
struct secpolicy *sp;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
if (mhp->ext[SADB_X_EXT_POLICY] == NULL ||
mhp->extlen[SADB_X_EXT_POLICY] < sizeof(struct sadb_x_policy)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n", __func__));
return key_senderror(so, m, EINVAL);
}
id = ((struct sadb_x_policy *)mhp->ext[SADB_X_EXT_POLICY])->sadb_x_policy_id;
/* Is there SP in SPD ? */
if ((sp = key_getspbyid(id)) == NULL) {
ipseclog((LOG_DEBUG, "%s: no SP found id:%u.\n", __func__, id));
return key_senderror(so, m, EINVAL);
}
key_unlink(sp);
KEY_FREESP(&sp);
{
struct mbuf *n, *nn;
struct sadb_msg *newmsg;
int off, len;
/* create new sadb_msg to reply. */
len = PFKEY_ALIGN8(sizeof(struct sadb_msg));
MGETHDR(n, M_NOWAIT, MT_DATA);
if (n && len > MHLEN) {
if (!(MCLGET(n, M_NOWAIT))) {
m_freem(n);
n = NULL;
}
}
if (!n)
return key_senderror(so, m, ENOBUFS);
n->m_len = len;
n->m_next = NULL;
off = 0;
m_copydata(m, 0, sizeof(struct sadb_msg), mtod(n, caddr_t) + off);
off += PFKEY_ALIGN8(sizeof(struct sadb_msg));
IPSEC_ASSERT(off == len, ("length inconsistency (off %u len %u)",
off, len));
n->m_next = m_copym(m, mhp->extoff[SADB_X_EXT_POLICY],
mhp->extlen[SADB_X_EXT_POLICY], M_NOWAIT);
if (!n->m_next) {
m_freem(n);
return key_senderror(so, m, ENOBUFS);
}
n->m_pkthdr.len = 0;
for (nn = n; nn; nn = nn->m_next)
n->m_pkthdr.len += nn->m_len;
newmsg = mtod(n, struct sadb_msg *);
newmsg->sadb_msg_errno = 0;
newmsg->sadb_msg_len = PFKEY_UNIT64(n->m_pkthdr.len);
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ALL);
}
}
/*
* SADB_X_SPDGET processing
* receive
* <base, policy(*)>
* from the user(?),
* and send,
* <base, address(SD), policy>
* to the ikmpd.
* policy(*) including direction of policy.
*
* m will always be freed.
*/
static int
key_spdget(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
u_int32_t id;
struct secpolicy *sp;
struct mbuf *n;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
if (mhp->ext[SADB_X_EXT_POLICY] == NULL ||
mhp->extlen[SADB_X_EXT_POLICY] < sizeof(struct sadb_x_policy)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
id = ((struct sadb_x_policy *)mhp->ext[SADB_X_EXT_POLICY])->sadb_x_policy_id;
/* Is there SP in SPD ? */
if ((sp = key_getspbyid(id)) == NULL) {
ipseclog((LOG_DEBUG, "%s: no SP found id:%u.\n", __func__, id));
return key_senderror(so, m, ENOENT);
}
n = key_setdumpsp(sp, SADB_X_SPDGET, mhp->msg->sadb_msg_seq,
mhp->msg->sadb_msg_pid);
KEY_FREESP(&sp);
if (n != NULL) {
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
} else
return key_senderror(so, m, ENOBUFS);
}
/*
* SADB_X_SPDACQUIRE processing.
* Acquire policy and SA(s) for a *OUTBOUND* packet.
* send
* <base, policy(*)>
* to KMD, and expect to receive
* <base> with SADB_X_SPDACQUIRE if error occured,
* or
* <base, policy>
* with SADB_X_SPDUPDATE from KMD by PF_KEY.
* policy(*) is without policy requests.
*
* 0 : succeed
* others: error number
*/
int
key_spdacquire(struct secpolicy *sp)
{
struct mbuf *result = NULL, *m;
struct secspacq *newspacq;
IPSEC_ASSERT(sp != NULL, ("null secpolicy"));
IPSEC_ASSERT(sp->req == NULL, ("policy exists"));
IPSEC_ASSERT(sp->policy == IPSEC_POLICY_IPSEC,
("policy not IPSEC %u", sp->policy));
/* Get an entry to check whether sent message or not. */
newspacq = key_getspacq(&sp->spidx);
if (newspacq != NULL) {
if (V_key_blockacq_count < newspacq->count) {
/* reset counter and do send message. */
newspacq->count = 0;
} else {
/* increment counter and do nothing. */
newspacq->count++;
SPACQ_UNLOCK();
return (0);
}
SPACQ_UNLOCK();
} else {
/* make new entry for blocking to send SADB_ACQUIRE. */
newspacq = key_newspacq(&sp->spidx);
if (newspacq == NULL)
return ENOBUFS;
}
/* create new sadb_msg to reply. */
m = key_setsadbmsg(SADB_X_SPDACQUIRE, 0, 0, 0, 0, 0);
if (!m)
return ENOBUFS;
result = m;
result->m_pkthdr.len = 0;
for (m = result; m; m = m->m_next)
result->m_pkthdr.len += m->m_len;
mtod(result, struct sadb_msg *)->sadb_msg_len =
PFKEY_UNIT64(result->m_pkthdr.len);
return key_sendup_mbuf(NULL, m, KEY_SENDUP_REGISTERED);
}
/*
* SADB_SPDFLUSH processing
* receive
* <base>
* from the user, and free all entries in secpctree.
* and send,
* <base>
* to the user.
* NOTE: what to do is only marking SADB_SASTATE_DEAD.
*
* m will always be freed.
*/
static int
key_spdflush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
TAILQ_HEAD(, secpolicy) drainq;
struct sadb_msg *newmsg;
struct secpolicy *sp, *nextsp;
u_int dir;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
if (m->m_len != PFKEY_ALIGN8(sizeof(struct sadb_msg)))
return key_senderror(so, m, EINVAL);
TAILQ_INIT(&drainq);
SPTREE_WLOCK();
for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
TAILQ_CONCAT(&drainq, &V_sptree[dir], chain);
}
/*
* We need to set state to DEAD for each policy to be sure,
* that another thread won't try to unlink it.
*/
TAILQ_FOREACH(sp, &drainq, chain)
sp->state = IPSEC_SPSTATE_DEAD;
SPTREE_WUNLOCK();
sp = TAILQ_FIRST(&drainq);
while (sp != NULL) {
nextsp = TAILQ_NEXT(sp, chain);
KEY_FREESP(&sp);
sp = nextsp;
}
if (sizeof(struct sadb_msg) > m->m_len + M_TRAILINGSPACE(m)) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
return key_senderror(so, m, ENOBUFS);
}
if (m->m_next)
m_freem(m->m_next);
m->m_next = NULL;
m->m_pkthdr.len = m->m_len = PFKEY_ALIGN8(sizeof(struct sadb_msg));
newmsg = mtod(m, struct sadb_msg *);
newmsg->sadb_msg_errno = 0;
newmsg->sadb_msg_len = PFKEY_UNIT64(m->m_pkthdr.len);
return key_sendup_mbuf(so, m, KEY_SENDUP_ALL);
}
/*
* SADB_SPDDUMP processing
* receive
* <base>
* from the user, and dump all SP leaves
* and send,
* <base> .....
* to the ikmpd.
*
* m will always be freed.
*/
static int
key_spddump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
SPTREE_RLOCK_TRACKER;
struct secpolicy *sp;
int cnt;
u_int dir;
struct mbuf *n;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* search SPD entry and get buffer size. */
cnt = 0;
SPTREE_RLOCK();
for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
TAILQ_FOREACH(sp, &V_sptree[dir], chain) {
cnt++;
}
}
if (cnt == 0) {
SPTREE_RUNLOCK();
return key_senderror(so, m, ENOENT);
}
for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
TAILQ_FOREACH(sp, &V_sptree[dir], chain) {
--cnt;
n = key_setdumpsp(sp, SADB_X_SPDDUMP, cnt,
mhp->msg->sadb_msg_pid);
if (n)
key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
}
}
SPTREE_RUNLOCK();
m_freem(m);
return 0;
}
static struct mbuf *
key_setdumpsp(struct secpolicy *sp, u_int8_t type, u_int32_t seq,
u_int32_t pid)
{
struct mbuf *result = NULL, *m;
struct seclifetime lt;
m = key_setsadbmsg(type, 0, SADB_SATYPE_UNSPEC, seq, pid, sp->refcnt);
if (!m)
goto fail;
result = m;
/*
* Note: do not send SADB_X_EXT_NAT_T_* here:
* we are sending traffic endpoints.
*/
m = key_setsadbaddr(SADB_EXT_ADDRESS_SRC,
&sp->spidx.src.sa, sp->spidx.prefs,
sp->spidx.ul_proto);
if (!m)
goto fail;
m_cat(result, m);
m = key_setsadbaddr(SADB_EXT_ADDRESS_DST,
&sp->spidx.dst.sa, sp->spidx.prefd,
sp->spidx.ul_proto);
if (!m)
goto fail;
m_cat(result, m);
m = key_sp2msg(sp);
if (!m)
goto fail;
m_cat(result, m);
if(sp->lifetime){
lt.addtime=sp->created;
lt.usetime= sp->lastused;
m = key_setlifetime(&lt, SADB_EXT_LIFETIME_CURRENT);
if (!m)
goto fail;
m_cat(result, m);
lt.addtime=sp->lifetime;
lt.usetime= sp->validtime;
m = key_setlifetime(&lt, SADB_EXT_LIFETIME_HARD);
if (!m)
goto fail;
m_cat(result, m);
}
if ((result->m_flags & M_PKTHDR) == 0)
goto fail;
if (result->m_len < sizeof(struct sadb_msg)) {
result = m_pullup(result, sizeof(struct sadb_msg));
if (result == NULL)
goto fail;
}
result->m_pkthdr.len = 0;
for (m = result; m; m = m->m_next)
result->m_pkthdr.len += m->m_len;
mtod(result, struct sadb_msg *)->sadb_msg_len =
PFKEY_UNIT64(result->m_pkthdr.len);
return result;
fail:
m_freem(result);
return NULL;
}
/*
* get PFKEY message length for security policy and request.
*/
static u_int
key_getspreqmsglen(struct secpolicy *sp)
{
u_int tlen;
tlen = sizeof(struct sadb_x_policy);
/* if is the policy for ipsec ? */
if (sp->policy != IPSEC_POLICY_IPSEC)
return tlen;
/* get length of ipsec requests */
{
struct ipsecrequest *isr;
int len;
for (isr = sp->req; isr != NULL; isr = isr->next) {
len = sizeof(struct sadb_x_ipsecrequest)
+ isr->saidx.src.sa.sa_len
+ isr->saidx.dst.sa.sa_len;
tlen += PFKEY_ALIGN8(len);
}
}
return tlen;
}
/*
* SADB_SPDEXPIRE processing
* send
* <base, address(SD), lifetime(CH), policy>
* to KMD by PF_KEY.
*
* OUT: 0 : succeed
* others : error number
*/
static int
key_spdexpire(struct secpolicy *sp)
{
struct mbuf *result = NULL, *m;
int len;
int error = -1;
struct sadb_lifetime *lt;
/* XXX: Why do we lock ? */
IPSEC_ASSERT(sp != NULL, ("null secpolicy"));
/* set msg header */
m = key_setsadbmsg(SADB_X_SPDEXPIRE, 0, 0, 0, 0, 0);
if (!m) {
error = ENOBUFS;
goto fail;
}
result = m;
/* create lifetime extension (current and hard) */
len = PFKEY_ALIGN8(sizeof(*lt)) * 2;
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL) {
error = ENOBUFS;
goto fail;
}
m_align(m, len);
m->m_len = len;
bzero(mtod(m, caddr_t), len);
lt = mtod(m, struct sadb_lifetime *);
lt->sadb_lifetime_len = PFKEY_UNIT64(sizeof(struct sadb_lifetime));
lt->sadb_lifetime_exttype = SADB_EXT_LIFETIME_CURRENT;
lt->sadb_lifetime_allocations = 0;
lt->sadb_lifetime_bytes = 0;
lt->sadb_lifetime_addtime = sp->created;
lt->sadb_lifetime_usetime = sp->lastused;
lt = (struct sadb_lifetime *)(mtod(m, caddr_t) + len / 2);
lt->sadb_lifetime_len = PFKEY_UNIT64(sizeof(struct sadb_lifetime));
lt->sadb_lifetime_exttype = SADB_EXT_LIFETIME_HARD;
lt->sadb_lifetime_allocations = 0;
lt->sadb_lifetime_bytes = 0;
lt->sadb_lifetime_addtime = sp->lifetime;
lt->sadb_lifetime_usetime = sp->validtime;
m_cat(result, m);
/*
* Note: do not send SADB_X_EXT_NAT_T_* here:
* we are sending traffic endpoints.
*/
/* set sadb_address for source */
m = key_setsadbaddr(SADB_EXT_ADDRESS_SRC,
&sp->spidx.src.sa,
sp->spidx.prefs, sp->spidx.ul_proto);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
/* set sadb_address for destination */
m = key_setsadbaddr(SADB_EXT_ADDRESS_DST,
&sp->spidx.dst.sa,
sp->spidx.prefd, sp->spidx.ul_proto);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
/* set secpolicy */
m = key_sp2msg(sp);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
if ((result->m_flags & M_PKTHDR) == 0) {
error = EINVAL;
goto fail;
}
if (result->m_len < sizeof(struct sadb_msg)) {
result = m_pullup(result, sizeof(struct sadb_msg));
if (result == NULL) {
error = ENOBUFS;
goto fail;
}
}
result->m_pkthdr.len = 0;
for (m = result; m; m = m->m_next)
result->m_pkthdr.len += m->m_len;
mtod(result, struct sadb_msg *)->sadb_msg_len =
PFKEY_UNIT64(result->m_pkthdr.len);
return key_sendup_mbuf(NULL, result, KEY_SENDUP_REGISTERED);
fail:
if (result)
m_freem(result);
return error;
}
/* %%% SAD management */
/*
* allocating a memory for new SA head, and copy from the values of mhp.
* OUT: NULL : failure due to the lack of memory.
* others : pointer to new SA head.
*/
static struct secashead *
key_newsah(struct secasindex *saidx)
{
struct secashead *newsah;
IPSEC_ASSERT(saidx != NULL, ("null saidx"));
newsah = malloc(sizeof(struct secashead), M_IPSEC_SAH, M_NOWAIT|M_ZERO);
if (newsah != NULL) {
int i;
for (i = 0; i < sizeof(newsah->savtree)/sizeof(newsah->savtree[0]); i++)
LIST_INIT(&newsah->savtree[i]);
newsah->saidx = *saidx;
/* add to saidxtree */
newsah->state = SADB_SASTATE_MATURE;
SAHTREE_LOCK();
LIST_INSERT_HEAD(&V_sahtree, newsah, chain);
SAHTREE_UNLOCK();
}
return(newsah);
}
/*
* delete SA index and all SA registerd.
*/
static void
key_delsah(struct secashead *sah)
{
struct secasvar *sav, *nextsav;
u_int stateidx;
int zombie = 0;
IPSEC_ASSERT(sah != NULL, ("NULL sah"));
SAHTREE_LOCK_ASSERT();
/* searching all SA registerd in the secindex. */
for (stateidx = 0;
stateidx < _ARRAYLEN(saorder_state_any);
stateidx++) {
u_int state = saorder_state_any[stateidx];
LIST_FOREACH_SAFE(sav, &sah->savtree[state], chain, nextsav) {
if (sav->refcnt == 0) {
/* sanity check */
KEY_CHKSASTATE(state, sav->state, __func__);
/*
* do NOT call KEY_FREESAV here:
* it will only delete the sav if refcnt == 1,
* where we already know that refcnt == 0
*/
key_delsav(sav);
} else {
/* give up to delete this sa */
zombie++;
}
}
}
if (!zombie) { /* delete only if there are savs */
/* remove from tree of SA index */
if (__LIST_CHAINED(sah))
LIST_REMOVE(sah, chain);
free(sah, M_IPSEC_SAH);
}
}
/*
* allocating a new SA with LARVAL state. key_add() and key_getspi() call,
* and copy the values of mhp into new buffer.
* When SAD message type is GETSPI:
* to set sequence number from acq_seq++,
* to set zero to SPI.
* not to call key_setsava().
* OUT: NULL : fail
* others : pointer to new secasvar.
*
* does not modify mbuf. does not free mbuf on error.
*/
static struct secasvar *
key_newsav(struct mbuf *m, const struct sadb_msghdr *mhp,
struct secashead *sah, int *errp, const char *where, int tag)
{
struct secasvar *newsav;
const struct sadb_sa *xsa;
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
IPSEC_ASSERT(sah != NULL, ("null secashead"));
newsav = malloc(sizeof(struct secasvar), M_IPSEC_SA, M_NOWAIT|M_ZERO);
if (newsav == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
*errp = ENOBUFS;
goto done;
}
switch (mhp->msg->sadb_msg_type) {
case SADB_GETSPI:
newsav->spi = 0;
#ifdef IPSEC_DOSEQCHECK
/* sync sequence number */
if (mhp->msg->sadb_msg_seq == 0)
newsav->seq =
(V_acq_seq = (V_acq_seq == ~0 ? 1 : ++V_acq_seq));
else
#endif
newsav->seq = mhp->msg->sadb_msg_seq;
break;
case SADB_ADD:
/* sanity check */
if (mhp->ext[SADB_EXT_SA] == NULL) {
free(newsav, M_IPSEC_SA);
newsav = NULL;
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
*errp = EINVAL;
goto done;
}
xsa = (const struct sadb_sa *)mhp->ext[SADB_EXT_SA];
newsav->spi = xsa->sadb_sa_spi;
newsav->seq = mhp->msg->sadb_msg_seq;
break;
default:
free(newsav, M_IPSEC_SA);
newsav = NULL;
*errp = EINVAL;
goto done;
}
/* copy sav values */
if (mhp->msg->sadb_msg_type != SADB_GETSPI) {
*errp = key_setsaval(newsav, m, mhp);
if (*errp) {
free(newsav, M_IPSEC_SA);
newsav = NULL;
goto done;
}
}
SECASVAR_LOCK_INIT(newsav);
/* reset created */
newsav->created = time_second;
newsav->pid = mhp->msg->sadb_msg_pid;
/* add to satree */
newsav->sah = sah;
sa_initref(newsav);
newsav->state = SADB_SASTATE_LARVAL;
SAHTREE_LOCK();
LIST_INSERT_TAIL(&sah->savtree[SADB_SASTATE_LARVAL], newsav,
secasvar, chain);
SAHTREE_UNLOCK();
done:
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s from %s:%u return SP:%p\n", __func__,
where, tag, newsav));
return newsav;
}
/*
* free() SA variable entry.
*/
static void
key_cleansav(struct secasvar *sav)
{
/*
* Cleanup xform state. Note that zeroize'ing causes the
* keys to be cleared; otherwise we must do it ourself.
*/
if (sav->tdb_xform != NULL) {
sav->tdb_xform->xf_zeroize(sav);
sav->tdb_xform = NULL;
} else {
KASSERT(sav->iv == NULL, ("iv but no xform"));
if (sav->key_auth != NULL)
bzero(sav->key_auth->key_data, _KEYLEN(sav->key_auth));
if (sav->key_enc != NULL)
bzero(sav->key_enc->key_data, _KEYLEN(sav->key_enc));
}
if (sav->key_auth != NULL) {
if (sav->key_auth->key_data != NULL)
free(sav->key_auth->key_data, M_IPSEC_MISC);
free(sav->key_auth, M_IPSEC_MISC);
sav->key_auth = NULL;
}
if (sav->key_enc != NULL) {
if (sav->key_enc->key_data != NULL)
free(sav->key_enc->key_data, M_IPSEC_MISC);
free(sav->key_enc, M_IPSEC_MISC);
sav->key_enc = NULL;
}
if (sav->sched) {
bzero(sav->sched, sav->schedlen);
free(sav->sched, M_IPSEC_MISC);
sav->sched = NULL;
}
if (sav->replay != NULL) {
free(sav->replay, M_IPSEC_MISC);
sav->replay = NULL;
}
if (sav->lft_c != NULL) {
free(sav->lft_c, M_IPSEC_MISC);
sav->lft_c = NULL;
}
if (sav->lft_h != NULL) {
free(sav->lft_h, M_IPSEC_MISC);
sav->lft_h = NULL;
}
if (sav->lft_s != NULL) {
free(sav->lft_s, M_IPSEC_MISC);
sav->lft_s = NULL;
}
}
/*
* free() SA variable entry.
*/
static void
key_delsav(struct secasvar *sav)
{
IPSEC_ASSERT(sav != NULL, ("null sav"));
IPSEC_ASSERT(sav->refcnt == 0, ("reference count %u > 0", sav->refcnt));
/* remove from SA header */
if (__LIST_CHAINED(sav))
LIST_REMOVE(sav, chain);
key_cleansav(sav);
SECASVAR_LOCK_DESTROY(sav);
free(sav, M_IPSEC_SA);
}
/*
* search SAD.
* OUT:
* NULL : not found
* others : found, pointer to a SA.
*/
static struct secashead *
key_getsah(struct secasindex *saidx)
{
struct secashead *sah;
SAHTREE_LOCK();
LIST_FOREACH(sah, &V_sahtree, chain) {
if (sah->state == SADB_SASTATE_DEAD)
continue;
if (key_cmpsaidx(&sah->saidx, saidx, CMP_REQID))
break;
}
SAHTREE_UNLOCK();
return sah;
}
/*
* check not to be duplicated SPI.
* NOTE: this function is too slow due to searching all SAD.
* OUT:
* NULL : not found
* others : found, pointer to a SA.
*/
static struct secasvar *
key_checkspidup(struct secasindex *saidx, u_int32_t spi)
{
struct secashead *sah;
struct secasvar *sav;
/* check address family */
if (saidx->src.sa.sa_family != saidx->dst.sa.sa_family) {
ipseclog((LOG_DEBUG, "%s: address family mismatched.\n",
__func__));
return NULL;
}
sav = NULL;
/* check all SAD */
SAHTREE_LOCK();
LIST_FOREACH(sah, &V_sahtree, chain) {
if (!key_ismyaddr((struct sockaddr *)&sah->saidx.dst))
continue;
sav = key_getsavbyspi(sah, spi);
if (sav != NULL)
break;
}
SAHTREE_UNLOCK();
return sav;
}
/*
* search SAD litmited alive SA, protocol, SPI.
* OUT:
* NULL : not found
* others : found, pointer to a SA.
*/
static struct secasvar *
key_getsavbyspi(struct secashead *sah, u_int32_t spi)
{
struct secasvar *sav;
u_int stateidx, state;
sav = NULL;
SAHTREE_LOCK_ASSERT();
/* search all status */
for (stateidx = 0;
stateidx < _ARRAYLEN(saorder_state_alive);
stateidx++) {
state = saorder_state_alive[stateidx];
LIST_FOREACH(sav, &sah->savtree[state], chain) {
/* sanity check */
if (sav->state != state) {
ipseclog((LOG_DEBUG, "%s: "
"invalid sav->state (queue: %d SA: %d)\n",
__func__, state, sav->state));
continue;
}
if (sav->spi == spi)
return sav;
}
}
return NULL;
}
/*
* copy SA values from PF_KEY message except *SPI, SEQ, PID, STATE and TYPE*.
* You must update these if need.
* OUT: 0: success.
* !0: failure.
*
* does not modify mbuf. does not free mbuf on error.
*/
static int
key_setsaval(struct secasvar *sav, struct mbuf *m,
const struct sadb_msghdr *mhp)
{
int error = 0;
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* initialization */
sav->replay = NULL;
sav->key_auth = NULL;
sav->key_enc = NULL;
sav->sched = NULL;
sav->schedlen = 0;
sav->iv = NULL;
sav->lft_c = NULL;
sav->lft_h = NULL;
sav->lft_s = NULL;
sav->tdb_xform = NULL; /* transform */
sav->tdb_encalgxform = NULL; /* encoding algorithm */
sav->tdb_authalgxform = NULL; /* authentication algorithm */
sav->tdb_compalgxform = NULL; /* compression algorithm */
/* Initialize even if NAT-T not compiled in: */
sav->natt_type = 0;
sav->natt_esp_frag_len = 0;
/* SA */
if (mhp->ext[SADB_EXT_SA] != NULL) {
const struct sadb_sa *sa0;
sa0 = (const struct sadb_sa *)mhp->ext[SADB_EXT_SA];
if (mhp->extlen[SADB_EXT_SA] < sizeof(*sa0)) {
error = EINVAL;
goto fail;
}
sav->alg_auth = sa0->sadb_sa_auth;
sav->alg_enc = sa0->sadb_sa_encrypt;
sav->flags = sa0->sadb_sa_flags;
/* replay window */
if ((sa0->sadb_sa_flags & SADB_X_EXT_OLD) == 0) {
sav->replay = (struct secreplay *)
malloc(sizeof(struct secreplay)+sa0->sadb_sa_replay, M_IPSEC_MISC, M_NOWAIT|M_ZERO);
if (sav->replay == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n",
__func__));
error = ENOBUFS;
goto fail;
}
if (sa0->sadb_sa_replay != 0)
sav->replay->bitmap = (caddr_t)(sav->replay+1);
sav->replay->wsize = sa0->sadb_sa_replay;
}
}
/* Authentication keys */
if (mhp->ext[SADB_EXT_KEY_AUTH] != NULL) {
const struct sadb_key *key0;
int len;
key0 = (const struct sadb_key *)mhp->ext[SADB_EXT_KEY_AUTH];
len = mhp->extlen[SADB_EXT_KEY_AUTH];
error = 0;
if (len < sizeof(*key0)) {
error = EINVAL;
goto fail;
}
switch (mhp->msg->sadb_msg_satype) {
case SADB_SATYPE_AH:
case SADB_SATYPE_ESP:
case SADB_X_SATYPE_TCPSIGNATURE:
if (len == PFKEY_ALIGN8(sizeof(struct sadb_key)) &&
sav->alg_auth != SADB_X_AALG_NULL)
error = EINVAL;
break;
case SADB_X_SATYPE_IPCOMP:
default:
error = EINVAL;
break;
}
if (error) {
ipseclog((LOG_DEBUG, "%s: invalid key_auth values.\n",
__func__));
goto fail;
}
sav->key_auth = (struct seckey *)key_dup_keymsg(key0, len,
M_IPSEC_MISC);
if (sav->key_auth == NULL ) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n",
__func__));
error = ENOBUFS;
goto fail;
}
}
/* Encryption key */
if (mhp->ext[SADB_EXT_KEY_ENCRYPT] != NULL) {
const struct sadb_key *key0;
int len;
key0 = (const struct sadb_key *)mhp->ext[SADB_EXT_KEY_ENCRYPT];
len = mhp->extlen[SADB_EXT_KEY_ENCRYPT];
error = 0;
if (len < sizeof(*key0)) {
error = EINVAL;
goto fail;
}
switch (mhp->msg->sadb_msg_satype) {
case SADB_SATYPE_ESP:
if (len == PFKEY_ALIGN8(sizeof(struct sadb_key)) &&
sav->alg_enc != SADB_EALG_NULL) {
error = EINVAL;
break;
}
sav->key_enc = (struct seckey *)key_dup_keymsg(key0,
len,
M_IPSEC_MISC);
if (sav->key_enc == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n",
__func__));
error = ENOBUFS;
goto fail;
}
break;
case SADB_X_SATYPE_IPCOMP:
if (len != PFKEY_ALIGN8(sizeof(struct sadb_key)))
error = EINVAL;
sav->key_enc = NULL; /*just in case*/
break;
case SADB_SATYPE_AH:
case SADB_X_SATYPE_TCPSIGNATURE:
default:
error = EINVAL;
break;
}
if (error) {
ipseclog((LOG_DEBUG, "%s: invalid key_enc value.\n",
__func__));
goto fail;
}
}
/* set iv */
sav->ivlen = 0;
switch (mhp->msg->sadb_msg_satype) {
case SADB_SATYPE_AH:
error = xform_init(sav, XF_AH);
break;
case SADB_SATYPE_ESP:
error = xform_init(sav, XF_ESP);
break;
case SADB_X_SATYPE_IPCOMP:
error = xform_init(sav, XF_IPCOMP);
break;
case SADB_X_SATYPE_TCPSIGNATURE:
error = xform_init(sav, XF_TCPSIGNATURE);
break;
}
if (error) {
ipseclog((LOG_DEBUG, "%s: unable to initialize SA type %u.\n",
__func__, mhp->msg->sadb_msg_satype));
goto fail;
}
/* reset created */
sav->created = time_second;
/* make lifetime for CURRENT */
sav->lft_c = malloc(sizeof(struct seclifetime), M_IPSEC_MISC, M_NOWAIT);
if (sav->lft_c == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
error = ENOBUFS;
goto fail;
}
sav->lft_c->allocations = 0;
sav->lft_c->bytes = 0;
sav->lft_c->addtime = time_second;
sav->lft_c->usetime = 0;
/* lifetimes for HARD and SOFT */
{
const struct sadb_lifetime *lft0;
lft0 = (struct sadb_lifetime *)mhp->ext[SADB_EXT_LIFETIME_HARD];
if (lft0 != NULL) {
if (mhp->extlen[SADB_EXT_LIFETIME_HARD] < sizeof(*lft0)) {
error = EINVAL;
goto fail;
}
sav->lft_h = key_dup_lifemsg(lft0, M_IPSEC_MISC);
if (sav->lft_h == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n",__func__));
error = ENOBUFS;
goto fail;
}
/* to be initialize ? */
}
lft0 = (struct sadb_lifetime *)mhp->ext[SADB_EXT_LIFETIME_SOFT];
if (lft0 != NULL) {
if (mhp->extlen[SADB_EXT_LIFETIME_SOFT] < sizeof(*lft0)) {
error = EINVAL;
goto fail;
}
sav->lft_s = key_dup_lifemsg(lft0, M_IPSEC_MISC);
if (sav->lft_s == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n",__func__));
error = ENOBUFS;
goto fail;
}
/* to be initialize ? */
}
}
return 0;
fail:
/* initialization */
key_cleansav(sav);
return error;
}
/*
* validation with a secasvar entry, and set SADB_SATYPE_MATURE.
* OUT: 0: valid
* other: errno
*/
static int
key_mature(struct secasvar *sav)
{
int error;
/* check SPI value */
switch (sav->sah->saidx.proto) {
case IPPROTO_ESP:
case IPPROTO_AH:
/*
* RFC 4302, 2.4. Security Parameters Index (SPI), SPI values
* 1-255 reserved by IANA for future use,
* 0 for implementation specific, local use.
*/
if (ntohl(sav->spi) <= 255) {
ipseclog((LOG_DEBUG, "%s: illegal range of SPI %u.\n",
__func__, (u_int32_t)ntohl(sav->spi)));
return EINVAL;
}
break;
}
/* check satype */
switch (sav->sah->saidx.proto) {
case IPPROTO_ESP:
/* check flags */
if ((sav->flags & (SADB_X_EXT_OLD|SADB_X_EXT_DERIV)) ==
(SADB_X_EXT_OLD|SADB_X_EXT_DERIV)) {
ipseclog((LOG_DEBUG, "%s: invalid flag (derived) "
"given to old-esp.\n", __func__));
return EINVAL;
}
error = xform_init(sav, XF_ESP);
break;
case IPPROTO_AH:
/* check flags */
if (sav->flags & SADB_X_EXT_DERIV) {
ipseclog((LOG_DEBUG, "%s: invalid flag (derived) "
"given to AH SA.\n", __func__));
return EINVAL;
}
if (sav->alg_enc != SADB_EALG_NONE) {
ipseclog((LOG_DEBUG, "%s: protocol and algorithm "
"mismated.\n", __func__));
return(EINVAL);
}
error = xform_init(sav, XF_AH);
break;
case IPPROTO_IPCOMP:
if (sav->alg_auth != SADB_AALG_NONE) {
ipseclog((LOG_DEBUG, "%s: protocol and algorithm "
"mismated.\n", __func__));
return(EINVAL);
}
if ((sav->flags & SADB_X_EXT_RAWCPI) == 0
&& ntohl(sav->spi) >= 0x10000) {
ipseclog((LOG_DEBUG, "%s: invalid cpi for IPComp.\n",
__func__));
return(EINVAL);
}
error = xform_init(sav, XF_IPCOMP);
break;
case IPPROTO_TCP:
if (sav->alg_enc != SADB_EALG_NONE) {
ipseclog((LOG_DEBUG, "%s: protocol and algorithm "
"mismated.\n", __func__));
return(EINVAL);
}
error = xform_init(sav, XF_TCPSIGNATURE);
break;
default:
ipseclog((LOG_DEBUG, "%s: Invalid satype.\n", __func__));
error = EPROTONOSUPPORT;
break;
}
if (error == 0) {
SAHTREE_LOCK();
key_sa_chgstate(sav, SADB_SASTATE_MATURE);
SAHTREE_UNLOCK();
}
return (error);
}
/*
* subroutine for SADB_GET and SADB_DUMP.
*/
static struct mbuf *
key_setdumpsa(struct secasvar *sav, u_int8_t type, u_int8_t satype,
u_int32_t seq, u_int32_t pid)
{
struct mbuf *result = NULL, *tres = NULL, *m;
int i;
int dumporder[] = {
SADB_EXT_SA, SADB_X_EXT_SA2,
SADB_EXT_LIFETIME_HARD, SADB_EXT_LIFETIME_SOFT,
SADB_EXT_LIFETIME_CURRENT, SADB_EXT_ADDRESS_SRC,
SADB_EXT_ADDRESS_DST, SADB_EXT_ADDRESS_PROXY, SADB_EXT_KEY_AUTH,
SADB_EXT_KEY_ENCRYPT, SADB_EXT_IDENTITY_SRC,
SADB_EXT_IDENTITY_DST, SADB_EXT_SENSITIVITY,
#ifdef IPSEC_NAT_T
SADB_X_EXT_NAT_T_TYPE,
SADB_X_EXT_NAT_T_SPORT, SADB_X_EXT_NAT_T_DPORT,
SADB_X_EXT_NAT_T_OAI, SADB_X_EXT_NAT_T_OAR,
SADB_X_EXT_NAT_T_FRAG,
#endif
};
m = key_setsadbmsg(type, 0, satype, seq, pid, sav->refcnt);
if (m == NULL)
goto fail;
result = m;
for (i = sizeof(dumporder)/sizeof(dumporder[0]) - 1; i >= 0; i--) {
m = NULL;
switch (dumporder[i]) {
case SADB_EXT_SA:
m = key_setsadbsa(sav);
if (!m)
goto fail;
break;
case SADB_X_EXT_SA2:
m = key_setsadbxsa2(sav->sah->saidx.mode,
sav->replay ? sav->replay->count : 0,
sav->sah->saidx.reqid);
if (!m)
goto fail;
break;
case SADB_EXT_ADDRESS_SRC:
m = key_setsadbaddr(SADB_EXT_ADDRESS_SRC,
&sav->sah->saidx.src.sa,
FULLMASK, IPSEC_ULPROTO_ANY);
if (!m)
goto fail;
break;
case SADB_EXT_ADDRESS_DST:
m = key_setsadbaddr(SADB_EXT_ADDRESS_DST,
&sav->sah->saidx.dst.sa,
FULLMASK, IPSEC_ULPROTO_ANY);
if (!m)
goto fail;
break;
case SADB_EXT_KEY_AUTH:
if (!sav->key_auth)
continue;
m = key_setkey(sav->key_auth, SADB_EXT_KEY_AUTH);
if (!m)
goto fail;
break;
case SADB_EXT_KEY_ENCRYPT:
if (!sav->key_enc)
continue;
m = key_setkey(sav->key_enc, SADB_EXT_KEY_ENCRYPT);
if (!m)
goto fail;
break;
case SADB_EXT_LIFETIME_CURRENT:
if (!sav->lft_c)
continue;
m = key_setlifetime(sav->lft_c,
SADB_EXT_LIFETIME_CURRENT);
if (!m)
goto fail;
break;
case SADB_EXT_LIFETIME_HARD:
if (!sav->lft_h)
continue;
m = key_setlifetime(sav->lft_h,
SADB_EXT_LIFETIME_HARD);
if (!m)
goto fail;
break;
case SADB_EXT_LIFETIME_SOFT:
if (!sav->lft_s)
continue;
m = key_setlifetime(sav->lft_s,
SADB_EXT_LIFETIME_SOFT);
if (!m)
goto fail;
break;
#ifdef IPSEC_NAT_T
case SADB_X_EXT_NAT_T_TYPE:
m = key_setsadbxtype(sav->natt_type);
if (!m)
goto fail;
break;
case SADB_X_EXT_NAT_T_DPORT:
m = key_setsadbxport(
KEY_PORTFROMSADDR(&sav->sah->saidx.dst),
SADB_X_EXT_NAT_T_DPORT);
if (!m)
goto fail;
break;
case SADB_X_EXT_NAT_T_SPORT:
m = key_setsadbxport(
KEY_PORTFROMSADDR(&sav->sah->saidx.src),
SADB_X_EXT_NAT_T_SPORT);
if (!m)
goto fail;
break;
case SADB_X_EXT_NAT_T_OAI:
case SADB_X_EXT_NAT_T_OAR:
case SADB_X_EXT_NAT_T_FRAG:
/* We do not (yet) support those. */
continue;
#endif
case SADB_EXT_ADDRESS_PROXY:
case SADB_EXT_IDENTITY_SRC:
case SADB_EXT_IDENTITY_DST:
/* XXX: should we brought from SPD ? */
case SADB_EXT_SENSITIVITY:
default:
continue;
}
if (!m)
goto fail;
if (tres)
m_cat(m, tres);
tres = m;
}
m_cat(result, tres);
if (result->m_len < sizeof(struct sadb_msg)) {
result = m_pullup(result, sizeof(struct sadb_msg));
if (result == NULL)
goto fail;
}
result->m_pkthdr.len = 0;
for (m = result; m; m = m->m_next)
result->m_pkthdr.len += m->m_len;
mtod(result, struct sadb_msg *)->sadb_msg_len =
PFKEY_UNIT64(result->m_pkthdr.len);
return result;
fail:
m_freem(result);
m_freem(tres);
return NULL;
}
/*
* set data into sadb_msg.
*/
static struct mbuf *
key_setsadbmsg(u_int8_t type, u_int16_t tlen, u_int8_t satype, u_int32_t seq,
pid_t pid, u_int16_t reserved)
{
struct mbuf *m;
struct sadb_msg *p;
int len;
len = PFKEY_ALIGN8(sizeof(struct sadb_msg));
if (len > MCLBYTES)
return NULL;
MGETHDR(m, M_NOWAIT, MT_DATA);
if (m && len > MHLEN) {
if (!(MCLGET(m, M_NOWAIT))) {
m_freem(m);
m = NULL;
}
}
if (!m)
return NULL;
m->m_pkthdr.len = m->m_len = len;
m->m_next = NULL;
p = mtod(m, struct sadb_msg *);
bzero(p, len);
p->sadb_msg_version = PF_KEY_V2;
p->sadb_msg_type = type;
p->sadb_msg_errno = 0;
p->sadb_msg_satype = satype;
p->sadb_msg_len = PFKEY_UNIT64(tlen);
p->sadb_msg_reserved = reserved;
p->sadb_msg_seq = seq;
p->sadb_msg_pid = (u_int32_t)pid;
return m;
}
/*
* copy secasvar data into sadb_address.
*/
static struct mbuf *
key_setsadbsa(struct secasvar *sav)
{
struct mbuf *m;
struct sadb_sa *p;
int len;
len = PFKEY_ALIGN8(sizeof(struct sadb_sa));
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return (NULL);
m_align(m, len);
m->m_len = len;
p = mtod(m, struct sadb_sa *);
bzero(p, len);
p->sadb_sa_len = PFKEY_UNIT64(len);
p->sadb_sa_exttype = SADB_EXT_SA;
p->sadb_sa_spi = sav->spi;
p->sadb_sa_replay = (sav->replay != NULL ? sav->replay->wsize : 0);
p->sadb_sa_state = sav->state;
p->sadb_sa_auth = sav->alg_auth;
p->sadb_sa_encrypt = sav->alg_enc;
p->sadb_sa_flags = sav->flags;
return m;
}
/*
* set data into sadb_address.
*/
static struct mbuf *
key_setsadbaddr(u_int16_t exttype, const struct sockaddr *saddr,
u_int8_t prefixlen, u_int16_t ul_proto)
{
struct mbuf *m;
struct sadb_address *p;
size_t len;
len = PFKEY_ALIGN8(sizeof(struct sadb_address)) +
PFKEY_ALIGN8(saddr->sa_len);
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return (NULL);
m_align(m, len);
m->m_len = len;
p = mtod(m, struct sadb_address *);
bzero(p, len);
p->sadb_address_len = PFKEY_UNIT64(len);
p->sadb_address_exttype = exttype;
p->sadb_address_proto = ul_proto;
if (prefixlen == FULLMASK) {
switch (saddr->sa_family) {
case AF_INET:
prefixlen = sizeof(struct in_addr) << 3;
break;
case AF_INET6:
prefixlen = sizeof(struct in6_addr) << 3;
break;
default:
; /*XXX*/
}
}
p->sadb_address_prefixlen = prefixlen;
p->sadb_address_reserved = 0;
bcopy(saddr,
mtod(m, caddr_t) + PFKEY_ALIGN8(sizeof(struct sadb_address)),
saddr->sa_len);
return m;
}
/*
* set data into sadb_x_sa2.
*/
static struct mbuf *
key_setsadbxsa2(u_int8_t mode, u_int32_t seq, u_int32_t reqid)
{
struct mbuf *m;
struct sadb_x_sa2 *p;
size_t len;
len = PFKEY_ALIGN8(sizeof(struct sadb_x_sa2));
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return (NULL);
m_align(m, len);
m->m_len = len;
p = mtod(m, struct sadb_x_sa2 *);
bzero(p, len);
p->sadb_x_sa2_len = PFKEY_UNIT64(len);
p->sadb_x_sa2_exttype = SADB_X_EXT_SA2;
p->sadb_x_sa2_mode = mode;
p->sadb_x_sa2_reserved1 = 0;
p->sadb_x_sa2_reserved2 = 0;
p->sadb_x_sa2_sequence = seq;
p->sadb_x_sa2_reqid = reqid;
return m;
}
#ifdef IPSEC_NAT_T
/*
* Set a type in sadb_x_nat_t_type.
*/
static struct mbuf *
key_setsadbxtype(u_int16_t type)
{
struct mbuf *m;
size_t len;
struct sadb_x_nat_t_type *p;
len = PFKEY_ALIGN8(sizeof(struct sadb_x_nat_t_type));
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return (NULL);
m_align(m, len);
m->m_len = len;
p = mtod(m, struct sadb_x_nat_t_type *);
bzero(p, len);
p->sadb_x_nat_t_type_len = PFKEY_UNIT64(len);
p->sadb_x_nat_t_type_exttype = SADB_X_EXT_NAT_T_TYPE;
p->sadb_x_nat_t_type_type = type;
return (m);
}
/*
* Set a port in sadb_x_nat_t_port.
* In contrast to default RFC 2367 behaviour, port is in network byte order.
*/
static struct mbuf *
key_setsadbxport(u_int16_t port, u_int16_t type)
{
struct mbuf *m;
size_t len;
struct sadb_x_nat_t_port *p;
len = PFKEY_ALIGN8(sizeof(struct sadb_x_nat_t_port));
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return (NULL);
m_align(m, len);
m->m_len = len;
p = mtod(m, struct sadb_x_nat_t_port *);
bzero(p, len);
p->sadb_x_nat_t_port_len = PFKEY_UNIT64(len);
p->sadb_x_nat_t_port_exttype = type;
p->sadb_x_nat_t_port_port = port;
return (m);
}
/*
* Get port from sockaddr. Port is in network byte order.
*/
u_int16_t
key_portfromsaddr(struct sockaddr *sa)
{
switch (sa->sa_family) {
#ifdef INET
case AF_INET:
return ((struct sockaddr_in *)sa)->sin_port;
#endif
#ifdef INET6
case AF_INET6:
return ((struct sockaddr_in6 *)sa)->sin6_port;
#endif
}
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s unexpected address family %d\n",
__func__, sa->sa_family));
return (0);
}
#endif /* IPSEC_NAT_T */
/*
* Set port in struct sockaddr. Port is in network byte order.
*/
static void
key_porttosaddr(struct sockaddr *sa, u_int16_t port)
{
switch (sa->sa_family) {
#ifdef INET
case AF_INET:
((struct sockaddr_in *)sa)->sin_port = port;
break;
#endif
#ifdef INET6
case AF_INET6:
((struct sockaddr_in6 *)sa)->sin6_port = port;
break;
#endif
default:
ipseclog((LOG_DEBUG, "%s: unexpected address family %d.\n",
__func__, sa->sa_family));
break;
}
}
/*
* set data into sadb_x_policy
*/
static struct mbuf *
key_setsadbxpolicy(u_int16_t type, u_int8_t dir, u_int32_t id)
{
struct mbuf *m;
struct sadb_x_policy *p;
size_t len;
len = PFKEY_ALIGN8(sizeof(struct sadb_x_policy));
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return (NULL);
m_align(m, len);
m->m_len = len;
p = mtod(m, struct sadb_x_policy *);
bzero(p, len);
p->sadb_x_policy_len = PFKEY_UNIT64(len);
p->sadb_x_policy_exttype = SADB_X_EXT_POLICY;
p->sadb_x_policy_type = type;
p->sadb_x_policy_dir = dir;
p->sadb_x_policy_id = id;
return m;
}
/* %%% utilities */
/* Take a key message (sadb_key) from the socket and turn it into one
* of the kernel's key structures (seckey).
*
* IN: pointer to the src
* OUT: NULL no more memory
*/
struct seckey *
key_dup_keymsg(const struct sadb_key *src, u_int len,
struct malloc_type *type)
{
struct seckey *dst;
dst = (struct seckey *)malloc(sizeof(struct seckey), type, M_NOWAIT);
if (dst != NULL) {
dst->bits = src->sadb_key_bits;
dst->key_data = (char *)malloc(len, type, M_NOWAIT);
if (dst->key_data != NULL) {
bcopy((const char *)src + sizeof(struct sadb_key),
dst->key_data, len);
} else {
ipseclog((LOG_DEBUG, "%s: No more memory.\n",
__func__));
free(dst, type);
dst = NULL;
}
} else {
ipseclog((LOG_DEBUG, "%s: No more memory.\n",
__func__));
}
return dst;
}
/* Take a lifetime message (sadb_lifetime) passed in on a socket and
* turn it into one of the kernel's lifetime structures (seclifetime).
*
* IN: pointer to the destination, source and malloc type
* OUT: NULL, no more memory
*/
static struct seclifetime *
key_dup_lifemsg(const struct sadb_lifetime *src, struct malloc_type *type)
{
struct seclifetime *dst = NULL;
dst = (struct seclifetime *)malloc(sizeof(struct seclifetime),
type, M_NOWAIT);
if (dst == NULL) {
/* XXX counter */
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
} else {
dst->allocations = src->sadb_lifetime_allocations;
dst->bytes = src->sadb_lifetime_bytes;
dst->addtime = src->sadb_lifetime_addtime;
dst->usetime = src->sadb_lifetime_usetime;
}
return dst;
}
/* compare my own address
* OUT: 1: true, i.e. my address.
* 0: false
*/
int
key_ismyaddr(struct sockaddr *sa)
{
IPSEC_ASSERT(sa != NULL, ("null sockaddr"));
switch (sa->sa_family) {
#ifdef INET
case AF_INET:
return (in_localip(satosin(sa)->sin_addr));
#endif
#ifdef INET6
case AF_INET6:
return key_ismyaddr6((struct sockaddr_in6 *)sa);
#endif
}
return 0;
}
#ifdef INET6
/*
* compare my own address for IPv6.
* 1: ours
* 0: other
*/
static int
key_ismyaddr6(struct sockaddr_in6 *sin6)
{
struct in6_addr in6;
if (!IN6_IS_SCOPE_LINKLOCAL(&sin6->sin6_addr))
return (in6_localip(&sin6->sin6_addr));
/* Convert address into kernel-internal form */
in6 = sin6->sin6_addr;
in6.s6_addr16[1] = htons(sin6->sin6_scope_id & 0xffff);
return (in6_localip(&in6));
}
#endif /*INET6*/
/*
* compare two secasindex structure.
* flag can specify to compare 2 saidxes.
* compare two secasindex structure without both mode and reqid.
* don't compare port.
* IN:
* saidx0: source, it can be in SAD.
* saidx1: object.
* OUT:
* 1 : equal
* 0 : not equal
*/
static int
key_cmpsaidx(const struct secasindex *saidx0, const struct secasindex *saidx1,
int flag)
{
int chkport = 0;
/* sanity */
if (saidx0 == NULL && saidx1 == NULL)
return 1;
if (saidx0 == NULL || saidx1 == NULL)
return 0;
if (saidx0->proto != saidx1->proto)
return 0;
if (flag == CMP_EXACTLY) {
if (saidx0->mode != saidx1->mode)
return 0;
if (saidx0->reqid != saidx1->reqid)
return 0;
if (bcmp(&saidx0->src, &saidx1->src, saidx0->src.sa.sa_len) != 0 ||
bcmp(&saidx0->dst, &saidx1->dst, saidx0->dst.sa.sa_len) != 0)
return 0;
} else {
/* CMP_MODE_REQID, CMP_REQID, CMP_HEAD */
if (flag == CMP_MODE_REQID
||flag == CMP_REQID) {
/*
* If reqid of SPD is non-zero, unique SA is required.
* The result must be of same reqid in this case.
*/
if (saidx1->reqid != 0 && saidx0->reqid != saidx1->reqid)
return 0;
}
if (flag == CMP_MODE_REQID) {
if (saidx0->mode != IPSEC_MODE_ANY
&& saidx0->mode != saidx1->mode)
return 0;
}
#ifdef IPSEC_NAT_T
/*
* If NAT-T is enabled, check ports for tunnel mode.
* Do not check ports if they are set to zero in the SPD.
* Also do not do it for native transport mode, as there
* is no port information available in the SP.
*/
if ((saidx1->mode == IPSEC_MODE_TUNNEL ||
(saidx1->mode == IPSEC_MODE_TRANSPORT &&
saidx1->proto == IPPROTO_ESP)) &&
saidx1->src.sa.sa_family == AF_INET &&
saidx1->dst.sa.sa_family == AF_INET &&
((const struct sockaddr_in *)(&saidx1->src))->sin_port &&
((const struct sockaddr_in *)(&saidx1->dst))->sin_port)
chkport = 1;
#endif /* IPSEC_NAT_T */
if (key_sockaddrcmp(&saidx0->src.sa, &saidx1->src.sa, chkport) != 0) {
return 0;
}
if (key_sockaddrcmp(&saidx0->dst.sa, &saidx1->dst.sa, chkport) != 0) {
return 0;
}
}
return 1;
}
/*
* compare two secindex structure exactly.
* IN:
* spidx0: source, it is often in SPD.
* spidx1: object, it is often from PFKEY message.
* OUT:
* 1 : equal
* 0 : not equal
*/
static int
key_cmpspidx_exactly(struct secpolicyindex *spidx0,
struct secpolicyindex *spidx1)
{
/* sanity */
if (spidx0 == NULL && spidx1 == NULL)
return 1;
if (spidx0 == NULL || spidx1 == NULL)
return 0;
if (spidx0->prefs != spidx1->prefs
|| spidx0->prefd != spidx1->prefd
|| spidx0->ul_proto != spidx1->ul_proto)
return 0;
return key_sockaddrcmp(&spidx0->src.sa, &spidx1->src.sa, 1) == 0 &&
key_sockaddrcmp(&spidx0->dst.sa, &spidx1->dst.sa, 1) == 0;
}
/*
* compare two secindex structure with mask.
* IN:
* spidx0: source, it is often in SPD.
* spidx1: object, it is often from IP header.
* OUT:
* 1 : equal
* 0 : not equal
*/
static int
key_cmpspidx_withmask(struct secpolicyindex *spidx0,
struct secpolicyindex *spidx1)
{
/* sanity */
if (spidx0 == NULL && spidx1 == NULL)
return 1;
if (spidx0 == NULL || spidx1 == NULL)
return 0;
if (spidx0->src.sa.sa_family != spidx1->src.sa.sa_family ||
spidx0->dst.sa.sa_family != spidx1->dst.sa.sa_family ||
spidx0->src.sa.sa_len != spidx1->src.sa.sa_len ||
spidx0->dst.sa.sa_len != spidx1->dst.sa.sa_len)
return 0;
/* if spidx.ul_proto == IPSEC_ULPROTO_ANY, ignore. */
if (spidx0->ul_proto != (u_int16_t)IPSEC_ULPROTO_ANY
&& spidx0->ul_proto != spidx1->ul_proto)
return 0;
switch (spidx0->src.sa.sa_family) {
case AF_INET:
if (spidx0->src.sin.sin_port != IPSEC_PORT_ANY
&& spidx0->src.sin.sin_port != spidx1->src.sin.sin_port)
return 0;
if (!key_bbcmp(&spidx0->src.sin.sin_addr,
&spidx1->src.sin.sin_addr, spidx0->prefs))
return 0;
break;
case AF_INET6:
if (spidx0->src.sin6.sin6_port != IPSEC_PORT_ANY
&& spidx0->src.sin6.sin6_port != spidx1->src.sin6.sin6_port)
return 0;
/*
* scope_id check. if sin6_scope_id is 0, we regard it
* as a wildcard scope, which matches any scope zone ID.
*/
if (spidx0->src.sin6.sin6_scope_id &&
spidx1->src.sin6.sin6_scope_id &&
spidx0->src.sin6.sin6_scope_id != spidx1->src.sin6.sin6_scope_id)
return 0;
if (!key_bbcmp(&spidx0->src.sin6.sin6_addr,
&spidx1->src.sin6.sin6_addr, spidx0->prefs))
return 0;
break;
default:
/* XXX */
if (bcmp(&spidx0->src, &spidx1->src, spidx0->src.sa.sa_len) != 0)
return 0;
break;
}
switch (spidx0->dst.sa.sa_family) {
case AF_INET:
if (spidx0->dst.sin.sin_port != IPSEC_PORT_ANY
&& spidx0->dst.sin.sin_port != spidx1->dst.sin.sin_port)
return 0;
if (!key_bbcmp(&spidx0->dst.sin.sin_addr,
&spidx1->dst.sin.sin_addr, spidx0->prefd))
return 0;
break;
case AF_INET6:
if (spidx0->dst.sin6.sin6_port != IPSEC_PORT_ANY
&& spidx0->dst.sin6.sin6_port != spidx1->dst.sin6.sin6_port)
return 0;
/*
* scope_id check. if sin6_scope_id is 0, we regard it
* as a wildcard scope, which matches any scope zone ID.
*/
if (spidx0->dst.sin6.sin6_scope_id &&
spidx1->dst.sin6.sin6_scope_id &&
spidx0->dst.sin6.sin6_scope_id != spidx1->dst.sin6.sin6_scope_id)
return 0;
if (!key_bbcmp(&spidx0->dst.sin6.sin6_addr,
&spidx1->dst.sin6.sin6_addr, spidx0->prefd))
return 0;
break;
default:
/* XXX */
if (bcmp(&spidx0->dst, &spidx1->dst, spidx0->dst.sa.sa_len) != 0)
return 0;
break;
}
/* XXX Do we check other field ? e.g. flowinfo */
return 1;
}
/* returns 0 on match */
static int
key_sockaddrcmp(const struct sockaddr *sa1, const struct sockaddr *sa2,
int port)
{
#ifdef satosin
#undef satosin
#endif
#define satosin(s) ((const struct sockaddr_in *)s)
#ifdef satosin6
#undef satosin6
#endif
#define satosin6(s) ((const struct sockaddr_in6 *)s)
if (sa1->sa_family != sa2->sa_family || sa1->sa_len != sa2->sa_len)
return 1;
switch (sa1->sa_family) {
case AF_INET:
if (sa1->sa_len != sizeof(struct sockaddr_in))
return 1;
if (satosin(sa1)->sin_addr.s_addr !=
satosin(sa2)->sin_addr.s_addr) {
return 1;
}
if (port && satosin(sa1)->sin_port != satosin(sa2)->sin_port)
return 1;
break;
case AF_INET6:
if (sa1->sa_len != sizeof(struct sockaddr_in6))
return 1; /*EINVAL*/
if (satosin6(sa1)->sin6_scope_id !=
satosin6(sa2)->sin6_scope_id) {
return 1;
}
if (!IN6_ARE_ADDR_EQUAL(&satosin6(sa1)->sin6_addr,
&satosin6(sa2)->sin6_addr)) {
return 1;
}
if (port &&
satosin6(sa1)->sin6_port != satosin6(sa2)->sin6_port) {
return 1;
}
break;
default:
if (bcmp(sa1, sa2, sa1->sa_len) != 0)
return 1;
break;
}
return 0;
#undef satosin
#undef satosin6
}
/*
* compare two buffers with mask.
* IN:
* addr1: source
* addr2: object
* bits: Number of bits to compare
* OUT:
* 1 : equal
* 0 : not equal
*/
static int
key_bbcmp(const void *a1, const void *a2, u_int bits)
{
const unsigned char *p1 = a1;
const unsigned char *p2 = a2;
/* XXX: This could be considerably faster if we compare a word
* at a time, but it is complicated on LSB Endian machines */
/* Handle null pointers */
if (p1 == NULL || p2 == NULL)
return (p1 == p2);
while (bits >= 8) {
if (*p1++ != *p2++)
return 0;
bits -= 8;
}
if (bits > 0) {
u_int8_t mask = ~((1<<(8-bits))-1);
if ((*p1 & mask) != (*p2 & mask))
return 0;
}
return 1; /* Match! */
}
static void
key_flush_spd(time_t now)
{
SPTREE_RLOCK_TRACKER;
struct secpolicy *sp;
u_int dir;
/* SPD */
for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
restart:
SPTREE_RLOCK();
TAILQ_FOREACH(sp, &V_sptree[dir], chain) {
if (sp->lifetime == 0 && sp->validtime == 0)
continue;
if ((sp->lifetime &&
now - sp->created > sp->lifetime) ||
(sp->validtime &&
now - sp->lastused > sp->validtime)) {
SP_ADDREF(sp);
SPTREE_RUNLOCK();
key_spdexpire(sp);
key_unlink(sp);
KEY_FREESP(&sp);
goto restart;
}
}
SPTREE_RUNLOCK();
}
}
static void
key_flush_sad(time_t now)
{
struct secashead *sah, *nextsah;
struct secasvar *sav, *nextsav;
/* SAD */
SAHTREE_LOCK();
LIST_FOREACH_SAFE(sah, &V_sahtree, chain, nextsah) {
/* if sah has been dead, then delete it and process next sah. */
if (sah->state == SADB_SASTATE_DEAD) {
key_delsah(sah);
continue;
}
/* if LARVAL entry doesn't become MATURE, delete it. */
LIST_FOREACH_SAFE(sav, &sah->savtree[SADB_SASTATE_LARVAL], chain, nextsav) {
/* Need to also check refcnt for a larval SA ??? */
if (now - sav->created > V_key_larval_lifetime)
KEY_FREESAV(&sav);
}
/*
* check MATURE entry to start to send expire message
* whether or not.
*/
LIST_FOREACH_SAFE(sav, &sah->savtree[SADB_SASTATE_MATURE], chain, nextsav) {
/* we don't need to check. */
if (sav->lft_s == NULL)
continue;
/* sanity check */
if (sav->lft_c == NULL) {
ipseclog((LOG_DEBUG,"%s: there is no CURRENT "
"time, why?\n", __func__));
continue;
}
/*
* RFC 2367:
* HARD lifetimes MUST take precedence over SOFT
* lifetimes, meaning if the HARD and SOFT lifetimes
* are the same, the HARD lifetime will appear on the
* EXPIRE message.
*/
/* check HARD lifetime */
if ((sav->lft_h->addtime != 0 &&
now - sav->created > sav->lft_h->addtime) ||
(sav->lft_h->bytes != 0 &&
sav->lft_h->bytes < sav->lft_c->bytes)) {
key_sa_chgstate(sav, SADB_SASTATE_DEAD);
key_expire(sav, 1);
KEY_FREESAV(&sav);
}
/* check SOFT lifetime */
else if ((sav->lft_s->addtime != 0 &&
now - sav->created > sav->lft_s->addtime) ||
(sav->lft_s->bytes != 0 &&
sav->lft_s->bytes < sav->lft_c->bytes)) {
key_sa_chgstate(sav, SADB_SASTATE_DYING);
key_expire(sav, 0);
}
}
/* check DYING entry to change status to DEAD. */
LIST_FOREACH_SAFE(sav, &sah->savtree[SADB_SASTATE_DYING], chain, nextsav) {
/* we don't need to check. */
if (sav->lft_h == NULL)
continue;
/* sanity check */
if (sav->lft_c == NULL) {
ipseclog((LOG_DEBUG, "%s: there is no CURRENT "
"time, why?\n", __func__));
continue;
}
if (sav->lft_h->addtime != 0 &&
now - sav->created > sav->lft_h->addtime) {
key_sa_chgstate(sav, SADB_SASTATE_DEAD);
key_expire(sav, 1);
KEY_FREESAV(&sav);
}
#if 0 /* XXX Should we keep to send expire message until HARD lifetime ? */
else if (sav->lft_s != NULL
&& sav->lft_s->addtime != 0
&& now - sav->created > sav->lft_s->addtime) {
/*
* XXX: should be checked to be
* installed the valid SA.
*/
/*
* If there is no SA then sending
* expire message.
*/
key_expire(sav, 0);
}
#endif
/* check HARD lifetime by bytes */
else if (sav->lft_h->bytes != 0 &&
sav->lft_h->bytes < sav->lft_c->bytes) {
key_sa_chgstate(sav, SADB_SASTATE_DEAD);
key_expire(sav, 1);
KEY_FREESAV(&sav);
}
}
/* delete entry in DEAD */
LIST_FOREACH_SAFE(sav, &sah->savtree[SADB_SASTATE_DEAD], chain, nextsav) {
/* sanity check */
if (sav->state != SADB_SASTATE_DEAD) {
ipseclog((LOG_DEBUG, "%s: invalid sav->state "
"(queue: %d SA: %d): kill it anyway\n",
__func__,
SADB_SASTATE_DEAD, sav->state));
}
/*
* do not call key_freesav() here.
* sav should already be freed, and sav->refcnt
* shows other references to sav
* (such as from SPD).
*/
}
}
SAHTREE_UNLOCK();
}
static void
key_flush_acq(time_t now)
{
struct secacq *acq, *nextacq;
/* ACQ tree */
ACQ_LOCK();
for (acq = LIST_FIRST(&V_acqtree); acq != NULL; acq = nextacq) {
nextacq = LIST_NEXT(acq, chain);
if (now - acq->created > V_key_blockacq_lifetime
&& __LIST_CHAINED(acq)) {
LIST_REMOVE(acq, chain);
free(acq, M_IPSEC_SAQ);
}
}
ACQ_UNLOCK();
}
static void
key_flush_spacq(time_t now)
{
struct secspacq *acq, *nextacq;
/* SP ACQ tree */
SPACQ_LOCK();
for (acq = LIST_FIRST(&V_spacqtree); acq != NULL; acq = nextacq) {
nextacq = LIST_NEXT(acq, chain);
if (now - acq->created > V_key_blockacq_lifetime
&& __LIST_CHAINED(acq)) {
LIST_REMOVE(acq, chain);
free(acq, M_IPSEC_SAQ);
}
}
SPACQ_UNLOCK();
}
/*
* time handler.
* scanning SPD and SAD to check status for each entries,
* and do to remove or to expire.
* XXX: year 2038 problem may remain.
*/
static void
key_timehandler(void *arg)
{
VNET_ITERATOR_DECL(vnet_iter);
time_t now = time_second;
VNET_LIST_RLOCK_NOSLEEP();
VNET_FOREACH(vnet_iter) {
CURVNET_SET(vnet_iter);
key_flush_spd(now);
key_flush_sad(now);
key_flush_acq(now);
key_flush_spacq(now);
CURVNET_RESTORE();
}
VNET_LIST_RUNLOCK_NOSLEEP();
#ifndef IPSEC_DEBUG2
/* do exchange to tick time !! */
callout_schedule(&key_timer, hz);
#endif /* IPSEC_DEBUG2 */
}
u_long
key_random()
{
u_long value;
key_randomfill(&value, sizeof(value));
return value;
}
void
key_randomfill(void *p, size_t l)
{
size_t n;
u_long v;
static int warn = 1;
n = 0;
n = (size_t)read_random(p, (u_int)l);
/* last resort */
while (n < l) {
v = random();
bcopy(&v, (u_int8_t *)p + n,
l - n < sizeof(v) ? l - n : sizeof(v));
n += sizeof(v);
if (warn) {
printf("WARNING: pseudo-random number generator "
"used for IPsec processing\n");
warn = 0;
}
}
}
/*
* map SADB_SATYPE_* to IPPROTO_*.
* if satype == SADB_SATYPE then satype is mapped to ~0.
* OUT:
* 0: invalid satype.
*/
static u_int16_t
key_satype2proto(u_int8_t satype)
{
switch (satype) {
case SADB_SATYPE_UNSPEC:
return IPSEC_PROTO_ANY;
case SADB_SATYPE_AH:
return IPPROTO_AH;
case SADB_SATYPE_ESP:
return IPPROTO_ESP;
case SADB_X_SATYPE_IPCOMP:
return IPPROTO_IPCOMP;
case SADB_X_SATYPE_TCPSIGNATURE:
return IPPROTO_TCP;
default:
return 0;
}
/* NOTREACHED */
}
/*
* map IPPROTO_* to SADB_SATYPE_*
* OUT:
* 0: invalid protocol type.
*/
static u_int8_t
key_proto2satype(u_int16_t proto)
{
switch (proto) {
case IPPROTO_AH:
return SADB_SATYPE_AH;
case IPPROTO_ESP:
return SADB_SATYPE_ESP;
case IPPROTO_IPCOMP:
return SADB_X_SATYPE_IPCOMP;
case IPPROTO_TCP:
return SADB_X_SATYPE_TCPSIGNATURE;
default:
return 0;
}
/* NOTREACHED */
}
/* %%% PF_KEY */
/*
* SADB_GETSPI processing is to receive
* <base, (SA2), src address, dst address, (SPI range)>
* from the IKMPd, to assign a unique spi value, to hang on the INBOUND
* tree with the status of LARVAL, and send
* <base, SA(*), address(SD)>
* to the IKMPd.
*
* IN: mhp: pointer to the pointer to each header.
* OUT: NULL if fail.
* other if success, return pointer to the message to send.
*/
static int
key_getspi(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct sadb_address *src0, *dst0;
struct secasindex saidx;
struct secashead *newsah;
struct secasvar *newsav;
u_int8_t proto;
u_int32_t spi;
u_int8_t mode;
u_int32_t reqid;
int error;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
if (mhp->ext[SADB_EXT_ADDRESS_SRC] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_DST] == NULL) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->extlen[SADB_EXT_ADDRESS_SRC] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_EXT_ADDRESS_DST] < sizeof(struct sadb_address)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_X_EXT_SA2] != NULL) {
mode = ((struct sadb_x_sa2 *)mhp->ext[SADB_X_EXT_SA2])->sadb_x_sa2_mode;
reqid = ((struct sadb_x_sa2 *)mhp->ext[SADB_X_EXT_SA2])->sadb_x_sa2_reqid;
} else {
mode = IPSEC_MODE_ANY;
reqid = 0;
}
src0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_SRC]);
dst0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_DST]);
/* map satype to proto */
if ((proto = key_satype2proto(mhp->msg->sadb_msg_satype)) == 0) {
ipseclog((LOG_DEBUG, "%s: invalid satype is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
/*
* Make sure the port numbers are zero.
* In case of NAT-T we will update them later if needed.
*/
switch (((struct sockaddr *)(src0 + 1))->sa_family) {
case AF_INET:
if (((struct sockaddr *)(src0 + 1))->sa_len !=
sizeof(struct sockaddr_in))
return key_senderror(so, m, EINVAL);
((struct sockaddr_in *)(src0 + 1))->sin_port = 0;
break;
case AF_INET6:
if (((struct sockaddr *)(src0 + 1))->sa_len !=
sizeof(struct sockaddr_in6))
return key_senderror(so, m, EINVAL);
((struct sockaddr_in6 *)(src0 + 1))->sin6_port = 0;
break;
default:
; /*???*/
}
switch (((struct sockaddr *)(dst0 + 1))->sa_family) {
case AF_INET:
if (((struct sockaddr *)(dst0 + 1))->sa_len !=
sizeof(struct sockaddr_in))
return key_senderror(so, m, EINVAL);
((struct sockaddr_in *)(dst0 + 1))->sin_port = 0;
break;
case AF_INET6:
if (((struct sockaddr *)(dst0 + 1))->sa_len !=
sizeof(struct sockaddr_in6))
return key_senderror(so, m, EINVAL);
((struct sockaddr_in6 *)(dst0 + 1))->sin6_port = 0;
break;
default:
; /*???*/
}
/* XXX boundary check against sa_len */
KEY_SETSECASIDX(proto, mode, reqid, src0 + 1, dst0 + 1, &saidx);
#ifdef IPSEC_NAT_T
/*
* Handle NAT-T info if present.
* We made sure the port numbers are zero above, so we do
* not have to worry in case we do not update them.
*/
if (mhp->ext[SADB_X_EXT_NAT_T_OAI] != NULL)
ipseclog((LOG_DEBUG, "%s: NAT-T OAi present\n", __func__));
if (mhp->ext[SADB_X_EXT_NAT_T_OAR] != NULL)
ipseclog((LOG_DEBUG, "%s: NAT-T OAr present\n", __func__));
if (mhp->ext[SADB_X_EXT_NAT_T_TYPE] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_SPORT] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_DPORT] != NULL) {
struct sadb_x_nat_t_type *type;
struct sadb_x_nat_t_port *sport, *dport;
if (mhp->extlen[SADB_X_EXT_NAT_T_TYPE] < sizeof(*type) ||
mhp->extlen[SADB_X_EXT_NAT_T_SPORT] < sizeof(*sport) ||
mhp->extlen[SADB_X_EXT_NAT_T_DPORT] < sizeof(*dport)) {
ipseclog((LOG_DEBUG, "%s: invalid nat-t message "
"passed.\n", __func__));
return key_senderror(so, m, EINVAL);
}
sport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_SPORT];
dport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_DPORT];
if (sport)
KEY_PORTTOSADDR(&saidx.src, sport->sadb_x_nat_t_port_port);
if (dport)
KEY_PORTTOSADDR(&saidx.dst, dport->sadb_x_nat_t_port_port);
}
#endif
/* SPI allocation */
spi = key_do_getnewspi((struct sadb_spirange *)mhp->ext[SADB_EXT_SPIRANGE],
&saidx);
if (spi == 0)
return key_senderror(so, m, EINVAL);
/* get a SA index */
if ((newsah = key_getsah(&saidx)) == NULL) {
/* create a new SA index */
if ((newsah = key_newsah(&saidx)) == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n",__func__));
return key_senderror(so, m, ENOBUFS);
}
}
/* get a new SA */
/* XXX rewrite */
newsav = KEY_NEWSAV(m, mhp, newsah, &error);
if (newsav == NULL) {
/* XXX don't free new SA index allocated in above. */
return key_senderror(so, m, error);
}
/* set spi */
newsav->spi = htonl(spi);
/* delete the entry in acqtree */
if (mhp->msg->sadb_msg_seq != 0) {
struct secacq *acq;
if ((acq = key_getacqbyseq(mhp->msg->sadb_msg_seq)) != NULL) {
/* reset counter in order to deletion by timehandler. */
acq->created = time_second;
acq->count = 0;
}
}
{
struct mbuf *n, *nn;
struct sadb_sa *m_sa;
struct sadb_msg *newmsg;
int off, len;
/* create new sadb_msg to reply. */
len = PFKEY_ALIGN8(sizeof(struct sadb_msg)) +
PFKEY_ALIGN8(sizeof(struct sadb_sa));
MGETHDR(n, M_NOWAIT, MT_DATA);
if (len > MHLEN) {
if (!(MCLGET(n, M_NOWAIT))) {
m_freem(n);
n = NULL;
}
}
if (!n)
return key_senderror(so, m, ENOBUFS);
n->m_len = len;
n->m_next = NULL;
off = 0;
m_copydata(m, 0, sizeof(struct sadb_msg), mtod(n, caddr_t) + off);
off += PFKEY_ALIGN8(sizeof(struct sadb_msg));
m_sa = (struct sadb_sa *)(mtod(n, caddr_t) + off);
m_sa->sadb_sa_len = PFKEY_UNIT64(sizeof(struct sadb_sa));
m_sa->sadb_sa_exttype = SADB_EXT_SA;
m_sa->sadb_sa_spi = htonl(spi);
off += PFKEY_ALIGN8(sizeof(struct sadb_sa));
IPSEC_ASSERT(off == len,
("length inconsistency (off %u len %u)", off, len));
n->m_next = key_gather_mbuf(m, mhp, 0, 2, SADB_EXT_ADDRESS_SRC,
SADB_EXT_ADDRESS_DST);
if (!n->m_next) {
m_freem(n);
return key_senderror(so, m, ENOBUFS);
}
if (n->m_len < sizeof(struct sadb_msg)) {
n = m_pullup(n, sizeof(struct sadb_msg));
if (n == NULL)
return key_sendup_mbuf(so, m, KEY_SENDUP_ONE);
}
n->m_pkthdr.len = 0;
for (nn = n; nn; nn = nn->m_next)
n->m_pkthdr.len += nn->m_len;
newmsg = mtod(n, struct sadb_msg *);
newmsg->sadb_msg_seq = newsav->seq;
newmsg->sadb_msg_errno = 0;
newmsg->sadb_msg_len = PFKEY_UNIT64(n->m_pkthdr.len);
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
}
}
/*
* allocating new SPI
* called by key_getspi().
* OUT:
* 0: failure.
* others: success.
*/
static u_int32_t
key_do_getnewspi(struct sadb_spirange *spirange, struct secasindex *saidx)
{
u_int32_t newspi;
u_int32_t min, max;
int count = V_key_spi_trycnt;
/* set spi range to allocate */
if (spirange != NULL) {
min = spirange->sadb_spirange_min;
max = spirange->sadb_spirange_max;
} else {
min = V_key_spi_minval;
max = V_key_spi_maxval;
}
/* IPCOMP needs 2-byte SPI */
if (saidx->proto == IPPROTO_IPCOMP) {
u_int32_t t;
if (min >= 0x10000)
min = 0xffff;
if (max >= 0x10000)
max = 0xffff;
if (min > max) {
t = min; min = max; max = t;
}
}
if (min == max) {
if (key_checkspidup(saidx, min) != NULL) {
ipseclog((LOG_DEBUG, "%s: SPI %u exists already.\n",
__func__, min));
return 0;
}
count--; /* taking one cost. */
newspi = min;
} else {
/* init SPI */
newspi = 0;
/* when requesting to allocate spi ranged */
while (count--) {
/* generate pseudo-random SPI value ranged. */
newspi = min + (key_random() % (max - min + 1));
if (key_checkspidup(saidx, newspi) == NULL)
break;
}
if (count == 0 || newspi == 0) {
ipseclog((LOG_DEBUG, "%s: to allocate spi is failed.\n",
__func__));
return 0;
}
}
/* statistics */
keystat.getspi_count =
(keystat.getspi_count + V_key_spi_trycnt - count) / 2;
return newspi;
}
/*
* SADB_UPDATE processing
* receive
* <base, SA, (SA2), (lifetime(HSC),) address(SD), (address(P),)
* key(AE), (identity(SD),) (sensitivity)>
* from the ikmpd, and update a secasvar entry whose status is SADB_SASTATE_LARVAL.
* and send
* <base, SA, (SA2), (lifetime(HSC),) address(SD), (address(P),)
* (identity(SD),) (sensitivity)>
* to the ikmpd.
*
* m will always be freed.
*/
static int
key_update(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct sadb_sa *sa0;
struct sadb_address *src0, *dst0;
#ifdef IPSEC_NAT_T
struct sadb_x_nat_t_type *type;
struct sadb_x_nat_t_port *sport, *dport;
struct sadb_address *iaddr, *raddr;
struct sadb_x_nat_t_frag *frag;
#endif
struct secasindex saidx;
struct secashead *sah;
struct secasvar *sav;
u_int16_t proto;
u_int8_t mode;
u_int32_t reqid;
int error;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* map satype to proto */
if ((proto = key_satype2proto(mhp->msg->sadb_msg_satype)) == 0) {
ipseclog((LOG_DEBUG, "%s: invalid satype is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_EXT_SA] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_SRC] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_DST] == NULL ||
(mhp->msg->sadb_msg_satype == SADB_SATYPE_ESP &&
mhp->ext[SADB_EXT_KEY_ENCRYPT] == NULL) ||
(mhp->msg->sadb_msg_satype == SADB_SATYPE_AH &&
mhp->ext[SADB_EXT_KEY_AUTH] == NULL) ||
(mhp->ext[SADB_EXT_LIFETIME_HARD] != NULL &&
mhp->ext[SADB_EXT_LIFETIME_SOFT] == NULL) ||
(mhp->ext[SADB_EXT_LIFETIME_HARD] == NULL &&
mhp->ext[SADB_EXT_LIFETIME_SOFT] != NULL)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->extlen[SADB_EXT_SA] < sizeof(struct sadb_sa) ||
mhp->extlen[SADB_EXT_ADDRESS_SRC] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_EXT_ADDRESS_DST] < sizeof(struct sadb_address)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_X_EXT_SA2] != NULL) {
mode = ((struct sadb_x_sa2 *)mhp->ext[SADB_X_EXT_SA2])->sadb_x_sa2_mode;
reqid = ((struct sadb_x_sa2 *)mhp->ext[SADB_X_EXT_SA2])->sadb_x_sa2_reqid;
} else {
mode = IPSEC_MODE_ANY;
reqid = 0;
}
/* XXX boundary checking for other extensions */
sa0 = (struct sadb_sa *)mhp->ext[SADB_EXT_SA];
src0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_SRC]);
dst0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_DST]);
/* XXX boundary check against sa_len */
KEY_SETSECASIDX(proto, mode, reqid, src0 + 1, dst0 + 1, &saidx);
/*
* Make sure the port numbers are zero.
* In case of NAT-T we will update them later if needed.
*/
KEY_PORTTOSADDR(&saidx.src, 0);
KEY_PORTTOSADDR(&saidx.dst, 0);
#ifdef IPSEC_NAT_T
/*
* Handle NAT-T info if present.
*/
if (mhp->ext[SADB_X_EXT_NAT_T_TYPE] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_SPORT] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_DPORT] != NULL) {
if (mhp->extlen[SADB_X_EXT_NAT_T_TYPE] < sizeof(*type) ||
mhp->extlen[SADB_X_EXT_NAT_T_SPORT] < sizeof(*sport) ||
mhp->extlen[SADB_X_EXT_NAT_T_DPORT] < sizeof(*dport)) {
ipseclog((LOG_DEBUG, "%s: invalid message.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
type = (struct sadb_x_nat_t_type *)
mhp->ext[SADB_X_EXT_NAT_T_TYPE];
sport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_SPORT];
dport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_DPORT];
} else {
type = 0;
sport = dport = 0;
}
if (mhp->ext[SADB_X_EXT_NAT_T_OAI] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_OAR] != NULL) {
if (mhp->extlen[SADB_X_EXT_NAT_T_OAI] < sizeof(*iaddr) ||
mhp->extlen[SADB_X_EXT_NAT_T_OAR] < sizeof(*raddr)) {
ipseclog((LOG_DEBUG, "%s: invalid message\n",
__func__));
return key_senderror(so, m, EINVAL);
}
iaddr = (struct sadb_address *)mhp->ext[SADB_X_EXT_NAT_T_OAI];
raddr = (struct sadb_address *)mhp->ext[SADB_X_EXT_NAT_T_OAR];
ipseclog((LOG_DEBUG, "%s: NAT-T OAi/r present\n", __func__));
} else {
iaddr = raddr = NULL;
}
if (mhp->ext[SADB_X_EXT_NAT_T_FRAG] != NULL) {
if (mhp->extlen[SADB_X_EXT_NAT_T_FRAG] < sizeof(*frag)) {
ipseclog((LOG_DEBUG, "%s: invalid message\n",
__func__));
return key_senderror(so, m, EINVAL);
}
frag = (struct sadb_x_nat_t_frag *)
mhp->ext[SADB_X_EXT_NAT_T_FRAG];
} else {
frag = 0;
}
#endif
/* get a SA header */
if ((sah = key_getsah(&saidx)) == NULL) {
ipseclog((LOG_DEBUG, "%s: no SA index found.\n", __func__));
return key_senderror(so, m, ENOENT);
}
/* set spidx if there */
/* XXX rewrite */
error = key_setident(sah, m, mhp);
if (error)
return key_senderror(so, m, error);
/* find a SA with sequence number. */
#ifdef IPSEC_DOSEQCHECK
if (mhp->msg->sadb_msg_seq != 0
&& (sav = key_getsavbyseq(sah, mhp->msg->sadb_msg_seq)) == NULL) {
ipseclog((LOG_DEBUG, "%s: no larval SA with sequence %u "
"exists.\n", __func__, mhp->msg->sadb_msg_seq));
return key_senderror(so, m, ENOENT);
}
#else
SAHTREE_LOCK();
sav = key_getsavbyspi(sah, sa0->sadb_sa_spi);
SAHTREE_UNLOCK();
if (sav == NULL) {
ipseclog((LOG_DEBUG, "%s: no such a SA found (spi:%u)\n",
__func__, (u_int32_t)ntohl(sa0->sadb_sa_spi)));
return key_senderror(so, m, EINVAL);
}
#endif
/* validity check */
if (sav->sah->saidx.proto != proto) {
ipseclog((LOG_DEBUG, "%s: protocol mismatched "
"(DB=%u param=%u)\n", __func__,
sav->sah->saidx.proto, proto));
return key_senderror(so, m, EINVAL);
}
#ifdef IPSEC_DOSEQCHECK
if (sav->spi != sa0->sadb_sa_spi) {
ipseclog((LOG_DEBUG, "%s: SPI mismatched (DB:%u param:%u)\n",
__func__,
(u_int32_t)ntohl(sav->spi),
(u_int32_t)ntohl(sa0->sadb_sa_spi)));
return key_senderror(so, m, EINVAL);
}
#endif
if (sav->pid != mhp->msg->sadb_msg_pid) {
ipseclog((LOG_DEBUG, "%s: pid mismatched (DB:%u param:%u)\n",
__func__, sav->pid, mhp->msg->sadb_msg_pid));
return key_senderror(so, m, EINVAL);
}
/* copy sav values */
error = key_setsaval(sav, m, mhp);
if (error) {
KEY_FREESAV(&sav);
return key_senderror(so, m, error);
}
#ifdef IPSEC_NAT_T
/*
* Handle more NAT-T info if present,
* now that we have a sav to fill.
*/
if (type)
sav->natt_type = type->sadb_x_nat_t_type_type;
if (sport)
KEY_PORTTOSADDR(&sav->sah->saidx.src,
sport->sadb_x_nat_t_port_port);
if (dport)
KEY_PORTTOSADDR(&sav->sah->saidx.dst,
dport->sadb_x_nat_t_port_port);
#if 0
/*
* In case SADB_X_EXT_NAT_T_FRAG was not given, leave it at 0.
* We should actually check for a minimum MTU here, if we
* want to support it in ip_output.
*/
if (frag)
sav->natt_esp_frag_len = frag->sadb_x_nat_t_frag_fraglen;
#endif
#endif
/* check SA values to be mature. */
if ((mhp->msg->sadb_msg_errno = key_mature(sav)) != 0) {
KEY_FREESAV(&sav);
return key_senderror(so, m, 0);
}
{
struct mbuf *n;
/* set msg buf from mhp */
n = key_getmsgbuf_x1(m, mhp);
if (n == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
return key_senderror(so, m, ENOBUFS);
}
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ALL);
}
}
/*
* search SAD with sequence for a SA which state is SADB_SASTATE_LARVAL.
* only called by key_update().
* OUT:
* NULL : not found
* others : found, pointer to a SA.
*/
#ifdef IPSEC_DOSEQCHECK
static struct secasvar *
key_getsavbyseq(struct secashead *sah, u_int32_t seq)
{
struct secasvar *sav;
u_int state;
state = SADB_SASTATE_LARVAL;
/* search SAD with sequence number ? */
LIST_FOREACH(sav, &sah->savtree[state], chain) {
KEY_CHKSASTATE(state, sav->state, __func__);
if (sav->seq == seq) {
sa_addref(sav);
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s cause refcnt++:%d SA:%p\n",
__func__, sav->refcnt, sav));
return sav;
}
}
return NULL;
}
#endif
/*
* SADB_ADD processing
* add an entry to SA database, when received
* <base, SA, (SA2), (lifetime(HSC),) address(SD), (address(P),)
* key(AE), (identity(SD),) (sensitivity)>
* from the ikmpd,
* and send
* <base, SA, (SA2), (lifetime(HSC),) address(SD), (address(P),)
* (identity(SD),) (sensitivity)>
* to the ikmpd.
*
* IGNORE identity and sensitivity messages.
*
* m will always be freed.
*/
static int
key_add(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct sadb_sa *sa0;
struct sadb_address *src0, *dst0;
#ifdef IPSEC_NAT_T
struct sadb_x_nat_t_type *type;
struct sadb_address *iaddr, *raddr;
struct sadb_x_nat_t_frag *frag;
#endif
struct secasindex saidx;
struct secashead *newsah;
struct secasvar *newsav;
u_int16_t proto;
u_int8_t mode;
u_int32_t reqid;
int error;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* map satype to proto */
if ((proto = key_satype2proto(mhp->msg->sadb_msg_satype)) == 0) {
ipseclog((LOG_DEBUG, "%s: invalid satype is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_EXT_SA] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_SRC] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_DST] == NULL ||
(mhp->msg->sadb_msg_satype == SADB_SATYPE_ESP &&
mhp->ext[SADB_EXT_KEY_ENCRYPT] == NULL) ||
(mhp->msg->sadb_msg_satype == SADB_SATYPE_AH &&
mhp->ext[SADB_EXT_KEY_AUTH] == NULL) ||
(mhp->ext[SADB_EXT_LIFETIME_HARD] != NULL &&
mhp->ext[SADB_EXT_LIFETIME_SOFT] == NULL) ||
(mhp->ext[SADB_EXT_LIFETIME_HARD] == NULL &&
mhp->ext[SADB_EXT_LIFETIME_SOFT] != NULL)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->extlen[SADB_EXT_SA] < sizeof(struct sadb_sa) ||
mhp->extlen[SADB_EXT_ADDRESS_SRC] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_EXT_ADDRESS_DST] < sizeof(struct sadb_address)) {
/* XXX need more */
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_X_EXT_SA2] != NULL) {
mode = ((struct sadb_x_sa2 *)mhp->ext[SADB_X_EXT_SA2])->sadb_x_sa2_mode;
reqid = ((struct sadb_x_sa2 *)mhp->ext[SADB_X_EXT_SA2])->sadb_x_sa2_reqid;
} else {
mode = IPSEC_MODE_ANY;
reqid = 0;
}
sa0 = (struct sadb_sa *)mhp->ext[SADB_EXT_SA];
src0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_SRC];
dst0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_DST];
/* XXX boundary check against sa_len */
KEY_SETSECASIDX(proto, mode, reqid, src0 + 1, dst0 + 1, &saidx);
/*
* Make sure the port numbers are zero.
* In case of NAT-T we will update them later if needed.
*/
KEY_PORTTOSADDR(&saidx.src, 0);
KEY_PORTTOSADDR(&saidx.dst, 0);
#ifdef IPSEC_NAT_T
/*
* Handle NAT-T info if present.
*/
if (mhp->ext[SADB_X_EXT_NAT_T_TYPE] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_SPORT] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_DPORT] != NULL) {
struct sadb_x_nat_t_port *sport, *dport;
if (mhp->extlen[SADB_X_EXT_NAT_T_TYPE] < sizeof(*type) ||
mhp->extlen[SADB_X_EXT_NAT_T_SPORT] < sizeof(*sport) ||
mhp->extlen[SADB_X_EXT_NAT_T_DPORT] < sizeof(*dport)) {
ipseclog((LOG_DEBUG, "%s: invalid message.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
type = (struct sadb_x_nat_t_type *)
mhp->ext[SADB_X_EXT_NAT_T_TYPE];
sport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_SPORT];
dport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_DPORT];
if (sport)
KEY_PORTTOSADDR(&saidx.src,
sport->sadb_x_nat_t_port_port);
if (dport)
KEY_PORTTOSADDR(&saidx.dst,
dport->sadb_x_nat_t_port_port);
} else {
type = 0;
}
if (mhp->ext[SADB_X_EXT_NAT_T_OAI] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_OAR] != NULL) {
if (mhp->extlen[SADB_X_EXT_NAT_T_OAI] < sizeof(*iaddr) ||
mhp->extlen[SADB_X_EXT_NAT_T_OAR] < sizeof(*raddr)) {
ipseclog((LOG_DEBUG, "%s: invalid message\n",
__func__));
return key_senderror(so, m, EINVAL);
}
iaddr = (struct sadb_address *)mhp->ext[SADB_X_EXT_NAT_T_OAI];
raddr = (struct sadb_address *)mhp->ext[SADB_X_EXT_NAT_T_OAR];
ipseclog((LOG_DEBUG, "%s: NAT-T OAi/r present\n", __func__));
} else {
iaddr = raddr = NULL;
}
if (mhp->ext[SADB_X_EXT_NAT_T_FRAG] != NULL) {
if (mhp->extlen[SADB_X_EXT_NAT_T_FRAG] < sizeof(*frag)) {
ipseclog((LOG_DEBUG, "%s: invalid message\n",
__func__));
return key_senderror(so, m, EINVAL);
}
frag = (struct sadb_x_nat_t_frag *)
mhp->ext[SADB_X_EXT_NAT_T_FRAG];
} else {
frag = 0;
}
#endif
/* get a SA header */
if ((newsah = key_getsah(&saidx)) == NULL) {
/* create a new SA header */
if ((newsah = key_newsah(&saidx)) == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n",__func__));
return key_senderror(so, m, ENOBUFS);
}
}
/* set spidx if there */
/* XXX rewrite */
error = key_setident(newsah, m, mhp);
if (error) {
return key_senderror(so, m, error);
}
/* create new SA entry. */
/* We can create new SA only if SPI is differenct. */
SAHTREE_LOCK();
newsav = key_getsavbyspi(newsah, sa0->sadb_sa_spi);
SAHTREE_UNLOCK();
if (newsav != NULL) {
ipseclog((LOG_DEBUG, "%s: SA already exists.\n", __func__));
return key_senderror(so, m, EEXIST);
}
newsav = KEY_NEWSAV(m, mhp, newsah, &error);
if (newsav == NULL) {
return key_senderror(so, m, error);
}
#ifdef IPSEC_NAT_T
/*
* Handle more NAT-T info if present,
* now that we have a sav to fill.
*/
if (type)
newsav->natt_type = type->sadb_x_nat_t_type_type;
#if 0
/*
* In case SADB_X_EXT_NAT_T_FRAG was not given, leave it at 0.
* We should actually check for a minimum MTU here, if we
* want to support it in ip_output.
*/
if (frag)
newsav->natt_esp_frag_len = frag->sadb_x_nat_t_frag_fraglen;
#endif
#endif
/* check SA values to be mature. */
if ((error = key_mature(newsav)) != 0) {
KEY_FREESAV(&newsav);
return key_senderror(so, m, error);
}
/*
* don't call key_freesav() here, as we would like to keep the SA
* in the database on success.
*/
{
struct mbuf *n;
/* set msg buf from mhp */
n = key_getmsgbuf_x1(m, mhp);
if (n == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
return key_senderror(so, m, ENOBUFS);
}
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ALL);
}
}
/* m is retained */
static int
key_setident(struct secashead *sah, struct mbuf *m,
const struct sadb_msghdr *mhp)
{
const struct sadb_ident *idsrc, *iddst;
int idsrclen, iddstlen;
IPSEC_ASSERT(sah != NULL, ("null secashead"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* don't make buffer if not there */
if (mhp->ext[SADB_EXT_IDENTITY_SRC] == NULL &&
mhp->ext[SADB_EXT_IDENTITY_DST] == NULL) {
sah->idents = NULL;
sah->identd = NULL;
return 0;
}
if (mhp->ext[SADB_EXT_IDENTITY_SRC] == NULL ||
mhp->ext[SADB_EXT_IDENTITY_DST] == NULL) {
ipseclog((LOG_DEBUG, "%s: invalid identity.\n", __func__));
return EINVAL;
}
idsrc = (const struct sadb_ident *)mhp->ext[SADB_EXT_IDENTITY_SRC];
iddst = (const struct sadb_ident *)mhp->ext[SADB_EXT_IDENTITY_DST];
idsrclen = mhp->extlen[SADB_EXT_IDENTITY_SRC];
iddstlen = mhp->extlen[SADB_EXT_IDENTITY_DST];
/* validity check */
if (idsrc->sadb_ident_type != iddst->sadb_ident_type) {
ipseclog((LOG_DEBUG, "%s: ident type mismatch.\n", __func__));
return EINVAL;
}
switch (idsrc->sadb_ident_type) {
case SADB_IDENTTYPE_PREFIX:
case SADB_IDENTTYPE_FQDN:
case SADB_IDENTTYPE_USERFQDN:
default:
/* XXX do nothing */
sah->idents = NULL;
sah->identd = NULL;
return 0;
}
/* make structure */
sah->idents = malloc(sizeof(struct secident), M_IPSEC_MISC, M_NOWAIT);
if (sah->idents == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
return ENOBUFS;
}
sah->identd = malloc(sizeof(struct secident), M_IPSEC_MISC, M_NOWAIT);
if (sah->identd == NULL) {
free(sah->idents, M_IPSEC_MISC);
sah->idents = NULL;
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
return ENOBUFS;
}
sah->idents->type = idsrc->sadb_ident_type;
sah->idents->id = idsrc->sadb_ident_id;
sah->identd->type = iddst->sadb_ident_type;
sah->identd->id = iddst->sadb_ident_id;
return 0;
}
/*
* m will not be freed on return.
* it is caller's responsibility to free the result.
*/
static struct mbuf *
key_getmsgbuf_x1(struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct mbuf *n;
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* create new sadb_msg to reply. */
n = key_gather_mbuf(m, mhp, 1, 9, SADB_EXT_RESERVED,
SADB_EXT_SA, SADB_X_EXT_SA2,
SADB_EXT_ADDRESS_SRC, SADB_EXT_ADDRESS_DST,
SADB_EXT_LIFETIME_HARD, SADB_EXT_LIFETIME_SOFT,
SADB_EXT_IDENTITY_SRC, SADB_EXT_IDENTITY_DST);
if (!n)
return NULL;
if (n->m_len < sizeof(struct sadb_msg)) {
n = m_pullup(n, sizeof(struct sadb_msg));
if (n == NULL)
return NULL;
}
mtod(n, struct sadb_msg *)->sadb_msg_errno = 0;
mtod(n, struct sadb_msg *)->sadb_msg_len =
PFKEY_UNIT64(n->m_pkthdr.len);
return n;
}
/*
* SADB_DELETE processing
* receive
* <base, SA(*), address(SD)>
* from the ikmpd, and set SADB_SASTATE_DEAD,
* and send,
* <base, SA(*), address(SD)>
* to the ikmpd.
*
* m will always be freed.
*/
static int
key_delete(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct sadb_sa *sa0;
struct sadb_address *src0, *dst0;
struct secasindex saidx;
struct secashead *sah;
struct secasvar *sav = NULL;
u_int16_t proto;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* map satype to proto */
if ((proto = key_satype2proto(mhp->msg->sadb_msg_satype)) == 0) {
ipseclog((LOG_DEBUG, "%s: invalid satype is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_EXT_ADDRESS_SRC] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_DST] == NULL) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->extlen[SADB_EXT_ADDRESS_SRC] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_EXT_ADDRESS_DST] < sizeof(struct sadb_address)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_EXT_SA] == NULL) {
/*
* Caller wants us to delete all non-LARVAL SAs
* that match the src/dst. This is used during
* IKE INITIAL-CONTACT.
*/
ipseclog((LOG_DEBUG, "%s: doing delete all.\n", __func__));
return key_delete_all(so, m, mhp, proto);
} else if (mhp->extlen[SADB_EXT_SA] < sizeof(struct sadb_sa)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
sa0 = (struct sadb_sa *)mhp->ext[SADB_EXT_SA];
src0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_SRC]);
dst0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_DST]);
/* XXX boundary check against sa_len */
KEY_SETSECASIDX(proto, IPSEC_MODE_ANY, 0, src0 + 1, dst0 + 1, &saidx);
/*
* Make sure the port numbers are zero.
* In case of NAT-T we will update them later if needed.
*/
KEY_PORTTOSADDR(&saidx.src, 0);
KEY_PORTTOSADDR(&saidx.dst, 0);
#ifdef IPSEC_NAT_T
/*
* Handle NAT-T info if present.
*/
if (mhp->ext[SADB_X_EXT_NAT_T_SPORT] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_DPORT] != NULL) {
struct sadb_x_nat_t_port *sport, *dport;
if (mhp->extlen[SADB_X_EXT_NAT_T_SPORT] < sizeof(*sport) ||
mhp->extlen[SADB_X_EXT_NAT_T_DPORT] < sizeof(*dport)) {
ipseclog((LOG_DEBUG, "%s: invalid message.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
sport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_SPORT];
dport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_DPORT];
if (sport)
KEY_PORTTOSADDR(&saidx.src,
sport->sadb_x_nat_t_port_port);
if (dport)
KEY_PORTTOSADDR(&saidx.dst,
dport->sadb_x_nat_t_port_port);
}
#endif
/* get a SA header */
SAHTREE_LOCK();
LIST_FOREACH(sah, &V_sahtree, chain) {
if (sah->state == SADB_SASTATE_DEAD)
continue;
if (key_cmpsaidx(&sah->saidx, &saidx, CMP_HEAD) == 0)
continue;
/* get a SA with SPI. */
sav = key_getsavbyspi(sah, sa0->sadb_sa_spi);
if (sav)
break;
}
if (sah == NULL) {
SAHTREE_UNLOCK();
ipseclog((LOG_DEBUG, "%s: no SA found.\n", __func__));
return key_senderror(so, m, ENOENT);
}
key_sa_chgstate(sav, SADB_SASTATE_DEAD);
KEY_FREESAV(&sav);
SAHTREE_UNLOCK();
{
struct mbuf *n;
struct sadb_msg *newmsg;
/* create new sadb_msg to reply. */
/* XXX-BZ NAT-T extensions? */
n = key_gather_mbuf(m, mhp, 1, 4, SADB_EXT_RESERVED,
SADB_EXT_SA, SADB_EXT_ADDRESS_SRC, SADB_EXT_ADDRESS_DST);
if (!n)
return key_senderror(so, m, ENOBUFS);
if (n->m_len < sizeof(struct sadb_msg)) {
n = m_pullup(n, sizeof(struct sadb_msg));
if (n == NULL)
return key_senderror(so, m, ENOBUFS);
}
newmsg = mtod(n, struct sadb_msg *);
newmsg->sadb_msg_errno = 0;
newmsg->sadb_msg_len = PFKEY_UNIT64(n->m_pkthdr.len);
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ALL);
}
}
/*
* delete all SAs for src/dst. Called from key_delete().
*/
static int
key_delete_all(struct socket *so, struct mbuf *m,
const struct sadb_msghdr *mhp, u_int16_t proto)
{
struct sadb_address *src0, *dst0;
struct secasindex saidx;
struct secashead *sah;
struct secasvar *sav, *nextsav;
u_int stateidx, state;
src0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_SRC]);
dst0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_DST]);
/* XXX boundary check against sa_len */
KEY_SETSECASIDX(proto, IPSEC_MODE_ANY, 0, src0 + 1, dst0 + 1, &saidx);
/*
* Make sure the port numbers are zero.
* In case of NAT-T we will update them later if needed.
*/
KEY_PORTTOSADDR(&saidx.src, 0);
KEY_PORTTOSADDR(&saidx.dst, 0);
#ifdef IPSEC_NAT_T
/*
* Handle NAT-T info if present.
*/
if (mhp->ext[SADB_X_EXT_NAT_T_SPORT] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_DPORT] != NULL) {
struct sadb_x_nat_t_port *sport, *dport;
if (mhp->extlen[SADB_X_EXT_NAT_T_SPORT] < sizeof(*sport) ||
mhp->extlen[SADB_X_EXT_NAT_T_DPORT] < sizeof(*dport)) {
ipseclog((LOG_DEBUG, "%s: invalid message.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
sport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_SPORT];
dport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_DPORT];
if (sport)
KEY_PORTTOSADDR(&saidx.src,
sport->sadb_x_nat_t_port_port);
if (dport)
KEY_PORTTOSADDR(&saidx.dst,
dport->sadb_x_nat_t_port_port);
}
#endif
SAHTREE_LOCK();
LIST_FOREACH(sah, &V_sahtree, chain) {
if (sah->state == SADB_SASTATE_DEAD)
continue;
if (key_cmpsaidx(&sah->saidx, &saidx, CMP_HEAD) == 0)
continue;
/* Delete all non-LARVAL SAs. */
for (stateidx = 0;
stateidx < _ARRAYLEN(saorder_state_alive);
stateidx++) {
state = saorder_state_alive[stateidx];
if (state == SADB_SASTATE_LARVAL)
continue;
for (sav = LIST_FIRST(&sah->savtree[state]);
sav != NULL; sav = nextsav) {
nextsav = LIST_NEXT(sav, chain);
/* sanity check */
if (sav->state != state) {
ipseclog((LOG_DEBUG, "%s: invalid "
"sav->state (queue %d SA %d)\n",
__func__, state, sav->state));
continue;
}
key_sa_chgstate(sav, SADB_SASTATE_DEAD);
KEY_FREESAV(&sav);
}
}
}
SAHTREE_UNLOCK();
{
struct mbuf *n;
struct sadb_msg *newmsg;
/* create new sadb_msg to reply. */
/* XXX-BZ NAT-T extensions? */
n = key_gather_mbuf(m, mhp, 1, 3, SADB_EXT_RESERVED,
SADB_EXT_ADDRESS_SRC, SADB_EXT_ADDRESS_DST);
if (!n)
return key_senderror(so, m, ENOBUFS);
if (n->m_len < sizeof(struct sadb_msg)) {
n = m_pullup(n, sizeof(struct sadb_msg));
if (n == NULL)
return key_senderror(so, m, ENOBUFS);
}
newmsg = mtod(n, struct sadb_msg *);
newmsg->sadb_msg_errno = 0;
newmsg->sadb_msg_len = PFKEY_UNIT64(n->m_pkthdr.len);
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ALL);
}
}
/*
* SADB_GET processing
* receive
* <base, SA(*), address(SD)>
* from the ikmpd, and get a SP and a SA to respond,
* and send,
* <base, SA, (lifetime(HSC),) address(SD), (address(P),) key(AE),
* (identity(SD),) (sensitivity)>
* to the ikmpd.
*
* m will always be freed.
*/
static int
key_get(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct sadb_sa *sa0;
struct sadb_address *src0, *dst0;
struct secasindex saidx;
struct secashead *sah;
struct secasvar *sav = NULL;
u_int16_t proto;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* map satype to proto */
if ((proto = key_satype2proto(mhp->msg->sadb_msg_satype)) == 0) {
ipseclog((LOG_DEBUG, "%s: invalid satype is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_EXT_SA] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_SRC] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_DST] == NULL) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->extlen[SADB_EXT_SA] < sizeof(struct sadb_sa) ||
mhp->extlen[SADB_EXT_ADDRESS_SRC] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_EXT_ADDRESS_DST] < sizeof(struct sadb_address)) {
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
sa0 = (struct sadb_sa *)mhp->ext[SADB_EXT_SA];
src0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_SRC];
dst0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_DST];
/* XXX boundary check against sa_len */
KEY_SETSECASIDX(proto, IPSEC_MODE_ANY, 0, src0 + 1, dst0 + 1, &saidx);
/*
* Make sure the port numbers are zero.
* In case of NAT-T we will update them later if needed.
*/
KEY_PORTTOSADDR(&saidx.src, 0);
KEY_PORTTOSADDR(&saidx.dst, 0);
#ifdef IPSEC_NAT_T
/*
* Handle NAT-T info if present.
*/
if (mhp->ext[SADB_X_EXT_NAT_T_SPORT] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_DPORT] != NULL) {
struct sadb_x_nat_t_port *sport, *dport;
if (mhp->extlen[SADB_X_EXT_NAT_T_SPORT] < sizeof(*sport) ||
mhp->extlen[SADB_X_EXT_NAT_T_DPORT] < sizeof(*dport)) {
ipseclog((LOG_DEBUG, "%s: invalid message.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
sport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_SPORT];
dport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_DPORT];
if (sport)
KEY_PORTTOSADDR(&saidx.src,
sport->sadb_x_nat_t_port_port);
if (dport)
KEY_PORTTOSADDR(&saidx.dst,
dport->sadb_x_nat_t_port_port);
}
#endif
/* get a SA header */
SAHTREE_LOCK();
LIST_FOREACH(sah, &V_sahtree, chain) {
if (sah->state == SADB_SASTATE_DEAD)
continue;
if (key_cmpsaidx(&sah->saidx, &saidx, CMP_HEAD) == 0)
continue;
/* get a SA with SPI. */
sav = key_getsavbyspi(sah, sa0->sadb_sa_spi);
if (sav)
break;
}
SAHTREE_UNLOCK();
if (sah == NULL) {
ipseclog((LOG_DEBUG, "%s: no SA found.\n", __func__));
return key_senderror(so, m, ENOENT);
}
{
struct mbuf *n;
u_int8_t satype;
/* map proto to satype */
if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
ipseclog((LOG_DEBUG, "%s: there was invalid proto in SAD.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
/* create new sadb_msg to reply. */
n = key_setdumpsa(sav, SADB_GET, satype, mhp->msg->sadb_msg_seq,
mhp->msg->sadb_msg_pid);
if (!n)
return key_senderror(so, m, ENOBUFS);
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
}
}
/* XXX make it sysctl-configurable? */
static void
key_getcomb_setlifetime(struct sadb_comb *comb)
{
comb->sadb_comb_soft_allocations = 1;
comb->sadb_comb_hard_allocations = 1;
comb->sadb_comb_soft_bytes = 0;
comb->sadb_comb_hard_bytes = 0;
comb->sadb_comb_hard_addtime = 86400; /* 1 day */
comb->sadb_comb_soft_addtime = comb->sadb_comb_soft_addtime * 80 / 100;
comb->sadb_comb_soft_usetime = 28800; /* 8 hours */
comb->sadb_comb_hard_usetime = comb->sadb_comb_hard_usetime * 80 / 100;
}
/*
* XXX reorder combinations by preference
* XXX no idea if the user wants ESP authentication or not
*/
static struct mbuf *
key_getcomb_esp()
{
struct sadb_comb *comb;
struct enc_xform *algo;
struct mbuf *result = NULL, *m, *n;
int encmin;
int i, off, o;
int totlen;
const int l = PFKEY_ALIGN8(sizeof(struct sadb_comb));
m = NULL;
for (i = 1; i <= SADB_EALG_MAX; i++) {
algo = esp_algorithm_lookup(i);
if (algo == NULL)
continue;
/* discard algorithms with key size smaller than system min */
if (_BITS(algo->maxkey) < V_ipsec_esp_keymin)
continue;
if (_BITS(algo->minkey) < V_ipsec_esp_keymin)
encmin = V_ipsec_esp_keymin;
else
encmin = _BITS(algo->minkey);
if (V_ipsec_esp_auth)
m = key_getcomb_ah();
else {
IPSEC_ASSERT(l <= MLEN,
("l=%u > MLEN=%lu", l, (u_long) MLEN));
MGET(m, M_NOWAIT, MT_DATA);
if (m) {
M_ALIGN(m, l);
m->m_len = l;
m->m_next = NULL;
bzero(mtod(m, caddr_t), m->m_len);
}
}
if (!m)
goto fail;
totlen = 0;
for (n = m; n; n = n->m_next)
totlen += n->m_len;
IPSEC_ASSERT((totlen % l) == 0, ("totlen=%u, l=%u", totlen, l));
for (off = 0; off < totlen; off += l) {
n = m_pulldown(m, off, l, &o);
if (!n) {
/* m is already freed */
goto fail;
}
comb = (struct sadb_comb *)(mtod(n, caddr_t) + o);
bzero(comb, sizeof(*comb));
key_getcomb_setlifetime(comb);
comb->sadb_comb_encrypt = i;
comb->sadb_comb_encrypt_minbits = encmin;
comb->sadb_comb_encrypt_maxbits = _BITS(algo->maxkey);
}
if (!result)
result = m;
else
m_cat(result, m);
}
return result;
fail:
if (result)
m_freem(result);
return NULL;
}
static void
key_getsizes_ah(const struct auth_hash *ah, int alg, u_int16_t* min,
u_int16_t* max)
{
*min = *max = ah->keysize;
if (ah->keysize == 0) {
/*
* Transform takes arbitrary key size but algorithm
* key size is restricted. Enforce this here.
*/
switch (alg) {
case SADB_X_AALG_MD5: *min = *max = 16; break;
case SADB_X_AALG_SHA: *min = *max = 20; break;
case SADB_X_AALG_NULL: *min = 1; *max = 256; break;
case SADB_X_AALG_SHA2_256: *min = *max = 32; break;
case SADB_X_AALG_SHA2_384: *min = *max = 48; break;
case SADB_X_AALG_SHA2_512: *min = *max = 64; break;
default:
DPRINTF(("%s: unknown AH algorithm %u\n",
__func__, alg));
break;
}
}
}
/*
* XXX reorder combinations by preference
*/
static struct mbuf *
key_getcomb_ah()
{
struct sadb_comb *comb;
struct auth_hash *algo;
struct mbuf *m;
u_int16_t minkeysize, maxkeysize;
int i;
const int l = PFKEY_ALIGN8(sizeof(struct sadb_comb));
m = NULL;
for (i = 1; i <= SADB_AALG_MAX; i++) {
#if 1
/* we prefer HMAC algorithms, not old algorithms */
if (i != SADB_AALG_SHA1HMAC &&
i != SADB_AALG_MD5HMAC &&
i != SADB_X_AALG_SHA2_256 &&
i != SADB_X_AALG_SHA2_384 &&
i != SADB_X_AALG_SHA2_512)
continue;
#endif
algo = ah_algorithm_lookup(i);
if (!algo)
continue;
key_getsizes_ah(algo, i, &minkeysize, &maxkeysize);
/* discard algorithms with key size smaller than system min */
if (_BITS(minkeysize) < V_ipsec_ah_keymin)
continue;
if (!m) {
IPSEC_ASSERT(l <= MLEN,
("l=%u > MLEN=%lu", l, (u_long) MLEN));
MGET(m, M_NOWAIT, MT_DATA);
if (m) {
M_ALIGN(m, l);
m->m_len = l;
m->m_next = NULL;
}
} else
M_PREPEND(m, l, M_NOWAIT);
if (!m)
return NULL;
comb = mtod(m, struct sadb_comb *);
bzero(comb, sizeof(*comb));
key_getcomb_setlifetime(comb);
comb->sadb_comb_auth = i;
comb->sadb_comb_auth_minbits = _BITS(minkeysize);
comb->sadb_comb_auth_maxbits = _BITS(maxkeysize);
}
return m;
}
/*
* not really an official behavior. discussed in pf_key@inner.net in Sep2000.
* XXX reorder combinations by preference
*/
static struct mbuf *
key_getcomb_ipcomp()
{
struct sadb_comb *comb;
struct comp_algo *algo;
struct mbuf *m;
int i;
const int l = PFKEY_ALIGN8(sizeof(struct sadb_comb));
m = NULL;
for (i = 1; i <= SADB_X_CALG_MAX; i++) {
algo = ipcomp_algorithm_lookup(i);
if (!algo)
continue;
if (!m) {
IPSEC_ASSERT(l <= MLEN,
("l=%u > MLEN=%lu", l, (u_long) MLEN));
MGET(m, M_NOWAIT, MT_DATA);
if (m) {
M_ALIGN(m, l);
m->m_len = l;
m->m_next = NULL;
}
} else
M_PREPEND(m, l, M_NOWAIT);
if (!m)
return NULL;
comb = mtod(m, struct sadb_comb *);
bzero(comb, sizeof(*comb));
key_getcomb_setlifetime(comb);
comb->sadb_comb_encrypt = i;
/* what should we set into sadb_comb_*_{min,max}bits? */
}
return m;
}
/*
* XXX no way to pass mode (transport/tunnel) to userland
* XXX replay checking?
* XXX sysctl interface to ipsec_{ah,esp}_keymin
*/
static struct mbuf *
key_getprop(const struct secasindex *saidx)
{
struct sadb_prop *prop;
struct mbuf *m, *n;
const int l = PFKEY_ALIGN8(sizeof(struct sadb_prop));
int totlen;
switch (saidx->proto) {
case IPPROTO_ESP:
m = key_getcomb_esp();
break;
case IPPROTO_AH:
m = key_getcomb_ah();
break;
case IPPROTO_IPCOMP:
m = key_getcomb_ipcomp();
break;
default:
return NULL;
}
if (!m)
return NULL;
M_PREPEND(m, l, M_NOWAIT);
if (!m)
return NULL;
totlen = 0;
for (n = m; n; n = n->m_next)
totlen += n->m_len;
prop = mtod(m, struct sadb_prop *);
bzero(prop, sizeof(*prop));
prop->sadb_prop_len = PFKEY_UNIT64(totlen);
prop->sadb_prop_exttype = SADB_EXT_PROPOSAL;
prop->sadb_prop_replay = 32; /* XXX */
return m;
}
/*
* SADB_ACQUIRE processing called by key_checkrequest() and key_acquire2().
* send
* <base, SA, address(SD), (address(P)), x_policy,
* (identity(SD),) (sensitivity,) proposal>
* to KMD, and expect to receive
* <base> with SADB_ACQUIRE if error occured,
* or
* <base, src address, dst address, (SPI range)> with SADB_GETSPI
* from KMD by PF_KEY.
*
* XXX x_policy is outside of RFC2367 (KAME extension).
* XXX sensitivity is not supported.
* XXX for ipcomp, RFC2367 does not define how to fill in proposal.
* see comment for key_getcomb_ipcomp().
*
* OUT:
* 0 : succeed
* others: error number
*/
static int
key_acquire(const struct secasindex *saidx, struct secpolicy *sp)
{
struct mbuf *result = NULL, *m;
struct secacq *newacq;
u_int8_t satype;
int error = -1;
u_int32_t seq;
IPSEC_ASSERT(saidx != NULL, ("null saidx"));
satype = key_proto2satype(saidx->proto);
IPSEC_ASSERT(satype != 0, ("null satype, protocol %u", saidx->proto));
/*
* We never do anything about acquirng SA. There is anather
* solution that kernel blocks to send SADB_ACQUIRE message until
* getting something message from IKEd. In later case, to be
* managed with ACQUIRING list.
*/
/* Get an entry to check whether sending message or not. */
if ((newacq = key_getacq(saidx)) != NULL) {
if (V_key_blockacq_count < newacq->count) {
/* reset counter and do send message. */
newacq->count = 0;
} else {
/* increment counter and do nothing. */
newacq->count++;
return 0;
}
} else {
/* make new entry for blocking to send SADB_ACQUIRE. */
if ((newacq = key_newacq(saidx)) == NULL)
return ENOBUFS;
}
seq = newacq->seq;
m = key_setsadbmsg(SADB_ACQUIRE, 0, satype, seq, 0, 0);
if (!m) {
error = ENOBUFS;
goto fail;
}
result = m;
/*
* No SADB_X_EXT_NAT_T_* here: we do not know
* anything related to NAT-T at this time.
*/
/* set sadb_address for saidx's. */
m = key_setsadbaddr(SADB_EXT_ADDRESS_SRC,
&saidx->src.sa, FULLMASK, IPSEC_ULPROTO_ANY);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
m = key_setsadbaddr(SADB_EXT_ADDRESS_DST,
&saidx->dst.sa, FULLMASK, IPSEC_ULPROTO_ANY);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
/* XXX proxy address (optional) */
/* set sadb_x_policy */
if (sp) {
m = key_setsadbxpolicy(sp->policy, sp->spidx.dir, sp->id);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
}
/* XXX identity (optional) */
#if 0
if (idexttype && fqdn) {
/* create identity extension (FQDN) */
struct sadb_ident *id;
int fqdnlen;
fqdnlen = strlen(fqdn) + 1; /* +1 for terminating-NUL */
id = (struct sadb_ident *)p;
bzero(id, sizeof(*id) + PFKEY_ALIGN8(fqdnlen));
id->sadb_ident_len = PFKEY_UNIT64(sizeof(*id) + PFKEY_ALIGN8(fqdnlen));
id->sadb_ident_exttype = idexttype;
id->sadb_ident_type = SADB_IDENTTYPE_FQDN;
bcopy(fqdn, id + 1, fqdnlen);
p += sizeof(struct sadb_ident) + PFKEY_ALIGN8(fqdnlen);
}
if (idexttype) {
/* create identity extension (USERFQDN) */
struct sadb_ident *id;
int userfqdnlen;
if (userfqdn) {
/* +1 for terminating-NUL */
userfqdnlen = strlen(userfqdn) + 1;
} else
userfqdnlen = 0;
id = (struct sadb_ident *)p;
bzero(id, sizeof(*id) + PFKEY_ALIGN8(userfqdnlen));
id->sadb_ident_len = PFKEY_UNIT64(sizeof(*id) + PFKEY_ALIGN8(userfqdnlen));
id->sadb_ident_exttype = idexttype;
id->sadb_ident_type = SADB_IDENTTYPE_USERFQDN;
/* XXX is it correct? */
if (curproc && curproc->p_cred)
id->sadb_ident_id = curproc->p_cred->p_ruid;
if (userfqdn && userfqdnlen)
bcopy(userfqdn, id + 1, userfqdnlen);
p += sizeof(struct sadb_ident) + PFKEY_ALIGN8(userfqdnlen);
}
#endif
/* XXX sensitivity (optional) */
/* create proposal/combination extension */
m = key_getprop(saidx);
#if 0
/*
* spec conformant: always attach proposal/combination extension,
* the problem is that we have no way to attach it for ipcomp,
* due to the way sadb_comb is declared in RFC2367.
*/
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
#else
/*
* outside of spec; make proposal/combination extension optional.
*/
if (m)
m_cat(result, m);
#endif
if ((result->m_flags & M_PKTHDR) == 0) {
error = EINVAL;
goto fail;
}
if (result->m_len < sizeof(struct sadb_msg)) {
result = m_pullup(result, sizeof(struct sadb_msg));
if (result == NULL) {
error = ENOBUFS;
goto fail;
}
}
result->m_pkthdr.len = 0;
for (m = result; m; m = m->m_next)
result->m_pkthdr.len += m->m_len;
mtod(result, struct sadb_msg *)->sadb_msg_len =
PFKEY_UNIT64(result->m_pkthdr.len);
return key_sendup_mbuf(NULL, result, KEY_SENDUP_REGISTERED);
fail:
if (result)
m_freem(result);
return error;
}
static struct secacq *
key_newacq(const struct secasindex *saidx)
{
struct secacq *newacq;
/* get new entry */
newacq = malloc(sizeof(struct secacq), M_IPSEC_SAQ, M_NOWAIT|M_ZERO);
if (newacq == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
return NULL;
}
/* copy secindex */
bcopy(saidx, &newacq->saidx, sizeof(newacq->saidx));
newacq->seq = (V_acq_seq == ~0 ? 1 : ++V_acq_seq);
newacq->created = time_second;
newacq->count = 0;
/* add to acqtree */
ACQ_LOCK();
LIST_INSERT_HEAD(&V_acqtree, newacq, chain);
ACQ_UNLOCK();
return newacq;
}
static struct secacq *
key_getacq(const struct secasindex *saidx)
{
struct secacq *acq;
ACQ_LOCK();
LIST_FOREACH(acq, &V_acqtree, chain) {
if (key_cmpsaidx(saidx, &acq->saidx, CMP_EXACTLY))
break;
}
ACQ_UNLOCK();
return acq;
}
static struct secacq *
key_getacqbyseq(u_int32_t seq)
{
struct secacq *acq;
ACQ_LOCK();
LIST_FOREACH(acq, &V_acqtree, chain) {
if (acq->seq == seq)
break;
}
ACQ_UNLOCK();
return acq;
}
static struct secspacq *
key_newspacq(struct secpolicyindex *spidx)
{
struct secspacq *acq;
/* get new entry */
acq = malloc(sizeof(struct secspacq), M_IPSEC_SAQ, M_NOWAIT|M_ZERO);
if (acq == NULL) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
return NULL;
}
/* copy secindex */
bcopy(spidx, &acq->spidx, sizeof(acq->spidx));
acq->created = time_second;
acq->count = 0;
/* add to spacqtree */
SPACQ_LOCK();
LIST_INSERT_HEAD(&V_spacqtree, acq, chain);
SPACQ_UNLOCK();
return acq;
}
static struct secspacq *
key_getspacq(struct secpolicyindex *spidx)
{
struct secspacq *acq;
SPACQ_LOCK();
LIST_FOREACH(acq, &V_spacqtree, chain) {
if (key_cmpspidx_exactly(spidx, &acq->spidx)) {
/* NB: return holding spacq_lock */
return acq;
}
}
SPACQ_UNLOCK();
return NULL;
}
/*
* SADB_ACQUIRE processing,
* in first situation, is receiving
* <base>
* from the ikmpd, and clear sequence of its secasvar entry.
*
* In second situation, is receiving
* <base, address(SD), (address(P),) (identity(SD),) (sensitivity,) proposal>
* from a user land process, and return
* <base, address(SD), (address(P),) (identity(SD),) (sensitivity,) proposal>
* to the socket.
*
* m will always be freed.
*/
static int
key_acquire2(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
const struct sadb_address *src0, *dst0;
struct secasindex saidx;
struct secashead *sah;
u_int16_t proto;
int error;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/*
* Error message from KMd.
* We assume that if error was occured in IKEd, the length of PFKEY
* message is equal to the size of sadb_msg structure.
* We do not raise error even if error occured in this function.
*/
if (mhp->msg->sadb_msg_len == PFKEY_UNIT64(sizeof(struct sadb_msg))) {
struct secacq *acq;
/* check sequence number */
if (mhp->msg->sadb_msg_seq == 0) {
ipseclog((LOG_DEBUG, "%s: must specify sequence "
"number.\n", __func__));
m_freem(m);
return 0;
}
if ((acq = key_getacqbyseq(mhp->msg->sadb_msg_seq)) == NULL) {
/*
* the specified larval SA is already gone, or we got
* a bogus sequence number. we can silently ignore it.
*/
m_freem(m);
return 0;
}
/* reset acq counter in order to deletion by timehander. */
acq->created = time_second;
acq->count = 0;
m_freem(m);
return 0;
}
/*
* This message is from user land.
*/
/* map satype to proto */
if ((proto = key_satype2proto(mhp->msg->sadb_msg_satype)) == 0) {
ipseclog((LOG_DEBUG, "%s: invalid satype is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->ext[SADB_EXT_ADDRESS_SRC] == NULL ||
mhp->ext[SADB_EXT_ADDRESS_DST] == NULL ||
mhp->ext[SADB_EXT_PROPOSAL] == NULL) {
/* error */
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
if (mhp->extlen[SADB_EXT_ADDRESS_SRC] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_EXT_ADDRESS_DST] < sizeof(struct sadb_address) ||
mhp->extlen[SADB_EXT_PROPOSAL] < sizeof(struct sadb_prop)) {
/* error */
ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
src0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_SRC];
dst0 = (struct sadb_address *)mhp->ext[SADB_EXT_ADDRESS_DST];
/* XXX boundary check against sa_len */
KEY_SETSECASIDX(proto, IPSEC_MODE_ANY, 0, src0 + 1, dst0 + 1, &saidx);
/*
* Make sure the port numbers are zero.
* In case of NAT-T we will update them later if needed.
*/
KEY_PORTTOSADDR(&saidx.src, 0);
KEY_PORTTOSADDR(&saidx.dst, 0);
#ifndef IPSEC_NAT_T
/*
* Handle NAT-T info if present.
*/
if (mhp->ext[SADB_X_EXT_NAT_T_SPORT] != NULL &&
mhp->ext[SADB_X_EXT_NAT_T_DPORT] != NULL) {
struct sadb_x_nat_t_port *sport, *dport;
if (mhp->extlen[SADB_X_EXT_NAT_T_SPORT] < sizeof(*sport) ||
mhp->extlen[SADB_X_EXT_NAT_T_DPORT] < sizeof(*dport)) {
ipseclog((LOG_DEBUG, "%s: invalid message.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
sport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_SPORT];
dport = (struct sadb_x_nat_t_port *)
mhp->ext[SADB_X_EXT_NAT_T_DPORT];
if (sport)
KEY_PORTTOSADDR(&saidx.src,
sport->sadb_x_nat_t_port_port);
if (dport)
KEY_PORTTOSADDR(&saidx.dst,
dport->sadb_x_nat_t_port_port);
}
#endif
/* get a SA index */
SAHTREE_LOCK();
LIST_FOREACH(sah, &V_sahtree, chain) {
if (sah->state == SADB_SASTATE_DEAD)
continue;
if (key_cmpsaidx(&sah->saidx, &saidx, CMP_MODE_REQID))
break;
}
SAHTREE_UNLOCK();
if (sah != NULL) {
ipseclog((LOG_DEBUG, "%s: a SA exists already.\n", __func__));
return key_senderror(so, m, EEXIST);
}
error = key_acquire(&saidx, NULL);
if (error != 0) {
ipseclog((LOG_DEBUG, "%s: error %d returned from key_acquire\n",
__func__, mhp->msg->sadb_msg_errno));
return key_senderror(so, m, error);
}
return key_sendup_mbuf(so, m, KEY_SENDUP_REGISTERED);
}
/*
* SADB_REGISTER processing.
* If SATYPE_UNSPEC has been passed as satype, only return sabd_supported.
* receive
* <base>
* from the ikmpd, and register a socket to send PF_KEY messages,
* and send
* <base, supported>
* to KMD by PF_KEY.
* If socket is detached, must free from regnode.
*
* m will always be freed.
*/
static int
key_register(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct secreg *reg, *newreg = 0;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* check for invalid register message */
if (mhp->msg->sadb_msg_satype >= sizeof(V_regtree)/sizeof(V_regtree[0]))
return key_senderror(so, m, EINVAL);
/* When SATYPE_UNSPEC is specified, only return sabd_supported. */
if (mhp->msg->sadb_msg_satype == SADB_SATYPE_UNSPEC)
goto setmsg;
/* check whether existing or not */
REGTREE_LOCK();
LIST_FOREACH(reg, &V_regtree[mhp->msg->sadb_msg_satype], chain) {
if (reg->so == so) {
REGTREE_UNLOCK();
ipseclog((LOG_DEBUG, "%s: socket exists already.\n",
__func__));
return key_senderror(so, m, EEXIST);
}
}
/* create regnode */
newreg = malloc(sizeof(struct secreg), M_IPSEC_SAR, M_NOWAIT|M_ZERO);
if (newreg == NULL) {
REGTREE_UNLOCK();
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
return key_senderror(so, m, ENOBUFS);
}
newreg->so = so;
((struct keycb *)sotorawcb(so))->kp_registered++;
/* add regnode to regtree. */
LIST_INSERT_HEAD(&V_regtree[mhp->msg->sadb_msg_satype], newreg, chain);
REGTREE_UNLOCK();
setmsg:
{
struct mbuf *n;
struct sadb_msg *newmsg;
struct sadb_supported *sup;
u_int len, alen, elen;
int off;
int i;
struct sadb_alg *alg;
/* create new sadb_msg to reply. */
alen = 0;
for (i = 1; i <= SADB_AALG_MAX; i++) {
if (ah_algorithm_lookup(i))
alen += sizeof(struct sadb_alg);
}
if (alen)
alen += sizeof(struct sadb_supported);
elen = 0;
for (i = 1; i <= SADB_EALG_MAX; i++) {
if (esp_algorithm_lookup(i))
elen += sizeof(struct sadb_alg);
}
if (elen)
elen += sizeof(struct sadb_supported);
len = sizeof(struct sadb_msg) + alen + elen;
if (len > MCLBYTES)
return key_senderror(so, m, ENOBUFS);
MGETHDR(n, M_NOWAIT, MT_DATA);
if (len > MHLEN) {
if (!(MCLGET(n, M_NOWAIT))) {
m_freem(n);
n = NULL;
}
}
if (!n)
return key_senderror(so, m, ENOBUFS);
n->m_pkthdr.len = n->m_len = len;
n->m_next = NULL;
off = 0;
m_copydata(m, 0, sizeof(struct sadb_msg), mtod(n, caddr_t) + off);
newmsg = mtod(n, struct sadb_msg *);
newmsg->sadb_msg_errno = 0;
newmsg->sadb_msg_len = PFKEY_UNIT64(len);
off += PFKEY_ALIGN8(sizeof(struct sadb_msg));
/* for authentication algorithm */
if (alen) {
sup = (struct sadb_supported *)(mtod(n, caddr_t) + off);
sup->sadb_supported_len = PFKEY_UNIT64(alen);
sup->sadb_supported_exttype = SADB_EXT_SUPPORTED_AUTH;
off += PFKEY_ALIGN8(sizeof(*sup));
for (i = 1; i <= SADB_AALG_MAX; i++) {
struct auth_hash *aalgo;
u_int16_t minkeysize, maxkeysize;
aalgo = ah_algorithm_lookup(i);
if (!aalgo)
continue;
alg = (struct sadb_alg *)(mtod(n, caddr_t) + off);
alg->sadb_alg_id = i;
alg->sadb_alg_ivlen = 0;
key_getsizes_ah(aalgo, i, &minkeysize, &maxkeysize);
alg->sadb_alg_minbits = _BITS(minkeysize);
alg->sadb_alg_maxbits = _BITS(maxkeysize);
off += PFKEY_ALIGN8(sizeof(*alg));
}
}
/* for encryption algorithm */
if (elen) {
sup = (struct sadb_supported *)(mtod(n, caddr_t) + off);
sup->sadb_supported_len = PFKEY_UNIT64(elen);
sup->sadb_supported_exttype = SADB_EXT_SUPPORTED_ENCRYPT;
off += PFKEY_ALIGN8(sizeof(*sup));
for (i = 1; i <= SADB_EALG_MAX; i++) {
struct enc_xform *ealgo;
ealgo = esp_algorithm_lookup(i);
if (!ealgo)
continue;
alg = (struct sadb_alg *)(mtod(n, caddr_t) + off);
alg->sadb_alg_id = i;
alg->sadb_alg_ivlen = ealgo->blocksize;
alg->sadb_alg_minbits = _BITS(ealgo->minkey);
alg->sadb_alg_maxbits = _BITS(ealgo->maxkey);
off += PFKEY_ALIGN8(sizeof(struct sadb_alg));
}
}
IPSEC_ASSERT(off == len,
("length assumption failed (off %u len %u)", off, len));
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_REGISTERED);
}
}
/*
* free secreg entry registered.
* XXX: I want to do free a socket marked done SADB_RESIGER to socket.
*/
void
key_freereg(struct socket *so)
{
struct secreg *reg;
int i;
IPSEC_ASSERT(so != NULL, ("NULL so"));
/*
* check whether existing or not.
* check all type of SA, because there is a potential that
* one socket is registered to multiple type of SA.
*/
REGTREE_LOCK();
for (i = 0; i <= SADB_SATYPE_MAX; i++) {
LIST_FOREACH(reg, &V_regtree[i], chain) {
if (reg->so == so && __LIST_CHAINED(reg)) {
LIST_REMOVE(reg, chain);
free(reg, M_IPSEC_SAR);
break;
}
}
}
REGTREE_UNLOCK();
}
/*
* SADB_EXPIRE processing
* send
* <base, SA, SA2, lifetime(C and one of HS), address(SD)>
* to KMD by PF_KEY.
* NOTE: We send only soft lifetime extension.
*
* OUT: 0 : succeed
* others : error number
*/
static int
key_expire(struct secasvar *sav, int hard)
{
int satype;
struct mbuf *result = NULL, *m;
int len;
int error = -1;
struct sadb_lifetime *lt;
IPSEC_ASSERT (sav != NULL, ("null sav"));
IPSEC_ASSERT (sav->sah != NULL, ("null sa header"));
/* set msg header */
satype = key_proto2satype(sav->sah->saidx.proto);
IPSEC_ASSERT(satype != 0, ("invalid proto, satype %u", satype));
m = key_setsadbmsg(SADB_EXPIRE, 0, satype, sav->seq, 0, sav->refcnt);
if (!m) {
error = ENOBUFS;
goto fail;
}
result = m;
/* create SA extension */
m = key_setsadbsa(sav);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
/* create SA extension */
m = key_setsadbxsa2(sav->sah->saidx.mode,
sav->replay ? sav->replay->count : 0,
sav->sah->saidx.reqid);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
/* create lifetime extension (current and soft) */
len = PFKEY_ALIGN8(sizeof(*lt)) * 2;
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL) {
error = ENOBUFS;
goto fail;
}
m_align(m, len);
m->m_len = len;
bzero(mtod(m, caddr_t), len);
lt = mtod(m, struct sadb_lifetime *);
lt->sadb_lifetime_len = PFKEY_UNIT64(sizeof(struct sadb_lifetime));
lt->sadb_lifetime_exttype = SADB_EXT_LIFETIME_CURRENT;
lt->sadb_lifetime_allocations = sav->lft_c->allocations;
lt->sadb_lifetime_bytes = sav->lft_c->bytes;
lt->sadb_lifetime_addtime = sav->lft_c->addtime;
lt->sadb_lifetime_usetime = sav->lft_c->usetime;
lt = (struct sadb_lifetime *)(mtod(m, caddr_t) + len / 2);
lt->sadb_lifetime_len = PFKEY_UNIT64(sizeof(struct sadb_lifetime));
if (hard) {
lt->sadb_lifetime_exttype = SADB_EXT_LIFETIME_HARD;
lt->sadb_lifetime_allocations = sav->lft_h->allocations;
lt->sadb_lifetime_bytes = sav->lft_h->bytes;
lt->sadb_lifetime_addtime = sav->lft_h->addtime;
lt->sadb_lifetime_usetime = sav->lft_h->usetime;
} else {
lt->sadb_lifetime_exttype = SADB_EXT_LIFETIME_SOFT;
lt->sadb_lifetime_allocations = sav->lft_s->allocations;
lt->sadb_lifetime_bytes = sav->lft_s->bytes;
lt->sadb_lifetime_addtime = sav->lft_s->addtime;
lt->sadb_lifetime_usetime = sav->lft_s->usetime;
}
m_cat(result, m);
/* set sadb_address for source */
m = key_setsadbaddr(SADB_EXT_ADDRESS_SRC,
&sav->sah->saidx.src.sa,
FULLMASK, IPSEC_ULPROTO_ANY);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
/* set sadb_address for destination */
m = key_setsadbaddr(SADB_EXT_ADDRESS_DST,
&sav->sah->saidx.dst.sa,
FULLMASK, IPSEC_ULPROTO_ANY);
if (!m) {
error = ENOBUFS;
goto fail;
}
m_cat(result, m);
/*
* XXX-BZ Handle NAT-T extensions here.
*/
if ((result->m_flags & M_PKTHDR) == 0) {
error = EINVAL;
goto fail;
}
if (result->m_len < sizeof(struct sadb_msg)) {
result = m_pullup(result, sizeof(struct sadb_msg));
if (result == NULL) {
error = ENOBUFS;
goto fail;
}
}
result->m_pkthdr.len = 0;
for (m = result; m; m = m->m_next)
result->m_pkthdr.len += m->m_len;
mtod(result, struct sadb_msg *)->sadb_msg_len =
PFKEY_UNIT64(result->m_pkthdr.len);
return key_sendup_mbuf(NULL, result, KEY_SENDUP_REGISTERED);
fail:
if (result)
m_freem(result);
return error;
}
/*
* SADB_FLUSH processing
* receive
* <base>
* from the ikmpd, and free all entries in secastree.
* and send,
* <base>
* to the ikmpd.
* NOTE: to do is only marking SADB_SASTATE_DEAD.
*
* m will always be freed.
*/
static int
key_flush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct sadb_msg *newmsg;
struct secashead *sah, *nextsah;
struct secasvar *sav, *nextsav;
u_int16_t proto;
u_int8_t state;
u_int stateidx;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* map satype to proto */
if ((proto = key_satype2proto(mhp->msg->sadb_msg_satype)) == 0) {
ipseclog((LOG_DEBUG, "%s: invalid satype is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
/* no SATYPE specified, i.e. flushing all SA. */
SAHTREE_LOCK();
for (sah = LIST_FIRST(&V_sahtree);
sah != NULL;
sah = nextsah) {
nextsah = LIST_NEXT(sah, chain);
if (mhp->msg->sadb_msg_satype != SADB_SATYPE_UNSPEC
&& proto != sah->saidx.proto)
continue;
for (stateidx = 0;
stateidx < _ARRAYLEN(saorder_state_alive);
stateidx++) {
state = saorder_state_any[stateidx];
for (sav = LIST_FIRST(&sah->savtree[state]);
sav != NULL;
sav = nextsav) {
nextsav = LIST_NEXT(sav, chain);
key_sa_chgstate(sav, SADB_SASTATE_DEAD);
KEY_FREESAV(&sav);
}
}
sah->state = SADB_SASTATE_DEAD;
}
SAHTREE_UNLOCK();
if (m->m_len < sizeof(struct sadb_msg) ||
sizeof(struct sadb_msg) > m->m_len + M_TRAILINGSPACE(m)) {
ipseclog((LOG_DEBUG, "%s: No more memory.\n", __func__));
return key_senderror(so, m, ENOBUFS);
}
if (m->m_next)
m_freem(m->m_next);
m->m_next = NULL;
m->m_pkthdr.len = m->m_len = sizeof(struct sadb_msg);
newmsg = mtod(m, struct sadb_msg *);
newmsg->sadb_msg_errno = 0;
newmsg->sadb_msg_len = PFKEY_UNIT64(m->m_pkthdr.len);
return key_sendup_mbuf(so, m, KEY_SENDUP_ALL);
}
/*
* SADB_DUMP processing
* dump all entries including status of DEAD in SAD.
* receive
* <base>
* from the ikmpd, and dump all secasvar leaves
* and send,
* <base> .....
* to the ikmpd.
*
* m will always be freed.
*/
static int
key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
struct secashead *sah;
struct secasvar *sav;
u_int16_t proto;
u_int stateidx;
u_int8_t satype;
u_int8_t state;
int cnt;
struct sadb_msg *newmsg;
struct mbuf *n;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
/* map satype to proto */
if ((proto = key_satype2proto(mhp->msg->sadb_msg_satype)) == 0) {
ipseclog((LOG_DEBUG, "%s: invalid satype is passed.\n",
__func__));
return key_senderror(so, m, EINVAL);
}
/* count sav entries to be sent to the userland. */
cnt = 0;
SAHTREE_LOCK();
LIST_FOREACH(sah, &V_sahtree, chain) {
if (mhp->msg->sadb_msg_satype != SADB_SATYPE_UNSPEC
&& proto != sah->saidx.proto)
continue;
for (stateidx = 0;
stateidx < _ARRAYLEN(saorder_state_any);
stateidx++) {
state = saorder_state_any[stateidx];
LIST_FOREACH(sav, &sah->savtree[state], chain) {
cnt++;
}
}
}
if (cnt == 0) {
SAHTREE_UNLOCK();
return key_senderror(so, m, ENOENT);
}
/* send this to the userland, one at a time. */
newmsg = NULL;
LIST_FOREACH(sah, &V_sahtree, chain) {
if (mhp->msg->sadb_msg_satype != SADB_SATYPE_UNSPEC
&& proto != sah->saidx.proto)
continue;
/* map proto to satype */
if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
SAHTREE_UNLOCK();
ipseclog((LOG_DEBUG, "%s: there was invalid proto in "
"SAD.\n", __func__));
return key_senderror(so, m, EINVAL);
}
for (stateidx = 0;
stateidx < _ARRAYLEN(saorder_state_any);
stateidx++) {
state = saorder_state_any[stateidx];
LIST_FOREACH(sav, &sah->savtree[state], chain) {
n = key_setdumpsa(sav, SADB_DUMP, satype,
--cnt, mhp->msg->sadb_msg_pid);
if (!n) {
SAHTREE_UNLOCK();
return key_senderror(so, m, ENOBUFS);
}
key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
}
}
}
SAHTREE_UNLOCK();
m_freem(m);
return 0;
}
/*
* SADB_X_PROMISC processing
*
* m will always be freed.
*/
static int
key_promisc(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
{
int olen;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
olen = PFKEY_UNUNIT64(mhp->msg->sadb_msg_len);
if (olen < sizeof(struct sadb_msg)) {
#if 1
return key_senderror(so, m, EINVAL);
#else
m_freem(m);
return 0;
#endif
} else if (olen == sizeof(struct sadb_msg)) {
/* enable/disable promisc mode */
struct keycb *kp;
if ((kp = (struct keycb *)sotorawcb(so)) == NULL)
return key_senderror(so, m, EINVAL);
mhp->msg->sadb_msg_errno = 0;
switch (mhp->msg->sadb_msg_satype) {
case 0:
case 1:
kp->kp_promisc = mhp->msg->sadb_msg_satype;
break;
default:
return key_senderror(so, m, EINVAL);
}
/* send the original message back to everyone */
mhp->msg->sadb_msg_errno = 0;
return key_sendup_mbuf(so, m, KEY_SENDUP_ALL);
} else {
/* send packet as is */
m_adj(m, PFKEY_ALIGN8(sizeof(struct sadb_msg)));
/* TODO: if sadb_msg_seq is specified, send to specific pid */
return key_sendup_mbuf(so, m, KEY_SENDUP_ALL);
}
}
static int (*key_typesw[])(struct socket *, struct mbuf *,
const struct sadb_msghdr *) = {
NULL, /* SADB_RESERVED */
key_getspi, /* SADB_GETSPI */
key_update, /* SADB_UPDATE */
key_add, /* SADB_ADD */
key_delete, /* SADB_DELETE */
key_get, /* SADB_GET */
key_acquire2, /* SADB_ACQUIRE */
key_register, /* SADB_REGISTER */
NULL, /* SADB_EXPIRE */
key_flush, /* SADB_FLUSH */
key_dump, /* SADB_DUMP */
key_promisc, /* SADB_X_PROMISC */
NULL, /* SADB_X_PCHANGE */
key_spdadd, /* SADB_X_SPDUPDATE */
key_spdadd, /* SADB_X_SPDADD */
key_spddelete, /* SADB_X_SPDDELETE */
key_spdget, /* SADB_X_SPDGET */
NULL, /* SADB_X_SPDACQUIRE */
key_spddump, /* SADB_X_SPDDUMP */
key_spdflush, /* SADB_X_SPDFLUSH */
key_spdadd, /* SADB_X_SPDSETIDX */
NULL, /* SADB_X_SPDEXPIRE */
key_spddelete2, /* SADB_X_SPDDELETE2 */
};
/*
* parse sadb_msg buffer to process PFKEYv2,
* and create a data to response if needed.
* I think to be dealed with mbuf directly.
* IN:
* msgp : pointer to pointer to a received buffer pulluped.
* This is rewrited to response.
* so : pointer to socket.
* OUT:
* length for buffer to send to user process.
*/
int
key_parse(struct mbuf *m, struct socket *so)
{
struct sadb_msg *msg;
struct sadb_msghdr mh;
u_int orglen;
int error;
int target;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
#if 0 /*kdebug_sadb assumes msg in linear buffer*/
KEYDEBUG(KEYDEBUG_KEY_DUMP,
ipseclog((LOG_DEBUG, "%s: passed sadb_msg\n", __func__));
kdebug_sadb(msg));
#endif
if (m->m_len < sizeof(struct sadb_msg)) {
m = m_pullup(m, sizeof(struct sadb_msg));
if (!m)
return ENOBUFS;
}
msg = mtod(m, struct sadb_msg *);
orglen = PFKEY_UNUNIT64(msg->sadb_msg_len);
target = KEY_SENDUP_ONE;
if ((m->m_flags & M_PKTHDR) == 0 ||
m->m_pkthdr.len != m->m_pkthdr.len) {
ipseclog((LOG_DEBUG, "%s: invalid message length.\n",__func__));
PFKEYSTAT_INC(out_invlen);
error = EINVAL;
goto senderror;
}
if (msg->sadb_msg_version != PF_KEY_V2) {
ipseclog((LOG_DEBUG, "%s: PF_KEY version %u is mismatched.\n",
__func__, msg->sadb_msg_version));
PFKEYSTAT_INC(out_invver);
error = EINVAL;
goto senderror;
}
if (msg->sadb_msg_type > SADB_MAX) {
ipseclog((LOG_DEBUG, "%s: invalid type %u is passed.\n",
__func__, msg->sadb_msg_type));
PFKEYSTAT_INC(out_invmsgtype);
error = EINVAL;
goto senderror;
}
/* for old-fashioned code - should be nuked */
if (m->m_pkthdr.len > MCLBYTES) {
m_freem(m);
return ENOBUFS;
}
if (m->m_next) {
struct mbuf *n;
MGETHDR(n, M_NOWAIT, MT_DATA);
if (n && m->m_pkthdr.len > MHLEN) {
if (!(MCLGET(n, M_NOWAIT))) {
m_free(n);
n = NULL;
}
}
if (!n) {
m_freem(m);
return ENOBUFS;
}
m_copydata(m, 0, m->m_pkthdr.len, mtod(n, caddr_t));
n->m_pkthdr.len = n->m_len = m->m_pkthdr.len;
n->m_next = NULL;
m_freem(m);
m = n;
}
/* align the mbuf chain so that extensions are in contiguous region. */
error = key_align(m, &mh);
if (error)
return error;
msg = mh.msg;
/* check SA type */
switch (msg->sadb_msg_satype) {
case SADB_SATYPE_UNSPEC:
switch (msg->sadb_msg_type) {
case SADB_GETSPI:
case SADB_UPDATE:
case SADB_ADD:
case SADB_DELETE:
case SADB_GET:
case SADB_ACQUIRE:
case SADB_EXPIRE:
ipseclog((LOG_DEBUG, "%s: must specify satype "
"when msg type=%u.\n", __func__,
msg->sadb_msg_type));
PFKEYSTAT_INC(out_invsatype);
error = EINVAL;
goto senderror;
}
break;
case SADB_SATYPE_AH:
case SADB_SATYPE_ESP:
case SADB_X_SATYPE_IPCOMP:
case SADB_X_SATYPE_TCPSIGNATURE:
switch (msg->sadb_msg_type) {
case SADB_X_SPDADD:
case SADB_X_SPDDELETE:
case SADB_X_SPDGET:
case SADB_X_SPDDUMP:
case SADB_X_SPDFLUSH:
case SADB_X_SPDSETIDX:
case SADB_X_SPDUPDATE:
case SADB_X_SPDDELETE2:
ipseclog((LOG_DEBUG, "%s: illegal satype=%u\n",
__func__, msg->sadb_msg_type));
PFKEYSTAT_INC(out_invsatype);
error = EINVAL;
goto senderror;
}
break;
case SADB_SATYPE_RSVP:
case SADB_SATYPE_OSPFV2:
case SADB_SATYPE_RIPV2:
case SADB_SATYPE_MIP:
ipseclog((LOG_DEBUG, "%s: type %u isn't supported.\n",
__func__, msg->sadb_msg_satype));
PFKEYSTAT_INC(out_invsatype);
error = EOPNOTSUPP;
goto senderror;
case 1: /* XXX: What does it do? */
if (msg->sadb_msg_type == SADB_X_PROMISC)
break;
/*FALLTHROUGH*/
default:
ipseclog((LOG_DEBUG, "%s: invalid type %u is passed.\n",
__func__, msg->sadb_msg_satype));
PFKEYSTAT_INC(out_invsatype);
error = EINVAL;
goto senderror;
}
/* check field of upper layer protocol and address family */
if (mh.ext[SADB_EXT_ADDRESS_SRC] != NULL
&& mh.ext[SADB_EXT_ADDRESS_DST] != NULL) {
struct sadb_address *src0, *dst0;
u_int plen;
src0 = (struct sadb_address *)(mh.ext[SADB_EXT_ADDRESS_SRC]);
dst0 = (struct sadb_address *)(mh.ext[SADB_EXT_ADDRESS_DST]);
/* check upper layer protocol */
if (src0->sadb_address_proto != dst0->sadb_address_proto) {
ipseclog((LOG_DEBUG, "%s: upper layer protocol "
"mismatched.\n", __func__));
PFKEYSTAT_INC(out_invaddr);
error = EINVAL;
goto senderror;
}
/* check family */
if (PFKEY_ADDR_SADDR(src0)->sa_family !=
PFKEY_ADDR_SADDR(dst0)->sa_family) {
ipseclog((LOG_DEBUG, "%s: address family mismatched.\n",
__func__));
PFKEYSTAT_INC(out_invaddr);
error = EINVAL;
goto senderror;
}
if (PFKEY_ADDR_SADDR(src0)->sa_len !=
PFKEY_ADDR_SADDR(dst0)->sa_len) {
ipseclog((LOG_DEBUG, "%s: address struct size "
"mismatched.\n", __func__));
PFKEYSTAT_INC(out_invaddr);
error = EINVAL;
goto senderror;
}
switch (PFKEY_ADDR_SADDR(src0)->sa_family) {
case AF_INET:
if (PFKEY_ADDR_SADDR(src0)->sa_len !=
sizeof(struct sockaddr_in)) {
PFKEYSTAT_INC(out_invaddr);
error = EINVAL;
goto senderror;
}
break;
case AF_INET6:
if (PFKEY_ADDR_SADDR(src0)->sa_len !=
sizeof(struct sockaddr_in6)) {
PFKEYSTAT_INC(out_invaddr);
error = EINVAL;
goto senderror;
}
break;
default:
ipseclog((LOG_DEBUG, "%s: unsupported address family\n",
__func__));
PFKEYSTAT_INC(out_invaddr);
error = EAFNOSUPPORT;
goto senderror;
}
switch (PFKEY_ADDR_SADDR(src0)->sa_family) {
case AF_INET:
plen = sizeof(struct in_addr) << 3;
break;
case AF_INET6:
plen = sizeof(struct in6_addr) << 3;
break;
default:
plen = 0; /*fool gcc*/
break;
}
/* check max prefix length */
if (src0->sadb_address_prefixlen > plen ||
dst0->sadb_address_prefixlen > plen) {
ipseclog((LOG_DEBUG, "%s: illegal prefixlen.\n",
__func__));
PFKEYSTAT_INC(out_invaddr);
error = EINVAL;
goto senderror;
}
/*
* prefixlen == 0 is valid because there can be a case when
* all addresses are matched.
*/
}
if (msg->sadb_msg_type >= sizeof(key_typesw)/sizeof(key_typesw[0]) ||
key_typesw[msg->sadb_msg_type] == NULL) {
PFKEYSTAT_INC(out_invmsgtype);
error = EINVAL;
goto senderror;
}
return (*key_typesw[msg->sadb_msg_type])(so, m, &mh);
senderror:
msg->sadb_msg_errno = error;
return key_sendup_mbuf(so, m, target);
}
static int
key_senderror(struct socket *so, struct mbuf *m, int code)
{
struct sadb_msg *msg;
IPSEC_ASSERT(m->m_len >= sizeof(struct sadb_msg),
("mbuf too small, len %u", m->m_len));
msg = mtod(m, struct sadb_msg *);
msg->sadb_msg_errno = code;
return key_sendup_mbuf(so, m, KEY_SENDUP_ONE);
}
/*
* set the pointer to each header into message buffer.
* m will be freed on error.
* XXX larger-than-MCLBYTES extension?
*/
static int
key_align(struct mbuf *m, struct sadb_msghdr *mhp)
{
struct mbuf *n;
struct sadb_ext *ext;
size_t off, end;
int extlen;
int toff;
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(m->m_len >= sizeof(struct sadb_msg),
("mbuf too small, len %u", m->m_len));
/* initialize */
bzero(mhp, sizeof(*mhp));
mhp->msg = mtod(m, struct sadb_msg *);
mhp->ext[0] = (struct sadb_ext *)mhp->msg; /*XXX backward compat */
end = PFKEY_UNUNIT64(mhp->msg->sadb_msg_len);
extlen = end; /*just in case extlen is not updated*/
for (off = sizeof(struct sadb_msg); off < end; off += extlen) {
n = m_pulldown(m, off, sizeof(struct sadb_ext), &toff);
if (!n) {
/* m is already freed */
return ENOBUFS;
}
ext = (struct sadb_ext *)(mtod(n, caddr_t) + toff);
/* set pointer */
switch (ext->sadb_ext_type) {
case SADB_EXT_SA:
case SADB_EXT_ADDRESS_SRC:
case SADB_EXT_ADDRESS_DST:
case SADB_EXT_ADDRESS_PROXY:
case SADB_EXT_LIFETIME_CURRENT:
case SADB_EXT_LIFETIME_HARD:
case SADB_EXT_LIFETIME_SOFT:
case SADB_EXT_KEY_AUTH:
case SADB_EXT_KEY_ENCRYPT:
case SADB_EXT_IDENTITY_SRC:
case SADB_EXT_IDENTITY_DST:
case SADB_EXT_SENSITIVITY:
case SADB_EXT_PROPOSAL:
case SADB_EXT_SUPPORTED_AUTH:
case SADB_EXT_SUPPORTED_ENCRYPT:
case SADB_EXT_SPIRANGE:
case SADB_X_EXT_POLICY:
case SADB_X_EXT_SA2:
#ifdef IPSEC_NAT_T
case SADB_X_EXT_NAT_T_TYPE:
case SADB_X_EXT_NAT_T_SPORT:
case SADB_X_EXT_NAT_T_DPORT:
case SADB_X_EXT_NAT_T_OAI:
case SADB_X_EXT_NAT_T_OAR:
case SADB_X_EXT_NAT_T_FRAG:
#endif
/* duplicate check */
/*
* XXX Are there duplication payloads of either
* KEY_AUTH or KEY_ENCRYPT ?
*/
if (mhp->ext[ext->sadb_ext_type] != NULL) {
ipseclog((LOG_DEBUG, "%s: duplicate ext_type "
"%u\n", __func__, ext->sadb_ext_type));
m_freem(m);
PFKEYSTAT_INC(out_dupext);
return EINVAL;
}
break;
default:
ipseclog((LOG_DEBUG, "%s: invalid ext_type %u\n",
__func__, ext->sadb_ext_type));
m_freem(m);
PFKEYSTAT_INC(out_invexttype);
return EINVAL;
}
extlen = PFKEY_UNUNIT64(ext->sadb_ext_len);
if (key_validate_ext(ext, extlen)) {
m_freem(m);
PFKEYSTAT_INC(out_invlen);
return EINVAL;
}
n = m_pulldown(m, off, extlen, &toff);
if (!n) {
/* m is already freed */
return ENOBUFS;
}
ext = (struct sadb_ext *)(mtod(n, caddr_t) + toff);
mhp->ext[ext->sadb_ext_type] = ext;
mhp->extoff[ext->sadb_ext_type] = off;
mhp->extlen[ext->sadb_ext_type] = extlen;
}
if (off != end) {
m_freem(m);
PFKEYSTAT_INC(out_invlen);
return EINVAL;
}
return 0;
}
static int
key_validate_ext(const struct sadb_ext *ext, int len)
{
const struct sockaddr *sa;
enum { NONE, ADDR } checktype = NONE;
int baselen = 0;
const int sal = offsetof(struct sockaddr, sa_len) + sizeof(sa->sa_len);
if (len != PFKEY_UNUNIT64(ext->sadb_ext_len))
return EINVAL;
/* if it does not match minimum/maximum length, bail */
if (ext->sadb_ext_type >= sizeof(minsize) / sizeof(minsize[0]) ||
ext->sadb_ext_type >= sizeof(maxsize) / sizeof(maxsize[0]))
return EINVAL;
if (!minsize[ext->sadb_ext_type] || len < minsize[ext->sadb_ext_type])
return EINVAL;
if (maxsize[ext->sadb_ext_type] && len > maxsize[ext->sadb_ext_type])
return EINVAL;
/* more checks based on sadb_ext_type XXX need more */
switch (ext->sadb_ext_type) {
case SADB_EXT_ADDRESS_SRC:
case SADB_EXT_ADDRESS_DST:
case SADB_EXT_ADDRESS_PROXY:
baselen = PFKEY_ALIGN8(sizeof(struct sadb_address));
checktype = ADDR;
break;
case SADB_EXT_IDENTITY_SRC:
case SADB_EXT_IDENTITY_DST:
if (((const struct sadb_ident *)ext)->sadb_ident_type ==
SADB_X_IDENTTYPE_ADDR) {
baselen = PFKEY_ALIGN8(sizeof(struct sadb_ident));
checktype = ADDR;
} else
checktype = NONE;
break;
default:
checktype = NONE;
break;
}
switch (checktype) {
case NONE:
break;
case ADDR:
sa = (const struct sockaddr *)(((const u_int8_t*)ext)+baselen);
if (len < baselen + sal)
return EINVAL;
if (baselen + PFKEY_ALIGN8(sa->sa_len) != len)
return EINVAL;
break;
}
return 0;
}
void
key_init(void)
{
int i;
for (i = 0; i < IPSEC_DIR_MAX; i++)
TAILQ_INIT(&V_sptree[i]);
LIST_INIT(&V_sahtree);
for (i = 0; i <= SADB_SATYPE_MAX; i++)
LIST_INIT(&V_regtree[i]);
LIST_INIT(&V_acqtree);
LIST_INIT(&V_spacqtree);
if (!IS_DEFAULT_VNET(curvnet))
return;
SPTREE_LOCK_INIT();
REGTREE_LOCK_INIT();
SAHTREE_LOCK_INIT();
ACQ_LOCK_INIT();
SPACQ_LOCK_INIT();
#ifndef IPSEC_DEBUG2
- callout_init(&key_timer, CALLOUT_MPSAFE);
+ callout_init(&key_timer, 1);
callout_reset(&key_timer, hz, key_timehandler, NULL);
#endif /*IPSEC_DEBUG2*/
/* initialize key statistics */
keystat.getspi_count = 1;
printf("IPsec: Initialized Security Association Processing.\n");
}
#ifdef VIMAGE
void
key_destroy(void)
{
TAILQ_HEAD(, secpolicy) drainq;
struct secpolicy *sp, *nextsp;
struct secacq *acq, *nextacq;
struct secspacq *spacq, *nextspacq;
struct secashead *sah, *nextsah;
struct secreg *reg;
int i;
TAILQ_INIT(&drainq);
SPTREE_WLOCK();
for (i = 0; i < IPSEC_DIR_MAX; i++) {
TAILQ_CONCAT(&drainq, &V_sptree[i], chain);
}
SPTREE_WUNLOCK();
sp = TAILQ_FIRST(&drainq);
while (sp != NULL) {
nextsp = TAILQ_NEXT(sp, chain);
KEY_FREESP(&sp);
sp = nextsp;
}
SAHTREE_LOCK();
for (sah = LIST_FIRST(&V_sahtree); sah != NULL; sah = nextsah) {
nextsah = LIST_NEXT(sah, chain);
if (__LIST_CHAINED(sah)) {
LIST_REMOVE(sah, chain);
free(sah, M_IPSEC_SAH);
}
}
SAHTREE_UNLOCK();
REGTREE_LOCK();
for (i = 0; i <= SADB_SATYPE_MAX; i++) {
LIST_FOREACH(reg, &V_regtree[i], chain) {
if (__LIST_CHAINED(reg)) {
LIST_REMOVE(reg, chain);
free(reg, M_IPSEC_SAR);
break;
}
}
}
REGTREE_UNLOCK();
ACQ_LOCK();
for (acq = LIST_FIRST(&V_acqtree); acq != NULL; acq = nextacq) {
nextacq = LIST_NEXT(acq, chain);
if (__LIST_CHAINED(acq)) {
LIST_REMOVE(acq, chain);
free(acq, M_IPSEC_SAQ);
}
}
ACQ_UNLOCK();
SPACQ_LOCK();
for (spacq = LIST_FIRST(&V_spacqtree); spacq != NULL;
spacq = nextspacq) {
nextspacq = LIST_NEXT(spacq, chain);
if (__LIST_CHAINED(spacq)) {
LIST_REMOVE(spacq, chain);
free(spacq, M_IPSEC_SAQ);
}
}
SPACQ_UNLOCK();
}
#endif
/*
* XXX: maybe This function is called after INBOUND IPsec processing.
*
* Special check for tunnel-mode packets.
* We must make some checks for consistency between inner and outer IP header.
*
* xxx more checks to be provided
*/
int
key_checktunnelsanity(struct secasvar *sav, u_int family, caddr_t src,
caddr_t dst)
{
IPSEC_ASSERT(sav->sah != NULL, ("null SA header"));
/* XXX: check inner IP header */
return 1;
}
/* record data transfer on SA, and update timestamps */
void
key_sa_recordxfer(struct secasvar *sav, struct mbuf *m)
{
IPSEC_ASSERT(sav != NULL, ("Null secasvar"));
IPSEC_ASSERT(m != NULL, ("Null mbuf"));
if (!sav->lft_c)
return;
/*
* XXX Currently, there is a difference of bytes size
* between inbound and outbound processing.
*/
sav->lft_c->bytes += m->m_pkthdr.len;
/* to check bytes lifetime is done in key_timehandler(). */
/*
* We use the number of packets as the unit of
* allocations. We increment the variable
* whenever {esp,ah}_{in,out}put is called.
*/
sav->lft_c->allocations++;
/* XXX check for expires? */
/*
* NOTE: We record CURRENT usetime by using wall clock,
* in seconds. HARD and SOFT lifetime are measured by the time
* difference (again in seconds) from usetime.
*
* usetime
* v expire expire
* -----+-----+--------+---> t
* <--------------> HARD
* <-----> SOFT
*/
sav->lft_c->usetime = time_second;
/* XXX check for expires? */
return;
}
static void
key_sa_chgstate(struct secasvar *sav, u_int8_t state)
{
IPSEC_ASSERT(sav != NULL, ("NULL sav"));
SAHTREE_LOCK_ASSERT();
if (sav->state != state) {
if (__LIST_CHAINED(sav))
LIST_REMOVE(sav, chain);
sav->state = state;
LIST_INSERT_HEAD(&sav->sah->savtree[state], sav, chain);
}
}
void
key_sa_stir_iv(struct secasvar *sav)
{
IPSEC_ASSERT(sav->iv != NULL, ("null IV"));
key_randomfill(sav->iv, sav->ivlen);
}
/*
* Take one of the kernel's security keys and convert it into a PF_KEY
* structure within an mbuf, suitable for sending up to a waiting
* application in user land.
*
* IN:
* src: A pointer to a kernel security key.
* exttype: Which type of key this is. Refer to the PF_KEY data structures.
* OUT:
* a valid mbuf or NULL indicating an error
*
*/
static struct mbuf *
key_setkey(struct seckey *src, u_int16_t exttype)
{
struct mbuf *m;
struct sadb_key *p;
int len;
if (src == NULL)
return NULL;
len = PFKEY_ALIGN8(sizeof(struct sadb_key) + _KEYLEN(src));
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return NULL;
m_align(m, len);
m->m_len = len;
p = mtod(m, struct sadb_key *);
bzero(p, len);
p->sadb_key_len = PFKEY_UNIT64(len);
p->sadb_key_exttype = exttype;
p->sadb_key_bits = src->bits;
bcopy(src->key_data, _KEYBUF(p), _KEYLEN(src));
return m;
}
/*
* Take one of the kernel's lifetime data structures and convert it
* into a PF_KEY structure within an mbuf, suitable for sending up to
* a waiting application in user land.
*
* IN:
* src: A pointer to a kernel lifetime structure.
* exttype: Which type of lifetime this is. Refer to the PF_KEY
* data structures for more information.
* OUT:
* a valid mbuf or NULL indicating an error
*
*/
static struct mbuf *
key_setlifetime(struct seclifetime *src, u_int16_t exttype)
{
struct mbuf *m = NULL;
struct sadb_lifetime *p;
int len = PFKEY_ALIGN8(sizeof(struct sadb_lifetime));
if (src == NULL)
return NULL;
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return m;
m_align(m, len);
m->m_len = len;
p = mtod(m, struct sadb_lifetime *);
bzero(p, len);
p->sadb_lifetime_len = PFKEY_UNIT64(len);
p->sadb_lifetime_exttype = exttype;
p->sadb_lifetime_allocations = src->allocations;
p->sadb_lifetime_bytes = src->bytes;
p->sadb_lifetime_addtime = src->addtime;
p->sadb_lifetime_usetime = src->usetime;
return m;
}
Index: head/sys/netpfil/ipfw/ip_dummynet.c
===================================================================
--- head/sys/netpfil/ipfw/ip_dummynet.c (revision 283290)
+++ head/sys/netpfil/ipfw/ip_dummynet.c (revision 283291)
@@ -1,2314 +1,2314 @@
/*-
* Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa
* Portions Copyright (c) 2000 Akamba Corp.
* All rights reserved
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/*
* Configuration and internal object management for dummynet.
*/
#include "opt_inet6.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/rwlock.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/time.h>
#include <sys/taskqueue.h>
#include <net/if.h> /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */
#include <netinet/in.h>
#include <netinet/ip_var.h> /* ip_output(), IP_FORWARDING */
#include <netinet/ip_fw.h>
#include <netinet/ip_dummynet.h>
#include <netpfil/ipfw/ip_fw_private.h>
#include <netpfil/ipfw/dn_heap.h>
#include <netpfil/ipfw/ip_dn_private.h>
#include <netpfil/ipfw/dn_sched.h>
/* which objects to copy */
#define DN_C_LINK 0x01
#define DN_C_SCH 0x02
#define DN_C_FLOW 0x04
#define DN_C_FS 0x08
#define DN_C_QUEUE 0x10
/* we use this argument in case of a schk_new */
struct schk_new_arg {
struct dn_alg *fp;
struct dn_sch *sch;
};
/*---- callout hooks. ----*/
static struct callout dn_timeout;
static struct task dn_task;
static struct taskqueue *dn_tq = NULL;
static void
dummynet(void *arg)
{
(void)arg; /* UNUSED */
taskqueue_enqueue_fast(dn_tq, &dn_task);
}
void
dn_reschedule(void)
{
callout_reset_sbt(&dn_timeout, tick_sbt, 0, dummynet, NULL,
C_HARDCLOCK | C_DIRECT_EXEC);
}
/*----- end of callout hooks -----*/
/* Return a scheduler descriptor given the type or name. */
static struct dn_alg *
find_sched_type(int type, char *name)
{
struct dn_alg *d;
SLIST_FOREACH(d, &dn_cfg.schedlist, next) {
if (d->type == type || (name && !strcasecmp(d->name, name)))
return d;
}
return NULL; /* not found */
}
int
ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg)
{
int oldv = *v;
const char *op = NULL;
if (dflt < lo)
dflt = lo;
if (dflt > hi)
dflt = hi;
if (oldv < lo) {
*v = dflt;
op = "Bump";
} else if (oldv > hi) {
*v = hi;
op = "Clamp";
} else
return *v;
if (op && msg)
printf("%s %s to %d (was %d)\n", op, msg, *v, oldv);
return *v;
}
/*---- flow_id mask, hash and compare functions ---*/
/*
* The flow_id includes the 5-tuple, the queue/pipe number
* which we store in the extra area in host order,
* and for ipv6 also the flow_id6.
* XXX see if we want the tos byte (can store in 'flags')
*/
static struct ipfw_flow_id *
flow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id)
{
int is_v6 = IS_IP6_FLOW_ID(id);
id->dst_port &= mask->dst_port;
id->src_port &= mask->src_port;
id->proto &= mask->proto;
id->extra &= mask->extra;
if (is_v6) {
APPLY_MASK(&id->dst_ip6, &mask->dst_ip6);
APPLY_MASK(&id->src_ip6, &mask->src_ip6);
id->flow_id6 &= mask->flow_id6;
} else {
id->dst_ip &= mask->dst_ip;
id->src_ip &= mask->src_ip;
}
return id;
}
/* computes an OR of two masks, result in dst and also returned */
static struct ipfw_flow_id *
flow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst)
{
int is_v6 = IS_IP6_FLOW_ID(dst);
dst->dst_port |= src->dst_port;
dst->src_port |= src->src_port;
dst->proto |= src->proto;
dst->extra |= src->extra;
if (is_v6) {
#define OR_MASK(_d, _s) \
(_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \
(_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \
(_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \
(_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3];
OR_MASK(&dst->dst_ip6, &src->dst_ip6);
OR_MASK(&dst->src_ip6, &src->src_ip6);
#undef OR_MASK
dst->flow_id6 |= src->flow_id6;
} else {
dst->dst_ip |= src->dst_ip;
dst->src_ip |= src->src_ip;
}
return dst;
}
static int
nonzero_mask(struct ipfw_flow_id *m)
{
if (m->dst_port || m->src_port || m->proto || m->extra)
return 1;
if (IS_IP6_FLOW_ID(m)) {
return
m->dst_ip6.__u6_addr.__u6_addr32[0] ||
m->dst_ip6.__u6_addr.__u6_addr32[1] ||
m->dst_ip6.__u6_addr.__u6_addr32[2] ||
m->dst_ip6.__u6_addr.__u6_addr32[3] ||
m->src_ip6.__u6_addr.__u6_addr32[0] ||
m->src_ip6.__u6_addr.__u6_addr32[1] ||
m->src_ip6.__u6_addr.__u6_addr32[2] ||
m->src_ip6.__u6_addr.__u6_addr32[3] ||
m->flow_id6;
} else {
return m->dst_ip || m->src_ip;
}
}
/* XXX we may want a better hash function */
static uint32_t
flow_id_hash(struct ipfw_flow_id *id)
{
uint32_t i;
if (IS_IP6_FLOW_ID(id)) {
uint32_t *d = (uint32_t *)&id->dst_ip6;
uint32_t *s = (uint32_t *)&id->src_ip6;
i = (d[0] ) ^ (d[1]) ^
(d[2] ) ^ (d[3]) ^
(d[0] >> 15) ^ (d[1] >> 15) ^
(d[2] >> 15) ^ (d[3] >> 15) ^
(s[0] << 1) ^ (s[1] << 1) ^
(s[2] << 1) ^ (s[3] << 1) ^
(s[0] << 16) ^ (s[1] << 16) ^
(s[2] << 16) ^ (s[3] << 16) ^
(id->dst_port << 1) ^ (id->src_port) ^
(id->extra) ^
(id->proto ) ^ (id->flow_id6);
} else {
i = (id->dst_ip) ^ (id->dst_ip >> 15) ^
(id->src_ip << 1) ^ (id->src_ip >> 16) ^
(id->extra) ^
(id->dst_port << 1) ^ (id->src_port) ^ (id->proto);
}
return i;
}
/* Like bcmp, returns 0 if ids match, 1 otherwise. */
static int
flow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2)
{
int is_v6 = IS_IP6_FLOW_ID(id1);
if (!is_v6) {
if (IS_IP6_FLOW_ID(id2))
return 1; /* different address families */
return (id1->dst_ip == id2->dst_ip &&
id1->src_ip == id2->src_ip &&
id1->dst_port == id2->dst_port &&
id1->src_port == id2->src_port &&
id1->proto == id2->proto &&
id1->extra == id2->extra) ? 0 : 1;
}
/* the ipv6 case */
return (
!bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) &&
!bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) &&
id1->dst_port == id2->dst_port &&
id1->src_port == id2->src_port &&
id1->proto == id2->proto &&
id1->extra == id2->extra &&
id1->flow_id6 == id2->flow_id6) ? 0 : 1;
}
/*--------- end of flow-id mask, hash and compare ---------*/
/*--- support functions for the qht hashtable ----
* Entries are hashed by flow-id
*/
static uint32_t
q_hash(uintptr_t key, int flags, void *arg)
{
/* compute the hash slot from the flow id */
struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ?
&((struct dn_queue *)key)->ni.fid :
(struct ipfw_flow_id *)key;
return flow_id_hash(id);
}
static int
q_match(void *obj, uintptr_t key, int flags, void *arg)
{
struct dn_queue *o = (struct dn_queue *)obj;
struct ipfw_flow_id *id2;
if (flags & DNHT_KEY_IS_OBJ) {
/* compare pointers */
id2 = &((struct dn_queue *)key)->ni.fid;
} else {
id2 = (struct ipfw_flow_id *)key;
}
return (0 == flow_id_cmp(&o->ni.fid, id2));
}
/*
* create a new queue instance for the given 'key'.
*/
static void *
q_new(uintptr_t key, int flags, void *arg)
{
struct dn_queue *q, *template = arg;
struct dn_fsk *fs = template->fs;
int size = sizeof(*q) + fs->sched->fp->q_datalen;
q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO);
if (q == NULL) {
D("no memory for new queue");
return NULL;
}
set_oid(&q->ni.oid, DN_QUEUE, size);
if (fs->fs.flags & DN_QHT_HASH)
q->ni.fid = *(struct ipfw_flow_id *)key;
q->fs = fs;
q->_si = template->_si;
q->_si->q_count++;
if (fs->sched->fp->new_queue)
fs->sched->fp->new_queue(q);
dn_cfg.queue_count++;
return q;
}
/*
* Notify schedulers that a queue is going away.
* If (flags & DN_DESTROY), also free the packets.
* The version for callbacks is called q_delete_cb().
*/
static void
dn_delete_queue(struct dn_queue *q, int flags)
{
struct dn_fsk *fs = q->fs;
// D("fs %p si %p\n", fs, q->_si);
/* notify the parent scheduler that the queue is going away */
if (fs && fs->sched->fp->free_queue)
fs->sched->fp->free_queue(q);
q->_si->q_count--;
q->_si = NULL;
if (flags & DN_DESTROY) {
if (q->mq.head)
dn_free_pkts(q->mq.head);
bzero(q, sizeof(*q)); // safety
free(q, M_DUMMYNET);
dn_cfg.queue_count--;
}
}
static int
q_delete_cb(void *q, void *arg)
{
int flags = (int)(uintptr_t)arg;
dn_delete_queue(q, flags);
return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0;
}
/*
* calls dn_delete_queue/q_delete_cb on all queues,
* which notifies the parent scheduler and possibly drains packets.
* flags & DN_DESTROY: drains queues and destroy qht;
*/
static void
qht_delete(struct dn_fsk *fs, int flags)
{
ND("fs %d start flags %d qht %p",
fs->fs.fs_nr, flags, fs->qht);
if (!fs->qht)
return;
if (fs->fs.flags & DN_QHT_HASH) {
dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags);
if (flags & DN_DESTROY) {
dn_ht_free(fs->qht, 0);
fs->qht = NULL;
}
} else {
dn_delete_queue((struct dn_queue *)(fs->qht), flags);
if (flags & DN_DESTROY)
fs->qht = NULL;
}
}
/*
* Find and possibly create the queue for a MULTIQUEUE scheduler.
* We never call it for !MULTIQUEUE (the queue is in the sch_inst).
*/
struct dn_queue *
ipdn_q_find(struct dn_fsk *fs, struct dn_sch_inst *si,
struct ipfw_flow_id *id)
{
struct dn_queue template;
template._si = si;
template.fs = fs;
if (fs->fs.flags & DN_QHT_HASH) {
struct ipfw_flow_id masked_id;
if (fs->qht == NULL) {
fs->qht = dn_ht_init(NULL, fs->fs.buckets,
offsetof(struct dn_queue, q_next),
q_hash, q_match, q_new);
if (fs->qht == NULL)
return NULL;
}
masked_id = *id;
flow_id_mask(&fs->fsk_mask, &masked_id);
return dn_ht_find(fs->qht, (uintptr_t)&masked_id,
DNHT_INSERT, &template);
} else {
if (fs->qht == NULL)
fs->qht = q_new(0, 0, &template);
return (struct dn_queue *)fs->qht;
}
}
/*--- end of queue hash table ---*/
/*--- support functions for the sch_inst hashtable ----
*
* These are hashed by flow-id
*/
static uint32_t
si_hash(uintptr_t key, int flags, void *arg)
{
/* compute the hash slot from the flow id */
struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ?
&((struct dn_sch_inst *)key)->ni.fid :
(struct ipfw_flow_id *)key;
return flow_id_hash(id);
}
static int
si_match(void *obj, uintptr_t key, int flags, void *arg)
{
struct dn_sch_inst *o = obj;
struct ipfw_flow_id *id2;
id2 = (flags & DNHT_KEY_IS_OBJ) ?
&((struct dn_sch_inst *)key)->ni.fid :
(struct ipfw_flow_id *)key;
return flow_id_cmp(&o->ni.fid, id2) == 0;
}
/*
* create a new instance for the given 'key'
* Allocate memory for instance, delay line and scheduler private data.
*/
static void *
si_new(uintptr_t key, int flags, void *arg)
{
struct dn_schk *s = arg;
struct dn_sch_inst *si;
int l = sizeof(*si) + s->fp->si_datalen;
si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO);
if (si == NULL)
goto error;
/* Set length only for the part passed up to userland. */
set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow));
set_oid(&(si->dline.oid), DN_DELAY_LINE,
sizeof(struct delay_line));
/* mark si and dline as outside the event queue */
si->ni.oid.id = si->dline.oid.id = -1;
si->sched = s;
si->dline.si = si;
if (s->fp->new_sched && s->fp->new_sched(si)) {
D("new_sched error");
goto error;
}
if (s->sch.flags & DN_HAVE_MASK)
si->ni.fid = *(struct ipfw_flow_id *)key;
dn_cfg.si_count++;
return si;
error:
if (si) {
bzero(si, sizeof(*si)); // safety
free(si, M_DUMMYNET);
}
return NULL;
}
/*
* Callback from siht to delete all scheduler instances. Remove
* si and delay line from the system heap, destroy all queues.
* We assume that all flowset have been notified and do not
* point to us anymore.
*/
static int
si_destroy(void *_si, void *arg)
{
struct dn_sch_inst *si = _si;
struct dn_schk *s = si->sched;
struct delay_line *dl = &si->dline;
if (dl->oid.subtype) /* remove delay line from event heap */
heap_extract(&dn_cfg.evheap, dl);
dn_free_pkts(dl->mq.head); /* drain delay line */
if (si->kflags & DN_ACTIVE) /* remove si from event heap */
heap_extract(&dn_cfg.evheap, si);
if (s->fp->free_sched)
s->fp->free_sched(si);
bzero(si, sizeof(*si)); /* safety */
free(si, M_DUMMYNET);
dn_cfg.si_count--;
return DNHT_SCAN_DEL;
}
/*
* Find the scheduler instance for this packet. If we need to apply
* a mask, do on a local copy of the flow_id to preserve the original.
* Assume siht is always initialized if we have a mask.
*/
struct dn_sch_inst *
ipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id)
{
if (s->sch.flags & DN_HAVE_MASK) {
struct ipfw_flow_id id_t = *id;
flow_id_mask(&s->sch.sched_mask, &id_t);
return dn_ht_find(s->siht, (uintptr_t)&id_t,
DNHT_INSERT, s);
}
if (!s->siht)
s->siht = si_new(0, 0, s);
return (struct dn_sch_inst *)s->siht;
}
/* callback to flush credit for the scheduler instance */
static int
si_reset_credit(void *_si, void *arg)
{
struct dn_sch_inst *si = _si;
struct dn_link *p = &si->sched->link;
si->credit = p->burst + (dn_cfg.io_fast ? p->bandwidth : 0);
return 0;
}
static void
schk_reset_credit(struct dn_schk *s)
{
if (s->sch.flags & DN_HAVE_MASK)
dn_ht_scan(s->siht, si_reset_credit, NULL);
else if (s->siht)
si_reset_credit(s->siht, NULL);
}
/*---- end of sch_inst hashtable ---------------------*/
/*-------------------------------------------------------
* flowset hash (fshash) support. Entries are hashed by fs_nr.
* New allocations are put in the fsunlinked list, from which
* they are removed when they point to a specific scheduler.
*/
static uint32_t
fsk_hash(uintptr_t key, int flags, void *arg)
{
uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key :
((struct dn_fsk *)key)->fs.fs_nr;
return ( (i>>8)^(i>>4)^i );
}
static int
fsk_match(void *obj, uintptr_t key, int flags, void *arg)
{
struct dn_fsk *fs = obj;
int i = !(flags & DNHT_KEY_IS_OBJ) ? key :
((struct dn_fsk *)key)->fs.fs_nr;
return (fs->fs.fs_nr == i);
}
static void *
fsk_new(uintptr_t key, int flags, void *arg)
{
struct dn_fsk *fs;
fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO);
if (fs) {
set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs));
dn_cfg.fsk_count++;
fs->drain_bucket = 0;
SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain);
}
return fs;
}
/*
* detach flowset from its current scheduler. Flags as follows:
* DN_DETACH removes from the fsk_list
* DN_DESTROY deletes individual queues
* DN_DELETE_FS destroys the flowset (otherwise goes in unlinked).
*/
static void
fsk_detach(struct dn_fsk *fs, int flags)
{
if (flags & DN_DELETE_FS)
flags |= DN_DESTROY;
ND("fs %d from sched %d flags %s %s %s",
fs->fs.fs_nr, fs->fs.sched_nr,
(flags & DN_DELETE_FS) ? "DEL_FS":"",
(flags & DN_DESTROY) ? "DEL":"",
(flags & DN_DETACH) ? "DET":"");
if (flags & DN_DETACH) { /* detach from the list */
struct dn_fsk_head *h;
h = fs->sched ? &fs->sched->fsk_list : &dn_cfg.fsu;
SLIST_REMOVE(h, fs, dn_fsk, sch_chain);
}
/* Free the RED parameters, they will be recomputed on
* subsequent attach if needed.
*/
if (fs->w_q_lookup)
free(fs->w_q_lookup, M_DUMMYNET);
fs->w_q_lookup = NULL;
qht_delete(fs, flags);
if (fs->sched && fs->sched->fp->free_fsk)
fs->sched->fp->free_fsk(fs);
fs->sched = NULL;
if (flags & DN_DELETE_FS) {
bzero(fs, sizeof(*fs)); /* safety */
free(fs, M_DUMMYNET);
dn_cfg.fsk_count--;
} else {
SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain);
}
}
/*
* Detach or destroy all flowsets in a list.
* flags specifies what to do:
* DN_DESTROY: flush all queues
* DN_DELETE_FS: DN_DESTROY + destroy flowset
* DN_DELETE_FS implies DN_DESTROY
*/
static void
fsk_detach_list(struct dn_fsk_head *h, int flags)
{
struct dn_fsk *fs;
int n = 0; /* only for stats */
ND("head %p flags %x", h, flags);
while ((fs = SLIST_FIRST(h))) {
SLIST_REMOVE_HEAD(h, sch_chain);
n++;
fsk_detach(fs, flags);
}
ND("done %d flowsets", n);
}
/*
* called on 'queue X delete' -- removes the flowset from fshash,
* deletes all queues for the flowset, and removes the flowset.
*/
static int
delete_fs(int i, int locked)
{
struct dn_fsk *fs;
int err = 0;
if (!locked)
DN_BH_WLOCK();
fs = dn_ht_find(dn_cfg.fshash, i, DNHT_REMOVE, NULL);
ND("fs %d found %p", i, fs);
if (fs) {
fsk_detach(fs, DN_DETACH | DN_DELETE_FS);
err = 0;
} else
err = EINVAL;
if (!locked)
DN_BH_WUNLOCK();
return err;
}
/*----- end of flowset hashtable support -------------*/
/*------------------------------------------------------------
* Scheduler hash. When searching by index we pass sched_nr,
* otherwise we pass struct dn_sch * which is the first field in
* struct dn_schk so we can cast between the two. We use this trick
* because in the create phase (but it should be fixed).
*/
static uint32_t
schk_hash(uintptr_t key, int flags, void *_arg)
{
uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key :
((struct dn_schk *)key)->sch.sched_nr;
return ( (i>>8)^(i>>4)^i );
}
static int
schk_match(void *obj, uintptr_t key, int flags, void *_arg)
{
struct dn_schk *s = (struct dn_schk *)obj;
int i = !(flags & DNHT_KEY_IS_OBJ) ? key :
((struct dn_schk *)key)->sch.sched_nr;
return (s->sch.sched_nr == i);
}
/*
* Create the entry and intialize with the sched hash if needed.
* Leave s->fp unset so we can tell whether a dn_ht_find() returns
* a new object or a previously existing one.
*/
static void *
schk_new(uintptr_t key, int flags, void *arg)
{
struct schk_new_arg *a = arg;
struct dn_schk *s;
int l = sizeof(*s) +a->fp->schk_datalen;
s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO);
if (s == NULL)
return NULL;
set_oid(&s->link.oid, DN_LINK, sizeof(s->link));
s->sch = *a->sch; // copy initial values
s->link.link_nr = s->sch.sched_nr;
SLIST_INIT(&s->fsk_list);
/* initialize the hash table or create the single instance */
s->fp = a->fp; /* si_new needs this */
s->drain_bucket = 0;
if (s->sch.flags & DN_HAVE_MASK) {
s->siht = dn_ht_init(NULL, s->sch.buckets,
offsetof(struct dn_sch_inst, si_next),
si_hash, si_match, si_new);
if (s->siht == NULL) {
free(s, M_DUMMYNET);
return NULL;
}
}
s->fp = NULL; /* mark as a new scheduler */
dn_cfg.schk_count++;
return s;
}
/*
* Callback for sched delete. Notify all attached flowsets to
* detach from the scheduler, destroy the internal flowset, and
* all instances. The scheduler goes away too.
* arg is 0 (only detach flowsets and destroy instances)
* DN_DESTROY (detach & delete queues, delete schk)
* or DN_DELETE_FS (delete queues and flowsets, delete schk)
*/
static int
schk_delete_cb(void *obj, void *arg)
{
struct dn_schk *s = obj;
#if 0
int a = (int)arg;
ND("sched %d arg %s%s",
s->sch.sched_nr,
a&DN_DESTROY ? "DEL ":"",
a&DN_DELETE_FS ? "DEL_FS":"");
#endif
fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0);
/* no more flowset pointing to us now */
if (s->sch.flags & DN_HAVE_MASK) {
dn_ht_scan(s->siht, si_destroy, NULL);
dn_ht_free(s->siht, 0);
} else if (s->siht)
si_destroy(s->siht, NULL);
if (s->profile) {
free(s->profile, M_DUMMYNET);
s->profile = NULL;
}
s->siht = NULL;
if (s->fp->destroy)
s->fp->destroy(s);
bzero(s, sizeof(*s)); // safety
free(obj, M_DUMMYNET);
dn_cfg.schk_count--;
return DNHT_SCAN_DEL;
}
/*
* called on a 'sched X delete' command. Deletes a single scheduler.
* This is done by removing from the schedhash, unlinking all
* flowsets and deleting their traffic.
*/
static int
delete_schk(int i)
{
struct dn_schk *s;
s = dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL);
ND("%d %p", i, s);
if (!s)
return EINVAL;
delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */
/* then detach flowsets, delete traffic */
schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY);
return 0;
}
/*--- end of schk hashtable support ---*/
static int
copy_obj(char **start, char *end, void *_o, const char *msg, int i)
{
struct dn_id *o = _o;
int have = end - *start;
if (have < o->len || o->len == 0 || o->type == 0) {
D("(WARN) type %d %s %d have %d need %d",
o->type, msg, i, have, o->len);
return 1;
}
ND("type %d %s %d len %d", o->type, msg, i, o->len);
bcopy(_o, *start, o->len);
if (o->type == DN_LINK) {
/* Adjust burst parameter for link */
struct dn_link *l = (struct dn_link *)*start;
l->burst = div64(l->burst, 8 * hz);
l->delay = l->delay * 1000 / hz;
} else if (o->type == DN_SCH) {
/* Set id->id to the number of instances */
struct dn_schk *s = _o;
struct dn_id *id = (struct dn_id *)(*start);
id->id = (s->sch.flags & DN_HAVE_MASK) ?
dn_ht_entries(s->siht) : (s->siht ? 1 : 0);
}
*start += o->len;
return 0;
}
/* Specific function to copy a queue.
* Copies only the user-visible part of a queue (which is in
* a struct dn_flow), and sets len accordingly.
*/
static int
copy_obj_q(char **start, char *end, void *_o, const char *msg, int i)
{
struct dn_id *o = _o;
int have = end - *start;
int len = sizeof(struct dn_flow); /* see above comment */
if (have < len || o->len == 0 || o->type != DN_QUEUE) {
D("ERROR type %d %s %d have %d need %d",
o->type, msg, i, have, len);
return 1;
}
ND("type %d %s %d len %d", o->type, msg, i, len);
bcopy(_o, *start, len);
((struct dn_id*)(*start))->len = len;
*start += len;
return 0;
}
static int
copy_q_cb(void *obj, void *arg)
{
struct dn_queue *q = obj;
struct copy_args *a = arg;
struct dn_flow *ni = (struct dn_flow *)(*a->start);
if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1))
return DNHT_SCAN_END;
ni->oid.type = DN_FLOW; /* override the DN_QUEUE */
ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL);
return 0;
}
static int
copy_q(struct copy_args *a, struct dn_fsk *fs, int flags)
{
if (!fs->qht)
return 0;
if (fs->fs.flags & DN_QHT_HASH)
dn_ht_scan(fs->qht, copy_q_cb, a);
else
copy_q_cb(fs->qht, a);
return 0;
}
/*
* This routine only copies the initial part of a profile ? XXX
*/
static int
copy_profile(struct copy_args *a, struct dn_profile *p)
{
int have = a->end - *a->start;
/* XXX here we check for max length */
int profile_len = sizeof(struct dn_profile) -
ED_MAX_SAMPLES_NO*sizeof(int);
if (p == NULL)
return 0;
if (have < profile_len) {
D("error have %d need %d", have, profile_len);
return 1;
}
bcopy(p, *a->start, profile_len);
((struct dn_id *)(*a->start))->len = profile_len;
*a->start += profile_len;
return 0;
}
static int
copy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags)
{
struct dn_fs *ufs = (struct dn_fs *)(*a->start);
if (!fs)
return 0;
ND("flowset %d", fs->fs.fs_nr);
if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr))
return DNHT_SCAN_END;
ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ?
dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0);
if (flags) { /* copy queues */
copy_q(a, fs, 0);
}
return 0;
}
static int
copy_si_cb(void *obj, void *arg)
{
struct dn_sch_inst *si = obj;
struct copy_args *a = arg;
struct dn_flow *ni = (struct dn_flow *)(*a->start);
if (copy_obj(a->start, a->end, &si->ni, "inst",
si->sched->sch.sched_nr))
return DNHT_SCAN_END;
ni->oid.type = DN_FLOW; /* override the DN_SCH_I */
ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL);
return 0;
}
static int
copy_si(struct copy_args *a, struct dn_schk *s, int flags)
{
if (s->sch.flags & DN_HAVE_MASK)
dn_ht_scan(s->siht, copy_si_cb, a);
else if (s->siht)
copy_si_cb(s->siht, a);
return 0;
}
/*
* compute a list of children of a scheduler and copy up
*/
static int
copy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags)
{
struct dn_fsk *fs;
struct dn_id *o;
uint32_t *p;
int n = 0, space = sizeof(*o);
SLIST_FOREACH(fs, &s->fsk_list, sch_chain) {
if (fs->fs.fs_nr < DN_MAX_ID)
n++;
}
space += n * sizeof(uint32_t);
DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n);
if (a->end - *(a->start) < space)
return DNHT_SCAN_END;
o = (struct dn_id *)(*(a->start));
o->len = space;
*a->start += o->len;
o->type = DN_TEXT;
p = (uint32_t *)(o+1);
SLIST_FOREACH(fs, &s->fsk_list, sch_chain)
if (fs->fs.fs_nr < DN_MAX_ID)
*p++ = fs->fs.fs_nr;
return 0;
}
static int
copy_data_helper(void *_o, void *_arg)
{
struct copy_args *a = _arg;
uint32_t *r = a->extra->r; /* start of first range */
uint32_t *lim; /* first invalid pointer */
int n;
lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len);
if (a->type == DN_LINK || a->type == DN_SCH) {
/* pipe|sched show, we receive a dn_schk */
struct dn_schk *s = _o;
n = s->sch.sched_nr;
if (a->type == DN_SCH && n >= DN_MAX_ID)
return 0; /* not a scheduler */
if (a->type == DN_LINK && n <= DN_MAX_ID)
return 0; /* not a pipe */
/* see if the object is within one of our ranges */
for (;r < lim; r += 2) {
if (n < r[0] || n > r[1])
continue;
/* Found a valid entry, copy and we are done */
if (a->flags & DN_C_LINK) {
if (copy_obj(a->start, a->end,
&s->link, "link", n))
return DNHT_SCAN_END;
if (copy_profile(a, s->profile))
return DNHT_SCAN_END;
if (copy_flowset(a, s->fs, 0))
return DNHT_SCAN_END;
}
if (a->flags & DN_C_SCH) {
if (copy_obj(a->start, a->end,
&s->sch, "sched", n))
return DNHT_SCAN_END;
/* list all attached flowsets */
if (copy_fsk_list(a, s, 0))
return DNHT_SCAN_END;
}
if (a->flags & DN_C_FLOW)
copy_si(a, s, 0);
break;
}
} else if (a->type == DN_FS) {
/* queue show, skip internal flowsets */
struct dn_fsk *fs = _o;
n = fs->fs.fs_nr;
if (n >= DN_MAX_ID)
return 0;
/* see if the object is within one of our ranges */
for (;r < lim; r += 2) {
if (n < r[0] || n > r[1])
continue;
if (copy_flowset(a, fs, 0))
return DNHT_SCAN_END;
copy_q(a, fs, 0);
break; /* we are done */
}
}
return 0;
}
static inline struct dn_schk *
locate_scheduler(int i)
{
return dn_ht_find(dn_cfg.schedhash, i, 0, NULL);
}
/*
* red parameters are in fixed point arithmetic.
*/
static int
config_red(struct dn_fsk *fs)
{
int64_t s, idle, weight, w0;
int t, i;
fs->w_q = fs->fs.w_q;
fs->max_p = fs->fs.max_p;
ND("called");
/* Doing stuff that was in userland */
i = fs->sched->link.bandwidth;
s = (i <= 0) ? 0 :
hz * dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i;
idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */
fs->lookup_step = div64(idle , dn_cfg.red_lookup_depth);
/* fs->lookup_step not scaled, */
if (!fs->lookup_step)
fs->lookup_step = 1;
w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled
for (t = fs->lookup_step; t > 1; --t)
weight = SCALE_MUL(weight, w0);
fs->lookup_weight = (int)(weight); // scaled
/* Now doing stuff that was in kerneland */
fs->min_th = SCALE(fs->fs.min_th);
fs->max_th = SCALE(fs->fs.max_th);
if (fs->fs.max_th == fs->fs.min_th)
fs->c_1 = fs->max_p;
else
fs->c_1 = SCALE((int64_t)(fs->max_p)) / (fs->fs.max_th - fs->fs.min_th);
fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th));
if (fs->fs.flags & DN_IS_GENTLE_RED) {
fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th;
fs->c_4 = SCALE(1) - 2 * fs->max_p;
}
/* If the lookup table already exist, free and create it again. */
if (fs->w_q_lookup) {
free(fs->w_q_lookup, M_DUMMYNET);
fs->w_q_lookup = NULL;
}
if (dn_cfg.red_lookup_depth == 0) {
printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth"
"must be > 0\n");
fs->fs.flags &= ~DN_IS_RED;
fs->fs.flags &= ~DN_IS_GENTLE_RED;
return (EINVAL);
}
fs->lookup_depth = dn_cfg.red_lookup_depth;
fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int),
M_DUMMYNET, M_NOWAIT);
if (fs->w_q_lookup == NULL) {
printf("dummynet: sorry, cannot allocate red lookup table\n");
fs->fs.flags &= ~DN_IS_RED;
fs->fs.flags &= ~DN_IS_GENTLE_RED;
return(ENOSPC);
}
/* Fill the lookup table with (1 - w_q)^x */
fs->w_q_lookup[0] = SCALE(1) - fs->w_q;
for (i = 1; i < fs->lookup_depth; i++)
fs->w_q_lookup[i] =
SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight);
if (dn_cfg.red_avg_pkt_size < 1)
dn_cfg.red_avg_pkt_size = 512;
fs->avg_pkt_size = dn_cfg.red_avg_pkt_size;
if (dn_cfg.red_max_pkt_size < 1)
dn_cfg.red_max_pkt_size = 1500;
fs->max_pkt_size = dn_cfg.red_max_pkt_size;
ND("exit");
return 0;
}
/* Scan all flowset attached to this scheduler and update red */
static void
update_red(struct dn_schk *s)
{
struct dn_fsk *fs;
SLIST_FOREACH(fs, &s->fsk_list, sch_chain) {
if (fs && (fs->fs.flags & DN_IS_RED))
config_red(fs);
}
}
/* attach flowset to scheduler s, possibly requeue */
static void
fsk_attach(struct dn_fsk *fs, struct dn_schk *s)
{
ND("remove fs %d from fsunlinked, link to sched %d",
fs->fs.fs_nr, s->sch.sched_nr);
SLIST_REMOVE(&dn_cfg.fsu, fs, dn_fsk, sch_chain);
fs->sched = s;
SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain);
if (s->fp->new_fsk)
s->fp->new_fsk(fs);
/* XXX compute fsk_mask */
fs->fsk_mask = fs->fs.flow_mask;
if (fs->sched->sch.flags & DN_HAVE_MASK)
flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask);
if (fs->qht) {
/*
* we must drain qht according to the old
* type, and reinsert according to the new one.
* The requeue is complex -- in general we need to
* reclassify every single packet.
* For the time being, let's hope qht is never set
* when we reach this point.
*/
D("XXX TODO requeue from fs %d to sch %d",
fs->fs.fs_nr, s->sch.sched_nr);
fs->qht = NULL;
}
/* set the new type for qht */
if (nonzero_mask(&fs->fsk_mask))
fs->fs.flags |= DN_QHT_HASH;
else
fs->fs.flags &= ~DN_QHT_HASH;
/* XXX config_red() can fail... */
if (fs->fs.flags & DN_IS_RED)
config_red(fs);
}
/* update all flowsets which may refer to this scheduler */
static void
update_fs(struct dn_schk *s)
{
struct dn_fsk *fs, *tmp;
SLIST_FOREACH_SAFE(fs, &dn_cfg.fsu, sch_chain, tmp) {
if (s->sch.sched_nr != fs->fs.sched_nr) {
D("fs %d for sch %d not %d still unlinked",
fs->fs.fs_nr, fs->fs.sched_nr,
s->sch.sched_nr);
continue;
}
fsk_attach(fs, s);
}
}
/*
* Configuration -- to preserve backward compatibility we use
* the following scheme (N is 65536)
* NUMBER SCHED LINK FLOWSET
* 1 .. N-1 (1)WFQ (2)WFQ (3)queue
* N+1 .. 2N-1 (4)FIFO (5)FIFO (6)FIFO for sched 1..N-1
* 2N+1 .. 3N-1 -- -- (7)FIFO for sched N+1..2N-1
*
* "pipe i config" configures #1, #2 and #3
* "sched i config" configures #1 and possibly #6
* "queue i config" configures #3
* #1 is configured with 'pipe i config' or 'sched i config'
* #2 is configured with 'pipe i config', and created if not
* existing with 'sched i config'
* #3 is configured with 'queue i config'
* #4 is automatically configured after #1, can only be FIFO
* #5 is automatically configured after #2
* #6 is automatically created when #1 is !MULTIQUEUE,
* and can be updated.
* #7 is automatically configured after #2
*/
/*
* configure a link (and its FIFO instance)
*/
static int
config_link(struct dn_link *p, struct dn_id *arg)
{
int i;
if (p->oid.len != sizeof(*p)) {
D("invalid pipe len %d", p->oid.len);
return EINVAL;
}
i = p->link_nr;
if (i <= 0 || i >= DN_MAX_ID)
return EINVAL;
/*
* The config program passes parameters as follows:
* bw = bits/second (0 means no limits),
* delay = ms, must be translated into ticks.
* qsize = slots/bytes
* burst ???
*/
p->delay = (p->delay * hz) / 1000;
/* Scale burst size: bytes -> bits * hz */
p->burst *= 8 * hz;
DN_BH_WLOCK();
/* do it twice, base link and FIFO link */
for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) {
struct dn_schk *s = locate_scheduler(i);
if (s == NULL) {
DN_BH_WUNLOCK();
D("sched %d not found", i);
return EINVAL;
}
/* remove profile if exists */
if (s->profile) {
free(s->profile, M_DUMMYNET);
s->profile = NULL;
}
/* copy all parameters */
s->link.oid = p->oid;
s->link.link_nr = i;
s->link.delay = p->delay;
if (s->link.bandwidth != p->bandwidth) {
/* XXX bandwidth changes, need to update red params */
s->link.bandwidth = p->bandwidth;
update_red(s);
}
s->link.burst = p->burst;
schk_reset_credit(s);
}
dn_cfg.id++;
DN_BH_WUNLOCK();
return 0;
}
/*
* configure a flowset. Can be called from inside with locked=1,
*/
static struct dn_fsk *
config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked)
{
int i;
struct dn_fsk *fs;
if (nfs->oid.len != sizeof(*nfs)) {
D("invalid flowset len %d", nfs->oid.len);
return NULL;
}
i = nfs->fs_nr;
if (i <= 0 || i >= 3*DN_MAX_ID)
return NULL;
ND("flowset %d", i);
/* XXX other sanity checks */
if (nfs->flags & DN_QSIZE_BYTES) {
ipdn_bound_var(&nfs->qsize, 16384,
1500, dn_cfg.byte_limit, NULL); // "queue byte size");
} else {
ipdn_bound_var(&nfs->qsize, 50,
1, dn_cfg.slot_limit, NULL); // "queue slot size");
}
if (nfs->flags & DN_HAVE_MASK) {
/* make sure we have some buckets */
ipdn_bound_var((int *)&nfs->buckets, dn_cfg.hash_size,
1, dn_cfg.max_hash_size, "flowset buckets");
} else {
nfs->buckets = 1; /* we only need 1 */
}
if (!locked)
DN_BH_WLOCK();
do { /* exit with break when done */
struct dn_schk *s;
int flags = nfs->sched_nr ? DNHT_INSERT : 0;
int j;
int oldc = dn_cfg.fsk_count;
fs = dn_ht_find(dn_cfg.fshash, i, flags, NULL);
if (fs == NULL) {
D("missing sched for flowset %d", i);
break;
}
/* grab some defaults from the existing one */
if (nfs->sched_nr == 0) /* reuse */
nfs->sched_nr = fs->fs.sched_nr;
for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) {
if (nfs->par[j] == -1) /* reuse */
nfs->par[j] = fs->fs.par[j];
}
if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) {
ND("flowset %d unchanged", i);
break; /* no change, nothing to do */
}
if (oldc != dn_cfg.fsk_count) /* new item */
dn_cfg.id++;
s = locate_scheduler(nfs->sched_nr);
/* detach from old scheduler if needed, preserving
* queues if we need to reattach. Then update the
* configuration, and possibly attach to the new sched.
*/
DX(2, "fs %d changed sched %d@%p to %d@%p",
fs->fs.fs_nr,
fs->fs.sched_nr, fs->sched, nfs->sched_nr, s);
if (fs->sched) {
int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY);
flags |= DN_DESTROY; /* XXX temporary */
fsk_detach(fs, flags);
}
fs->fs = *nfs; /* copy configuration */
if (s != NULL)
fsk_attach(fs, s);
} while (0);
if (!locked)
DN_BH_WUNLOCK();
return fs;
}
/*
* config/reconfig a scheduler and its FIFO variant.
* For !MULTIQUEUE schedulers, also set up the flowset.
*
* On reconfigurations (detected because s->fp is set),
* detach existing flowsets preserving traffic, preserve link,
* and delete the old scheduler creating a new one.
*/
static int
config_sched(struct dn_sch *_nsch, struct dn_id *arg)
{
struct dn_schk *s;
struct schk_new_arg a; /* argument for schk_new */
int i;
struct dn_link p; /* copy of oldlink */
struct dn_profile *pf = NULL; /* copy of old link profile */
/* Used to preserv mask parameter */
struct ipfw_flow_id new_mask;
int new_buckets = 0;
int new_flags = 0;
int pipe_cmd;
int err = ENOMEM;
a.sch = _nsch;
if (a.sch->oid.len != sizeof(*a.sch)) {
D("bad sched len %d", a.sch->oid.len);
return EINVAL;
}
i = a.sch->sched_nr;
if (i <= 0 || i >= DN_MAX_ID)
return EINVAL;
/* make sure we have some buckets */
if (a.sch->flags & DN_HAVE_MASK)
ipdn_bound_var((int *)&a.sch->buckets, dn_cfg.hash_size,
1, dn_cfg.max_hash_size, "sched buckets");
/* XXX other sanity checks */
bzero(&p, sizeof(p));
pipe_cmd = a.sch->flags & DN_PIPE_CMD;
a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set?
if (pipe_cmd) {
/* Copy mask parameter */
new_mask = a.sch->sched_mask;
new_buckets = a.sch->buckets;
new_flags = a.sch->flags;
}
DN_BH_WLOCK();
again: /* run twice, for wfq and fifo */
/*
* lookup the type. If not supplied, use the previous one
* or default to WF2Q+. Otherwise, return an error.
*/
dn_cfg.id++;
a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name);
if (a.fp != NULL) {
/* found. Lookup or create entry */
s = dn_ht_find(dn_cfg.schedhash, i, DNHT_INSERT, &a);
} else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) {
/* No type. search existing s* or retry with WF2Q+ */
s = dn_ht_find(dn_cfg.schedhash, i, 0, &a);
if (s != NULL) {
a.fp = s->fp;
/* Scheduler exists, skip to FIFO scheduler
* if command was pipe config...
*/
if (pipe_cmd)
goto next;
} else {
/* New scheduler, create a wf2q+ with no mask
* if command was pipe config...
*/
if (pipe_cmd) {
/* clear mask parameter */
bzero(&a.sch->sched_mask, sizeof(new_mask));
a.sch->buckets = 0;
a.sch->flags &= ~DN_HAVE_MASK;
}
a.sch->oid.subtype = DN_SCHED_WF2QP;
goto again;
}
} else {
D("invalid scheduler type %d %s",
a.sch->oid.subtype, a.sch->name);
err = EINVAL;
goto error;
}
/* normalize name and subtype */
a.sch->oid.subtype = a.fp->type;
bzero(a.sch->name, sizeof(a.sch->name));
strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name));
if (s == NULL) {
D("cannot allocate scheduler %d", i);
goto error;
}
/* restore existing link if any */
if (p.link_nr) {
s->link = p;
if (!pf || pf->link_nr != p.link_nr) { /* no saved value */
s->profile = NULL; /* XXX maybe not needed */
} else {
s->profile = malloc(sizeof(struct dn_profile),
M_DUMMYNET, M_NOWAIT | M_ZERO);
if (s->profile == NULL) {
D("cannot allocate profile");
goto error; //XXX
}
bcopy(pf, s->profile, sizeof(*pf));
}
}
p.link_nr = 0;
if (s->fp == NULL) {
DX(2, "sched %d new type %s", i, a.fp->name);
} else if (s->fp != a.fp ||
bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) {
/* already existing. */
DX(2, "sched %d type changed from %s to %s",
i, s->fp->name, a.fp->name);
DX(4, " type/sub %d/%d -> %d/%d",
s->sch.oid.type, s->sch.oid.subtype,
a.sch->oid.type, a.sch->oid.subtype);
if (s->link.link_nr == 0)
D("XXX WARNING link 0 for sched %d", i);
p = s->link; /* preserve link */
if (s->profile) {/* preserve profile */
if (!pf)
pf = malloc(sizeof(*pf),
M_DUMMYNET, M_NOWAIT | M_ZERO);
if (pf) /* XXX should issue a warning otherwise */
bcopy(s->profile, pf, sizeof(*pf));
}
/* remove from the hash */
dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL);
/* Detach flowsets, preserve queues. */
// schk_delete_cb(s, NULL);
// XXX temporarily, kill queues
schk_delete_cb(s, (void *)DN_DESTROY);
goto again;
} else {
DX(4, "sched %d unchanged type %s", i, a.fp->name);
}
/* complete initialization */
s->sch = *a.sch;
s->fp = a.fp;
s->cfg = arg;
// XXX schk_reset_credit(s);
/* create the internal flowset if needed,
* trying to reuse existing ones if available
*/
if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) {
s->fs = dn_ht_find(dn_cfg.fshash, i, 0, NULL);
if (!s->fs) {
struct dn_fs fs;
bzero(&fs, sizeof(fs));
set_oid(&fs.oid, DN_FS, sizeof(fs));
fs.fs_nr = i + DN_MAX_ID;
fs.sched_nr = i;
s->fs = config_fs(&fs, NULL, 1 /* locked */);
}
if (!s->fs) {
schk_delete_cb(s, (void *)DN_DESTROY);
D("error creating internal fs for %d", i);
goto error;
}
}
/* call init function after the flowset is created */
if (s->fp->config)
s->fp->config(s);
update_fs(s);
next:
if (i < DN_MAX_ID) { /* now configure the FIFO instance */
i += DN_MAX_ID;
if (pipe_cmd) {
/* Restore mask parameter for FIFO */
a.sch->sched_mask = new_mask;
a.sch->buckets = new_buckets;
a.sch->flags = new_flags;
} else {
/* sched config shouldn't modify the FIFO scheduler */
if (dn_ht_find(dn_cfg.schedhash, i, 0, &a) != NULL) {
/* FIFO already exist, don't touch it */
err = 0; /* and this is not an error */
goto error;
}
}
a.sch->sched_nr = i;
a.sch->oid.subtype = DN_SCHED_FIFO;
bzero(a.sch->name, sizeof(a.sch->name));
goto again;
}
err = 0;
error:
DN_BH_WUNLOCK();
if (pf)
free(pf, M_DUMMYNET);
return err;
}
/*
* attach a profile to a link
*/
static int
config_profile(struct dn_profile *pf, struct dn_id *arg)
{
struct dn_schk *s;
int i, olen, err = 0;
if (pf->oid.len < sizeof(*pf)) {
D("short profile len %d", pf->oid.len);
return EINVAL;
}
i = pf->link_nr;
if (i <= 0 || i >= DN_MAX_ID)
return EINVAL;
/* XXX other sanity checks */
DN_BH_WLOCK();
for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) {
s = locate_scheduler(i);
if (s == NULL) {
err = EINVAL;
break;
}
dn_cfg.id++;
/*
* If we had a profile and the new one does not fit,
* or it is deleted, then we need to free memory.
*/
if (s->profile && (pf->samples_no == 0 ||
s->profile->oid.len < pf->oid.len)) {
free(s->profile, M_DUMMYNET);
s->profile = NULL;
}
if (pf->samples_no == 0)
continue;
/*
* new profile, possibly allocate memory
* and copy data.
*/
if (s->profile == NULL)
s->profile = malloc(pf->oid.len,
M_DUMMYNET, M_NOWAIT | M_ZERO);
if (s->profile == NULL) {
D("no memory for profile %d", i);
err = ENOMEM;
break;
}
/* preserve larger length XXX double check */
olen = s->profile->oid.len;
if (olen < pf->oid.len)
olen = pf->oid.len;
bcopy(pf, s->profile, pf->oid.len);
s->profile->oid.len = olen;
}
DN_BH_WUNLOCK();
return err;
}
/*
* Delete all objects:
*/
static void
dummynet_flush(void)
{
/* delete all schedulers and related links/queues/flowsets */
dn_ht_scan(dn_cfg.schedhash, schk_delete_cb,
(void *)(uintptr_t)DN_DELETE_FS);
/* delete all remaining (unlinked) flowsets */
DX(4, "still %d unlinked fs", dn_cfg.fsk_count);
dn_ht_free(dn_cfg.fshash, DNHT_REMOVE);
fsk_detach_list(&dn_cfg.fsu, DN_DELETE_FS);
/* Reinitialize system heap... */
heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id));
}
/*
* Main handler for configuration. We are guaranteed to be called
* with an oid which is at least a dn_id.
* - the first object is the command (config, delete, flush, ...)
* - config_link must be issued after the corresponding config_sched
* - parameters (DN_TXT) for an object must preceed the object
* processed on a config_sched.
*/
int
do_config(void *p, int l)
{
struct dn_id *next, *o;
int err = 0, err2 = 0;
struct dn_id *arg = NULL;
uintptr_t *a;
o = p;
if (o->id != DN_API_VERSION) {
D("invalid api version got %d need %d",
o->id, DN_API_VERSION);
return EINVAL;
}
for (; l >= sizeof(*o); o = next) {
struct dn_id *prev = arg;
if (o->len < sizeof(*o) || l < o->len) {
D("bad len o->len %d len %d", o->len, l);
err = EINVAL;
break;
}
l -= o->len;
next = (struct dn_id *)((char *)o + o->len);
err = 0;
switch (o->type) {
default:
D("cmd %d not implemented", o->type);
break;
#ifdef EMULATE_SYSCTL
/* sysctl emulation.
* if we recognize the command, jump to the correct
* handler and return
*/
case DN_SYSCTL_SET:
err = kesysctl_emu_set(p, l);
return err;
#endif
case DN_CMD_CONFIG: /* simply a header */
break;
case DN_CMD_DELETE:
/* the argument is in the first uintptr_t after o */
a = (uintptr_t *)(o+1);
if (o->len < sizeof(*o) + sizeof(*a)) {
err = EINVAL;
break;
}
switch (o->subtype) {
case DN_LINK:
/* delete base and derived schedulers */
DN_BH_WLOCK();
err = delete_schk(*a);
err2 = delete_schk(*a + DN_MAX_ID);
DN_BH_WUNLOCK();
if (!err)
err = err2;
break;
default:
D("invalid delete type %d",
o->subtype);
err = EINVAL;
break;
case DN_FS:
err = (*a <1 || *a >= DN_MAX_ID) ?
EINVAL : delete_fs(*a, 0) ;
break;
}
break;
case DN_CMD_FLUSH:
DN_BH_WLOCK();
dummynet_flush();
DN_BH_WUNLOCK();
break;
case DN_TEXT: /* store argument the next block */
prev = NULL;
arg = o;
break;
case DN_LINK:
err = config_link((struct dn_link *)o, arg);
break;
case DN_PROFILE:
err = config_profile((struct dn_profile *)o, arg);
break;
case DN_SCH:
err = config_sched((struct dn_sch *)o, arg);
break;
case DN_FS:
err = (NULL==config_fs((struct dn_fs *)o, arg, 0));
break;
}
if (prev)
arg = NULL;
if (err != 0)
break;
}
return err;
}
static int
compute_space(struct dn_id *cmd, struct copy_args *a)
{
int x = 0, need = 0;
int profile_size = sizeof(struct dn_profile) -
ED_MAX_SAMPLES_NO*sizeof(int);
/* NOTE about compute space:
* NP = dn_cfg.schk_count
* NSI = dn_cfg.si_count
* NF = dn_cfg.fsk_count
* NQ = dn_cfg.queue_count
* - ipfw pipe show
* (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler
* link, scheduler template, flowset
* integrated in scheduler and header
* for flowset list
* (NSI)*(dn_flow) all scheduler instance (includes
* the queue instance)
* - ipfw sched show
* (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler
* link, scheduler template, flowset
* integrated in scheduler and header
* for flowset list
* (NSI * dn_flow) all scheduler instances
* (NF * sizeof(uint_32)) space for flowset list linked to scheduler
* (NQ * dn_queue) all queue [XXXfor now not listed]
* - ipfw queue show
* (NF * dn_fs) all flowset
* (NQ * dn_queue) all queues
*/
switch (cmd->subtype) {
default:
return -1;
/* XXX where do LINK and SCH differ ? */
/* 'ipfw sched show' could list all queues associated to
* a scheduler. This feature for now is disabled
*/
case DN_LINK: /* pipe show */
x = DN_C_LINK | DN_C_SCH | DN_C_FLOW;
need += dn_cfg.schk_count *
(sizeof(struct dn_fs) + profile_size) / 2;
need += dn_cfg.fsk_count * sizeof(uint32_t);
break;
case DN_SCH: /* sched show */
need += dn_cfg.schk_count *
(sizeof(struct dn_fs) + profile_size) / 2;
need += dn_cfg.fsk_count * sizeof(uint32_t);
x = DN_C_SCH | DN_C_LINK | DN_C_FLOW;
break;
case DN_FS: /* queue show */
x = DN_C_FS | DN_C_QUEUE;
break;
case DN_GET_COMPAT: /* compatibility mode */
need = dn_compat_calc_size();
break;
}
a->flags = x;
if (x & DN_C_SCH) {
need += dn_cfg.schk_count * sizeof(struct dn_sch) / 2;
/* NOT also, each fs might be attached to a sched */
need += dn_cfg.schk_count * sizeof(struct dn_id) / 2;
}
if (x & DN_C_FS)
need += dn_cfg.fsk_count * sizeof(struct dn_fs);
if (x & DN_C_LINK) {
need += dn_cfg.schk_count * sizeof(struct dn_link) / 2;
}
/*
* When exporting a queue to userland, only pass up the
* struct dn_flow, which is the only visible part.
*/
if (x & DN_C_QUEUE)
need += dn_cfg.queue_count * sizeof(struct dn_flow);
if (x & DN_C_FLOW)
need += dn_cfg.si_count * (sizeof(struct dn_flow));
return need;
}
/*
* If compat != NULL dummynet_get is called in compatibility mode.
* *compat will be the pointer to the buffer to pass to ipfw
*/
int
dummynet_get(struct sockopt *sopt, void **compat)
{
int have, i, need, error;
char *start = NULL, *buf;
size_t sopt_valsize;
struct dn_id *cmd;
struct copy_args a;
struct copy_range r;
int l = sizeof(struct dn_id);
bzero(&a, sizeof(a));
bzero(&r, sizeof(r));
/* save and restore original sopt_valsize around copyin */
sopt_valsize = sopt->sopt_valsize;
cmd = &r.o;
if (!compat) {
/* copy at least an oid, and possibly a full object */
error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd));
sopt->sopt_valsize = sopt_valsize;
if (error)
goto done;
l = cmd->len;
#ifdef EMULATE_SYSCTL
/* sysctl emulation. */
if (cmd->type == DN_SYSCTL_GET)
return kesysctl_emu_get(sopt);
#endif
if (l > sizeof(r)) {
/* request larger than default, allocate buffer */
cmd = malloc(l, M_DUMMYNET, M_WAITOK);
error = sooptcopyin(sopt, cmd, l, l);
sopt->sopt_valsize = sopt_valsize;
if (error)
goto done;
}
} else { /* compatibility */
error = 0;
cmd->type = DN_CMD_GET;
cmd->len = sizeof(struct dn_id);
cmd->subtype = DN_GET_COMPAT;
// cmd->id = sopt_valsize;
D("compatibility mode");
}
a.extra = (struct copy_range *)cmd;
if (cmd->len == sizeof(*cmd)) { /* no range, create a default */
uint32_t *rp = (uint32_t *)(cmd + 1);
cmd->len += 2* sizeof(uint32_t);
rp[0] = 1;
rp[1] = DN_MAX_ID - 1;
if (cmd->subtype == DN_LINK) {
rp[0] += DN_MAX_ID;
rp[1] += DN_MAX_ID;
}
}
/* Count space (under lock) and allocate (outside lock).
* Exit with lock held if we manage to get enough buffer.
* Try a few times then give up.
*/
for (have = 0, i = 0; i < 10; i++) {
DN_BH_WLOCK();
need = compute_space(cmd, &a);
/* if there is a range, ignore value from compute_space() */
if (l > sizeof(*cmd))
need = sopt_valsize - sizeof(*cmd);
if (need < 0) {
DN_BH_WUNLOCK();
error = EINVAL;
goto done;
}
need += sizeof(*cmd);
cmd->id = need;
if (have >= need)
break;
DN_BH_WUNLOCK();
if (start)
free(start, M_DUMMYNET);
start = NULL;
if (need > sopt_valsize)
break;
have = need;
start = malloc(have, M_DUMMYNET, M_WAITOK | M_ZERO);
}
if (start == NULL) {
if (compat) {
*compat = NULL;
error = 1; // XXX
} else {
error = sooptcopyout(sopt, cmd, sizeof(*cmd));
}
goto done;
}
ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, "
"%d:%d si %d, %d:%d queues %d",
dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH,
dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK,
dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS,
dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I,
dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE);
sopt->sopt_valsize = sopt_valsize;
a.type = cmd->subtype;
if (compat == NULL) {
bcopy(cmd, start, sizeof(*cmd));
((struct dn_id*)(start))->len = sizeof(struct dn_id);
buf = start + sizeof(*cmd);
} else
buf = start;
a.start = &buf;
a.end = start + have;
/* start copying other objects */
if (compat) {
a.type = DN_COMPAT_PIPE;
dn_ht_scan(dn_cfg.schedhash, copy_data_helper_compat, &a);
a.type = DN_COMPAT_QUEUE;
dn_ht_scan(dn_cfg.fshash, copy_data_helper_compat, &a);
} else if (a.type == DN_FS) {
dn_ht_scan(dn_cfg.fshash, copy_data_helper, &a);
} else {
dn_ht_scan(dn_cfg.schedhash, copy_data_helper, &a);
}
DN_BH_WUNLOCK();
if (compat) {
*compat = start;
sopt->sopt_valsize = buf - start;
/* free() is done by ip_dummynet_compat() */
start = NULL; //XXX hack
} else {
error = sooptcopyout(sopt, start, buf - start);
}
done:
if (cmd && cmd != &r.o)
free(cmd, M_DUMMYNET);
if (start)
free(start, M_DUMMYNET);
return error;
}
/* Callback called on scheduler instance to delete it if idle */
static int
drain_scheduler_cb(void *_si, void *arg)
{
struct dn_sch_inst *si = _si;
if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL)
return 0;
if (si->sched->fp->flags & DN_MULTIQUEUE) {
if (si->q_count == 0)
return si_destroy(si, NULL);
else
return 0;
} else { /* !DN_MULTIQUEUE */
if ((si+1)->ni.length == 0)
return si_destroy(si, NULL);
else
return 0;
}
return 0; /* unreachable */
}
/* Callback called on scheduler to check if it has instances */
static int
drain_scheduler_sch_cb(void *_s, void *arg)
{
struct dn_schk *s = _s;
if (s->sch.flags & DN_HAVE_MASK) {
dn_ht_scan_bucket(s->siht, &s->drain_bucket,
drain_scheduler_cb, NULL);
s->drain_bucket++;
} else {
if (s->siht) {
if (drain_scheduler_cb(s->siht, NULL) == DNHT_SCAN_DEL)
s->siht = NULL;
}
}
return 0;
}
/* Called every tick, try to delete a 'bucket' of scheduler */
void
dn_drain_scheduler(void)
{
dn_ht_scan_bucket(dn_cfg.schedhash, &dn_cfg.drain_sch,
drain_scheduler_sch_cb, NULL);
dn_cfg.drain_sch++;
}
/* Callback called on queue to delete if it is idle */
static int
drain_queue_cb(void *_q, void *arg)
{
struct dn_queue *q = _q;
if (q->ni.length == 0) {
dn_delete_queue(q, DN_DESTROY);
return DNHT_SCAN_DEL; /* queue is deleted */
}
return 0; /* queue isn't deleted */
}
/* Callback called on flowset used to check if it has queues */
static int
drain_queue_fs_cb(void *_fs, void *arg)
{
struct dn_fsk *fs = _fs;
if (fs->fs.flags & DN_QHT_HASH) {
/* Flowset has a hash table for queues */
dn_ht_scan_bucket(fs->qht, &fs->drain_bucket,
drain_queue_cb, NULL);
fs->drain_bucket++;
} else {
/* No hash table for this flowset, null the pointer
* if the queue is deleted
*/
if (fs->qht) {
if (drain_queue_cb(fs->qht, NULL) == DNHT_SCAN_DEL)
fs->qht = NULL;
}
}
return 0;
}
/* Called every tick, try to delete a 'bucket' of queue */
void
dn_drain_queue(void)
{
/* scan a bucket of flowset */
dn_ht_scan_bucket(dn_cfg.fshash, &dn_cfg.drain_fs,
drain_queue_fs_cb, NULL);
dn_cfg.drain_fs++;
}
/*
* Handler for the various dummynet socket options
*/
static int
ip_dn_ctl(struct sockopt *sopt)
{
void *p = NULL;
int error, l;
error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET);
if (error)
return (error);
/* Disallow sets in really-really secure mode. */
if (sopt->sopt_dir == SOPT_SET) {
error = securelevel_ge(sopt->sopt_td->td_ucred, 3);
if (error)
return (error);
}
switch (sopt->sopt_name) {
default :
D("dummynet: unknown option %d", sopt->sopt_name);
error = EINVAL;
break;
case IP_DUMMYNET_FLUSH:
case IP_DUMMYNET_CONFIGURE:
case IP_DUMMYNET_DEL: /* remove a pipe or queue */
case IP_DUMMYNET_GET:
D("dummynet: compat option %d", sopt->sopt_name);
error = ip_dummynet_compat(sopt);
break;
case IP_DUMMYNET3 :
if (sopt->sopt_dir == SOPT_GET) {
error = dummynet_get(sopt, NULL);
break;
}
l = sopt->sopt_valsize;
if (l < sizeof(struct dn_id) || l > 12000) {
D("argument len %d invalid", l);
break;
}
p = malloc(l, M_TEMP, M_WAITOK); // XXX can it fail ?
error = sooptcopyin(sopt, p, l, l);
if (error)
break ;
error = do_config(p, l);
break;
}
if (p != NULL)
free(p, M_TEMP);
return error ;
}
static void
ip_dn_init(void)
{
if (dn_cfg.init_done)
return;
printf("DUMMYNET %p with IPv6 initialized (100409)\n", curvnet);
dn_cfg.init_done = 1;
/* Set defaults here. MSVC does not accept initializers,
* and this is also useful for vimages
*/
/* queue limits */
dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */
dn_cfg.byte_limit = 1024 * 1024;
dn_cfg.expire = 1;
/* RED parameters */
dn_cfg.red_lookup_depth = 256; /* default lookup table depth */
dn_cfg.red_avg_pkt_size = 512; /* default medium packet size */
dn_cfg.red_max_pkt_size = 1500; /* default max packet size */
/* hash tables */
dn_cfg.max_hash_size = 65536; /* max in the hash tables */
dn_cfg.hash_size = 64; /* default hash size */
/* create hash tables for schedulers and flowsets.
* In both we search by key and by pointer.
*/
dn_cfg.schedhash = dn_ht_init(NULL, dn_cfg.hash_size,
offsetof(struct dn_schk, schk_next),
schk_hash, schk_match, schk_new);
dn_cfg.fshash = dn_ht_init(NULL, dn_cfg.hash_size,
offsetof(struct dn_fsk, fsk_next),
fsk_hash, fsk_match, fsk_new);
/* bucket index to drain object */
dn_cfg.drain_fs = 0;
dn_cfg.drain_sch = 0;
heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id));
SLIST_INIT(&dn_cfg.fsu);
SLIST_INIT(&dn_cfg.schedlist);
DN_LOCK_INIT();
TASK_INIT(&dn_task, 0, dummynet_task, curvnet);
dn_tq = taskqueue_create_fast("dummynet", M_WAITOK,
taskqueue_thread_enqueue, &dn_tq);
taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet");
- callout_init(&dn_timeout, CALLOUT_MPSAFE);
+ callout_init(&dn_timeout, 1);
dn_reschedule();
/* Initialize curr_time adjustment mechanics. */
getmicrouptime(&dn_cfg.prev_t);
}
static void
ip_dn_destroy(int last)
{
callout_drain(&dn_timeout);
DN_BH_WLOCK();
if (last) {
ND("removing last instance\n");
ip_dn_ctl_ptr = NULL;
ip_dn_io_ptr = NULL;
}
dummynet_flush();
DN_BH_WUNLOCK();
taskqueue_drain(dn_tq, &dn_task);
taskqueue_free(dn_tq);
dn_ht_free(dn_cfg.schedhash, 0);
dn_ht_free(dn_cfg.fshash, 0);
heap_free(&dn_cfg.evheap);
DN_LOCK_DESTROY();
}
static int
dummynet_modevent(module_t mod, int type, void *data)
{
if (type == MOD_LOAD) {
if (ip_dn_io_ptr) {
printf("DUMMYNET already loaded\n");
return EEXIST ;
}
ip_dn_init();
ip_dn_ctl_ptr = ip_dn_ctl;
ip_dn_io_ptr = dummynet_io;
return 0;
} else if (type == MOD_UNLOAD) {
ip_dn_destroy(1 /* last */);
return 0;
} else
return EOPNOTSUPP;
}
/* modevent helpers for the modules */
static int
load_dn_sched(struct dn_alg *d)
{
struct dn_alg *s;
if (d == NULL)
return 1; /* error */
ip_dn_init(); /* just in case, we need the lock */
/* Check that mandatory funcs exists */
if (d->enqueue == NULL || d->dequeue == NULL) {
D("missing enqueue or dequeue for %s", d->name);
return 1;
}
/* Search if scheduler already exists */
DN_BH_WLOCK();
SLIST_FOREACH(s, &dn_cfg.schedlist, next) {
if (strcmp(s->name, d->name) == 0) {
D("%s already loaded", d->name);
break; /* scheduler already exists */
}
}
if (s == NULL)
SLIST_INSERT_HEAD(&dn_cfg.schedlist, d, next);
DN_BH_WUNLOCK();
D("dn_sched %s %sloaded", d->name, s ? "not ":"");
return s ? 1 : 0;
}
static int
unload_dn_sched(struct dn_alg *s)
{
struct dn_alg *tmp, *r;
int err = EINVAL;
ND("called for %s", s->name);
DN_BH_WLOCK();
SLIST_FOREACH_SAFE(r, &dn_cfg.schedlist, next, tmp) {
if (strcmp(s->name, r->name) != 0)
continue;
ND("ref_count = %d", r->ref_count);
err = (r->ref_count != 0) ? EBUSY : 0;
if (err == 0)
SLIST_REMOVE(&dn_cfg.schedlist, r, dn_alg, next);
break;
}
DN_BH_WUNLOCK();
D("dn_sched %s %sunloaded", s->name, err ? "not ":"");
return err;
}
int
dn_sched_modevent(module_t mod, int cmd, void *arg)
{
struct dn_alg *sch = arg;
if (cmd == MOD_LOAD)
return load_dn_sched(sch);
else if (cmd == MOD_UNLOAD)
return unload_dn_sched(sch);
else
return EINVAL;
}
static moduledata_t dummynet_mod = {
"dummynet", dummynet_modevent, NULL
};
#define DN_SI_SUB SI_SUB_PROTO_IFATTACHDOMAIN
#define DN_MODEV_ORD (SI_ORDER_ANY - 128) /* after ipfw */
DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD);
MODULE_DEPEND(dummynet, ipfw, 3, 3, 3);
MODULE_VERSION(dummynet, 3);
/*
* Starting up. Done in order after dummynet_modevent() has been called.
* VNET_SYSINIT is also called for each existing vnet and each new vnet.
*/
//VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_init, NULL);
/*
* Shutdown handlers up shop. These are done in REVERSE ORDER, but still
* after dummynet_modevent() has been called. Not called on reboot.
* VNET_SYSUNINIT is also called for each exiting vnet as it exits.
* or when the module is unloaded.
*/
//VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_destroy, NULL);
/* end of file */
Index: head/sys/netpfil/ipfw/ip_fw_dynamic.c
===================================================================
--- head/sys/netpfil/ipfw/ip_fw_dynamic.c (revision 283290)
+++ head/sys/netpfil/ipfw/ip_fw_dynamic.c (revision 283291)
@@ -1,1605 +1,1605 @@
/*-
* Copyright (c) 2002 Luigi Rizzo, Universita` di Pisa
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#define DEB(x)
#define DDB(x) x
/*
* Dynamic rule support for ipfw
*/
#include "opt_ipfw.h"
#include "opt_inet.h"
#ifndef INET
#error IPFIREWALL requires INET.
#endif /* INET */
#include "opt_inet6.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/rmlock.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <net/ethernet.h> /* for ETHERTYPE_IP */
#include <net/if.h>
#include <net/if_var.h>
#include <net/vnet.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/ip_var.h> /* ip_defttl */
#include <netinet/ip_fw.h>
#include <netinet/tcp_var.h>
#include <netinet/udp.h>
#include <netinet/ip6.h> /* IN6_ARE_ADDR_EQUAL */
#ifdef INET6
#include <netinet6/in6_var.h>
#include <netinet6/ip6_var.h>
#endif
#include <netpfil/ipfw/ip_fw_private.h>
#include <machine/in_cksum.h> /* XXX for in_cksum */
#ifdef MAC
#include <security/mac/mac_framework.h>
#endif
/*
* Description of dynamic rules.
*
* Dynamic rules are stored in lists accessed through a hash table
* (ipfw_dyn_v) whose size is curr_dyn_buckets. This value can
* be modified through the sysctl variable dyn_buckets which is
* updated when the table becomes empty.
*
* XXX currently there is only one list, ipfw_dyn.
*
* When a packet is received, its address fields are first masked
* with the mask defined for the rule, then hashed, then matched
* against the entries in the corresponding list.
* Dynamic rules can be used for different purposes:
* + stateful rules;
* + enforcing limits on the number of sessions;
* + in-kernel NAT (not implemented yet)
*
* The lifetime of dynamic rules is regulated by dyn_*_lifetime,
* measured in seconds and depending on the flags.
*
* The total number of dynamic rules is equal to UMA zone items count.
* The max number of dynamic rules is dyn_max. When we reach
* the maximum number of rules we do not create anymore. This is
* done to avoid consuming too much memory, but also too much
* time when searching on each packet (ideally, we should try instead
* to put a limit on the length of the list on each bucket...).
*
* Each dynamic rule holds a pointer to the parent ipfw rule so
* we know what action to perform. Dynamic rules are removed when
* the parent rule is deleted. This can be changed by dyn_keep_states
* sysctl.
*
* There are some limitations with dynamic rules -- we do not
* obey the 'randomized match', and we do not do multiple
* passes through the firewall. XXX check the latter!!!
*/
struct ipfw_dyn_bucket {
struct mtx mtx; /* Bucket protecting lock */
ipfw_dyn_rule *head; /* Pointer to first rule */
};
/*
* Static variables followed by global ones
*/
static VNET_DEFINE(struct ipfw_dyn_bucket *, ipfw_dyn_v);
static VNET_DEFINE(u_int32_t, dyn_buckets_max);
static VNET_DEFINE(u_int32_t, curr_dyn_buckets);
static VNET_DEFINE(struct callout, ipfw_timeout);
#define V_ipfw_dyn_v VNET(ipfw_dyn_v)
#define V_dyn_buckets_max VNET(dyn_buckets_max)
#define V_curr_dyn_buckets VNET(curr_dyn_buckets)
#define V_ipfw_timeout VNET(ipfw_timeout)
static VNET_DEFINE(uma_zone_t, ipfw_dyn_rule_zone);
#define V_ipfw_dyn_rule_zone VNET(ipfw_dyn_rule_zone)
#define IPFW_BUCK_LOCK_INIT(b) \
mtx_init(&(b)->mtx, "IPFW dynamic bucket", NULL, MTX_DEF)
#define IPFW_BUCK_LOCK_DESTROY(b) \
mtx_destroy(&(b)->mtx)
#define IPFW_BUCK_LOCK(i) mtx_lock(&V_ipfw_dyn_v[(i)].mtx)
#define IPFW_BUCK_UNLOCK(i) mtx_unlock(&V_ipfw_dyn_v[(i)].mtx)
#define IPFW_BUCK_ASSERT(i) mtx_assert(&V_ipfw_dyn_v[(i)].mtx, MA_OWNED)
static VNET_DEFINE(int, dyn_keep_states);
#define V_dyn_keep_states VNET(dyn_keep_states)
/*
* Timeouts for various events in handing dynamic rules.
*/
static VNET_DEFINE(u_int32_t, dyn_ack_lifetime);
static VNET_DEFINE(u_int32_t, dyn_syn_lifetime);
static VNET_DEFINE(u_int32_t, dyn_fin_lifetime);
static VNET_DEFINE(u_int32_t, dyn_rst_lifetime);
static VNET_DEFINE(u_int32_t, dyn_udp_lifetime);
static VNET_DEFINE(u_int32_t, dyn_short_lifetime);
#define V_dyn_ack_lifetime VNET(dyn_ack_lifetime)
#define V_dyn_syn_lifetime VNET(dyn_syn_lifetime)
#define V_dyn_fin_lifetime VNET(dyn_fin_lifetime)
#define V_dyn_rst_lifetime VNET(dyn_rst_lifetime)
#define V_dyn_udp_lifetime VNET(dyn_udp_lifetime)
#define V_dyn_short_lifetime VNET(dyn_short_lifetime)
/*
* Keepalives are sent if dyn_keepalive is set. They are sent every
* dyn_keepalive_period seconds, in the last dyn_keepalive_interval
* seconds of lifetime of a rule.
* dyn_rst_lifetime and dyn_fin_lifetime should be strictly lower
* than dyn_keepalive_period.
*/
static VNET_DEFINE(u_int32_t, dyn_keepalive_interval);
static VNET_DEFINE(u_int32_t, dyn_keepalive_period);
static VNET_DEFINE(u_int32_t, dyn_keepalive);
static VNET_DEFINE(time_t, dyn_keepalive_last);
#define V_dyn_keepalive_interval VNET(dyn_keepalive_interval)
#define V_dyn_keepalive_period VNET(dyn_keepalive_period)
#define V_dyn_keepalive VNET(dyn_keepalive)
#define V_dyn_keepalive_last VNET(dyn_keepalive_last)
static VNET_DEFINE(u_int32_t, dyn_max); /* max # of dynamic rules */
#define DYN_COUNT uma_zone_get_cur(V_ipfw_dyn_rule_zone)
#define V_dyn_max VNET(dyn_max)
/* for userspace, we emulate the uma_zone_counter with ipfw_dyn_count */
static int ipfw_dyn_count; /* number of objects */
#ifdef USERSPACE /* emulation of UMA object counters for userspace */
#define uma_zone_get_cur(x) ipfw_dyn_count
#endif /* USERSPACE */
static int last_log; /* Log ratelimiting */
static void ipfw_dyn_tick(void *vnetx);
static void check_dyn_rules(struct ip_fw_chain *, ipfw_range_tlv *, int, int);
#ifdef SYSCTL_NODE
static int sysctl_ipfw_dyn_count(SYSCTL_HANDLER_ARGS);
static int sysctl_ipfw_dyn_max(SYSCTL_HANDLER_ARGS);
SYSBEGIN(f2)
SYSCTL_DECL(_net_inet_ip_fw);
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, dyn_buckets,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_buckets_max), 0,
"Max number of dyn. buckets");
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, curr_dyn_buckets,
CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(curr_dyn_buckets), 0,
"Current Number of dyn. buckets");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_count,
CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RD, 0, 0, sysctl_ipfw_dyn_count, "IU",
"Number of dyn. rules");
SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, dyn_max,
CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW, 0, 0, sysctl_ipfw_dyn_max, "IU",
"Max number of dyn. rules");
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, dyn_ack_lifetime,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_ack_lifetime), 0,
"Lifetime of dyn. rules for acks");
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, dyn_syn_lifetime,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_syn_lifetime), 0,
"Lifetime of dyn. rules for syn");
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, dyn_fin_lifetime,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_fin_lifetime), 0,
"Lifetime of dyn. rules for fin");
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, dyn_rst_lifetime,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_rst_lifetime), 0,
"Lifetime of dyn. rules for rst");
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, dyn_udp_lifetime,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_udp_lifetime), 0,
"Lifetime of dyn. rules for UDP");
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, dyn_short_lifetime,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_short_lifetime), 0,
"Lifetime of dyn. rules for other situations");
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, dyn_keepalive,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_keepalive), 0,
"Enable keepalives for dyn. rules");
SYSCTL_UINT(_net_inet_ip_fw, OID_AUTO, dyn_keep_states,
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(dyn_keep_states), 0,
"Do not flush dynamic states on rule deletion");
SYSEND
#endif /* SYSCTL_NODE */
#ifdef INET6
static __inline int
hash_packet6(struct ipfw_flow_id *id)
{
u_int32_t i;
i = (id->dst_ip6.__u6_addr.__u6_addr32[2]) ^
(id->dst_ip6.__u6_addr.__u6_addr32[3]) ^
(id->src_ip6.__u6_addr.__u6_addr32[2]) ^
(id->src_ip6.__u6_addr.__u6_addr32[3]) ^
(id->dst_port) ^ (id->src_port);
return i;
}
#endif
/*
* IMPORTANT: the hash function for dynamic rules must be commutative
* in source and destination (ip,port), because rules are bidirectional
* and we want to find both in the same bucket.
*/
static __inline int
hash_packet(struct ipfw_flow_id *id, int buckets)
{
u_int32_t i;
#ifdef INET6
if (IS_IP6_FLOW_ID(id))
i = hash_packet6(id);
else
#endif /* INET6 */
i = (id->dst_ip) ^ (id->src_ip) ^ (id->dst_port) ^ (id->src_port);
i &= (buckets - 1);
return i;
}
/**
* Print customizable flow id description via log(9) facility.
*/
static void
print_dyn_rule_flags(struct ipfw_flow_id *id, int dyn_type, int log_flags,
char *prefix, char *postfix)
{
struct in_addr da;
#ifdef INET6
char src[INET6_ADDRSTRLEN], dst[INET6_ADDRSTRLEN];
#else
char src[INET_ADDRSTRLEN], dst[INET_ADDRSTRLEN];
#endif
#ifdef INET6
if (IS_IP6_FLOW_ID(id)) {
ip6_sprintf(src, &id->src_ip6);
ip6_sprintf(dst, &id->dst_ip6);
} else
#endif
{
da.s_addr = htonl(id->src_ip);
inet_ntop(AF_INET, &da, src, sizeof(src));
da.s_addr = htonl(id->dst_ip);
inet_ntop(AF_INET, &da, dst, sizeof(dst));
}
log(log_flags, "ipfw: %s type %d %s %d -> %s %d, %d %s\n",
prefix, dyn_type, src, id->src_port, dst,
id->dst_port, DYN_COUNT, postfix);
}
#define print_dyn_rule(id, dtype, prefix, postfix) \
print_dyn_rule_flags(id, dtype, LOG_DEBUG, prefix, postfix)
#define TIME_LEQ(a,b) ((int)((a)-(b)) <= 0)
#define TIME_LE(a,b) ((int)((a)-(b)) < 0)
/*
* Lookup a dynamic rule, locked version.
*/
static ipfw_dyn_rule *
lookup_dyn_rule_locked(struct ipfw_flow_id *pkt, int i, int *match_direction,
struct tcphdr *tcp)
{
/*
* Stateful ipfw extensions.
* Lookup into dynamic session queue.
*/
#define MATCH_REVERSE 0
#define MATCH_FORWARD 1
#define MATCH_NONE 2
#define MATCH_UNKNOWN 3
int dir = MATCH_NONE;
ipfw_dyn_rule *prev, *q = NULL;
IPFW_BUCK_ASSERT(i);
for (prev = NULL, q = V_ipfw_dyn_v[i].head; q; prev = q, q = q->next) {
if (q->dyn_type == O_LIMIT_PARENT && q->count)
continue;
if (pkt->proto != q->id.proto || q->dyn_type == O_LIMIT_PARENT)
continue;
if (IS_IP6_FLOW_ID(pkt)) {
if (IN6_ARE_ADDR_EQUAL(&pkt->src_ip6, &q->id.src_ip6) &&
IN6_ARE_ADDR_EQUAL(&pkt->dst_ip6, &q->id.dst_ip6) &&
pkt->src_port == q->id.src_port &&
pkt->dst_port == q->id.dst_port) {
dir = MATCH_FORWARD;
break;
}
if (IN6_ARE_ADDR_EQUAL(&pkt->src_ip6, &q->id.dst_ip6) &&
IN6_ARE_ADDR_EQUAL(&pkt->dst_ip6, &q->id.src_ip6) &&
pkt->src_port == q->id.dst_port &&
pkt->dst_port == q->id.src_port) {
dir = MATCH_REVERSE;
break;
}
} else {
if (pkt->src_ip == q->id.src_ip &&
pkt->dst_ip == q->id.dst_ip &&
pkt->src_port == q->id.src_port &&
pkt->dst_port == q->id.dst_port) {
dir = MATCH_FORWARD;
break;
}
if (pkt->src_ip == q->id.dst_ip &&
pkt->dst_ip == q->id.src_ip &&
pkt->src_port == q->id.dst_port &&
pkt->dst_port == q->id.src_port) {
dir = MATCH_REVERSE;
break;
}
}
}
if (q == NULL)
goto done; /* q = NULL, not found */
if (prev != NULL) { /* found and not in front */
prev->next = q->next;
q->next = V_ipfw_dyn_v[i].head;
V_ipfw_dyn_v[i].head = q;
}
if (pkt->proto == IPPROTO_TCP) { /* update state according to flags */
uint32_t ack;
u_char flags = pkt->_flags & (TH_FIN | TH_SYN | TH_RST);
#define BOTH_SYN (TH_SYN | (TH_SYN << 8))
#define BOTH_FIN (TH_FIN | (TH_FIN << 8))
#define TCP_FLAGS (TH_FLAGS | (TH_FLAGS << 8))
#define ACK_FWD 0x10000 /* fwd ack seen */
#define ACK_REV 0x20000 /* rev ack seen */
q->state |= (dir == MATCH_FORWARD) ? flags : (flags << 8);
switch (q->state & TCP_FLAGS) {
case TH_SYN: /* opening */
q->expire = time_uptime + V_dyn_syn_lifetime;
break;
case BOTH_SYN: /* move to established */
case BOTH_SYN | TH_FIN: /* one side tries to close */
case BOTH_SYN | (TH_FIN << 8):
#define _SEQ_GE(a,b) ((int)(a) - (int)(b) >= 0)
if (tcp == NULL)
break;
ack = ntohl(tcp->th_ack);
if (dir == MATCH_FORWARD) {
if (q->ack_fwd == 0 ||
_SEQ_GE(ack, q->ack_fwd)) {
q->ack_fwd = ack;
q->state |= ACK_FWD;
}
} else {
if (q->ack_rev == 0 ||
_SEQ_GE(ack, q->ack_rev)) {
q->ack_rev = ack;
q->state |= ACK_REV;
}
}
if ((q->state & (ACK_FWD | ACK_REV)) ==
(ACK_FWD | ACK_REV)) {
q->expire = time_uptime + V_dyn_ack_lifetime;
q->state &= ~(ACK_FWD | ACK_REV);
}
break;
case BOTH_SYN | BOTH_FIN: /* both sides closed */
if (V_dyn_fin_lifetime >= V_dyn_keepalive_period)
V_dyn_fin_lifetime = V_dyn_keepalive_period - 1;
q->expire = time_uptime + V_dyn_fin_lifetime;
break;
default:
#if 0
/*
* reset or some invalid combination, but can also
* occur if we use keep-state the wrong way.
*/
if ( (q->state & ((TH_RST << 8)|TH_RST)) == 0)
printf("invalid state: 0x%x\n", q->state);
#endif
if (V_dyn_rst_lifetime >= V_dyn_keepalive_period)
V_dyn_rst_lifetime = V_dyn_keepalive_period - 1;
q->expire = time_uptime + V_dyn_rst_lifetime;
break;
}
} else if (pkt->proto == IPPROTO_UDP) {
q->expire = time_uptime + V_dyn_udp_lifetime;
} else {
/* other protocols */
q->expire = time_uptime + V_dyn_short_lifetime;
}
done:
if (match_direction != NULL)
*match_direction = dir;
return (q);
}
ipfw_dyn_rule *
ipfw_lookup_dyn_rule(struct ipfw_flow_id *pkt, int *match_direction,
struct tcphdr *tcp)
{
ipfw_dyn_rule *q;
int i;
i = hash_packet(pkt, V_curr_dyn_buckets);
IPFW_BUCK_LOCK(i);
q = lookup_dyn_rule_locked(pkt, i, match_direction, tcp);
if (q == NULL)
IPFW_BUCK_UNLOCK(i);
/* NB: return table locked when q is not NULL */
return q;
}
/*
* Unlock bucket mtx
* @p - pointer to dynamic rule
*/
void
ipfw_dyn_unlock(ipfw_dyn_rule *q)
{
IPFW_BUCK_UNLOCK(q->bucket);
}
static int
resize_dynamic_table(struct ip_fw_chain *chain, int nbuckets)
{
int i, k, nbuckets_old;
ipfw_dyn_rule *q;
struct ipfw_dyn_bucket *dyn_v, *dyn_v_old;
/* Check if given number is power of 2 and less than 64k */
if ((nbuckets > 65536) || (!powerof2(nbuckets)))
return 1;
CTR3(KTR_NET, "%s: resize dynamic hash: %d -> %d", __func__,
V_curr_dyn_buckets, nbuckets);
/* Allocate and initialize new hash */
dyn_v = malloc(nbuckets * sizeof(ipfw_dyn_rule), M_IPFW,
M_WAITOK | M_ZERO);
for (i = 0 ; i < nbuckets; i++)
IPFW_BUCK_LOCK_INIT(&dyn_v[i]);
/*
* Call upper half lock, as get_map() do to ease
* read-only access to dynamic rules hash from sysctl
*/
IPFW_UH_WLOCK(chain);
/*
* Acquire chain write lock to permit hash access
* for main traffic path without additional locks
*/
IPFW_WLOCK(chain);
/* Save old values */
nbuckets_old = V_curr_dyn_buckets;
dyn_v_old = V_ipfw_dyn_v;
/* Skip relinking if array is not set up */
if (V_ipfw_dyn_v == NULL)
V_curr_dyn_buckets = 0;
/* Re-link all dynamic states */
for (i = 0 ; i < V_curr_dyn_buckets ; i++) {
while (V_ipfw_dyn_v[i].head != NULL) {
/* Remove from current chain */
q = V_ipfw_dyn_v[i].head;
V_ipfw_dyn_v[i].head = q->next;
/* Get new hash value */
k = hash_packet(&q->id, nbuckets);
q->bucket = k;
/* Add to the new head */
q->next = dyn_v[k].head;
dyn_v[k].head = q;
}
}
/* Update current pointers/buckets values */
V_curr_dyn_buckets = nbuckets;
V_ipfw_dyn_v = dyn_v;
IPFW_WUNLOCK(chain);
IPFW_UH_WUNLOCK(chain);
/* Start periodic callout on initial creation */
if (dyn_v_old == NULL) {
callout_reset_on(&V_ipfw_timeout, hz, ipfw_dyn_tick, curvnet, 0);
return (0);
}
/* Destroy all mutexes */
for (i = 0 ; i < nbuckets_old ; i++)
IPFW_BUCK_LOCK_DESTROY(&dyn_v_old[i]);
/* Free old hash */
free(dyn_v_old, M_IPFW);
return 0;
}
/**
* Install state of type 'type' for a dynamic session.
* The hash table contains two type of rules:
* - regular rules (O_KEEP_STATE)
* - rules for sessions with limited number of sess per user
* (O_LIMIT). When they are created, the parent is
* increased by 1, and decreased on delete. In this case,
* the third parameter is the parent rule and not the chain.
* - "parent" rules for the above (O_LIMIT_PARENT).
*/
static ipfw_dyn_rule *
add_dyn_rule(struct ipfw_flow_id *id, int i, u_int8_t dyn_type, struct ip_fw *rule)
{
ipfw_dyn_rule *r;
IPFW_BUCK_ASSERT(i);
r = uma_zalloc(V_ipfw_dyn_rule_zone, M_NOWAIT | M_ZERO);
if (r == NULL) {
if (last_log != time_uptime) {
last_log = time_uptime;
log(LOG_DEBUG,
"ipfw: Cannot allocate dynamic state, "
"consider increasing net.inet.ip.fw.dyn_max\n");
}
return NULL;
}
ipfw_dyn_count++;
/*
* refcount on parent is already incremented, so
* it is safe to use parent unlocked.
*/
if (dyn_type == O_LIMIT) {
ipfw_dyn_rule *parent = (ipfw_dyn_rule *)rule;
if ( parent->dyn_type != O_LIMIT_PARENT)
panic("invalid parent");
r->parent = parent;
rule = parent->rule;
}
r->id = *id;
r->expire = time_uptime + V_dyn_syn_lifetime;
r->rule = rule;
r->dyn_type = dyn_type;
IPFW_ZERO_DYN_COUNTER(r);
r->count = 0;
r->bucket = i;
r->next = V_ipfw_dyn_v[i].head;
V_ipfw_dyn_v[i].head = r;
DEB(print_dyn_rule(id, dyn_type, "add dyn entry", "total");)
return r;
}
/**
* lookup dynamic parent rule using pkt and rule as search keys.
* If the lookup fails, then install one.
*/
static ipfw_dyn_rule *
lookup_dyn_parent(struct ipfw_flow_id *pkt, int *pindex, struct ip_fw *rule)
{
ipfw_dyn_rule *q;
int i, is_v6;
is_v6 = IS_IP6_FLOW_ID(pkt);
i = hash_packet( pkt, V_curr_dyn_buckets );
*pindex = i;
IPFW_BUCK_LOCK(i);
for (q = V_ipfw_dyn_v[i].head ; q != NULL ; q=q->next)
if (q->dyn_type == O_LIMIT_PARENT &&
rule== q->rule &&
pkt->proto == q->id.proto &&
pkt->src_port == q->id.src_port &&
pkt->dst_port == q->id.dst_port &&
(
(is_v6 &&
IN6_ARE_ADDR_EQUAL(&(pkt->src_ip6),
&(q->id.src_ip6)) &&
IN6_ARE_ADDR_EQUAL(&(pkt->dst_ip6),
&(q->id.dst_ip6))) ||
(!is_v6 &&
pkt->src_ip == q->id.src_ip &&
pkt->dst_ip == q->id.dst_ip)
)
) {
q->expire = time_uptime + V_dyn_short_lifetime;
DEB(print_dyn_rule(pkt, q->dyn_type,
"lookup_dyn_parent found", "");)
return q;
}
/* Add virtual limiting rule */
return add_dyn_rule(pkt, i, O_LIMIT_PARENT, rule);
}
/**
* Install dynamic state for rule type cmd->o.opcode
*
* Returns 1 (failure) if state is not installed because of errors or because
* session limitations are enforced.
*/
int
ipfw_install_state(struct ip_fw_chain *chain, struct ip_fw *rule,
ipfw_insn_limit *cmd, struct ip_fw_args *args, uint32_t tablearg)
{
ipfw_dyn_rule *q;
int i;
DEB(print_dyn_rule(&args->f_id, cmd->o.opcode, "install_state", "");)
i = hash_packet(&args->f_id, V_curr_dyn_buckets);
IPFW_BUCK_LOCK(i);
q = lookup_dyn_rule_locked(&args->f_id, i, NULL, NULL);
if (q != NULL) { /* should never occur */
DEB(
if (last_log != time_uptime) {
last_log = time_uptime;
printf("ipfw: %s: entry already present, done\n",
__func__);
})
IPFW_BUCK_UNLOCK(i);
return (0);
}
/*
* State limiting is done via uma(9) zone limiting.
* Save pointer to newly-installed rule and reject
* packet if add_dyn_rule() returned NULL.
* Note q is currently set to NULL.
*/
switch (cmd->o.opcode) {
case O_KEEP_STATE: /* bidir rule */
q = add_dyn_rule(&args->f_id, i, O_KEEP_STATE, rule);
break;
case O_LIMIT: { /* limit number of sessions */
struct ipfw_flow_id id;
ipfw_dyn_rule *parent;
uint32_t conn_limit;
uint16_t limit_mask = cmd->limit_mask;
int pindex;
conn_limit = IP_FW_ARG_TABLEARG(chain, cmd->conn_limit, limit);
DEB(
if (cmd->conn_limit == IP_FW_TARG)
printf("ipfw: %s: O_LIMIT rule, conn_limit: %u "
"(tablearg)\n", __func__, conn_limit);
else
printf("ipfw: %s: O_LIMIT rule, conn_limit: %u\n",
__func__, conn_limit);
)
id.dst_ip = id.src_ip = id.dst_port = id.src_port = 0;
id.proto = args->f_id.proto;
id.addr_type = args->f_id.addr_type;
id.fib = M_GETFIB(args->m);
if (IS_IP6_FLOW_ID (&(args->f_id))) {
if (limit_mask & DYN_SRC_ADDR)
id.src_ip6 = args->f_id.src_ip6;
if (limit_mask & DYN_DST_ADDR)
id.dst_ip6 = args->f_id.dst_ip6;
} else {
if (limit_mask & DYN_SRC_ADDR)
id.src_ip = args->f_id.src_ip;
if (limit_mask & DYN_DST_ADDR)
id.dst_ip = args->f_id.dst_ip;
}
if (limit_mask & DYN_SRC_PORT)
id.src_port = args->f_id.src_port;
if (limit_mask & DYN_DST_PORT)
id.dst_port = args->f_id.dst_port;
/*
* We have to release lock for previous bucket to
* avoid possible deadlock
*/
IPFW_BUCK_UNLOCK(i);
if ((parent = lookup_dyn_parent(&id, &pindex, rule)) == NULL) {
printf("ipfw: %s: add parent failed\n", __func__);
IPFW_BUCK_UNLOCK(pindex);
return (1);
}
if (parent->count >= conn_limit) {
if (V_fw_verbose && last_log != time_uptime) {
last_log = time_uptime;
char sbuf[24];
last_log = time_uptime;
snprintf(sbuf, sizeof(sbuf),
"%d drop session",
parent->rule->rulenum);
print_dyn_rule_flags(&args->f_id,
cmd->o.opcode,
LOG_SECURITY | LOG_DEBUG,
sbuf, "too many entries");
}
IPFW_BUCK_UNLOCK(pindex);
return (1);
}
/* Increment counter on parent */
parent->count++;
IPFW_BUCK_UNLOCK(pindex);
IPFW_BUCK_LOCK(i);
q = add_dyn_rule(&args->f_id, i, O_LIMIT, (struct ip_fw *)parent);
if (q == NULL) {
/* Decrement index and notify caller */
IPFW_BUCK_UNLOCK(i);
IPFW_BUCK_LOCK(pindex);
parent->count--;
IPFW_BUCK_UNLOCK(pindex);
return (1);
}
break;
}
default:
printf("ipfw: %s: unknown dynamic rule type %u\n",
__func__, cmd->o.opcode);
}
if (q == NULL) {
IPFW_BUCK_UNLOCK(i);
return (1); /* Notify caller about failure */
}
/* XXX just set lifetime */
lookup_dyn_rule_locked(&args->f_id, i, NULL, NULL);
IPFW_BUCK_UNLOCK(i);
return (0);
}
/*
* Generate a TCP packet, containing either a RST or a keepalive.
* When flags & TH_RST, we are sending a RST packet, because of a
* "reset" action matched the packet.
* Otherwise we are sending a keepalive, and flags & TH_
* The 'replyto' mbuf is the mbuf being replied to, if any, and is required
* so that MAC can label the reply appropriately.
*/
struct mbuf *
ipfw_send_pkt(struct mbuf *replyto, struct ipfw_flow_id *id, u_int32_t seq,
u_int32_t ack, int flags)
{
struct mbuf *m = NULL; /* stupid compiler */
int len, dir;
struct ip *h = NULL; /* stupid compiler */
#ifdef INET6
struct ip6_hdr *h6 = NULL;
#endif
struct tcphdr *th = NULL;
MGETHDR(m, M_NOWAIT, MT_DATA);
if (m == NULL)
return (NULL);
M_SETFIB(m, id->fib);
#ifdef MAC
if (replyto != NULL)
mac_netinet_firewall_reply(replyto, m);
else
mac_netinet_firewall_send(m);
#else
(void)replyto; /* don't warn about unused arg */
#endif
switch (id->addr_type) {
case 4:
len = sizeof(struct ip) + sizeof(struct tcphdr);
break;
#ifdef INET6
case 6:
len = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
break;
#endif
default:
/* XXX: log me?!? */
FREE_PKT(m);
return (NULL);
}
dir = ((flags & (TH_SYN | TH_RST)) == TH_SYN);
m->m_data += max_linkhdr;
m->m_flags |= M_SKIP_FIREWALL;
m->m_pkthdr.len = m->m_len = len;
m->m_pkthdr.rcvif = NULL;
bzero(m->m_data, len);
switch (id->addr_type) {
case 4:
h = mtod(m, struct ip *);
/* prepare for checksum */
h->ip_p = IPPROTO_TCP;
h->ip_len = htons(sizeof(struct tcphdr));
if (dir) {
h->ip_src.s_addr = htonl(id->src_ip);
h->ip_dst.s_addr = htonl(id->dst_ip);
} else {
h->ip_src.s_addr = htonl(id->dst_ip);
h->ip_dst.s_addr = htonl(id->src_ip);
}
th = (struct tcphdr *)(h + 1);
break;
#ifdef INET6
case 6:
h6 = mtod(m, struct ip6_hdr *);
/* prepare for checksum */
h6->ip6_nxt = IPPROTO_TCP;
h6->ip6_plen = htons(sizeof(struct tcphdr));
if (dir) {
h6->ip6_src = id->src_ip6;
h6->ip6_dst = id->dst_ip6;
} else {
h6->ip6_src = id->dst_ip6;
h6->ip6_dst = id->src_ip6;
}
th = (struct tcphdr *)(h6 + 1);
break;
#endif
}
if (dir) {
th->th_sport = htons(id->src_port);
th->th_dport = htons(id->dst_port);
} else {
th->th_sport = htons(id->dst_port);
th->th_dport = htons(id->src_port);
}
th->th_off = sizeof(struct tcphdr) >> 2;
if (flags & TH_RST) {
if (flags & TH_ACK) {
th->th_seq = htonl(ack);
th->th_flags = TH_RST;
} else {
if (flags & TH_SYN)
seq++;
th->th_ack = htonl(seq);
th->th_flags = TH_RST | TH_ACK;
}
} else {
/*
* Keepalive - use caller provided sequence numbers
*/
th->th_seq = htonl(seq);
th->th_ack = htonl(ack);
th->th_flags = TH_ACK;
}
switch (id->addr_type) {
case 4:
th->th_sum = in_cksum(m, len);
/* finish the ip header */
h->ip_v = 4;
h->ip_hl = sizeof(*h) >> 2;
h->ip_tos = IPTOS_LOWDELAY;
h->ip_off = htons(0);
h->ip_len = htons(len);
h->ip_ttl = V_ip_defttl;
h->ip_sum = 0;
break;
#ifdef INET6
case 6:
th->th_sum = in6_cksum(m, IPPROTO_TCP, sizeof(*h6),
sizeof(struct tcphdr));
/* finish the ip6 header */
h6->ip6_vfc |= IPV6_VERSION;
h6->ip6_hlim = IPV6_DEFHLIM;
break;
#endif
}
return (m);
}
/*
* Queue keepalive packets for given dynamic rule
*/
static struct mbuf **
ipfw_dyn_send_ka(struct mbuf **mtailp, ipfw_dyn_rule *q)
{
struct mbuf *m_rev, *m_fwd;
m_rev = (q->state & ACK_REV) ? NULL :
ipfw_send_pkt(NULL, &(q->id), q->ack_rev - 1, q->ack_fwd, TH_SYN);
m_fwd = (q->state & ACK_FWD) ? NULL :
ipfw_send_pkt(NULL, &(q->id), q->ack_fwd - 1, q->ack_rev, 0);
if (m_rev != NULL) {
*mtailp = m_rev;
mtailp = &(*mtailp)->m_nextpkt;
}
if (m_fwd != NULL) {
*mtailp = m_fwd;
mtailp = &(*mtailp)->m_nextpkt;
}
return (mtailp);
}
/*
* This procedure is used to perform various maintance
* on dynamic hash list. Currently it is called every second.
*/
static void
ipfw_dyn_tick(void * vnetx)
{
struct ip_fw_chain *chain;
int check_ka = 0;
#ifdef VIMAGE
struct vnet *vp = vnetx;
#endif
CURVNET_SET(vp);
chain = &V_layer3_chain;
/* Run keepalive checks every keepalive_period iff ka is enabled */
if ((V_dyn_keepalive_last + V_dyn_keepalive_period <= time_uptime) &&
(V_dyn_keepalive != 0)) {
V_dyn_keepalive_last = time_uptime;
check_ka = 1;
}
check_dyn_rules(chain, NULL, check_ka, 1);
callout_reset_on(&V_ipfw_timeout, hz, ipfw_dyn_tick, vnetx, 0);
CURVNET_RESTORE();
}
/*
* Walk thru all dynamic states doing generic maintance:
* 1) free expired states
* 2) free all states based on deleted rule / set
* 3) send keepalives for states if needed
*
* @chain - pointer to current ipfw rules chain
* @rule - delete all states originated by given rule if != NULL
* @set - delete all states originated by any rule in set @set if != RESVD_SET
* @check_ka - perform checking/sending keepalives
* @timer - indicate call from timer routine.
*
* Timer routine must call this function unlocked to permit
* sending keepalives/resizing table.
*
* Others has to call function with IPFW_UH_WLOCK held.
* Additionally, function assume that dynamic rule/set is
* ALREADY deleted so no new states can be generated by
* 'deleted' rules.
*
* Write lock is needed to ensure that unused parent rules
* are not freed by other instance (see stage 2, 3)
*/
static void
check_dyn_rules(struct ip_fw_chain *chain, ipfw_range_tlv *rt,
int check_ka, int timer)
{
struct mbuf *m0, *m, *mnext, **mtailp;
struct ip *h;
int i, dyn_count, new_buckets = 0, max_buckets;
int expired = 0, expired_limits = 0, parents = 0, total = 0;
ipfw_dyn_rule *q, *q_prev, *q_next;
ipfw_dyn_rule *exp_head, **exptailp;
ipfw_dyn_rule *exp_lhead, **expltailp;
KASSERT(V_ipfw_dyn_v != NULL, ("%s: dynamic table not allocated",
__func__));
/* Avoid possible LOR */
KASSERT(!check_ka || timer, ("%s: keepalive check with lock held",
__func__));
/*
* Do not perform any checks if we currently have no dynamic states
*/
if (DYN_COUNT == 0)
return;
/* Expired states */
exp_head = NULL;
exptailp = &exp_head;
/* Expired limit states */
exp_lhead = NULL;
expltailp = &exp_lhead;
/*
* We make a chain of packets to go out here -- not deferring
* until after we drop the IPFW dynamic rule lock would result
* in a lock order reversal with the normal packet input -> ipfw
* call stack.
*/
m0 = NULL;
mtailp = &m0;
/* Protect from hash resizing */
if (timer != 0)
IPFW_UH_WLOCK(chain);
else
IPFW_UH_WLOCK_ASSERT(chain);
#define NEXT_RULE() { q_prev = q; q = q->next ; continue; }
/* Stage 1: perform requested deletion */
for (i = 0 ; i < V_curr_dyn_buckets ; i++) {
IPFW_BUCK_LOCK(i);
for (q = V_ipfw_dyn_v[i].head, q_prev = q; q ; ) {
/* account every rule */
total++;
/* Skip parent rules at all */
if (q->dyn_type == O_LIMIT_PARENT) {
parents++;
NEXT_RULE();
}
/*
* Remove rules which are:
* 1) expired
* 2) matches deletion range
*/
if ((TIME_LEQ(q->expire, time_uptime)) ||
(rt != NULL && ipfw_match_range(q->rule, rt))) {
if (TIME_LE(time_uptime, q->expire) &&
q->dyn_type == O_KEEP_STATE &&
V_dyn_keep_states != 0) {
/*
* Do not delete state if
* it is not expired and
* dyn_keep_states is ON.
* However we need to re-link it
* to any other stable rule
*/
q->rule = chain->default_rule;
NEXT_RULE();
}
/* Unlink q from current list */
q_next = q->next;
if (q == V_ipfw_dyn_v[i].head)
V_ipfw_dyn_v[i].head = q_next;
else
q_prev->next = q_next;
q->next = NULL;
/* queue q to expire list */
if (q->dyn_type != O_LIMIT) {
*exptailp = q;
exptailp = &(*exptailp)->next;
DEB(print_dyn_rule(&q->id, q->dyn_type,
"unlink entry", "left");
)
} else {
/* Separate list for limit rules */
*expltailp = q;
expltailp = &(*expltailp)->next;
expired_limits++;
DEB(print_dyn_rule(&q->id, q->dyn_type,
"unlink limit entry", "left");
)
}
q = q_next;
expired++;
continue;
}
/*
* Check if we need to send keepalive:
* we need to ensure if is time to do KA,
* this is established TCP session, and
* expire time is within keepalive interval
*/
if ((check_ka != 0) && (q->id.proto == IPPROTO_TCP) &&
((q->state & BOTH_SYN) == BOTH_SYN) &&
(TIME_LEQ(q->expire, time_uptime +
V_dyn_keepalive_interval)))
mtailp = ipfw_dyn_send_ka(mtailp, q);
NEXT_RULE();
}
IPFW_BUCK_UNLOCK(i);
}
/* Stage 2: decrement counters from O_LIMIT parents */
if (expired_limits != 0) {
/*
* XXX: Note that deleting set with more than one
* heavily-used LIMIT rules can result in overwhelming
* locking due to lack of per-hash value sorting
*
* We should probably think about:
* 1) pre-allocating hash of size, say,
* MAX(16, V_curr_dyn_buckets / 1024)
* 2) checking if expired_limits is large enough
* 3) If yes, init hash (or its part), re-link
* current list and start decrementing procedure in
* each bucket separately
*/
/*
* Small optimization: do not unlock bucket until
* we see the next item resides in different bucket
*/
if (exp_lhead != NULL) {
i = exp_lhead->parent->bucket;
IPFW_BUCK_LOCK(i);
}
for (q = exp_lhead; q != NULL; q = q->next) {
if (i != q->parent->bucket) {
IPFW_BUCK_UNLOCK(i);
i = q->parent->bucket;
IPFW_BUCK_LOCK(i);
}
/* Decrease parent refcount */
q->parent->count--;
}
if (exp_lhead != NULL)
IPFW_BUCK_UNLOCK(i);
}
/*
* We protectet ourselves from unused parent deletion
* (from the timer function) by holding UH write lock.
*/
/* Stage 3: remove unused parent rules */
if ((parents != 0) && (expired != 0)) {
for (i = 0 ; i < V_curr_dyn_buckets ; i++) {
IPFW_BUCK_LOCK(i);
for (q = V_ipfw_dyn_v[i].head, q_prev = q ; q ; ) {
if (q->dyn_type != O_LIMIT_PARENT)
NEXT_RULE();
if (q->count != 0)
NEXT_RULE();
/* Parent rule without consumers */
/* Unlink q from current list */
q_next = q->next;
if (q == V_ipfw_dyn_v[i].head)
V_ipfw_dyn_v[i].head = q_next;
else
q_prev->next = q_next;
q->next = NULL;
/* Add to expired list */
*exptailp = q;
exptailp = &(*exptailp)->next;
DEB(print_dyn_rule(&q->id, q->dyn_type,
"unlink parent entry", "left");
)
expired++;
q = q_next;
}
IPFW_BUCK_UNLOCK(i);
}
}
#undef NEXT_RULE
if (timer != 0) {
/*
* Check if we need to resize hash:
* if current number of states exceeds number of buckes in hash,
* grow hash size to the minimum power of 2 which is bigger than
* current states count. Limit hash size by 64k.
*/
max_buckets = (V_dyn_buckets_max > 65536) ?
65536 : V_dyn_buckets_max;
dyn_count = DYN_COUNT;
if ((dyn_count > V_curr_dyn_buckets * 2) &&
(dyn_count < max_buckets)) {
new_buckets = V_curr_dyn_buckets;
while (new_buckets < dyn_count) {
new_buckets *= 2;
if (new_buckets >= max_buckets)
break;
}
}
IPFW_UH_WUNLOCK(chain);
}
/* Finally delete old states ad limits if any */
for (q = exp_head; q != NULL; q = q_next) {
q_next = q->next;
uma_zfree(V_ipfw_dyn_rule_zone, q);
ipfw_dyn_count--;
}
for (q = exp_lhead; q != NULL; q = q_next) {
q_next = q->next;
uma_zfree(V_ipfw_dyn_rule_zone, q);
ipfw_dyn_count--;
}
/*
* The rest code MUST be called from timer routine only
* without holding any locks
*/
if (timer == 0)
return;
/* Send keepalive packets if any */
for (m = m0; m != NULL; m = mnext) {
mnext = m->m_nextpkt;
m->m_nextpkt = NULL;
h = mtod(m, struct ip *);
if (h->ip_v == 4)
ip_output(m, NULL, NULL, 0, NULL, NULL);
#ifdef INET6
else
ip6_output(m, NULL, NULL, 0, NULL, NULL, NULL);
#endif
}
/* Run table resize without holding any locks */
if (new_buckets != 0)
resize_dynamic_table(chain, new_buckets);
}
/*
* Deletes all dynamic rules originated by given rule or all rules in
* given set. Specify RESVD_SET to indicate set should not be used.
* @chain - pointer to current ipfw rules chain
* @rr - delete all states originated by rules in matched range.
*
* Function has to be called with IPFW_UH_WLOCK held.
* Additionally, function assume that dynamic rule/set is
* ALREADY deleted so no new states can be generated by
* 'deleted' rules.
*/
void
ipfw_expire_dyn_rules(struct ip_fw_chain *chain, ipfw_range_tlv *rt)
{
check_dyn_rules(chain, rt, 0, 0);
}
/*
* Check if rule contains at least one dynamic opcode.
*
* Returns 1 if such opcode is found, 0 otherwise.
*/
int
ipfw_is_dyn_rule(struct ip_fw *rule)
{
int cmdlen, l;
ipfw_insn *cmd;
l = rule->cmd_len;
cmd = rule->cmd;
cmdlen = 0;
for ( ; l > 0 ; l -= cmdlen, cmd += cmdlen) {
cmdlen = F_LEN(cmd);
switch (cmd->opcode) {
case O_LIMIT:
case O_KEEP_STATE:
case O_PROBE_STATE:
case O_CHECK_STATE:
return (1);
}
}
return (0);
}
void
ipfw_dyn_init(struct ip_fw_chain *chain)
{
V_ipfw_dyn_v = NULL;
V_dyn_buckets_max = 256; /* must be power of 2 */
V_curr_dyn_buckets = 256; /* must be power of 2 */
V_dyn_ack_lifetime = 300;
V_dyn_syn_lifetime = 20;
V_dyn_fin_lifetime = 1;
V_dyn_rst_lifetime = 1;
V_dyn_udp_lifetime = 10;
V_dyn_short_lifetime = 5;
V_dyn_keepalive_interval = 20;
V_dyn_keepalive_period = 5;
V_dyn_keepalive = 1; /* do send keepalives */
V_dyn_keepalive_last = time_uptime;
V_dyn_max = 16384; /* max # of dynamic rules */
V_ipfw_dyn_rule_zone = uma_zcreate("IPFW dynamic rule",
sizeof(ipfw_dyn_rule), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0);
/* Enforce limit on dynamic rules */
uma_zone_set_max(V_ipfw_dyn_rule_zone, V_dyn_max);
- callout_init(&V_ipfw_timeout, CALLOUT_MPSAFE);
+ callout_init(&V_ipfw_timeout, 1);
/*
* This can potentially be done on first dynamic rule
* being added to chain.
*/
resize_dynamic_table(chain, V_curr_dyn_buckets);
}
void
ipfw_dyn_uninit(int pass)
{
int i;
if (pass == 0) {
callout_drain(&V_ipfw_timeout);
return;
}
if (V_ipfw_dyn_v != NULL) {
/*
* Skip deleting all dynamic states -
* uma_zdestroy() does this more efficiently;
*/
/* Destroy all mutexes */
for (i = 0 ; i < V_curr_dyn_buckets ; i++)
IPFW_BUCK_LOCK_DESTROY(&V_ipfw_dyn_v[i]);
free(V_ipfw_dyn_v, M_IPFW);
V_ipfw_dyn_v = NULL;
}
uma_zdestroy(V_ipfw_dyn_rule_zone);
}
#ifdef SYSCTL_NODE
/*
* Get/set maximum number of dynamic states in given VNET instance.
*/
static int
sysctl_ipfw_dyn_max(SYSCTL_HANDLER_ARGS)
{
int error;
unsigned int nstates;
nstates = V_dyn_max;
error = sysctl_handle_int(oidp, &nstates, 0, req);
/* Read operation or some error */
if ((error != 0) || (req->newptr == NULL))
return (error);
V_dyn_max = nstates;
uma_zone_set_max(V_ipfw_dyn_rule_zone, V_dyn_max);
return (0);
}
/*
* Get current number of dynamic states in given VNET instance.
*/
static int
sysctl_ipfw_dyn_count(SYSCTL_HANDLER_ARGS)
{
int error;
unsigned int nstates;
nstates = DYN_COUNT;
error = sysctl_handle_int(oidp, &nstates, 0, req);
return (error);
}
#endif
/*
* Returns size of dynamic states in legacy format
*/
int
ipfw_dyn_len(void)
{
return (V_ipfw_dyn_v == NULL) ? 0 :
(DYN_COUNT * sizeof(ipfw_dyn_rule));
}
/*
* Returns number of dynamic states.
* Used by dump format v1 (current).
*/
int
ipfw_dyn_get_count(void)
{
return (V_ipfw_dyn_v == NULL) ? 0 : DYN_COUNT;
}
static void
export_dyn_rule(ipfw_dyn_rule *src, ipfw_dyn_rule *dst)
{
memcpy(dst, src, sizeof(*src));
memcpy(&(dst->rule), &(src->rule->rulenum), sizeof(src->rule->rulenum));
/*
* store set number into high word of
* dst->rule pointer.
*/
memcpy((char *)&dst->rule + sizeof(src->rule->rulenum),
&(src->rule->set), sizeof(src->rule->set));
/*
* store a non-null value in "next".
* The userland code will interpret a
* NULL here as a marker
* for the last dynamic rule.
*/
memcpy(&dst->next, &dst, sizeof(dst));
dst->expire =
TIME_LEQ(dst->expire, time_uptime) ? 0 : dst->expire - time_uptime;
}
/*
* Fills int buffer given by @sd with dynamic states.
* Used by dump format v1 (current).
*
* Returns 0 on success.
*/
int
ipfw_dump_states(struct ip_fw_chain *chain, struct sockopt_data *sd)
{
ipfw_dyn_rule *p;
ipfw_obj_dyntlv *dst, *last;
ipfw_obj_ctlv *ctlv;
int i;
size_t sz;
if (V_ipfw_dyn_v == NULL)
return (0);
IPFW_UH_RLOCK_ASSERT(chain);
ctlv = (ipfw_obj_ctlv *)ipfw_get_sopt_space(sd, sizeof(*ctlv));
if (ctlv == NULL)
return (ENOMEM);
sz = sizeof(ipfw_obj_dyntlv);
ctlv->head.type = IPFW_TLV_DYNSTATE_LIST;
ctlv->objsize = sz;
last = NULL;
for (i = 0 ; i < V_curr_dyn_buckets; i++) {
IPFW_BUCK_LOCK(i);
for (p = V_ipfw_dyn_v[i].head ; p != NULL; p = p->next) {
dst = (ipfw_obj_dyntlv *)ipfw_get_sopt_space(sd, sz);
if (dst == NULL) {
IPFW_BUCK_UNLOCK(i);
return (ENOMEM);
}
export_dyn_rule(p, &dst->state);
dst->head.length = sz;
dst->head.type = IPFW_TLV_DYN_ENT;
last = dst;
}
IPFW_BUCK_UNLOCK(i);
}
if (last != NULL) /* mark last dynamic rule */
last->head.flags = IPFW_DF_LAST;
return (0);
}
/*
* Fill given buffer with dynamic states (legacy format).
* IPFW_UH_RLOCK has to be held while calling.
*/
void
ipfw_get_dynamic(struct ip_fw_chain *chain, char **pbp, const char *ep)
{
ipfw_dyn_rule *p, *last = NULL;
char *bp;
int i;
if (V_ipfw_dyn_v == NULL)
return;
bp = *pbp;
IPFW_UH_RLOCK_ASSERT(chain);
for (i = 0 ; i < V_curr_dyn_buckets; i++) {
IPFW_BUCK_LOCK(i);
for (p = V_ipfw_dyn_v[i].head ; p != NULL; p = p->next) {
if (bp + sizeof *p <= ep) {
ipfw_dyn_rule *dst =
(ipfw_dyn_rule *)bp;
export_dyn_rule(p, dst);
last = dst;
bp += sizeof(ipfw_dyn_rule);
}
}
IPFW_BUCK_UNLOCK(i);
}
if (last != NULL) /* mark last dynamic rule */
bzero(&last->next, sizeof(last));
*pbp = bp;
}
/* end of file */
Index: head/sys/netpfil/pf/if_pfsync.c
===================================================================
--- head/sys/netpfil/pf/if_pfsync.c (revision 283290)
+++ head/sys/netpfil/pf/if_pfsync.c (revision 283291)
@@ -1,2421 +1,2421 @@
/*-
* Copyright (c) 2002 Michael Shalayeff
* Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR OR HIS RELATIVES BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF MIND, USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
/*-
* Copyright (c) 2009 David Gwynne <dlg@openbsd.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
* $OpenBSD: if_pfsync.c,v 1.110 2009/02/24 05:39:19 dlg Exp $
*
* Revisions picked from OpenBSD after revision 1.110 import:
* 1.119 - don't m_copydata() beyond the len of mbuf in pfsync_input()
* 1.118, 1.124, 1.148, 1.149, 1.151, 1.171 - fixes to bulk updates
* 1.120, 1.175 - use monotonic time_uptime
* 1.122 - reduce number of updates for non-TCP sessions
* 1.125, 1.127 - rewrite merge or stale processing
* 1.128 - cleanups
* 1.146 - bzero() mbuf before sparsely filling it with data
* 1.170 - SIOCSIFMTU checks
* 1.126, 1.142 - deferred packets processing
* 1.173 - correct expire time processing
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_pf.h"
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/endian.h>
#include <sys/interrupt.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/priv.h>
#include <sys/protosw.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <net/bpf.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_clone.h>
#include <net/if_types.h>
#include <net/vnet.h>
#include <net/pfvar.h>
#include <net/if_pfsync.h>
#include <netinet/if_ether.h>
#include <netinet/in.h>
#include <netinet/in_var.h>
#include <netinet/ip.h>
#include <netinet/ip_carp.h>
#include <netinet/ip_var.h>
#include <netinet/tcp.h>
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_seq.h>
#define PFSYNC_MINPKT ( \
sizeof(struct ip) + \
sizeof(struct pfsync_header) + \
sizeof(struct pfsync_subheader) )
struct pfsync_pkt {
struct ip *ip;
struct in_addr src;
u_int8_t flags;
};
static int pfsync_upd_tcp(struct pf_state *, struct pfsync_state_peer *,
struct pfsync_state_peer *);
static int pfsync_in_clr(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_ins(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_iack(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_upd(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_upd_c(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_ureq(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_del(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_del_c(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_bus(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_tdb(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_eof(struct pfsync_pkt *, struct mbuf *, int, int);
static int pfsync_in_error(struct pfsync_pkt *, struct mbuf *, int, int);
static int (*pfsync_acts[])(struct pfsync_pkt *, struct mbuf *, int, int) = {
pfsync_in_clr, /* PFSYNC_ACT_CLR */
pfsync_in_ins, /* PFSYNC_ACT_INS */
pfsync_in_iack, /* PFSYNC_ACT_INS_ACK */
pfsync_in_upd, /* PFSYNC_ACT_UPD */
pfsync_in_upd_c, /* PFSYNC_ACT_UPD_C */
pfsync_in_ureq, /* PFSYNC_ACT_UPD_REQ */
pfsync_in_del, /* PFSYNC_ACT_DEL */
pfsync_in_del_c, /* PFSYNC_ACT_DEL_C */
pfsync_in_error, /* PFSYNC_ACT_INS_F */
pfsync_in_error, /* PFSYNC_ACT_DEL_F */
pfsync_in_bus, /* PFSYNC_ACT_BUS */
pfsync_in_tdb, /* PFSYNC_ACT_TDB */
pfsync_in_eof /* PFSYNC_ACT_EOF */
};
struct pfsync_q {
void (*write)(struct pf_state *, void *);
size_t len;
u_int8_t action;
};
/* we have one of these for every PFSYNC_S_ */
static void pfsync_out_state(struct pf_state *, void *);
static void pfsync_out_iack(struct pf_state *, void *);
static void pfsync_out_upd_c(struct pf_state *, void *);
static void pfsync_out_del(struct pf_state *, void *);
static struct pfsync_q pfsync_qs[] = {
{ pfsync_out_state, sizeof(struct pfsync_state), PFSYNC_ACT_INS },
{ pfsync_out_iack, sizeof(struct pfsync_ins_ack), PFSYNC_ACT_INS_ACK },
{ pfsync_out_state, sizeof(struct pfsync_state), PFSYNC_ACT_UPD },
{ pfsync_out_upd_c, sizeof(struct pfsync_upd_c), PFSYNC_ACT_UPD_C },
{ pfsync_out_del, sizeof(struct pfsync_del_c), PFSYNC_ACT_DEL_C }
};
static void pfsync_q_ins(struct pf_state *, int);
static void pfsync_q_del(struct pf_state *);
static void pfsync_update_state(struct pf_state *);
struct pfsync_upd_req_item {
TAILQ_ENTRY(pfsync_upd_req_item) ur_entry;
struct pfsync_upd_req ur_msg;
};
struct pfsync_deferral {
struct pfsync_softc *pd_sc;
TAILQ_ENTRY(pfsync_deferral) pd_entry;
u_int pd_refs;
struct callout pd_tmo;
struct pf_state *pd_st;
struct mbuf *pd_m;
};
struct pfsync_softc {
/* Configuration */
struct ifnet *sc_ifp;
struct ifnet *sc_sync_if;
struct ip_moptions sc_imo;
struct in_addr sc_sync_peer;
uint32_t sc_flags;
#define PFSYNCF_OK 0x00000001
#define PFSYNCF_DEFER 0x00000002
#define PFSYNCF_PUSH 0x00000004
uint8_t sc_maxupdates;
struct ip sc_template;
struct callout sc_tmo;
struct mtx sc_mtx;
/* Queued data */
size_t sc_len;
TAILQ_HEAD(, pf_state) sc_qs[PFSYNC_S_COUNT];
TAILQ_HEAD(, pfsync_upd_req_item) sc_upd_req_list;
TAILQ_HEAD(, pfsync_deferral) sc_deferrals;
u_int sc_deferred;
void *sc_plus;
size_t sc_pluslen;
/* Bulk update info */
struct mtx sc_bulk_mtx;
uint32_t sc_ureq_sent;
int sc_bulk_tries;
uint32_t sc_ureq_received;
int sc_bulk_hashid;
uint64_t sc_bulk_stateid;
uint32_t sc_bulk_creatorid;
struct callout sc_bulk_tmo;
struct callout sc_bulkfail_tmo;
};
#define PFSYNC_LOCK(sc) mtx_lock(&(sc)->sc_mtx)
#define PFSYNC_UNLOCK(sc) mtx_unlock(&(sc)->sc_mtx)
#define PFSYNC_LOCK_ASSERT(sc) mtx_assert(&(sc)->sc_mtx, MA_OWNED)
#define PFSYNC_BLOCK(sc) mtx_lock(&(sc)->sc_bulk_mtx)
#define PFSYNC_BUNLOCK(sc) mtx_unlock(&(sc)->sc_bulk_mtx)
#define PFSYNC_BLOCK_ASSERT(sc) mtx_assert(&(sc)->sc_bulk_mtx, MA_OWNED)
static const char pfsyncname[] = "pfsync";
static MALLOC_DEFINE(M_PFSYNC, pfsyncname, "pfsync(4) data");
static VNET_DEFINE(struct pfsync_softc *, pfsyncif) = NULL;
#define V_pfsyncif VNET(pfsyncif)
static VNET_DEFINE(void *, pfsync_swi_cookie) = NULL;
#define V_pfsync_swi_cookie VNET(pfsync_swi_cookie)
static VNET_DEFINE(struct pfsyncstats, pfsyncstats);
#define V_pfsyncstats VNET(pfsyncstats)
static VNET_DEFINE(int, pfsync_carp_adj) = CARP_MAXSKEW;
#define V_pfsync_carp_adj VNET(pfsync_carp_adj)
static void pfsync_timeout(void *);
static void pfsync_push(struct pfsync_softc *);
static void pfsyncintr(void *);
static int pfsync_multicast_setup(struct pfsync_softc *, struct ifnet *,
void *);
static void pfsync_multicast_cleanup(struct pfsync_softc *);
static void pfsync_pointers_init(void);
static void pfsync_pointers_uninit(void);
static int pfsync_init(void);
static void pfsync_uninit(void);
SYSCTL_NODE(_net, OID_AUTO, pfsync, CTLFLAG_RW, 0, "PFSYNC");
SYSCTL_STRUCT(_net_pfsync, OID_AUTO, stats, CTLFLAG_VNET | CTLFLAG_RW,
&VNET_NAME(pfsyncstats), pfsyncstats,
"PFSYNC statistics (struct pfsyncstats, net/if_pfsync.h)");
SYSCTL_INT(_net_pfsync, OID_AUTO, carp_demotion_factor, CTLFLAG_RW,
&VNET_NAME(pfsync_carp_adj), 0, "pfsync's CARP demotion factor adjustment");
static int pfsync_clone_create(struct if_clone *, int, caddr_t);
static void pfsync_clone_destroy(struct ifnet *);
static int pfsync_alloc_scrub_memory(struct pfsync_state_peer *,
struct pf_state_peer *);
static int pfsyncoutput(struct ifnet *, struct mbuf *,
const struct sockaddr *, struct route *);
static int pfsyncioctl(struct ifnet *, u_long, caddr_t);
static int pfsync_defer(struct pf_state *, struct mbuf *);
static void pfsync_undefer(struct pfsync_deferral *, int);
static void pfsync_undefer_state(struct pf_state *, int);
static void pfsync_defer_tmo(void *);
static void pfsync_request_update(u_int32_t, u_int64_t);
static void pfsync_update_state_req(struct pf_state *);
static void pfsync_drop(struct pfsync_softc *);
static void pfsync_sendout(int);
static void pfsync_send_plus(void *, size_t);
static void pfsync_bulk_start(void);
static void pfsync_bulk_status(u_int8_t);
static void pfsync_bulk_update(void *);
static void pfsync_bulk_fail(void *);
#ifdef IPSEC
static void pfsync_update_net_tdb(struct pfsync_tdb *);
#endif
#define PFSYNC_MAX_BULKTRIES 12
VNET_DEFINE(struct if_clone *, pfsync_cloner);
#define V_pfsync_cloner VNET(pfsync_cloner)
static int
pfsync_clone_create(struct if_clone *ifc, int unit, caddr_t param)
{
struct pfsync_softc *sc;
struct ifnet *ifp;
int q;
if (unit != 0)
return (EINVAL);
sc = malloc(sizeof(struct pfsync_softc), M_PFSYNC, M_WAITOK | M_ZERO);
sc->sc_flags |= PFSYNCF_OK;
for (q = 0; q < PFSYNC_S_COUNT; q++)
TAILQ_INIT(&sc->sc_qs[q]);
TAILQ_INIT(&sc->sc_upd_req_list);
TAILQ_INIT(&sc->sc_deferrals);
sc->sc_len = PFSYNC_MINPKT;
sc->sc_maxupdates = 128;
ifp = sc->sc_ifp = if_alloc(IFT_PFSYNC);
if (ifp == NULL) {
free(sc, M_PFSYNC);
return (ENOSPC);
}
if_initname(ifp, pfsyncname, unit);
ifp->if_softc = sc;
ifp->if_ioctl = pfsyncioctl;
ifp->if_output = pfsyncoutput;
ifp->if_type = IFT_PFSYNC;
ifp->if_snd.ifq_maxlen = ifqmaxlen;
ifp->if_hdrlen = sizeof(struct pfsync_header);
ifp->if_mtu = ETHERMTU;
mtx_init(&sc->sc_mtx, pfsyncname, NULL, MTX_DEF);
mtx_init(&sc->sc_bulk_mtx, "pfsync bulk", NULL, MTX_DEF);
- callout_init(&sc->sc_tmo, CALLOUT_MPSAFE);
+ callout_init(&sc->sc_tmo, 1);
callout_init_mtx(&sc->sc_bulk_tmo, &sc->sc_bulk_mtx, 0);
callout_init_mtx(&sc->sc_bulkfail_tmo, &sc->sc_bulk_mtx, 0);
if_attach(ifp);
bpfattach(ifp, DLT_PFSYNC, PFSYNC_HDRLEN);
V_pfsyncif = sc;
return (0);
}
static void
pfsync_clone_destroy(struct ifnet *ifp)
{
struct pfsync_softc *sc = ifp->if_softc;
/*
* At this stage, everything should have already been
* cleared by pfsync_uninit(), and we have only to
* drain callouts.
*/
while (sc->sc_deferred > 0) {
struct pfsync_deferral *pd = TAILQ_FIRST(&sc->sc_deferrals);
TAILQ_REMOVE(&sc->sc_deferrals, pd, pd_entry);
sc->sc_deferred--;
if (callout_stop(&pd->pd_tmo)) {
pf_release_state(pd->pd_st);
m_freem(pd->pd_m);
free(pd, M_PFSYNC);
} else {
pd->pd_refs++;
callout_drain(&pd->pd_tmo);
free(pd, M_PFSYNC);
}
}
callout_drain(&sc->sc_tmo);
callout_drain(&sc->sc_bulkfail_tmo);
callout_drain(&sc->sc_bulk_tmo);
if (!(sc->sc_flags & PFSYNCF_OK) && carp_demote_adj_p)
(*carp_demote_adj_p)(-V_pfsync_carp_adj, "pfsync destroy");
bpfdetach(ifp);
if_detach(ifp);
pfsync_drop(sc);
if_free(ifp);
if (sc->sc_imo.imo_membership)
pfsync_multicast_cleanup(sc);
mtx_destroy(&sc->sc_mtx);
mtx_destroy(&sc->sc_bulk_mtx);
free(sc, M_PFSYNC);
V_pfsyncif = NULL;
}
static int
pfsync_alloc_scrub_memory(struct pfsync_state_peer *s,
struct pf_state_peer *d)
{
if (s->scrub.scrub_flag && d->scrub == NULL) {
d->scrub = uma_zalloc(V_pf_state_scrub_z, M_NOWAIT | M_ZERO);
if (d->scrub == NULL)
return (ENOMEM);
}
return (0);
}
static int
pfsync_state_import(struct pfsync_state *sp, u_int8_t flags)
{
struct pfsync_softc *sc = V_pfsyncif;
#ifndef __NO_STRICT_ALIGNMENT
struct pfsync_state_key key[2];
#endif
struct pfsync_state_key *kw, *ks;
struct pf_state *st = NULL;
struct pf_state_key *skw = NULL, *sks = NULL;
struct pf_rule *r = NULL;
struct pfi_kif *kif;
int error;
PF_RULES_RASSERT();
if (sp->creatorid == 0) {
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("%s: invalid creator id: %08x\n", __func__,
ntohl(sp->creatorid));
return (EINVAL);
}
if ((kif = pfi_kif_find(sp->ifname)) == NULL) {
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("%s: unknown interface: %s\n", __func__,
sp->ifname);
if (flags & PFSYNC_SI_IOCTL)
return (EINVAL);
return (0); /* skip this state */
}
/*
* If the ruleset checksums match or the state is coming from the ioctl,
* it's safe to associate the state with the rule of that number.
*/
if (sp->rule != htonl(-1) && sp->anchor == htonl(-1) &&
(flags & (PFSYNC_SI_IOCTL | PFSYNC_SI_CKSUM)) && ntohl(sp->rule) <
pf_main_ruleset.rules[PF_RULESET_FILTER].active.rcount)
r = pf_main_ruleset.rules[
PF_RULESET_FILTER].active.ptr_array[ntohl(sp->rule)];
else
r = &V_pf_default_rule;
if ((r->max_states &&
counter_u64_fetch(r->states_cur) >= r->max_states))
goto cleanup;
/*
* XXXGL: consider M_WAITOK in ioctl path after.
*/
if ((st = uma_zalloc(V_pf_state_z, M_NOWAIT | M_ZERO)) == NULL)
goto cleanup;
if ((skw = uma_zalloc(V_pf_state_key_z, M_NOWAIT)) == NULL)
goto cleanup;
#ifndef __NO_STRICT_ALIGNMENT
bcopy(&sp->key, key, sizeof(struct pfsync_state_key) * 2);
kw = &key[PF_SK_WIRE];
ks = &key[PF_SK_STACK];
#else
kw = &sp->key[PF_SK_WIRE];
ks = &sp->key[PF_SK_STACK];
#endif
if (PF_ANEQ(&kw->addr[0], &ks->addr[0], sp->af) ||
PF_ANEQ(&kw->addr[1], &ks->addr[1], sp->af) ||
kw->port[0] != ks->port[0] ||
kw->port[1] != ks->port[1]) {
sks = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
if (sks == NULL)
goto cleanup;
} else
sks = skw;
/* allocate memory for scrub info */
if (pfsync_alloc_scrub_memory(&sp->src, &st->src) ||
pfsync_alloc_scrub_memory(&sp->dst, &st->dst))
goto cleanup;
/* Copy to state key(s). */
skw->addr[0] = kw->addr[0];
skw->addr[1] = kw->addr[1];
skw->port[0] = kw->port[0];
skw->port[1] = kw->port[1];
skw->proto = sp->proto;
skw->af = sp->af;
if (sks != skw) {
sks->addr[0] = ks->addr[0];
sks->addr[1] = ks->addr[1];
sks->port[0] = ks->port[0];
sks->port[1] = ks->port[1];
sks->proto = sp->proto;
sks->af = sp->af;
}
/* copy to state */
bcopy(&sp->rt_addr, &st->rt_addr, sizeof(st->rt_addr));
st->creation = time_uptime - ntohl(sp->creation);
st->expire = time_uptime;
if (sp->expire) {
uint32_t timeout;
timeout = r->timeout[sp->timeout];
if (!timeout)
timeout = V_pf_default_rule.timeout[sp->timeout];
/* sp->expire may have been adaptively scaled by export. */
st->expire -= timeout - ntohl(sp->expire);
}
st->direction = sp->direction;
st->log = sp->log;
st->timeout = sp->timeout;
st->state_flags = sp->state_flags;
st->id = sp->id;
st->creatorid = sp->creatorid;
pf_state_peer_ntoh(&sp->src, &st->src);
pf_state_peer_ntoh(&sp->dst, &st->dst);
st->rule.ptr = r;
st->nat_rule.ptr = NULL;
st->anchor.ptr = NULL;
st->rt_kif = NULL;
st->pfsync_time = time_uptime;
st->sync_state = PFSYNC_S_NONE;
if (!(flags & PFSYNC_SI_IOCTL))
st->state_flags |= PFSTATE_NOSYNC;
if ((error = pf_state_insert(kif, skw, sks, st)) != 0)
goto cleanup_state;
/* XXX when we have nat_rule/anchors, use STATE_INC_COUNTERS */
counter_u64_add(r->states_cur, 1);
counter_u64_add(r->states_tot, 1);
if (!(flags & PFSYNC_SI_IOCTL)) {
st->state_flags &= ~PFSTATE_NOSYNC;
if (st->state_flags & PFSTATE_ACK) {
pfsync_q_ins(st, PFSYNC_S_IACK);
pfsync_push(sc);
}
}
st->state_flags &= ~PFSTATE_ACK;
PF_STATE_UNLOCK(st);
return (0);
cleanup:
error = ENOMEM;
if (skw == sks)
sks = NULL;
if (skw != NULL)
uma_zfree(V_pf_state_key_z, skw);
if (sks != NULL)
uma_zfree(V_pf_state_key_z, sks);
cleanup_state: /* pf_state_insert() frees the state keys. */
if (st) {
if (st->dst.scrub)
uma_zfree(V_pf_state_scrub_z, st->dst.scrub);
if (st->src.scrub)
uma_zfree(V_pf_state_scrub_z, st->src.scrub);
uma_zfree(V_pf_state_z, st);
}
return (error);
}
static int
pfsync_input(struct mbuf **mp, int *offp __unused, int proto __unused)
{
struct pfsync_softc *sc = V_pfsyncif;
struct pfsync_pkt pkt;
struct mbuf *m = *mp;
struct ip *ip = mtod(m, struct ip *);
struct pfsync_header *ph;
struct pfsync_subheader subh;
int offset, len;
int rv;
uint16_t count;
*mp = NULL;
V_pfsyncstats.pfsyncs_ipackets++;
/* Verify that we have a sync interface configured. */
if (!sc || !sc->sc_sync_if || !V_pf_status.running ||
(sc->sc_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
goto done;
/* verify that the packet came in on the right interface */
if (sc->sc_sync_if != m->m_pkthdr.rcvif) {
V_pfsyncstats.pfsyncs_badif++;
goto done;
}
if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1);
if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len);
/* verify that the IP TTL is 255. */
if (ip->ip_ttl != PFSYNC_DFLTTL) {
V_pfsyncstats.pfsyncs_badttl++;
goto done;
}
offset = ip->ip_hl << 2;
if (m->m_pkthdr.len < offset + sizeof(*ph)) {
V_pfsyncstats.pfsyncs_hdrops++;
goto done;
}
if (offset + sizeof(*ph) > m->m_len) {
if (m_pullup(m, offset + sizeof(*ph)) == NULL) {
V_pfsyncstats.pfsyncs_hdrops++;
return (IPPROTO_DONE);
}
ip = mtod(m, struct ip *);
}
ph = (struct pfsync_header *)((char *)ip + offset);
/* verify the version */
if (ph->version != PFSYNC_VERSION) {
V_pfsyncstats.pfsyncs_badver++;
goto done;
}
len = ntohs(ph->len) + offset;
if (m->m_pkthdr.len < len) {
V_pfsyncstats.pfsyncs_badlen++;
goto done;
}
/* Cheaper to grab this now than having to mess with mbufs later */
pkt.ip = ip;
pkt.src = ip->ip_src;
pkt.flags = 0;
/*
* Trusting pf_chksum during packet processing, as well as seeking
* in interface name tree, require holding PF_RULES_RLOCK().
*/
PF_RULES_RLOCK();
if (!bcmp(&ph->pfcksum, &V_pf_status.pf_chksum, PF_MD5_DIGEST_LENGTH))
pkt.flags |= PFSYNC_SI_CKSUM;
offset += sizeof(*ph);
while (offset <= len - sizeof(subh)) {
m_copydata(m, offset, sizeof(subh), (caddr_t)&subh);
offset += sizeof(subh);
if (subh.action >= PFSYNC_ACT_MAX) {
V_pfsyncstats.pfsyncs_badact++;
PF_RULES_RUNLOCK();
goto done;
}
count = ntohs(subh.count);
V_pfsyncstats.pfsyncs_iacts[subh.action] += count;
rv = (*pfsync_acts[subh.action])(&pkt, m, offset, count);
if (rv == -1) {
PF_RULES_RUNLOCK();
return (IPPROTO_DONE);
}
offset += rv;
}
PF_RULES_RUNLOCK();
done:
m_freem(m);
return (IPPROTO_DONE);
}
static int
pfsync_in_clr(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
struct pfsync_clr *clr;
struct mbuf *mp;
int len = sizeof(*clr) * count;
int i, offp;
u_int32_t creatorid;
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
clr = (struct pfsync_clr *)(mp->m_data + offp);
for (i = 0; i < count; i++) {
creatorid = clr[i].creatorid;
if (clr[i].ifname[0] != '\0' &&
pfi_kif_find(clr[i].ifname) == NULL)
continue;
for (int i = 0; i <= pf_hashmask; i++) {
struct pf_idhash *ih = &V_pf_idhash[i];
struct pf_state *s;
relock:
PF_HASHROW_LOCK(ih);
LIST_FOREACH(s, &ih->states, entry) {
if (s->creatorid == creatorid) {
s->state_flags |= PFSTATE_NOSYNC;
pf_unlink_state(s, PF_ENTER_LOCKED);
goto relock;
}
}
PF_HASHROW_UNLOCK(ih);
}
}
return (len);
}
static int
pfsync_in_ins(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
struct mbuf *mp;
struct pfsync_state *sa, *sp;
int len = sizeof(*sp) * count;
int i, offp;
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
sa = (struct pfsync_state *)(mp->m_data + offp);
for (i = 0; i < count; i++) {
sp = &sa[i];
/* Check for invalid values. */
if (sp->timeout >= PFTM_MAX ||
sp->src.state > PF_TCPS_PROXY_DST ||
sp->dst.state > PF_TCPS_PROXY_DST ||
sp->direction > PF_OUT ||
(sp->af != AF_INET && sp->af != AF_INET6)) {
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("%s: invalid value\n", __func__);
V_pfsyncstats.pfsyncs_badval++;
continue;
}
if (pfsync_state_import(sp, pkt->flags) == ENOMEM)
/* Drop out, but process the rest of the actions. */
break;
}
return (len);
}
static int
pfsync_in_iack(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
struct pfsync_ins_ack *ia, *iaa;
struct pf_state *st;
struct mbuf *mp;
int len = count * sizeof(*ia);
int offp, i;
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
iaa = (struct pfsync_ins_ack *)(mp->m_data + offp);
for (i = 0; i < count; i++) {
ia = &iaa[i];
st = pf_find_state_byid(ia->id, ia->creatorid);
if (st == NULL)
continue;
if (st->state_flags & PFSTATE_ACK) {
PFSYNC_LOCK(V_pfsyncif);
pfsync_undefer_state(st, 0);
PFSYNC_UNLOCK(V_pfsyncif);
}
PF_STATE_UNLOCK(st);
}
/*
* XXX this is not yet implemented, but we know the size of the
* message so we can skip it.
*/
return (count * sizeof(struct pfsync_ins_ack));
}
static int
pfsync_upd_tcp(struct pf_state *st, struct pfsync_state_peer *src,
struct pfsync_state_peer *dst)
{
int sync = 0;
PF_STATE_LOCK_ASSERT(st);
/*
* The state should never go backwards except
* for syn-proxy states. Neither should the
* sequence window slide backwards.
*/
if ((st->src.state > src->state &&
(st->src.state < PF_TCPS_PROXY_SRC ||
src->state >= PF_TCPS_PROXY_SRC)) ||
(st->src.state == src->state &&
SEQ_GT(st->src.seqlo, ntohl(src->seqlo))))
sync++;
else
pf_state_peer_ntoh(src, &st->src);
if ((st->dst.state > dst->state) ||
(st->dst.state >= TCPS_SYN_SENT &&
SEQ_GT(st->dst.seqlo, ntohl(dst->seqlo))))
sync++;
else
pf_state_peer_ntoh(dst, &st->dst);
return (sync);
}
static int
pfsync_in_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
struct pfsync_softc *sc = V_pfsyncif;
struct pfsync_state *sa, *sp;
struct pf_state *st;
int sync;
struct mbuf *mp;
int len = count * sizeof(*sp);
int offp, i;
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
sa = (struct pfsync_state *)(mp->m_data + offp);
for (i = 0; i < count; i++) {
sp = &sa[i];
/* check for invalid values */
if (sp->timeout >= PFTM_MAX ||
sp->src.state > PF_TCPS_PROXY_DST ||
sp->dst.state > PF_TCPS_PROXY_DST) {
if (V_pf_status.debug >= PF_DEBUG_MISC) {
printf("pfsync_input: PFSYNC_ACT_UPD: "
"invalid value\n");
}
V_pfsyncstats.pfsyncs_badval++;
continue;
}
st = pf_find_state_byid(sp->id, sp->creatorid);
if (st == NULL) {
/* insert the update */
if (pfsync_state_import(sp, 0))
V_pfsyncstats.pfsyncs_badstate++;
continue;
}
if (st->state_flags & PFSTATE_ACK) {
PFSYNC_LOCK(sc);
pfsync_undefer_state(st, 1);
PFSYNC_UNLOCK(sc);
}
if (st->key[PF_SK_WIRE]->proto == IPPROTO_TCP)
sync = pfsync_upd_tcp(st, &sp->src, &sp->dst);
else {
sync = 0;
/*
* Non-TCP protocol state machine always go
* forwards
*/
if (st->src.state > sp->src.state)
sync++;
else
pf_state_peer_ntoh(&sp->src, &st->src);
if (st->dst.state > sp->dst.state)
sync++;
else
pf_state_peer_ntoh(&sp->dst, &st->dst);
}
if (sync < 2) {
pfsync_alloc_scrub_memory(&sp->dst, &st->dst);
pf_state_peer_ntoh(&sp->dst, &st->dst);
st->expire = time_uptime;
st->timeout = sp->timeout;
}
st->pfsync_time = time_uptime;
if (sync) {
V_pfsyncstats.pfsyncs_stale++;
pfsync_update_state(st);
PF_STATE_UNLOCK(st);
PFSYNC_LOCK(sc);
pfsync_push(sc);
PFSYNC_UNLOCK(sc);
continue;
}
PF_STATE_UNLOCK(st);
}
return (len);
}
static int
pfsync_in_upd_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
struct pfsync_softc *sc = V_pfsyncif;
struct pfsync_upd_c *ua, *up;
struct pf_state *st;
int len = count * sizeof(*up);
int sync;
struct mbuf *mp;
int offp, i;
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
ua = (struct pfsync_upd_c *)(mp->m_data + offp);
for (i = 0; i < count; i++) {
up = &ua[i];
/* check for invalid values */
if (up->timeout >= PFTM_MAX ||
up->src.state > PF_TCPS_PROXY_DST ||
up->dst.state > PF_TCPS_PROXY_DST) {
if (V_pf_status.debug >= PF_DEBUG_MISC) {
printf("pfsync_input: "
"PFSYNC_ACT_UPD_C: "
"invalid value\n");
}
V_pfsyncstats.pfsyncs_badval++;
continue;
}
st = pf_find_state_byid(up->id, up->creatorid);
if (st == NULL) {
/* We don't have this state. Ask for it. */
PFSYNC_LOCK(sc);
pfsync_request_update(up->creatorid, up->id);
PFSYNC_UNLOCK(sc);
continue;
}
if (st->state_flags & PFSTATE_ACK) {
PFSYNC_LOCK(sc);
pfsync_undefer_state(st, 1);
PFSYNC_UNLOCK(sc);
}
if (st->key[PF_SK_WIRE]->proto == IPPROTO_TCP)
sync = pfsync_upd_tcp(st, &up->src, &up->dst);
else {
sync = 0;
/*
* Non-TCP protocol state machine always go
* forwards
*/
if (st->src.state > up->src.state)
sync++;
else
pf_state_peer_ntoh(&up->src, &st->src);
if (st->dst.state > up->dst.state)
sync++;
else
pf_state_peer_ntoh(&up->dst, &st->dst);
}
if (sync < 2) {
pfsync_alloc_scrub_memory(&up->dst, &st->dst);
pf_state_peer_ntoh(&up->dst, &st->dst);
st->expire = time_uptime;
st->timeout = up->timeout;
}
st->pfsync_time = time_uptime;
if (sync) {
V_pfsyncstats.pfsyncs_stale++;
pfsync_update_state(st);
PF_STATE_UNLOCK(st);
PFSYNC_LOCK(sc);
pfsync_push(sc);
PFSYNC_UNLOCK(sc);
continue;
}
PF_STATE_UNLOCK(st);
}
return (len);
}
static int
pfsync_in_ureq(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
struct pfsync_upd_req *ur, *ura;
struct mbuf *mp;
int len = count * sizeof(*ur);
int i, offp;
struct pf_state *st;
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
ura = (struct pfsync_upd_req *)(mp->m_data + offp);
for (i = 0; i < count; i++) {
ur = &ura[i];
if (ur->id == 0 && ur->creatorid == 0)
pfsync_bulk_start();
else {
st = pf_find_state_byid(ur->id, ur->creatorid);
if (st == NULL) {
V_pfsyncstats.pfsyncs_badstate++;
continue;
}
if (st->state_flags & PFSTATE_NOSYNC) {
PF_STATE_UNLOCK(st);
continue;
}
pfsync_update_state_req(st);
PF_STATE_UNLOCK(st);
}
}
return (len);
}
static int
pfsync_in_del(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
struct mbuf *mp;
struct pfsync_state *sa, *sp;
struct pf_state *st;
int len = count * sizeof(*sp);
int offp, i;
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
sa = (struct pfsync_state *)(mp->m_data + offp);
for (i = 0; i < count; i++) {
sp = &sa[i];
st = pf_find_state_byid(sp->id, sp->creatorid);
if (st == NULL) {
V_pfsyncstats.pfsyncs_badstate++;
continue;
}
st->state_flags |= PFSTATE_NOSYNC;
pf_unlink_state(st, PF_ENTER_LOCKED);
}
return (len);
}
static int
pfsync_in_del_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
struct mbuf *mp;
struct pfsync_del_c *sa, *sp;
struct pf_state *st;
int len = count * sizeof(*sp);
int offp, i;
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
sa = (struct pfsync_del_c *)(mp->m_data + offp);
for (i = 0; i < count; i++) {
sp = &sa[i];
st = pf_find_state_byid(sp->id, sp->creatorid);
if (st == NULL) {
V_pfsyncstats.pfsyncs_badstate++;
continue;
}
st->state_flags |= PFSTATE_NOSYNC;
pf_unlink_state(st, PF_ENTER_LOCKED);
}
return (len);
}
static int
pfsync_in_bus(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
struct pfsync_softc *sc = V_pfsyncif;
struct pfsync_bus *bus;
struct mbuf *mp;
int len = count * sizeof(*bus);
int offp;
PFSYNC_BLOCK(sc);
/* If we're not waiting for a bulk update, who cares. */
if (sc->sc_ureq_sent == 0) {
PFSYNC_BUNLOCK(sc);
return (len);
}
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
PFSYNC_BUNLOCK(sc);
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
bus = (struct pfsync_bus *)(mp->m_data + offp);
switch (bus->status) {
case PFSYNC_BUS_START:
callout_reset(&sc->sc_bulkfail_tmo, 4 * hz +
V_pf_limits[PF_LIMIT_STATES].limit /
((sc->sc_ifp->if_mtu - PFSYNC_MINPKT) /
sizeof(struct pfsync_state)),
pfsync_bulk_fail, sc);
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("pfsync: received bulk update start\n");
break;
case PFSYNC_BUS_END:
if (time_uptime - ntohl(bus->endtime) >=
sc->sc_ureq_sent) {
/* that's it, we're happy */
sc->sc_ureq_sent = 0;
sc->sc_bulk_tries = 0;
callout_stop(&sc->sc_bulkfail_tmo);
if (!(sc->sc_flags & PFSYNCF_OK) && carp_demote_adj_p)
(*carp_demote_adj_p)(-V_pfsync_carp_adj,
"pfsync bulk done");
sc->sc_flags |= PFSYNCF_OK;
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("pfsync: received valid "
"bulk update end\n");
} else {
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("pfsync: received invalid "
"bulk update end: bad timestamp\n");
}
break;
}
PFSYNC_BUNLOCK(sc);
return (len);
}
static int
pfsync_in_tdb(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
int len = count * sizeof(struct pfsync_tdb);
#if defined(IPSEC)
struct pfsync_tdb *tp;
struct mbuf *mp;
int offp;
int i;
int s;
mp = m_pulldown(m, offset, len, &offp);
if (mp == NULL) {
V_pfsyncstats.pfsyncs_badlen++;
return (-1);
}
tp = (struct pfsync_tdb *)(mp->m_data + offp);
for (i = 0; i < count; i++)
pfsync_update_net_tdb(&tp[i]);
#endif
return (len);
}
#if defined(IPSEC)
/* Update an in-kernel tdb. Silently fail if no tdb is found. */
static void
pfsync_update_net_tdb(struct pfsync_tdb *pt)
{
struct tdb *tdb;
int s;
/* check for invalid values */
if (ntohl(pt->spi) <= SPI_RESERVED_MAX ||
(pt->dst.sa.sa_family != AF_INET &&
pt->dst.sa.sa_family != AF_INET6))
goto bad;
tdb = gettdb(pt->spi, &pt->dst, pt->sproto);
if (tdb) {
pt->rpl = ntohl(pt->rpl);
pt->cur_bytes = (unsigned long long)be64toh(pt->cur_bytes);
/* Neither replay nor byte counter should ever decrease. */
if (pt->rpl < tdb->tdb_rpl ||
pt->cur_bytes < tdb->tdb_cur_bytes) {
goto bad;
}
tdb->tdb_rpl = pt->rpl;
tdb->tdb_cur_bytes = pt->cur_bytes;
}
return;
bad:
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("pfsync_insert: PFSYNC_ACT_TDB_UPD: "
"invalid value\n");
V_pfsyncstats.pfsyncs_badstate++;
return;
}
#endif
static int
pfsync_in_eof(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
/* check if we are at the right place in the packet */
if (offset != m->m_pkthdr.len)
V_pfsyncstats.pfsyncs_badlen++;
/* we're done. free and let the caller return */
m_freem(m);
return (-1);
}
static int
pfsync_in_error(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
{
V_pfsyncstats.pfsyncs_badact++;
m_freem(m);
return (-1);
}
static int
pfsyncoutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
struct route *rt)
{
m_freem(m);
return (0);
}
/* ARGSUSED */
static int
pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
struct pfsync_softc *sc = ifp->if_softc;
struct ifreq *ifr = (struct ifreq *)data;
struct pfsyncreq pfsyncr;
int error;
switch (cmd) {
case SIOCSIFFLAGS:
PFSYNC_LOCK(sc);
if (ifp->if_flags & IFF_UP) {
ifp->if_drv_flags |= IFF_DRV_RUNNING;
PFSYNC_UNLOCK(sc);
pfsync_pointers_init();
} else {
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
PFSYNC_UNLOCK(sc);
pfsync_pointers_uninit();
}
break;
case SIOCSIFMTU:
if (!sc->sc_sync_if ||
ifr->ifr_mtu <= PFSYNC_MINPKT ||
ifr->ifr_mtu > sc->sc_sync_if->if_mtu)
return (EINVAL);
if (ifr->ifr_mtu < ifp->if_mtu) {
PFSYNC_LOCK(sc);
if (sc->sc_len > PFSYNC_MINPKT)
pfsync_sendout(1);
PFSYNC_UNLOCK(sc);
}
ifp->if_mtu = ifr->ifr_mtu;
break;
case SIOCGETPFSYNC:
bzero(&pfsyncr, sizeof(pfsyncr));
PFSYNC_LOCK(sc);
if (sc->sc_sync_if) {
strlcpy(pfsyncr.pfsyncr_syncdev,
sc->sc_sync_if->if_xname, IFNAMSIZ);
}
pfsyncr.pfsyncr_syncpeer = sc->sc_sync_peer;
pfsyncr.pfsyncr_maxupdates = sc->sc_maxupdates;
pfsyncr.pfsyncr_defer = (PFSYNCF_DEFER ==
(sc->sc_flags & PFSYNCF_DEFER));
PFSYNC_UNLOCK(sc);
return (copyout(&pfsyncr, ifr->ifr_data, sizeof(pfsyncr)));
case SIOCSETPFSYNC:
{
struct ip_moptions *imo = &sc->sc_imo;
struct ifnet *sifp;
struct ip *ip;
void *mship = NULL;
if ((error = priv_check(curthread, PRIV_NETINET_PF)) != 0)
return (error);
if ((error = copyin(ifr->ifr_data, &pfsyncr, sizeof(pfsyncr))))
return (error);
if (pfsyncr.pfsyncr_maxupdates > 255)
return (EINVAL);
if (pfsyncr.pfsyncr_syncdev[0] == 0)
sifp = NULL;
else if ((sifp = ifunit_ref(pfsyncr.pfsyncr_syncdev)) == NULL)
return (EINVAL);
if (sifp != NULL && (
pfsyncr.pfsyncr_syncpeer.s_addr == 0 ||
pfsyncr.pfsyncr_syncpeer.s_addr ==
htonl(INADDR_PFSYNC_GROUP)))
mship = malloc((sizeof(struct in_multi *) *
IP_MIN_MEMBERSHIPS), M_PFSYNC, M_WAITOK | M_ZERO);
PFSYNC_LOCK(sc);
if (pfsyncr.pfsyncr_syncpeer.s_addr == 0)
sc->sc_sync_peer.s_addr = htonl(INADDR_PFSYNC_GROUP);
else
sc->sc_sync_peer.s_addr =
pfsyncr.pfsyncr_syncpeer.s_addr;
sc->sc_maxupdates = pfsyncr.pfsyncr_maxupdates;
if (pfsyncr.pfsyncr_defer) {
sc->sc_flags |= PFSYNCF_DEFER;
pfsync_defer_ptr = pfsync_defer;
} else {
sc->sc_flags &= ~PFSYNCF_DEFER;
pfsync_defer_ptr = NULL;
}
if (sifp == NULL) {
if (sc->sc_sync_if)
if_rele(sc->sc_sync_if);
sc->sc_sync_if = NULL;
if (imo->imo_membership)
pfsync_multicast_cleanup(sc);
PFSYNC_UNLOCK(sc);
break;
}
if (sc->sc_len > PFSYNC_MINPKT &&
(sifp->if_mtu < sc->sc_ifp->if_mtu ||
(sc->sc_sync_if != NULL &&
sifp->if_mtu < sc->sc_sync_if->if_mtu) ||
sifp->if_mtu < MCLBYTES - sizeof(struct ip)))
pfsync_sendout(1);
if (imo->imo_membership)
pfsync_multicast_cleanup(sc);
if (sc->sc_sync_peer.s_addr == htonl(INADDR_PFSYNC_GROUP)) {
error = pfsync_multicast_setup(sc, sifp, mship);
if (error) {
if_rele(sifp);
free(mship, M_PFSYNC);
return (error);
}
}
if (sc->sc_sync_if)
if_rele(sc->sc_sync_if);
sc->sc_sync_if = sifp;
ip = &sc->sc_template;
bzero(ip, sizeof(*ip));
ip->ip_v = IPVERSION;
ip->ip_hl = sizeof(sc->sc_template) >> 2;
ip->ip_tos = IPTOS_LOWDELAY;
/* len and id are set later. */
ip->ip_off = htons(IP_DF);
ip->ip_ttl = PFSYNC_DFLTTL;
ip->ip_p = IPPROTO_PFSYNC;
ip->ip_src.s_addr = INADDR_ANY;
ip->ip_dst.s_addr = sc->sc_sync_peer.s_addr;
/* Request a full state table update. */
if ((sc->sc_flags & PFSYNCF_OK) && carp_demote_adj_p)
(*carp_demote_adj_p)(V_pfsync_carp_adj,
"pfsync bulk start");
sc->sc_flags &= ~PFSYNCF_OK;
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("pfsync: requesting bulk update\n");
pfsync_request_update(0, 0);
PFSYNC_UNLOCK(sc);
PFSYNC_BLOCK(sc);
sc->sc_ureq_sent = time_uptime;
callout_reset(&sc->sc_bulkfail_tmo, 5 * hz, pfsync_bulk_fail,
sc);
PFSYNC_BUNLOCK(sc);
break;
}
default:
return (ENOTTY);
}
return (0);
}
static void
pfsync_out_state(struct pf_state *st, void *buf)
{
struct pfsync_state *sp = buf;
pfsync_state_export(sp, st);
}
static void
pfsync_out_iack(struct pf_state *st, void *buf)
{
struct pfsync_ins_ack *iack = buf;
iack->id = st->id;
iack->creatorid = st->creatorid;
}
static void
pfsync_out_upd_c(struct pf_state *st, void *buf)
{
struct pfsync_upd_c *up = buf;
bzero(up, sizeof(*up));
up->id = st->id;
pf_state_peer_hton(&st->src, &up->src);
pf_state_peer_hton(&st->dst, &up->dst);
up->creatorid = st->creatorid;
up->timeout = st->timeout;
}
static void
pfsync_out_del(struct pf_state *st, void *buf)
{
struct pfsync_del_c *dp = buf;
dp->id = st->id;
dp->creatorid = st->creatorid;
st->state_flags |= PFSTATE_NOSYNC;
}
static void
pfsync_drop(struct pfsync_softc *sc)
{
struct pf_state *st, *next;
struct pfsync_upd_req_item *ur;
int q;
for (q = 0; q < PFSYNC_S_COUNT; q++) {
if (TAILQ_EMPTY(&sc->sc_qs[q]))
continue;
TAILQ_FOREACH_SAFE(st, &sc->sc_qs[q], sync_list, next) {
KASSERT(st->sync_state == q,
("%s: st->sync_state == q",
__func__));
st->sync_state = PFSYNC_S_NONE;
pf_release_state(st);
}
TAILQ_INIT(&sc->sc_qs[q]);
}
while ((ur = TAILQ_FIRST(&sc->sc_upd_req_list)) != NULL) {
TAILQ_REMOVE(&sc->sc_upd_req_list, ur, ur_entry);
free(ur, M_PFSYNC);
}
sc->sc_plus = NULL;
sc->sc_len = PFSYNC_MINPKT;
}
static void
pfsync_sendout(int schedswi)
{
struct pfsync_softc *sc = V_pfsyncif;
struct ifnet *ifp = sc->sc_ifp;
struct mbuf *m;
struct ip *ip;
struct pfsync_header *ph;
struct pfsync_subheader *subh;
struct pf_state *st;
struct pfsync_upd_req_item *ur;
int offset;
int q, count = 0;
KASSERT(sc != NULL, ("%s: null sc", __func__));
KASSERT(sc->sc_len > PFSYNC_MINPKT,
("%s: sc_len %zu", __func__, sc->sc_len));
PFSYNC_LOCK_ASSERT(sc);
if (ifp->if_bpf == NULL && sc->sc_sync_if == NULL) {
pfsync_drop(sc);
return;
}
m = m_get2(max_linkhdr + sc->sc_len, M_NOWAIT, MT_DATA, M_PKTHDR);
if (m == NULL) {
if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1);
V_pfsyncstats.pfsyncs_onomem++;
return;
}
m->m_data += max_linkhdr;
m->m_len = m->m_pkthdr.len = sc->sc_len;
/* build the ip header */
ip = (struct ip *)m->m_data;
bcopy(&sc->sc_template, ip, sizeof(*ip));
offset = sizeof(*ip);
ip->ip_len = htons(m->m_pkthdr.len);
ip_fillid(ip);
/* build the pfsync header */
ph = (struct pfsync_header *)(m->m_data + offset);
bzero(ph, sizeof(*ph));
offset += sizeof(*ph);
ph->version = PFSYNC_VERSION;
ph->len = htons(sc->sc_len - sizeof(*ip));
bcopy(V_pf_status.pf_chksum, ph->pfcksum, PF_MD5_DIGEST_LENGTH);
/* walk the queues */
for (q = 0; q < PFSYNC_S_COUNT; q++) {
if (TAILQ_EMPTY(&sc->sc_qs[q]))
continue;
subh = (struct pfsync_subheader *)(m->m_data + offset);
offset += sizeof(*subh);
count = 0;
TAILQ_FOREACH(st, &sc->sc_qs[q], sync_list) {
KASSERT(st->sync_state == q,
("%s: st->sync_state == q",
__func__));
/*
* XXXGL: some of write methods do unlocked reads
* of state data :(
*/
pfsync_qs[q].write(st, m->m_data + offset);
offset += pfsync_qs[q].len;
st->sync_state = PFSYNC_S_NONE;
pf_release_state(st);
count++;
}
TAILQ_INIT(&sc->sc_qs[q]);
bzero(subh, sizeof(*subh));
subh->action = pfsync_qs[q].action;
subh->count = htons(count);
V_pfsyncstats.pfsyncs_oacts[pfsync_qs[q].action] += count;
}
if (!TAILQ_EMPTY(&sc->sc_upd_req_list)) {
subh = (struct pfsync_subheader *)(m->m_data + offset);
offset += sizeof(*subh);
count = 0;
while ((ur = TAILQ_FIRST(&sc->sc_upd_req_list)) != NULL) {
TAILQ_REMOVE(&sc->sc_upd_req_list, ur, ur_entry);
bcopy(&ur->ur_msg, m->m_data + offset,
sizeof(ur->ur_msg));
offset += sizeof(ur->ur_msg);
free(ur, M_PFSYNC);
count++;
}
bzero(subh, sizeof(*subh));
subh->action = PFSYNC_ACT_UPD_REQ;
subh->count = htons(count);
V_pfsyncstats.pfsyncs_oacts[PFSYNC_ACT_UPD_REQ] += count;
}
/* has someone built a custom region for us to add? */
if (sc->sc_plus != NULL) {
bcopy(sc->sc_plus, m->m_data + offset, sc->sc_pluslen);
offset += sc->sc_pluslen;
sc->sc_plus = NULL;
}
subh = (struct pfsync_subheader *)(m->m_data + offset);
offset += sizeof(*subh);
bzero(subh, sizeof(*subh));
subh->action = PFSYNC_ACT_EOF;
subh->count = htons(1);
V_pfsyncstats.pfsyncs_oacts[PFSYNC_ACT_EOF]++;
/* we're done, let's put it on the wire */
if (ifp->if_bpf) {
m->m_data += sizeof(*ip);
m->m_len = m->m_pkthdr.len = sc->sc_len - sizeof(*ip);
BPF_MTAP(ifp, m);
m->m_data -= sizeof(*ip);
m->m_len = m->m_pkthdr.len = sc->sc_len;
}
if (sc->sc_sync_if == NULL) {
sc->sc_len = PFSYNC_MINPKT;
m_freem(m);
return;
}
if_inc_counter(sc->sc_ifp, IFCOUNTER_OPACKETS, 1);
if_inc_counter(sc->sc_ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len);
sc->sc_len = PFSYNC_MINPKT;
if (!_IF_QFULL(&sc->sc_ifp->if_snd))
_IF_ENQUEUE(&sc->sc_ifp->if_snd, m);
else {
m_freem(m);
if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1);
}
if (schedswi)
swi_sched(V_pfsync_swi_cookie, 0);
}
static void
pfsync_insert_state(struct pf_state *st)
{
struct pfsync_softc *sc = V_pfsyncif;
if (st->state_flags & PFSTATE_NOSYNC)
return;
if ((st->rule.ptr->rule_flag & PFRULE_NOSYNC) ||
st->key[PF_SK_WIRE]->proto == IPPROTO_PFSYNC) {
st->state_flags |= PFSTATE_NOSYNC;
return;
}
KASSERT(st->sync_state == PFSYNC_S_NONE,
("%s: st->sync_state %u", __func__, st->sync_state));
PFSYNC_LOCK(sc);
if (sc->sc_len == PFSYNC_MINPKT)
callout_reset(&sc->sc_tmo, 1 * hz, pfsync_timeout, V_pfsyncif);
pfsync_q_ins(st, PFSYNC_S_INS);
PFSYNC_UNLOCK(sc);
st->sync_updates = 0;
}
static int
pfsync_defer(struct pf_state *st, struct mbuf *m)
{
struct pfsync_softc *sc = V_pfsyncif;
struct pfsync_deferral *pd;
if (m->m_flags & (M_BCAST|M_MCAST))
return (0);
PFSYNC_LOCK(sc);
if (sc == NULL || !(sc->sc_ifp->if_flags & IFF_DRV_RUNNING) ||
!(sc->sc_flags & PFSYNCF_DEFER)) {
PFSYNC_UNLOCK(sc);
return (0);
}
if (sc->sc_deferred >= 128)
pfsync_undefer(TAILQ_FIRST(&sc->sc_deferrals), 0);
pd = malloc(sizeof(*pd), M_PFSYNC, M_NOWAIT);
if (pd == NULL)
return (0);
sc->sc_deferred++;
m->m_flags |= M_SKIP_FIREWALL;
st->state_flags |= PFSTATE_ACK;
pd->pd_sc = sc;
pd->pd_refs = 0;
pd->pd_st = st;
pf_ref_state(st);
pd->pd_m = m;
TAILQ_INSERT_TAIL(&sc->sc_deferrals, pd, pd_entry);
callout_init_mtx(&pd->pd_tmo, &sc->sc_mtx, CALLOUT_RETURNUNLOCKED);
callout_reset(&pd->pd_tmo, 10, pfsync_defer_tmo, pd);
pfsync_push(sc);
return (1);
}
static void
pfsync_undefer(struct pfsync_deferral *pd, int drop)
{
struct pfsync_softc *sc = pd->pd_sc;
struct mbuf *m = pd->pd_m;
struct pf_state *st = pd->pd_st;
PFSYNC_LOCK_ASSERT(sc);
TAILQ_REMOVE(&sc->sc_deferrals, pd, pd_entry);
sc->sc_deferred--;
pd->pd_st->state_flags &= ~PFSTATE_ACK; /* XXX: locking! */
free(pd, M_PFSYNC);
pf_release_state(st);
if (drop)
m_freem(m);
else {
_IF_ENQUEUE(&sc->sc_ifp->if_snd, m);
pfsync_push(sc);
}
}
static void
pfsync_defer_tmo(void *arg)
{
struct pfsync_deferral *pd = arg;
struct pfsync_softc *sc = pd->pd_sc;
struct mbuf *m = pd->pd_m;
struct pf_state *st = pd->pd_st;
PFSYNC_LOCK_ASSERT(sc);
CURVNET_SET(m->m_pkthdr.rcvif->if_vnet);
TAILQ_REMOVE(&sc->sc_deferrals, pd, pd_entry);
sc->sc_deferred--;
pd->pd_st->state_flags &= ~PFSTATE_ACK; /* XXX: locking! */
if (pd->pd_refs == 0)
free(pd, M_PFSYNC);
PFSYNC_UNLOCK(sc);
ip_output(m, NULL, NULL, 0, NULL, NULL);
pf_release_state(st);
CURVNET_RESTORE();
}
static void
pfsync_undefer_state(struct pf_state *st, int drop)
{
struct pfsync_softc *sc = V_pfsyncif;
struct pfsync_deferral *pd;
PFSYNC_LOCK_ASSERT(sc);
TAILQ_FOREACH(pd, &sc->sc_deferrals, pd_entry) {
if (pd->pd_st == st) {
if (callout_stop(&pd->pd_tmo))
pfsync_undefer(pd, drop);
return;
}
}
panic("%s: unable to find deferred state", __func__);
}
static void
pfsync_update_state(struct pf_state *st)
{
struct pfsync_softc *sc = V_pfsyncif;
int sync = 0;
PF_STATE_LOCK_ASSERT(st);
PFSYNC_LOCK(sc);
if (st->state_flags & PFSTATE_ACK)
pfsync_undefer_state(st, 0);
if (st->state_flags & PFSTATE_NOSYNC) {
if (st->sync_state != PFSYNC_S_NONE)
pfsync_q_del(st);
PFSYNC_UNLOCK(sc);
return;
}
if (sc->sc_len == PFSYNC_MINPKT)
callout_reset(&sc->sc_tmo, 1 * hz, pfsync_timeout, V_pfsyncif);
switch (st->sync_state) {
case PFSYNC_S_UPD_C:
case PFSYNC_S_UPD:
case PFSYNC_S_INS:
/* we're already handling it */
if (st->key[PF_SK_WIRE]->proto == IPPROTO_TCP) {
st->sync_updates++;
if (st->sync_updates >= sc->sc_maxupdates)
sync = 1;
}
break;
case PFSYNC_S_IACK:
pfsync_q_del(st);
case PFSYNC_S_NONE:
pfsync_q_ins(st, PFSYNC_S_UPD_C);
st->sync_updates = 0;
break;
default:
panic("%s: unexpected sync state %d", __func__, st->sync_state);
}
if (sync || (time_uptime - st->pfsync_time) < 2)
pfsync_push(sc);
PFSYNC_UNLOCK(sc);
}
static void
pfsync_request_update(u_int32_t creatorid, u_int64_t id)
{
struct pfsync_softc *sc = V_pfsyncif;
struct pfsync_upd_req_item *item;
size_t nlen = sizeof(struct pfsync_upd_req);
PFSYNC_LOCK_ASSERT(sc);
/*
* This code does a bit to prevent multiple update requests for the
* same state being generated. It searches current subheader queue,
* but it doesn't lookup into queue of already packed datagrams.
*/
TAILQ_FOREACH(item, &sc->sc_upd_req_list, ur_entry)
if (item->ur_msg.id == id &&
item->ur_msg.creatorid == creatorid)
return;
item = malloc(sizeof(*item), M_PFSYNC, M_NOWAIT);
if (item == NULL)
return; /* XXX stats */
item->ur_msg.id = id;
item->ur_msg.creatorid = creatorid;
if (TAILQ_EMPTY(&sc->sc_upd_req_list))
nlen += sizeof(struct pfsync_subheader);
if (sc->sc_len + nlen > sc->sc_ifp->if_mtu) {
pfsync_sendout(1);
nlen = sizeof(struct pfsync_subheader) +
sizeof(struct pfsync_upd_req);
}
TAILQ_INSERT_TAIL(&sc->sc_upd_req_list, item, ur_entry);
sc->sc_len += nlen;
}
static void
pfsync_update_state_req(struct pf_state *st)
{
struct pfsync_softc *sc = V_pfsyncif;
PF_STATE_LOCK_ASSERT(st);
PFSYNC_LOCK(sc);
if (st->state_flags & PFSTATE_NOSYNC) {
if (st->sync_state != PFSYNC_S_NONE)
pfsync_q_del(st);
PFSYNC_UNLOCK(sc);
return;
}
switch (st->sync_state) {
case PFSYNC_S_UPD_C:
case PFSYNC_S_IACK:
pfsync_q_del(st);
case PFSYNC_S_NONE:
pfsync_q_ins(st, PFSYNC_S_UPD);
pfsync_push(sc);
break;
case PFSYNC_S_INS:
case PFSYNC_S_UPD:
case PFSYNC_S_DEL:
/* we're already handling it */
break;
default:
panic("%s: unexpected sync state %d", __func__, st->sync_state);
}
PFSYNC_UNLOCK(sc);
}
static void
pfsync_delete_state(struct pf_state *st)
{
struct pfsync_softc *sc = V_pfsyncif;
PFSYNC_LOCK(sc);
if (st->state_flags & PFSTATE_ACK)
pfsync_undefer_state(st, 1);
if (st->state_flags & PFSTATE_NOSYNC) {
if (st->sync_state != PFSYNC_S_NONE)
pfsync_q_del(st);
PFSYNC_UNLOCK(sc);
return;
}
if (sc->sc_len == PFSYNC_MINPKT)
callout_reset(&sc->sc_tmo, 1 * hz, pfsync_timeout, V_pfsyncif);
switch (st->sync_state) {
case PFSYNC_S_INS:
/* We never got to tell the world so just forget about it. */
pfsync_q_del(st);
break;
case PFSYNC_S_UPD_C:
case PFSYNC_S_UPD:
case PFSYNC_S_IACK:
pfsync_q_del(st);
/* FALLTHROUGH to putting it on the del list */
case PFSYNC_S_NONE:
pfsync_q_ins(st, PFSYNC_S_DEL);
break;
default:
panic("%s: unexpected sync state %d", __func__, st->sync_state);
}
PFSYNC_UNLOCK(sc);
}
static void
pfsync_clear_states(u_int32_t creatorid, const char *ifname)
{
struct pfsync_softc *sc = V_pfsyncif;
struct {
struct pfsync_subheader subh;
struct pfsync_clr clr;
} __packed r;
bzero(&r, sizeof(r));
r.subh.action = PFSYNC_ACT_CLR;
r.subh.count = htons(1);
V_pfsyncstats.pfsyncs_oacts[PFSYNC_ACT_CLR]++;
strlcpy(r.clr.ifname, ifname, sizeof(r.clr.ifname));
r.clr.creatorid = creatorid;
PFSYNC_LOCK(sc);
pfsync_send_plus(&r, sizeof(r));
PFSYNC_UNLOCK(sc);
}
static void
pfsync_q_ins(struct pf_state *st, int q)
{
struct pfsync_softc *sc = V_pfsyncif;
size_t nlen = pfsync_qs[q].len;
PFSYNC_LOCK_ASSERT(sc);
KASSERT(st->sync_state == PFSYNC_S_NONE,
("%s: st->sync_state %u", __func__, st->sync_state));
KASSERT(sc->sc_len >= PFSYNC_MINPKT, ("pfsync pkt len is too low %zu",
sc->sc_len));
if (TAILQ_EMPTY(&sc->sc_qs[q]))
nlen += sizeof(struct pfsync_subheader);
if (sc->sc_len + nlen > sc->sc_ifp->if_mtu) {
pfsync_sendout(1);
nlen = sizeof(struct pfsync_subheader) + pfsync_qs[q].len;
}
sc->sc_len += nlen;
TAILQ_INSERT_TAIL(&sc->sc_qs[q], st, sync_list);
st->sync_state = q;
pf_ref_state(st);
}
static void
pfsync_q_del(struct pf_state *st)
{
struct pfsync_softc *sc = V_pfsyncif;
int q = st->sync_state;
PFSYNC_LOCK_ASSERT(sc);
KASSERT(st->sync_state != PFSYNC_S_NONE,
("%s: st->sync_state != PFSYNC_S_NONE", __func__));
sc->sc_len -= pfsync_qs[q].len;
TAILQ_REMOVE(&sc->sc_qs[q], st, sync_list);
st->sync_state = PFSYNC_S_NONE;
pf_release_state(st);
if (TAILQ_EMPTY(&sc->sc_qs[q]))
sc->sc_len -= sizeof(struct pfsync_subheader);
}
static void
pfsync_bulk_start(void)
{
struct pfsync_softc *sc = V_pfsyncif;
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("pfsync: received bulk update request\n");
PFSYNC_BLOCK(sc);
sc->sc_ureq_received = time_uptime;
sc->sc_bulk_hashid = 0;
sc->sc_bulk_stateid = 0;
pfsync_bulk_status(PFSYNC_BUS_START);
callout_reset(&sc->sc_bulk_tmo, 1, pfsync_bulk_update, sc);
PFSYNC_BUNLOCK(sc);
}
static void
pfsync_bulk_update(void *arg)
{
struct pfsync_softc *sc = arg;
struct pf_state *s;
int i, sent = 0;
PFSYNC_BLOCK_ASSERT(sc);
CURVNET_SET(sc->sc_ifp->if_vnet);
/*
* Start with last state from previous invocation.
* It may had gone, in this case start from the
* hash slot.
*/
s = pf_find_state_byid(sc->sc_bulk_stateid, sc->sc_bulk_creatorid);
if (s != NULL)
i = PF_IDHASH(s);
else
i = sc->sc_bulk_hashid;
for (; i <= pf_hashmask; i++) {
struct pf_idhash *ih = &V_pf_idhash[i];
if (s != NULL)
PF_HASHROW_ASSERT(ih);
else {
PF_HASHROW_LOCK(ih);
s = LIST_FIRST(&ih->states);
}
for (; s; s = LIST_NEXT(s, entry)) {
if (sent > 1 && (sc->sc_ifp->if_mtu - sc->sc_len) <
sizeof(struct pfsync_state)) {
/* We've filled a packet. */
sc->sc_bulk_hashid = i;
sc->sc_bulk_stateid = s->id;
sc->sc_bulk_creatorid = s->creatorid;
PF_HASHROW_UNLOCK(ih);
callout_reset(&sc->sc_bulk_tmo, 1,
pfsync_bulk_update, sc);
goto full;
}
if (s->sync_state == PFSYNC_S_NONE &&
s->timeout < PFTM_MAX &&
s->pfsync_time <= sc->sc_ureq_received) {
pfsync_update_state_req(s);
sent++;
}
}
PF_HASHROW_UNLOCK(ih);
}
/* We're done. */
pfsync_bulk_status(PFSYNC_BUS_END);
full:
CURVNET_RESTORE();
}
static void
pfsync_bulk_status(u_int8_t status)
{
struct {
struct pfsync_subheader subh;
struct pfsync_bus bus;
} __packed r;
struct pfsync_softc *sc = V_pfsyncif;
bzero(&r, sizeof(r));
r.subh.action = PFSYNC_ACT_BUS;
r.subh.count = htons(1);
V_pfsyncstats.pfsyncs_oacts[PFSYNC_ACT_BUS]++;
r.bus.creatorid = V_pf_status.hostid;
r.bus.endtime = htonl(time_uptime - sc->sc_ureq_received);
r.bus.status = status;
PFSYNC_LOCK(sc);
pfsync_send_plus(&r, sizeof(r));
PFSYNC_UNLOCK(sc);
}
static void
pfsync_bulk_fail(void *arg)
{
struct pfsync_softc *sc = arg;
CURVNET_SET(sc->sc_ifp->if_vnet);
PFSYNC_BLOCK_ASSERT(sc);
if (sc->sc_bulk_tries++ < PFSYNC_MAX_BULKTRIES) {
/* Try again */
callout_reset(&sc->sc_bulkfail_tmo, 5 * hz,
pfsync_bulk_fail, V_pfsyncif);
PFSYNC_LOCK(sc);
pfsync_request_update(0, 0);
PFSYNC_UNLOCK(sc);
} else {
/* Pretend like the transfer was ok. */
sc->sc_ureq_sent = 0;
sc->sc_bulk_tries = 0;
PFSYNC_LOCK(sc);
if (!(sc->sc_flags & PFSYNCF_OK) && carp_demote_adj_p)
(*carp_demote_adj_p)(-V_pfsync_carp_adj,
"pfsync bulk fail");
sc->sc_flags |= PFSYNCF_OK;
PFSYNC_UNLOCK(sc);
if (V_pf_status.debug >= PF_DEBUG_MISC)
printf("pfsync: failed to receive bulk update\n");
}
CURVNET_RESTORE();
}
static void
pfsync_send_plus(void *plus, size_t pluslen)
{
struct pfsync_softc *sc = V_pfsyncif;
PFSYNC_LOCK_ASSERT(sc);
if (sc->sc_len + pluslen > sc->sc_ifp->if_mtu)
pfsync_sendout(1);
sc->sc_plus = plus;
sc->sc_len += (sc->sc_pluslen = pluslen);
pfsync_sendout(1);
}
static void
pfsync_timeout(void *arg)
{
struct pfsync_softc *sc = arg;
CURVNET_SET(sc->sc_ifp->if_vnet);
PFSYNC_LOCK(sc);
pfsync_push(sc);
PFSYNC_UNLOCK(sc);
CURVNET_RESTORE();
}
static void
pfsync_push(struct pfsync_softc *sc)
{
PFSYNC_LOCK_ASSERT(sc);
sc->sc_flags |= PFSYNCF_PUSH;
swi_sched(V_pfsync_swi_cookie, 0);
}
static void
pfsyncintr(void *arg)
{
struct pfsync_softc *sc = arg;
struct mbuf *m, *n;
CURVNET_SET(sc->sc_ifp->if_vnet);
PFSYNC_LOCK(sc);
if ((sc->sc_flags & PFSYNCF_PUSH) && sc->sc_len > PFSYNC_MINPKT) {
pfsync_sendout(0);
sc->sc_flags &= ~PFSYNCF_PUSH;
}
_IF_DEQUEUE_ALL(&sc->sc_ifp->if_snd, m);
PFSYNC_UNLOCK(sc);
for (; m != NULL; m = n) {
n = m->m_nextpkt;
m->m_nextpkt = NULL;
/*
* We distinguish between a deferral packet and our
* own pfsync packet based on M_SKIP_FIREWALL
* flag. This is XXX.
*/
if (m->m_flags & M_SKIP_FIREWALL)
ip_output(m, NULL, NULL, 0, NULL, NULL);
else if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo,
NULL) == 0)
V_pfsyncstats.pfsyncs_opackets++;
else
V_pfsyncstats.pfsyncs_oerrors++;
}
CURVNET_RESTORE();
}
static int
pfsync_multicast_setup(struct pfsync_softc *sc, struct ifnet *ifp, void *mship)
{
struct ip_moptions *imo = &sc->sc_imo;
int error;
if (!(ifp->if_flags & IFF_MULTICAST))
return (EADDRNOTAVAIL);
imo->imo_membership = (struct in_multi **)mship;
imo->imo_max_memberships = IP_MIN_MEMBERSHIPS;
imo->imo_multicast_vif = -1;
if ((error = in_joingroup(ifp, &sc->sc_sync_peer, NULL,
&imo->imo_membership[0])) != 0) {
imo->imo_membership = NULL;
return (error);
}
imo->imo_num_memberships++;
imo->imo_multicast_ifp = ifp;
imo->imo_multicast_ttl = PFSYNC_DFLTTL;
imo->imo_multicast_loop = 0;
return (0);
}
static void
pfsync_multicast_cleanup(struct pfsync_softc *sc)
{
struct ip_moptions *imo = &sc->sc_imo;
in_leavegroup(imo->imo_membership[0], NULL);
free(imo->imo_membership, M_PFSYNC);
imo->imo_membership = NULL;
imo->imo_multicast_ifp = NULL;
}
#ifdef INET
extern struct domain inetdomain;
static struct protosw in_pfsync_protosw = {
.pr_type = SOCK_RAW,
.pr_domain = &inetdomain,
.pr_protocol = IPPROTO_PFSYNC,
.pr_flags = PR_ATOMIC|PR_ADDR,
.pr_input = pfsync_input,
.pr_output = rip_output,
.pr_ctloutput = rip_ctloutput,
.pr_usrreqs = &rip_usrreqs
};
#endif
static void
pfsync_pointers_init()
{
PF_RULES_WLOCK();
pfsync_state_import_ptr = pfsync_state_import;
pfsync_insert_state_ptr = pfsync_insert_state;
pfsync_update_state_ptr = pfsync_update_state;
pfsync_delete_state_ptr = pfsync_delete_state;
pfsync_clear_states_ptr = pfsync_clear_states;
pfsync_defer_ptr = pfsync_defer;
PF_RULES_WUNLOCK();
}
static void
pfsync_pointers_uninit()
{
PF_RULES_WLOCK();
pfsync_state_import_ptr = NULL;
pfsync_insert_state_ptr = NULL;
pfsync_update_state_ptr = NULL;
pfsync_delete_state_ptr = NULL;
pfsync_clear_states_ptr = NULL;
pfsync_defer_ptr = NULL;
PF_RULES_WUNLOCK();
}
static int
pfsync_init()
{
VNET_ITERATOR_DECL(vnet_iter);
int error = 0;
VNET_LIST_RLOCK();
VNET_FOREACH(vnet_iter) {
CURVNET_SET(vnet_iter);
V_pfsync_cloner = if_clone_simple(pfsyncname,
pfsync_clone_create, pfsync_clone_destroy, 1);
error = swi_add(NULL, pfsyncname, pfsyncintr, V_pfsyncif,
SWI_NET, INTR_MPSAFE, &V_pfsync_swi_cookie);
CURVNET_RESTORE();
if (error)
goto fail_locked;
}
VNET_LIST_RUNLOCK();
#ifdef INET
error = pf_proto_register(PF_INET, &in_pfsync_protosw);
if (error)
goto fail;
error = ipproto_register(IPPROTO_PFSYNC);
if (error) {
pf_proto_unregister(PF_INET, IPPROTO_PFSYNC, SOCK_RAW);
goto fail;
}
#endif
pfsync_pointers_init();
return (0);
fail:
VNET_LIST_RLOCK();
fail_locked:
VNET_FOREACH(vnet_iter) {
CURVNET_SET(vnet_iter);
if (V_pfsync_swi_cookie) {
swi_remove(V_pfsync_swi_cookie);
if_clone_detach(V_pfsync_cloner);
}
CURVNET_RESTORE();
}
VNET_LIST_RUNLOCK();
return (error);
}
static void
pfsync_uninit()
{
VNET_ITERATOR_DECL(vnet_iter);
pfsync_pointers_uninit();
ipproto_unregister(IPPROTO_PFSYNC);
pf_proto_unregister(PF_INET, IPPROTO_PFSYNC, SOCK_RAW);
VNET_LIST_RLOCK();
VNET_FOREACH(vnet_iter) {
CURVNET_SET(vnet_iter);
if_clone_detach(V_pfsync_cloner);
swi_remove(V_pfsync_swi_cookie);
CURVNET_RESTORE();
}
VNET_LIST_RUNLOCK();
}
static int
pfsync_modevent(module_t mod, int type, void *data)
{
int error = 0;
switch (type) {
case MOD_LOAD:
error = pfsync_init();
break;
case MOD_QUIESCE:
/*
* Module should not be unloaded due to race conditions.
*/
error = EBUSY;
break;
case MOD_UNLOAD:
pfsync_uninit();
break;
default:
error = EINVAL;
break;
}
return (error);
}
static moduledata_t pfsync_mod = {
pfsyncname,
pfsync_modevent,
0
};
#define PFSYNC_MODVER 1
DECLARE_MODULE(pfsync, pfsync_mod, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY);
MODULE_VERSION(pfsync, PFSYNC_MODVER);
MODULE_DEPEND(pfsync, pf, PF_MODVER, PF_MODVER, PF_MODVER);
Index: head/sys/ofed/include/linux/timer.h
===================================================================
--- head/sys/ofed/include/linux/timer.h (revision 283290)
+++ head/sys/ofed/include/linux/timer.h (revision 283291)
@@ -1,72 +1,72 @@
/*-
* Copyright (c) 2010 Isilon Systems, Inc.
* Copyright (c) 2010 iX Systems, Inc.
* Copyright (c) 2010 Panasas, Inc.
* Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice unmodified, this list of conditions, and the following
* disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _LINUX_TIMER_H_
#define _LINUX_TIMER_H_
#include <linux/types.h>
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/callout.h>
struct timer_list {
struct callout timer_callout;
void (*function) (unsigned long);
unsigned long data;
unsigned long expires;
};
extern unsigned long linux_timer_hz_mask;
#define setup_timer(timer, func, dat) \
do { \
(timer)->function = (func); \
(timer)->data = (dat); \
- callout_init(&(timer)->timer_callout, CALLOUT_MPSAFE); \
+ callout_init(&(timer)->timer_callout, 1); \
} while (0)
#define init_timer(timer) \
do { \
(timer)->function = NULL; \
(timer)->data = 0; \
- callout_init(&(timer)->timer_callout, CALLOUT_MPSAFE); \
+ callout_init(&(timer)->timer_callout, 1); \
} while (0)
extern void mod_timer(struct timer_list *, unsigned long);
extern void add_timer(struct timer_list *);
#define del_timer(timer) callout_stop(&(timer)->timer_callout)
#define del_timer_sync(timer) callout_drain(&(timer)->timer_callout)
#define timer_pending(timer) callout_pending(&(timer)->timer_callout)
#define round_jiffies(j) \
((unsigned long)(((j) + linux_timer_hz_mask) & ~linux_timer_hz_mask))
#define round_jiffies_relative(j) \
round_jiffies(j)
#endif /* _LINUX_TIMER_H_ */
Index: head/sys/ofed/include/linux/workqueue.h
===================================================================
--- head/sys/ofed/include/linux/workqueue.h (revision 283290)
+++ head/sys/ofed/include/linux/workqueue.h (revision 283291)
@@ -1,223 +1,223 @@
/*-
* Copyright (c) 2010 Isilon Systems, Inc.
* Copyright (c) 2010 iX Systems, Inc.
* Copyright (c) 2010 Panasas, Inc.
* Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice unmodified, this list of conditions, and the following
* disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _LINUX_WORKQUEUE_H_
#define _LINUX_WORKQUEUE_H_
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/slab.h>
#include <sys/taskqueue.h>
struct workqueue_struct {
struct taskqueue *taskqueue;
};
struct work_struct {
struct task work_task;
struct taskqueue *taskqueue;
void (*fn)(struct work_struct *);
};
struct delayed_work {
struct work_struct work;
struct callout timer;
};
static inline struct delayed_work *
to_delayed_work(struct work_struct *work)
{
return container_of(work, struct delayed_work, work);
}
static inline void
_work_fn(void *context, int pending)
{
struct work_struct *work;
work = context;
work->fn(work);
}
#define INIT_WORK(work, func) \
do { \
(work)->fn = (func); \
(work)->taskqueue = NULL; \
TASK_INIT(&(work)->work_task, 0, _work_fn, (work)); \
} while (0)
#define INIT_DELAYED_WORK(_work, func) \
do { \
INIT_WORK(&(_work)->work, func); \
- callout_init(&(_work)->timer, CALLOUT_MPSAFE); \
+ callout_init(&(_work)->timer, 1); \
} while (0)
#define INIT_DEFERRABLE_WORK INIT_DELAYED_WORK
#define schedule_work(work) \
do { \
(work)->taskqueue = taskqueue_thread; \
taskqueue_enqueue(taskqueue_thread, &(work)->work_task); \
} while (0)
#define flush_scheduled_work() flush_taskqueue(taskqueue_thread)
static inline int queue_work (struct workqueue_struct *q, struct work_struct *work)
{
(work)->taskqueue = (q)->taskqueue;
/* Return opposite val to align with Linux logic */
return !taskqueue_enqueue((q)->taskqueue, &(work)->work_task);
}
static inline void
_delayed_work_fn(void *arg)
{
struct delayed_work *work;
work = arg;
taskqueue_enqueue(work->work.taskqueue, &work->work.work_task);
}
static inline int
queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *work,
unsigned long delay)
{
int pending;
pending = work->work.work_task.ta_pending;
work->work.taskqueue = wq->taskqueue;
if (delay != 0)
callout_reset(&work->timer, delay, _delayed_work_fn, work);
else
_delayed_work_fn((void *)work);
return (!pending);
}
static inline bool schedule_delayed_work(struct delayed_work *dwork,
unsigned long delay)
{
struct workqueue_struct wq;
wq.taskqueue = taskqueue_thread;
return queue_delayed_work(&wq, dwork, delay);
}
static inline struct workqueue_struct *
_create_workqueue_common(char *name, int cpus)
{
struct workqueue_struct *wq;
wq = kmalloc(sizeof(*wq), M_WAITOK);
wq->taskqueue = taskqueue_create((name), M_WAITOK,
taskqueue_thread_enqueue, &wq->taskqueue);
taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name);
return (wq);
}
#define create_singlethread_workqueue(name) \
_create_workqueue_common(name, 1)
#define create_workqueue(name) \
_create_workqueue_common(name, MAXCPU)
static inline void
destroy_workqueue(struct workqueue_struct *wq)
{
taskqueue_free(wq->taskqueue);
kfree(wq);
}
#define flush_workqueue(wq) flush_taskqueue((wq)->taskqueue)
static inline void
_flush_fn(void *context, int pending)
{
}
static inline void
flush_taskqueue(struct taskqueue *tq)
{
struct task flushtask;
PHOLD(curproc);
TASK_INIT(&flushtask, 0, _flush_fn, NULL);
taskqueue_enqueue(tq, &flushtask);
taskqueue_drain(tq, &flushtask);
PRELE(curproc);
}
static inline int
cancel_work_sync(struct work_struct *work)
{
if (work->taskqueue &&
taskqueue_cancel(work->taskqueue, &work->work_task, NULL))
taskqueue_drain(work->taskqueue, &work->work_task);
return 0;
}
/*
* This may leave work running on another CPU as it does on Linux.
*/
static inline int
cancel_delayed_work(struct delayed_work *work)
{
callout_stop(&work->timer);
if (work->work.taskqueue)
return (taskqueue_cancel(work->work.taskqueue,
&work->work.work_task, NULL) == 0);
return 0;
}
static inline int
cancel_delayed_work_sync(struct delayed_work *work)
{
callout_drain(&work->timer);
if (work->work.taskqueue &&
taskqueue_cancel(work->work.taskqueue, &work->work.work_task, NULL))
taskqueue_drain(work->work.taskqueue, &work->work.work_task);
return 0;
}
static inline bool
mod_delayed_work(struct workqueue_struct *wq, struct delayed_work *dwork,
unsigned long delay)
{
cancel_delayed_work(dwork);
queue_delayed_work(wq, dwork, delay);
return false;
}
#endif /* _LINUX_WORKQUEUE_H_ */
Index: head/sys/powerpc/mambo/mambo_console.c
===================================================================
--- head/sys/powerpc/mambo/mambo_console.c (revision 283290)
+++ head/sys/powerpc/mambo/mambo_console.c (revision 283291)
@@ -1,185 +1,185 @@
/*-
* Copyright (C) 2008 by Nathan Whitehorn. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/priv.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/cons.h>
#include <sys/consio.h>
#include <sys/tty.h>
#include <dev/ofw/openfirm.h>
#include <ddb/ddb.h>
#include "mambocall.h"
#define MAMBOBURSTLEN 128 /* max number of bytes to write in one chunk */
#define MAMBO_CONSOLE_WRITE 0
#define MAMBO_CONSOLE_READ 60
static tsw_outwakeup_t mambotty_outwakeup;
static struct ttydevsw mambo_ttydevsw = {
.tsw_flags = TF_NOPREFIX,
.tsw_outwakeup = mambotty_outwakeup,
};
static int polltime;
static struct callout mambo_callout;
static struct tty *tp = NULL;
#if defined(KDB)
static int alt_break_state;
#endif
static void mambo_timeout(void *);
static cn_probe_t mambo_cnprobe;
static cn_init_t mambo_cninit;
static cn_term_t mambo_cnterm;
static cn_getc_t mambo_cngetc;
static cn_putc_t mambo_cnputc;
static cn_grab_t mambo_cngrab;
static cn_ungrab_t mambo_cnungrab;
CONSOLE_DRIVER(mambo);
static void
cn_drvinit(void *unused)
{
if (mambo_consdev.cn_pri != CN_DEAD &&
mambo_consdev.cn_name[0] != '\0') {
if (OF_finddevice("/mambo") == -1)
return;
tp = tty_alloc(&mambo_ttydevsw, NULL);
tty_init_console(tp, 0);
tty_makedev(tp, NULL, "%s", "mambocons");
polltime = 1;
- callout_init(&mambo_callout, CALLOUT_MPSAFE);
+ callout_init(&mambo_callout, 1);
callout_reset(&mambo_callout, polltime, mambo_timeout, NULL);
}
}
SYSINIT(cndev, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE, cn_drvinit, NULL);
static void
mambotty_outwakeup(struct tty *tp)
{
int len;
u_char buf[MAMBOBURSTLEN];
for (;;) {
len = ttydisc_getc(tp, buf, sizeof buf);
if (len == 0)
break;
mambocall(MAMBO_CONSOLE_WRITE, buf, (register_t)len, 1UL);
}
}
static void
mambo_timeout(void *v)
{
int c;
tty_lock(tp);
while ((c = mambo_cngetc(NULL)) != -1)
ttydisc_rint(tp, c, 0);
ttydisc_rint_done(tp);
tty_unlock(tp);
callout_reset(&mambo_callout, polltime, mambo_timeout, NULL);
}
static void
mambo_cnprobe(struct consdev *cp)
{
if (OF_finddevice("/mambo") == -1) {
cp->cn_pri = CN_DEAD;
return;
}
cp->cn_pri = CN_NORMAL;
}
static void
mambo_cninit(struct consdev *cp)
{
/* XXX: This is the alias, but that should be good enough */
strcpy(cp->cn_name, "mambocons");
}
static void
mambo_cnterm(struct consdev *cp)
{
}
static void
mambo_cngrab(struct consdev *cp)
{
}
static void
mambo_cnungrab(struct consdev *cp)
{
}
static int
mambo_cngetc(struct consdev *cp)
{
int ch;
ch = mambocall(MAMBO_CONSOLE_READ);
if (ch > 0 && ch < 0xff) {
#if defined(KDB)
kdb_alt_break(ch, &alt_break_state);
#endif
return (ch);
}
return (-1);
}
static void
mambo_cnputc(struct consdev *cp, int c)
{
char cbuf;
cbuf = c;
mambocall(MAMBO_CONSOLE_WRITE, &cbuf, 1UL, 1UL);
}
Index: head/sys/powerpc/pseries/phyp_console.c
===================================================================
--- head/sys/powerpc/pseries/phyp_console.c (revision 283290)
+++ head/sys/powerpc/pseries/phyp_console.c (revision 283291)
@@ -1,433 +1,433 @@
/*-
* Copyright (C) 2011 by Nathan Whitehorn. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/priv.h>
#include <sys/systm.h>
#include <sys/module.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/cons.h>
#include <sys/tty.h>
#include <machine/bus.h>
#include <dev/ofw/openfirm.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#include <dev/uart/uart.h>
#include <dev/uart/uart_cpu.h>
#include <dev/uart/uart_bus.h>
#include "phyp-hvcall.h"
#include "uart_if.h"
struct uart_phyp_softc {
device_t dev;
phandle_t node;
int vtermid;
struct tty *tp;
struct resource *irqres;
int irqrid;
struct callout callout;
void *sc_icookie;
int polltime;
struct mtx sc_mtx;
int protocol;
union {
uint64_t u64[2];
char str[16];
} phyp_inbuf;
uint64_t inbuflen;
uint8_t outseqno;
};
static struct uart_phyp_softc *console_sc = NULL;
#if defined(KDB)
static int alt_break_state;
#endif
enum {
HVTERM1, HVTERMPROT
};
#define VS_DATA_PACKET_HEADER 0xff
#define VS_CONTROL_PACKET_HEADER 0xfe
#define VSV_SET_MODEM_CTL 0x01
#define VSV_MODEM_CTL_UPDATE 0x02
#define VSV_RENEGOTIATE_CONNECTION 0x03
#define VS_QUERY_PACKET_HEADER 0xfd
#define VSV_SEND_VERSION_NUMBER 0x01
#define VSV_SEND_MODEM_CTL_STATUS 0x02
#define VS_QUERY_RESPONSE_PACKET_HEADER 0xfc
static int uart_phyp_probe(device_t dev);
static int uart_phyp_attach(device_t dev);
static void uart_phyp_intr(void *v);
static device_method_t uart_phyp_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, uart_phyp_probe),
DEVMETHOD(device_attach, uart_phyp_attach),
DEVMETHOD_END
};
static driver_t uart_phyp_driver = {
"uart",
uart_phyp_methods,
sizeof(struct uart_phyp_softc),
};
DRIVER_MODULE(uart_phyp, vdevice, uart_phyp_driver, uart_devclass, 0, 0);
static cn_probe_t uart_phyp_cnprobe;
static cn_init_t uart_phyp_cninit;
static cn_term_t uart_phyp_cnterm;
static cn_getc_t uart_phyp_cngetc;
static cn_putc_t uart_phyp_cnputc;
static cn_grab_t uart_phyp_cngrab;
static cn_ungrab_t uart_phyp_cnungrab;
CONSOLE_DRIVER(uart_phyp);
static void uart_phyp_ttyoutwakeup(struct tty *tp);
static struct ttydevsw uart_phyp_tty_class = {
.tsw_flags = TF_INITLOCK|TF_CALLOUT,
.tsw_outwakeup = uart_phyp_ttyoutwakeup,
};
static int
uart_phyp_probe_node(struct uart_phyp_softc *sc)
{
phandle_t node = sc->node;
uint32_t reg;
char buf[64];
sc->inbuflen = 0;
sc->outseqno = 0;
if (OF_getprop(node, "name", buf, sizeof(buf)) <= 0)
return (ENXIO);
if (strcmp(buf, "vty") != 0)
return (ENXIO);
if (OF_getprop(node, "device_type", buf, sizeof(buf)) <= 0)
return (ENXIO);
if (strcmp(buf, "serial") != 0)
return (ENXIO);
reg = -1;
OF_getprop(node, "reg", &reg, sizeof(reg));
if (reg == -1)
return (ENXIO);
sc->vtermid = reg;
sc->node = node;
if (OF_getprop(node, "compatible", buf, sizeof(buf)) <= 0)
return (ENXIO);
if (strcmp(buf, "hvterm1") == 0) {
sc->protocol = HVTERM1;
return (0);
} else if (strcmp(buf, "hvterm-protocol") == 0) {
sc->protocol = HVTERMPROT;
return (0);
}
return (ENXIO);
}
static int
uart_phyp_probe(device_t dev)
{
const char *name;
struct uart_phyp_softc sc;
int err;
name = ofw_bus_get_name(dev);
if (name == NULL || strcmp(name, "vty") != 0)
return (ENXIO);
sc.node = ofw_bus_get_node(dev);
err = uart_phyp_probe_node(&sc);
if (err != 0)
return (err);
device_set_desc(dev, "POWER Hypervisor Virtual Serial Port");
return (err);
}
static void
uart_phyp_cnprobe(struct consdev *cp)
{
char buf[64];
ihandle_t stdout;
phandle_t input, chosen;
static struct uart_phyp_softc sc;
if ((chosen = OF_finddevice("/chosen")) == -1)
goto fail;
/* Check if OF has an active stdin/stdout */
input = -1;
if (OF_getprop(chosen, "stdout", &stdout,
sizeof(stdout)) == sizeof(stdout) && stdout != 0)
input = OF_instance_to_package(stdout);
if (input == -1)
goto fail;
if (OF_getprop(input, "device_type", buf, sizeof(buf)) == -1)
goto fail;
if (strcmp(buf, "serial") != 0)
goto fail;
sc.node = input;
if (uart_phyp_probe_node(&sc) != 0)
goto fail;
mtx_init(&sc.sc_mtx, "uart_phyp", NULL, MTX_SPIN | MTX_QUIET |
MTX_NOWITNESS);
cp->cn_pri = CN_NORMAL;
console_sc = &sc;
return;
fail:
cp->cn_pri = CN_DEAD;
return;
}
static int
uart_phyp_attach(device_t dev)
{
struct uart_phyp_softc *sc;
int unit;
sc = device_get_softc(dev);
sc->dev = dev;
sc->node = ofw_bus_get_node(dev);
uart_phyp_probe_node(sc);
unit = device_get_unit(dev);
sc->tp = tty_alloc(&uart_phyp_tty_class, sc);
mtx_init(&sc->sc_mtx, device_get_nameunit(dev), NULL,
MTX_SPIN | MTX_QUIET | MTX_NOWITNESS);
if (console_sc != NULL && console_sc->vtermid == sc->vtermid) {
sc->outseqno = console_sc->outseqno;
console_sc = sc;
sprintf(uart_phyp_consdev.cn_name, "ttyu%r", unit);
tty_init_console(sc->tp, 0);
}
sc->irqrid = 0;
sc->irqres = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irqrid,
RF_ACTIVE | RF_SHAREABLE);
if (sc->irqres != NULL) {
bus_setup_intr(dev, sc->irqres, INTR_TYPE_TTY | INTR_MPSAFE,
NULL, uart_phyp_intr, sc, &sc->sc_icookie);
} else {
- callout_init(&sc->callout, CALLOUT_MPSAFE);
+ callout_init(&sc->callout, 1);
sc->polltime = hz / 20;
if (sc->polltime < 1)
sc->polltime = 1;
callout_reset(&sc->callout, sc->polltime, uart_phyp_intr, sc);
}
tty_makedev(sc->tp, NULL, "u%r", unit);
return (0);
}
static void
uart_phyp_cninit(struct consdev *cp)
{
strcpy(cp->cn_name, "phypcons");
}
static void
uart_phyp_cnterm(struct consdev *cp)
{
}
static int
uart_phyp_get(struct uart_phyp_softc *sc, void *buffer, size_t bufsize)
{
int err;
int hdr = 0;
uart_lock(&sc->sc_mtx);
if (sc->inbuflen == 0) {
err = phyp_pft_hcall(H_GET_TERM_CHAR, sc->vtermid,
0, 0, 0, &sc->inbuflen, &sc->phyp_inbuf.u64[0],
&sc->phyp_inbuf.u64[1]);
if (err != H_SUCCESS) {
uart_unlock(&sc->sc_mtx);
return (-1);
}
hdr = 1;
}
if (sc->inbuflen == 0) {
uart_unlock(&sc->sc_mtx);
return (0);
}
if (bufsize > sc->inbuflen)
bufsize = sc->inbuflen;
if ((sc->protocol == HVTERMPROT) && (hdr == 1)) {
sc->inbuflen = sc->inbuflen - 4;
/* The VTERM protocol has a 4 byte header, skip it here. */
memmove(&sc->phyp_inbuf.str[0], &sc->phyp_inbuf.str[4],
sc->inbuflen);
}
memcpy(buffer, sc->phyp_inbuf.str, bufsize);
sc->inbuflen -= bufsize;
if (sc->inbuflen > 0)
memmove(&sc->phyp_inbuf.str[0], &sc->phyp_inbuf.str[bufsize],
sc->inbuflen);
uart_unlock(&sc->sc_mtx);
return (bufsize);
}
static int
uart_phyp_put(struct uart_phyp_softc *sc, void *buffer, size_t bufsize)
{
uint16_t seqno;
uint64_t len = 0;
int err;
union {
uint64_t u64[2];
char bytes[16];
} cbuf;
uart_lock(&sc->sc_mtx);
switch (sc->protocol) {
case HVTERM1:
if (bufsize > 16)
bufsize = 16;
memcpy(&cbuf, buffer, bufsize);
len = bufsize;
break;
case HVTERMPROT:
if (bufsize > 12)
bufsize = 12;
seqno = sc->outseqno++;
cbuf.bytes[0] = VS_DATA_PACKET_HEADER;
cbuf.bytes[1] = 4 + bufsize; /* total length, max 16 bytes */
cbuf.bytes[2] = (seqno >> 8) & 0xff;
cbuf.bytes[3] = seqno & 0xff;
memcpy(&cbuf.bytes[4], buffer, bufsize);
len = 4 + bufsize;
break;
}
do {
err = phyp_hcall(H_PUT_TERM_CHAR, sc->vtermid, len, cbuf.u64[0],
cbuf.u64[1]);
DELAY(100);
} while (err == H_BUSY);
uart_unlock(&sc->sc_mtx);
return (bufsize);
}
static int
uart_phyp_cngetc(struct consdev *cp)
{
unsigned char c;
int retval;
retval = uart_phyp_get(console_sc, &c, 1);
if (retval != 1)
return (-1);
#if defined(KDB)
kdb_alt_break(c, &alt_break_state);
#endif
return (c);
}
static void
uart_phyp_cnputc(struct consdev *cp, int c)
{
unsigned char ch = c;
uart_phyp_put(console_sc, &ch, 1);
}
static void
uart_phyp_cngrab(struct consdev *cp)
{
}
static void
uart_phyp_cnungrab(struct consdev *cp)
{
}
static void
uart_phyp_ttyoutwakeup(struct tty *tp)
{
struct uart_phyp_softc *sc;
char buffer[8];
int len;
sc = tty_softc(tp);
while ((len = ttydisc_getc(tp, buffer, sizeof(buffer))) != 0)
uart_phyp_put(sc, buffer, len);
}
static void
uart_phyp_intr(void *v)
{
struct uart_phyp_softc *sc = v;
struct tty *tp = sc->tp;
unsigned char c;
int len;
tty_lock(tp);
while ((len = uart_phyp_get(sc, &c, 1)) > 0)
ttydisc_rint(tp, c, 0);
ttydisc_rint_done(tp);
tty_unlock(tp);
if (sc->irqres == NULL)
callout_reset(&sc->callout, sc->polltime, uart_phyp_intr, sc);
}
Index: head/sys/sys/callout.h
===================================================================
--- head/sys/sys/callout.h (revision 283290)
+++ head/sys/sys/callout.h (revision 283291)
@@ -1,128 +1,128 @@
/*-
* Copyright (c) 1990, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)callout.h 8.2 (Berkeley) 1/21/94
* $FreeBSD$
*/
#ifndef _SYS_CALLOUT_H_
#define _SYS_CALLOUT_H_
#include <sys/_callout.h>
#define CALLOUT_LOCAL_ALLOC 0x0001 /* was allocated from callfree */
#define CALLOUT_ACTIVE 0x0002 /* callout is currently active */
#define CALLOUT_PENDING 0x0004 /* callout is waiting for timeout */
-#define CALLOUT_MPSAFE 0x0008 /* callout handler is mp safe */
+#define CALLOUT_MPSAFE 0x0008 /* deprecated */
#define CALLOUT_RETURNUNLOCKED 0x0010 /* handler returns with mtx unlocked */
#define CALLOUT_SHAREDLOCK 0x0020 /* callout lock held in shared mode */
#define CALLOUT_DFRMIGRATION 0x0040 /* callout in deferred migration mode */
#define CALLOUT_PROCESSED 0x0080 /* callout in wheel or processing list? */
#define CALLOUT_DIRECT 0x0100 /* allow exec from hw int context */
#define C_DIRECT_EXEC 0x0001 /* direct execution of callout */
#define C_PRELBITS 7
#define C_PRELRANGE ((1 << C_PRELBITS) - 1)
#define C_PREL(x) (((x) + 1) << 1)
#define C_PRELGET(x) (int)((((x) >> 1) & C_PRELRANGE) - 1)
#define C_HARDCLOCK 0x0100 /* align to hardclock() calls */
#define C_ABSOLUTE 0x0200 /* event time is absolute. */
struct callout_handle {
struct callout *callout;
};
#ifdef _KERNEL
/*
* Note the flags field is actually *two* fields. The c_flags
* field is the one that caller operations that may, or may not have
* a lock touches i.e. callout_deactivate(). The other, the c_iflags,
* is the internal flags that *must* be kept correct on which the
* callout system depend on e.g. callout_pending().
* The c_iflag is used internally by the callout system to determine which
* list the callout is on and track internal state. Callers *should not*
* use the c_flags field directly but should use the macros provided.
*
* The c_iflags field holds internal flags that are protected by internal
* locks of the callout subsystem. The c_flags field holds external flags.
* The caller must hold its own lock while manipulating or reading external
* flags via callout_active(), callout_deactivate(), callout_reset*(), or
* callout_stop() to avoid races.
*/
#define callout_active(c) ((c)->c_flags & CALLOUT_ACTIVE)
#define callout_deactivate(c) ((c)->c_flags &= ~CALLOUT_ACTIVE)
#define callout_drain(c) _callout_stop_safe(c, 1)
void callout_init(struct callout *, int);
void _callout_init_lock(struct callout *, struct lock_object *, int);
#define callout_init_mtx(c, mtx, flags) \
_callout_init_lock((c), ((mtx) != NULL) ? &(mtx)->lock_object : \
NULL, (flags))
#define callout_init_rm(c, rm, flags) \
_callout_init_lock((c), ((rm) != NULL) ? &(rm)->lock_object : \
NULL, (flags))
#define callout_init_rw(c, rw, flags) \
_callout_init_lock((c), ((rw) != NULL) ? &(rw)->lock_object : \
NULL, (flags))
#define callout_pending(c) ((c)->c_iflags & CALLOUT_PENDING)
int callout_reset_sbt_on(struct callout *, sbintime_t, sbintime_t,
void (*)(void *), void *, int, int);
#define callout_reset_sbt(c, sbt, pr, fn, arg, flags) \
callout_reset_sbt_on((c), (sbt), (pr), (fn), (arg), -1, (flags))
#define callout_reset_sbt_curcpu(c, sbt, pr, fn, arg, flags) \
callout_reset_sbt_on((c), (sbt), (pr), (fn), (arg), PCPU_GET(cpuid),\
(flags))
#define callout_reset_on(c, to_ticks, fn, arg, cpu) \
callout_reset_sbt_on((c), tick_sbt * (to_ticks), 0, (fn), (arg), \
(cpu), C_HARDCLOCK)
#define callout_reset(c, on_tick, fn, arg) \
callout_reset_on((c), (on_tick), (fn), (arg), -1)
#define callout_reset_curcpu(c, on_tick, fn, arg) \
callout_reset_on((c), (on_tick), (fn), (arg), PCPU_GET(cpuid))
#define callout_schedule_sbt_on(c, sbt, pr, cpu, flags) \
callout_reset_sbt_on((c), (sbt), (pr), (c)->c_func, (c)->c_arg, \
(cpu), (flags))
#define callout_schedule_sbt(c, sbt, pr, flags) \
callout_schedule_sbt_on((c), (sbt), (pr), -1, (flags))
#define callout_schedule_sbt_curcpu(c, sbt, pr, flags) \
callout_schedule_sbt_on((c), (sbt), (pr), PCPU_GET(cpuid), (flags))
int callout_schedule(struct callout *, int);
int callout_schedule_on(struct callout *, int, int);
#define callout_schedule_curcpu(c, on_tick) \
callout_schedule_on((c), (on_tick), PCPU_GET(cpuid))
#define callout_stop(c) _callout_stop_safe(c, 0)
int _callout_stop_safe(struct callout *, int);
void callout_process(sbintime_t now);
#endif
#endif /* _SYS_CALLOUT_H_ */
Index: head/sys/vm/uma_core.c
===================================================================
--- head/sys/vm/uma_core.c (revision 283290)
+++ head/sys/vm/uma_core.c (revision 283291)
@@ -1,3655 +1,3655 @@
/*-
* Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
* Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
* Copyright (c) 2004-2006 Robert N. M. Watson
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice unmodified, this list of conditions, and the following
* disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* uma_core.c Implementation of the Universal Memory allocator
*
* This allocator is intended to replace the multitude of similar object caches
* in the standard FreeBSD kernel. The intent is to be flexible as well as
* effecient. A primary design goal is to return unused memory to the rest of
* the system. This will make the system as a whole more flexible due to the
* ability to move memory to subsystems which most need it instead of leaving
* pools of reserved memory unused.
*
* The basic ideas stem from similar slab/zone based allocators whose algorithms
* are well known.
*
*/
/*
* TODO:
* - Improve memory usage for large allocations
* - Investigate cache size adjustments
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/* I should really use ktr.. */
/*
#define UMA_DEBUG 1
#define UMA_DEBUG_ALLOC 1
#define UMA_DEBUG_ALLOC_1 1
*/
#include "opt_ddb.h"
#include "opt_param.h"
#include "opt_vm.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bitset.h>
#include <sys/kernel.h>
#include <sys/types.h>
#include <sys/queue.h>
#include <sys/malloc.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/sysctl.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/random.h>
#include <sys/rwlock.h>
#include <sys/sbuf.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/vmmeter.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_param.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
#include <vm/uma.h>
#include <vm/uma_int.h>
#include <vm/uma_dbg.h>
#include <ddb/ddb.h>
#ifdef DEBUG_MEMGUARD
#include <vm/memguard.h>
#endif
/*
* This is the zone and keg from which all zones are spawned. The idea is that
* even the zone & keg heads are allocated from the allocator, so we use the
* bss section to bootstrap us.
*/
static struct uma_keg masterkeg;
static struct uma_zone masterzone_k;
static struct uma_zone masterzone_z;
static uma_zone_t kegs = &masterzone_k;
static uma_zone_t zones = &masterzone_z;
/* This is the zone from which all of uma_slab_t's are allocated. */
static uma_zone_t slabzone;
static uma_zone_t slabrefzone; /* With refcounters (for UMA_ZONE_REFCNT) */
/*
* The initial hash tables come out of this zone so they can be allocated
* prior to malloc coming up.
*/
static uma_zone_t hashzone;
/* The boot-time adjusted value for cache line alignment. */
int uma_align_cache = 64 - 1;
static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
/*
* Are we allowed to allocate buckets?
*/
static int bucketdisable = 1;
/* Linked list of all kegs in the system */
static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
/* Linked list of all cache-only zones in the system */
static LIST_HEAD(,uma_zone) uma_cachezones =
LIST_HEAD_INITIALIZER(uma_cachezones);
/* This RW lock protects the keg list */
static struct rwlock_padalign uma_rwlock;
/* Linked list of boot time pages */
static LIST_HEAD(,uma_slab) uma_boot_pages =
LIST_HEAD_INITIALIZER(uma_boot_pages);
/* This mutex protects the boot time pages list */
static struct mtx_padalign uma_boot_pages_mtx;
static struct sx uma_drain_lock;
/* Is the VM done starting up? */
static int booted = 0;
#define UMA_STARTUP 1
#define UMA_STARTUP2 2
/*
* Only mbuf clusters use ref zones. Just provide enough references
* to support the one user. New code should not use the ref facility.
*/
static const u_int uma_max_ipers_ref = PAGE_SIZE / MCLBYTES;
/*
* This is the handle used to schedule events that need to happen
* outside of the allocation fast path.
*/
static struct callout uma_callout;
#define UMA_TIMEOUT 20 /* Seconds for callout interval. */
/*
* This structure is passed as the zone ctor arg so that I don't have to create
* a special allocation function just for zones.
*/
struct uma_zctor_args {
const char *name;
size_t size;
uma_ctor ctor;
uma_dtor dtor;
uma_init uminit;
uma_fini fini;
uma_import import;
uma_release release;
void *arg;
uma_keg_t keg;
int align;
uint32_t flags;
};
struct uma_kctor_args {
uma_zone_t zone;
size_t size;
uma_init uminit;
uma_fini fini;
int align;
uint32_t flags;
};
struct uma_bucket_zone {
uma_zone_t ubz_zone;
char *ubz_name;
int ubz_entries; /* Number of items it can hold. */
int ubz_maxsize; /* Maximum allocation size per-item. */
};
/*
* Compute the actual number of bucket entries to pack them in power
* of two sizes for more efficient space utilization.
*/
#define BUCKET_SIZE(n) \
(((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *))
#define BUCKET_MAX BUCKET_SIZE(256)
struct uma_bucket_zone bucket_zones[] = {
{ NULL, "4 Bucket", BUCKET_SIZE(4), 4096 },
{ NULL, "6 Bucket", BUCKET_SIZE(6), 3072 },
{ NULL, "8 Bucket", BUCKET_SIZE(8), 2048 },
{ NULL, "12 Bucket", BUCKET_SIZE(12), 1536 },
{ NULL, "16 Bucket", BUCKET_SIZE(16), 1024 },
{ NULL, "32 Bucket", BUCKET_SIZE(32), 512 },
{ NULL, "64 Bucket", BUCKET_SIZE(64), 256 },
{ NULL, "128 Bucket", BUCKET_SIZE(128), 128 },
{ NULL, "256 Bucket", BUCKET_SIZE(256), 64 },
{ NULL, NULL, 0}
};
/*
* Flags and enumerations to be passed to internal functions.
*/
enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
/* Prototypes.. */
static void *noobj_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
static void *page_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
static void *startup_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
static void page_free(void *, vm_size_t, uint8_t);
static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
static void cache_drain(uma_zone_t);
static void bucket_drain(uma_zone_t, uma_bucket_t);
static void bucket_cache_drain(uma_zone_t zone);
static int keg_ctor(void *, int, void *, int);
static void keg_dtor(void *, int, void *);
static int zone_ctor(void *, int, void *, int);
static void zone_dtor(void *, int, void *);
static int zero_init(void *, int, int);
static void keg_small_init(uma_keg_t keg);
static void keg_large_init(uma_keg_t keg);
static void zone_foreach(void (*zfunc)(uma_zone_t));
static void zone_timeout(uma_zone_t zone);
static int hash_alloc(struct uma_hash *);
static int hash_expand(struct uma_hash *, struct uma_hash *);
static void hash_free(struct uma_hash *hash);
static void uma_timeout(void *);
static void uma_startup3(void);
static void *zone_alloc_item(uma_zone_t, void *, int);
static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
static void bucket_enable(void);
static void bucket_init(void);
static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
static void bucket_zone_drain(void);
static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, void *, int flags);
static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
uma_fini fini, int align, uint32_t flags);
static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
static void zone_release(uma_zone_t zone, void **bucket, int cnt);
static void uma_zero_item(void *item, uma_zone_t zone);
void uma_print_zone(uma_zone_t);
void uma_print_stats(void);
static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL);
SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT,
0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones");
SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats");
static int zone_warnings = 1;
SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RWTUN, &zone_warnings, 0,
"Warn when UMA zones becomes full");
/*
* This routine checks to see whether or not it's safe to enable buckets.
*/
static void
bucket_enable(void)
{
bucketdisable = vm_page_count_min();
}
/*
* Initialize bucket_zones, the array of zones of buckets of various sizes.
*
* For each zone, calculate the memory required for each bucket, consisting
* of the header and an array of pointers.
*/
static void
bucket_init(void)
{
struct uma_bucket_zone *ubz;
int size;
for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) {
size = roundup(sizeof(struct uma_bucket), sizeof(void *));
size += sizeof(void *) * ubz->ubz_entries;
ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET);
}
}
/*
* Given a desired number of entries for a bucket, return the zone from which
* to allocate the bucket.
*/
static struct uma_bucket_zone *
bucket_zone_lookup(int entries)
{
struct uma_bucket_zone *ubz;
for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
if (ubz->ubz_entries >= entries)
return (ubz);
ubz--;
return (ubz);
}
static int
bucket_select(int size)
{
struct uma_bucket_zone *ubz;
ubz = &bucket_zones[0];
if (size > ubz->ubz_maxsize)
return MAX((ubz->ubz_maxsize * ubz->ubz_entries) / size, 1);
for (; ubz->ubz_entries != 0; ubz++)
if (ubz->ubz_maxsize < size)
break;
ubz--;
return (ubz->ubz_entries);
}
static uma_bucket_t
bucket_alloc(uma_zone_t zone, void *udata, int flags)
{
struct uma_bucket_zone *ubz;
uma_bucket_t bucket;
/*
* This is to stop us from allocating per cpu buckets while we're
* running out of vm.boot_pages. Otherwise, we would exhaust the
* boot pages. This also prevents us from allocating buckets in
* low memory situations.
*/
if (bucketdisable)
return (NULL);
/*
* To limit bucket recursion we store the original zone flags
* in a cookie passed via zalloc_arg/zfree_arg. This allows the
* NOVM flag to persist even through deep recursions. We also
* store ZFLAG_BUCKET once we have recursed attempting to allocate
* a bucket for a bucket zone so we do not allow infinite bucket
* recursion. This cookie will even persist to frees of unused
* buckets via the allocation path or bucket allocations in the
* free path.
*/
if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
udata = (void *)(uintptr_t)zone->uz_flags;
else {
if ((uintptr_t)udata & UMA_ZFLAG_BUCKET)
return (NULL);
udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET);
}
if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
flags |= M_NOVM;
ubz = bucket_zone_lookup(zone->uz_count);
if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
ubz++;
bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
if (bucket) {
#ifdef INVARIANTS
bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
#endif
bucket->ub_cnt = 0;
bucket->ub_entries = ubz->ubz_entries;
}
return (bucket);
}
static void
bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata)
{
struct uma_bucket_zone *ubz;
KASSERT(bucket->ub_cnt == 0,
("bucket_free: Freeing a non free bucket."));
if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
udata = (void *)(uintptr_t)zone->uz_flags;
ubz = bucket_zone_lookup(bucket->ub_entries);
uma_zfree_arg(ubz->ubz_zone, bucket, udata);
}
static void
bucket_zone_drain(void)
{
struct uma_bucket_zone *ubz;
for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
zone_drain(ubz->ubz_zone);
}
static void
zone_log_warning(uma_zone_t zone)
{
static const struct timeval warninterval = { 300, 0 };
if (!zone_warnings || zone->uz_warning == NULL)
return;
if (ratecheck(&zone->uz_ratecheck, &warninterval))
printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
}
static void
zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
{
uma_klink_t klink;
LIST_FOREACH(klink, &zone->uz_kegs, kl_link)
kegfn(klink->kl_keg);
}
/*
* Routine called by timeout which is used to fire off some time interval
* based calculations. (stats, hash size, etc.)
*
* Arguments:
* arg Unused
*
* Returns:
* Nothing
*/
static void
uma_timeout(void *unused)
{
bucket_enable();
zone_foreach(zone_timeout);
/* Reschedule this event */
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
}
/*
* Routine to perform timeout driven calculations. This expands the
* hashes and does per cpu statistics aggregation.
*
* Returns nothing.
*/
static void
keg_timeout(uma_keg_t keg)
{
KEG_LOCK(keg);
/*
* Expand the keg hash table.
*
* This is done if the number of slabs is larger than the hash size.
* What I'm trying to do here is completely reduce collisions. This
* may be a little aggressive. Should I allow for two collisions max?
*/
if (keg->uk_flags & UMA_ZONE_HASH &&
keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
struct uma_hash newhash;
struct uma_hash oldhash;
int ret;
/*
* This is so involved because allocating and freeing
* while the keg lock is held will lead to deadlock.
* I have to do everything in stages and check for
* races.
*/
newhash = keg->uk_hash;
KEG_UNLOCK(keg);
ret = hash_alloc(&newhash);
KEG_LOCK(keg);
if (ret) {
if (hash_expand(&keg->uk_hash, &newhash)) {
oldhash = keg->uk_hash;
keg->uk_hash = newhash;
} else
oldhash = newhash;
KEG_UNLOCK(keg);
hash_free(&oldhash);
return;
}
}
KEG_UNLOCK(keg);
}
static void
zone_timeout(uma_zone_t zone)
{
zone_foreach_keg(zone, &keg_timeout);
}
/*
* Allocate and zero fill the next sized hash table from the appropriate
* backing store.
*
* Arguments:
* hash A new hash structure with the old hash size in uh_hashsize
*
* Returns:
* 1 on sucess and 0 on failure.
*/
static int
hash_alloc(struct uma_hash *hash)
{
int oldsize;
int alloc;
oldsize = hash->uh_hashsize;
/* We're just going to go to a power of two greater */
if (oldsize) {
hash->uh_hashsize = oldsize * 2;
alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize;
hash->uh_slab_hash = (struct slabhead *)malloc(alloc,
M_UMAHASH, M_NOWAIT);
} else {
alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
M_WAITOK);
hash->uh_hashsize = UMA_HASH_SIZE_INIT;
}
if (hash->uh_slab_hash) {
bzero(hash->uh_slab_hash, alloc);
hash->uh_hashmask = hash->uh_hashsize - 1;
return (1);
}
return (0);
}
/*
* Expands the hash table for HASH zones. This is done from zone_timeout
* to reduce collisions. This must not be done in the regular allocation
* path, otherwise, we can recurse on the vm while allocating pages.
*
* Arguments:
* oldhash The hash you want to expand
* newhash The hash structure for the new table
*
* Returns:
* Nothing
*
* Discussion:
*/
static int
hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash)
{
uma_slab_t slab;
int hval;
int i;
if (!newhash->uh_slab_hash)
return (0);
if (oldhash->uh_hashsize >= newhash->uh_hashsize)
return (0);
/*
* I need to investigate hash algorithms for resizing without a
* full rehash.
*/
for (i = 0; i < oldhash->uh_hashsize; i++)
while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) {
slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]);
SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink);
hval = UMA_HASH(newhash, slab->us_data);
SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval],
slab, us_hlink);
}
return (1);
}
/*
* Free the hash bucket to the appropriate backing store.
*
* Arguments:
* slab_hash The hash bucket we're freeing
* hashsize The number of entries in that hash bucket
*
* Returns:
* Nothing
*/
static void
hash_free(struct uma_hash *hash)
{
if (hash->uh_slab_hash == NULL)
return;
if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
else
free(hash->uh_slab_hash, M_UMAHASH);
}
/*
* Frees all outstanding items in a bucket
*
* Arguments:
* zone The zone to free to, must be unlocked.
* bucket The free/alloc bucket with items, cpu queue must be locked.
*
* Returns:
* Nothing
*/
static void
bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
{
int i;
if (bucket == NULL)
return;
if (zone->uz_fini)
for (i = 0; i < bucket->ub_cnt; i++)
zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
bucket->ub_cnt = 0;
}
/*
* Drains the per cpu caches for a zone.
*
* NOTE: This may only be called while the zone is being turn down, and not
* during normal operation. This is necessary in order that we do not have
* to migrate CPUs to drain the per-CPU caches.
*
* Arguments:
* zone The zone to drain, must be unlocked.
*
* Returns:
* Nothing
*/
static void
cache_drain(uma_zone_t zone)
{
uma_cache_t cache;
int cpu;
/*
* XXX: It is safe to not lock the per-CPU caches, because we're
* tearing down the zone anyway. I.e., there will be no further use
* of the caches at this point.
*
* XXX: It would good to be able to assert that the zone is being
* torn down to prevent improper use of cache_drain().
*
* XXX: We lock the zone before passing into bucket_cache_drain() as
* it is used elsewhere. Should the tear-down path be made special
* there in some form?
*/
CPU_FOREACH(cpu) {
cache = &zone->uz_cpu[cpu];
bucket_drain(zone, cache->uc_allocbucket);
bucket_drain(zone, cache->uc_freebucket);
if (cache->uc_allocbucket != NULL)
bucket_free(zone, cache->uc_allocbucket, NULL);
if (cache->uc_freebucket != NULL)
bucket_free(zone, cache->uc_freebucket, NULL);
cache->uc_allocbucket = cache->uc_freebucket = NULL;
}
ZONE_LOCK(zone);
bucket_cache_drain(zone);
ZONE_UNLOCK(zone);
}
static void
cache_shrink(uma_zone_t zone)
{
if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
return;
ZONE_LOCK(zone);
zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
ZONE_UNLOCK(zone);
}
static void
cache_drain_safe_cpu(uma_zone_t zone)
{
uma_cache_t cache;
uma_bucket_t b1, b2;
if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
return;
b1 = b2 = NULL;
ZONE_LOCK(zone);
critical_enter();
cache = &zone->uz_cpu[curcpu];
if (cache->uc_allocbucket) {
if (cache->uc_allocbucket->ub_cnt != 0)
LIST_INSERT_HEAD(&zone->uz_buckets,
cache->uc_allocbucket, ub_link);
else
b1 = cache->uc_allocbucket;
cache->uc_allocbucket = NULL;
}
if (cache->uc_freebucket) {
if (cache->uc_freebucket->ub_cnt != 0)
LIST_INSERT_HEAD(&zone->uz_buckets,
cache->uc_freebucket, ub_link);
else
b2 = cache->uc_freebucket;
cache->uc_freebucket = NULL;
}
critical_exit();
ZONE_UNLOCK(zone);
if (b1)
bucket_free(zone, b1, NULL);
if (b2)
bucket_free(zone, b2, NULL);
}
/*
* Safely drain per-CPU caches of a zone(s) to alloc bucket.
* This is an expensive call because it needs to bind to all CPUs
* one by one and enter a critical section on each of them in order
* to safely access their cache buckets.
* Zone lock must not be held on call this function.
*/
static void
cache_drain_safe(uma_zone_t zone)
{
int cpu;
/*
* Polite bucket sizes shrinking was not enouth, shrink aggressively.
*/
if (zone)
cache_shrink(zone);
else
zone_foreach(cache_shrink);
CPU_FOREACH(cpu) {
thread_lock(curthread);
sched_bind(curthread, cpu);
thread_unlock(curthread);
if (zone)
cache_drain_safe_cpu(zone);
else
zone_foreach(cache_drain_safe_cpu);
}
thread_lock(curthread);
sched_unbind(curthread);
thread_unlock(curthread);
}
/*
* Drain the cached buckets from a zone. Expects a locked zone on entry.
*/
static void
bucket_cache_drain(uma_zone_t zone)
{
uma_bucket_t bucket;
/*
* Drain the bucket queues and free the buckets, we just keep two per
* cpu (alloc/free).
*/
while ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
LIST_REMOVE(bucket, ub_link);
ZONE_UNLOCK(zone);
bucket_drain(zone, bucket);
bucket_free(zone, bucket, NULL);
ZONE_LOCK(zone);
}
/*
* Shrink further bucket sizes. Price of single zone lock collision
* is probably lower then price of global cache drain.
*/
if (zone->uz_count > zone->uz_count_min)
zone->uz_count--;
}
static void
keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)
{
uint8_t *mem;
int i;
uint8_t flags;
mem = slab->us_data;
flags = slab->us_flags;
i = start;
if (keg->uk_fini != NULL) {
for (i--; i > -1; i--)
keg->uk_fini(slab->us_data + (keg->uk_rsize * i),
keg->uk_size);
}
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
#ifdef UMA_DEBUG
printf("%s: Returning %d bytes.\n", keg->uk_name,
PAGE_SIZE * keg->uk_ppera);
#endif
keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
}
/*
* Frees pages from a keg back to the system. This is done on demand from
* the pageout daemon.
*
* Returns nothing.
*/
static void
keg_drain(uma_keg_t keg)
{
struct slabhead freeslabs = { 0 };
uma_slab_t slab;
uma_slab_t n;
/*
* We don't want to take pages from statically allocated kegs at this
* time
*/
if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
return;
#ifdef UMA_DEBUG
printf("%s free items: %u\n", keg->uk_name, keg->uk_free);
#endif
KEG_LOCK(keg);
if (keg->uk_free == 0)
goto finished;
slab = LIST_FIRST(&keg->uk_free_slab);
while (slab) {
n = LIST_NEXT(slab, us_link);
/* We have no where to free these to */
if (slab->us_flags & UMA_SLAB_BOOT) {
slab = n;
continue;
}
LIST_REMOVE(slab, us_link);
keg->uk_pages -= keg->uk_ppera;
keg->uk_free -= keg->uk_ipers;
if (keg->uk_flags & UMA_ZONE_HASH)
UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data);
SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink);
slab = n;
}
finished:
KEG_UNLOCK(keg);
while ((slab = SLIST_FIRST(&freeslabs)) != NULL) {
SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink);
keg_free_slab(keg, slab, keg->uk_ipers);
}
}
static void
zone_drain_wait(uma_zone_t zone, int waitok)
{
/*
* Set draining to interlock with zone_dtor() so we can release our
* locks as we go. Only dtor() should do a WAITOK call since it
* is the only call that knows the structure will still be available
* when it wakes up.
*/
ZONE_LOCK(zone);
while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
if (waitok == M_NOWAIT)
goto out;
msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1);
}
zone->uz_flags |= UMA_ZFLAG_DRAINING;
bucket_cache_drain(zone);
ZONE_UNLOCK(zone);
/*
* The DRAINING flag protects us from being freed while
* we're running. Normally the uma_rwlock would protect us but we
* must be able to release and acquire the right lock for each keg.
*/
zone_foreach_keg(zone, &keg_drain);
ZONE_LOCK(zone);
zone->uz_flags &= ~UMA_ZFLAG_DRAINING;
wakeup(zone);
out:
ZONE_UNLOCK(zone);
}
void
zone_drain(uma_zone_t zone)
{
zone_drain_wait(zone, M_NOWAIT);
}
/*
* Allocate a new slab for a keg. This does not insert the slab onto a list.
*
* Arguments:
* wait Shall we wait?
*
* Returns:
* The slab that was allocated or NULL if there is no memory and the
* caller specified M_NOWAIT.
*/
static uma_slab_t
keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
{
uma_slabrefcnt_t slabref;
uma_alloc allocf;
uma_slab_t slab;
uint8_t *mem;
uint8_t flags;
int i;
mtx_assert(&keg->uk_lock, MA_OWNED);
slab = NULL;
mem = NULL;
#ifdef UMA_DEBUG
printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name);
#endif
allocf = keg->uk_allocf;
KEG_UNLOCK(keg);
if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
if (slab == NULL)
goto out;
}
/*
* This reproduces the old vm_zone behavior of zero filling pages the
* first time they are added to a zone.
*
* Malloced items are zeroed in uma_zalloc.
*/
if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
wait |= M_ZERO;
else
wait &= ~M_ZERO;
if (keg->uk_flags & UMA_ZONE_NODUMP)
wait |= M_NODUMP;
/* zone is passed for legacy reasons. */
mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
if (mem == NULL) {
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
slab = NULL;
goto out;
}
/* Point the slab into the allocated memory */
if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
slab = (uma_slab_t )(mem + keg->uk_pgoff);
if (keg->uk_flags & UMA_ZONE_VTOSLAB)
for (i = 0; i < keg->uk_ppera; i++)
vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab);
slab->us_keg = keg;
slab->us_data = mem;
slab->us_freecount = keg->uk_ipers;
slab->us_flags = flags;
BIT_FILL(SLAB_SETSIZE, &slab->us_free);
#ifdef INVARIANTS
BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree);
#endif
if (keg->uk_flags & UMA_ZONE_REFCNT) {
slabref = (uma_slabrefcnt_t)slab;
for (i = 0; i < keg->uk_ipers; i++)
slabref->us_refcnt[i] = 0;
}
if (keg->uk_init != NULL) {
for (i = 0; i < keg->uk_ipers; i++)
if (keg->uk_init(slab->us_data + (keg->uk_rsize * i),
keg->uk_size, wait) != 0)
break;
if (i != keg->uk_ipers) {
keg_free_slab(keg, slab, i);
slab = NULL;
goto out;
}
}
out:
KEG_LOCK(keg);
if (slab != NULL) {
if (keg->uk_flags & UMA_ZONE_HASH)
UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
keg->uk_pages += keg->uk_ppera;
keg->uk_free += keg->uk_ipers;
}
return (slab);
}
/*
* This function is intended to be used early on in place of page_alloc() so
* that we may use the boot time page cache to satisfy allocations before
* the VM is ready.
*/
static void *
startup_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
{
uma_keg_t keg;
uma_slab_t tmps;
int pages, check_pages;
keg = zone_first_keg(zone);
pages = howmany(bytes, PAGE_SIZE);
check_pages = pages - 1;
KASSERT(pages > 0, ("startup_alloc can't reserve 0 pages\n"));
/*
* Check our small startup cache to see if it has pages remaining.
*/
mtx_lock(&uma_boot_pages_mtx);
/* First check if we have enough room. */
tmps = LIST_FIRST(&uma_boot_pages);
while (tmps != NULL && check_pages-- > 0)
tmps = LIST_NEXT(tmps, us_link);
if (tmps != NULL) {
/*
* It's ok to lose tmps references. The last one will
* have tmps->us_data pointing to the start address of
* "pages" contiguous pages of memory.
*/
while (pages-- > 0) {
tmps = LIST_FIRST(&uma_boot_pages);
LIST_REMOVE(tmps, us_link);
}
mtx_unlock(&uma_boot_pages_mtx);
*pflag = tmps->us_flags;
return (tmps->us_data);
}
mtx_unlock(&uma_boot_pages_mtx);
if (booted < UMA_STARTUP2)
panic("UMA: Increase vm.boot_pages");
/*
* Now that we've booted reset these users to their real allocator.
*/
#ifdef UMA_MD_SMALL_ALLOC
keg->uk_allocf = (keg->uk_ppera > 1) ? page_alloc : uma_small_alloc;
#else
keg->uk_allocf = page_alloc;
#endif
return keg->uk_allocf(zone, bytes, pflag, wait);
}
/*
* Allocates a number of pages from the system
*
* Arguments:
* bytes The number of bytes requested
* wait Shall we wait?
*
* Returns:
* A pointer to the alloced memory or possibly
* NULL if M_NOWAIT is set.
*/
static void *
page_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
{
void *p; /* Returned page */
*pflag = UMA_SLAB_KMEM;
p = (void *) kmem_malloc(kmem_arena, bytes, wait);
return (p);
}
/*
* Allocates a number of pages from within an object
*
* Arguments:
* bytes The number of bytes requested
* wait Shall we wait?
*
* Returns:
* A pointer to the alloced memory or possibly
* NULL if M_NOWAIT is set.
*/
static void *
noobj_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
{
TAILQ_HEAD(, vm_page) alloctail;
u_long npages;
vm_offset_t retkva, zkva;
vm_page_t p, p_next;
uma_keg_t keg;
TAILQ_INIT(&alloctail);
keg = zone_first_keg(zone);
npages = howmany(bytes, PAGE_SIZE);
while (npages > 0) {
p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
if (p != NULL) {
/*
* Since the page does not belong to an object, its
* listq is unused.
*/
TAILQ_INSERT_TAIL(&alloctail, p, listq);
npages--;
continue;
}
if (wait & M_WAITOK) {
VM_WAIT;
continue;
}
/*
* Page allocation failed, free intermediate pages and
* exit.
*/
TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
vm_page_unwire(p, PQ_INACTIVE);
vm_page_free(p);
}
return (NULL);
}
*flags = UMA_SLAB_PRIV;
zkva = keg->uk_kva +
atomic_fetchadd_long(&keg->uk_offset, round_page(bytes));
retkva = zkva;
TAILQ_FOREACH(p, &alloctail, listq) {
pmap_qenter(zkva, &p, 1);
zkva += PAGE_SIZE;
}
return ((void *)retkva);
}
/*
* Frees a number of pages to the system
*
* Arguments:
* mem A pointer to the memory to be freed
* size The size of the memory being freed
* flags The original p->us_flags field
*
* Returns:
* Nothing
*/
static void
page_free(void *mem, vm_size_t size, uint8_t flags)
{
struct vmem *vmem;
if (flags & UMA_SLAB_KMEM)
vmem = kmem_arena;
else if (flags & UMA_SLAB_KERNEL)
vmem = kernel_arena;
else
panic("UMA: page_free used with invalid flags %d", flags);
kmem_free(vmem, (vm_offset_t)mem, size);
}
/*
* Zero fill initializer
*
* Arguments/Returns follow uma_init specifications
*/
static int
zero_init(void *mem, int size, int flags)
{
bzero(mem, size);
return (0);
}
/*
* Finish creating a small uma keg. This calculates ipers, and the keg size.
*
* Arguments
* keg The zone we should initialize
*
* Returns
* Nothing
*/
static void
keg_small_init(uma_keg_t keg)
{
u_int rsize;
u_int memused;
u_int wastedspace;
u_int shsize;
if (keg->uk_flags & UMA_ZONE_PCPU) {
u_int ncpus = mp_ncpus ? mp_ncpus : MAXCPU;
keg->uk_slabsize = sizeof(struct pcpu);
keg->uk_ppera = howmany(ncpus * sizeof(struct pcpu),
PAGE_SIZE);
} else {
keg->uk_slabsize = UMA_SLAB_SIZE;
keg->uk_ppera = 1;
}
/*
* Calculate the size of each allocation (rsize) according to
* alignment. If the requested size is smaller than we have
* allocation bits for we round it up.
*/
rsize = keg->uk_size;
if (rsize < keg->uk_slabsize / SLAB_SETSIZE)
rsize = keg->uk_slabsize / SLAB_SETSIZE;
if (rsize & keg->uk_align)
rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
keg->uk_rsize = rsize;
KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
keg->uk_rsize < sizeof(struct pcpu),
("%s: size %u too large", __func__, keg->uk_rsize));
if (keg->uk_flags & UMA_ZONE_REFCNT)
rsize += sizeof(uint32_t);
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
shsize = 0;
else
shsize = sizeof(struct uma_slab);
keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize;
KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
memused = keg->uk_ipers * rsize + shsize;
wastedspace = keg->uk_slabsize - memused;
/*
* We can't do OFFPAGE if we're internal or if we've been
* asked to not go to the VM for buckets. If we do this we
* may end up going to the VM for slabs which we do not
* want to do if we're UMA_ZFLAG_CACHEONLY as a result
* of UMA_ZONE_VM, which clearly forbids it.
*/
if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
(keg->uk_flags & UMA_ZFLAG_CACHEONLY))
return;
/*
* See if using an OFFPAGE slab will limit our waste. Only do
* this if it permits more items per-slab.
*
* XXX We could try growing slabsize to limit max waste as well.
* Historically this was not done because the VM could not
* efficiently handle contiguous allocations.
*/
if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) &&
(keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) {
keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize;
KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
#ifdef UMA_DEBUG
printf("UMA decided we need offpage slab headers for "
"keg: %s, calculated wastedspace = %d, "
"maximum wasted space allowed = %d, "
"calculated ipers = %d, "
"new wasted space = %d\n", keg->uk_name, wastedspace,
keg->uk_slabsize / UMA_MAX_WASTE, keg->uk_ipers,
keg->uk_slabsize - keg->uk_ipers * keg->uk_rsize);
#endif
keg->uk_flags |= UMA_ZONE_OFFPAGE;
}
if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
(keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
keg->uk_flags |= UMA_ZONE_HASH;
}
/*
* Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do
* OFFPAGE for now. When I can allow for more dynamic slab sizes this will be
* more complicated.
*
* Arguments
* keg The keg we should initialize
*
* Returns
* Nothing
*/
static void
keg_large_init(uma_keg_t keg)
{
u_int shsize;
KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__));
keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE);
keg->uk_slabsize = keg->uk_ppera * PAGE_SIZE;
keg->uk_ipers = 1;
keg->uk_rsize = keg->uk_size;
/* We can't do OFFPAGE if we're internal, bail out here. */
if (keg->uk_flags & UMA_ZFLAG_INTERNAL)
return;
/* Check whether we have enough space to not do OFFPAGE. */
if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0) {
shsize = sizeof(struct uma_slab);
if (keg->uk_flags & UMA_ZONE_REFCNT)
shsize += keg->uk_ipers * sizeof(uint32_t);
if (shsize & UMA_ALIGN_PTR)
shsize = (shsize & ~UMA_ALIGN_PTR) +
(UMA_ALIGN_PTR + 1);
if ((PAGE_SIZE * keg->uk_ppera) - keg->uk_rsize < shsize)
keg->uk_flags |= UMA_ZONE_OFFPAGE;
}
if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
(keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
keg->uk_flags |= UMA_ZONE_HASH;
}
static void
keg_cachespread_init(uma_keg_t keg)
{
int alignsize;
int trailer;
int pages;
int rsize;
KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__));
alignsize = keg->uk_align + 1;
rsize = keg->uk_size;
/*
* We want one item to start on every align boundary in a page. To
* do this we will span pages. We will also extend the item by the
* size of align if it is an even multiple of align. Otherwise, it
* would fall on the same boundary every time.
*/
if (rsize & keg->uk_align)
rsize = (rsize & ~keg->uk_align) + alignsize;
if ((rsize & alignsize) == 0)
rsize += alignsize;
trailer = rsize - keg->uk_size;
pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
keg->uk_rsize = rsize;
keg->uk_ppera = pages;
keg->uk_slabsize = UMA_SLAB_SIZE;
keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
KASSERT(keg->uk_ipers <= SLAB_SETSIZE,
("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
keg->uk_ipers));
}
/*
* Keg header ctor. This initializes all fields, locks, etc. And inserts
* the keg onto the global keg list.
*
* Arguments/Returns follow uma_ctor specifications
* udata Actually uma_kctor_args
*/
static int
keg_ctor(void *mem, int size, void *udata, int flags)
{
struct uma_kctor_args *arg = udata;
uma_keg_t keg = mem;
uma_zone_t zone;
bzero(keg, size);
keg->uk_size = arg->size;
keg->uk_init = arg->uminit;
keg->uk_fini = arg->fini;
keg->uk_align = arg->align;
keg->uk_free = 0;
keg->uk_reserve = 0;
keg->uk_pages = 0;
keg->uk_flags = arg->flags;
keg->uk_allocf = page_alloc;
keg->uk_freef = page_free;
keg->uk_slabzone = NULL;
/*
* The master zone is passed to us at keg-creation time.
*/
zone = arg->zone;
keg->uk_name = zone->uz_name;
if (arg->flags & UMA_ZONE_VM)
keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
if (arg->flags & UMA_ZONE_ZINIT)
keg->uk_init = zero_init;
if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC)
keg->uk_flags |= UMA_ZONE_VTOSLAB;
if (arg->flags & UMA_ZONE_PCPU)
#ifdef SMP
keg->uk_flags |= UMA_ZONE_OFFPAGE;
#else
keg->uk_flags &= ~UMA_ZONE_PCPU;
#endif
if (keg->uk_flags & UMA_ZONE_CACHESPREAD) {
keg_cachespread_init(keg);
} else if (keg->uk_flags & UMA_ZONE_REFCNT) {
if (keg->uk_size >
(UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) -
sizeof(uint32_t)))
keg_large_init(keg);
else
keg_small_init(keg);
} else {
if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab)))
keg_large_init(keg);
else
keg_small_init(keg);
}
if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
if (keg->uk_flags & UMA_ZONE_REFCNT) {
if (keg->uk_ipers > uma_max_ipers_ref)
panic("Too many ref items per zone: %d > %d\n",
keg->uk_ipers, uma_max_ipers_ref);
keg->uk_slabzone = slabrefzone;
} else
keg->uk_slabzone = slabzone;
}
/*
* If we haven't booted yet we need allocations to go through the
* startup cache until the vm is ready.
*/
if (keg->uk_ppera == 1) {
#ifdef UMA_MD_SMALL_ALLOC
keg->uk_allocf = uma_small_alloc;
keg->uk_freef = uma_small_free;
if (booted < UMA_STARTUP)
keg->uk_allocf = startup_alloc;
#else
if (booted < UMA_STARTUP2)
keg->uk_allocf = startup_alloc;
#endif
} else if (booted < UMA_STARTUP2 &&
(keg->uk_flags & UMA_ZFLAG_INTERNAL))
keg->uk_allocf = startup_alloc;
/*
* Initialize keg's lock
*/
KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS));
/*
* If we're putting the slab header in the actual page we need to
* figure out where in each page it goes. This calculates a right
* justified offset into the memory on an ALIGN_PTR boundary.
*/
if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) {
u_int totsize;
/* Size of the slab struct and free list */
totsize = sizeof(struct uma_slab);
/* Size of the reference counts. */
if (keg->uk_flags & UMA_ZONE_REFCNT)
totsize += keg->uk_ipers * sizeof(uint32_t);
if (totsize & UMA_ALIGN_PTR)
totsize = (totsize & ~UMA_ALIGN_PTR) +
(UMA_ALIGN_PTR + 1);
keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize;
/*
* The only way the following is possible is if with our
* UMA_ALIGN_PTR adjustments we are now bigger than
* UMA_SLAB_SIZE. I haven't checked whether this is
* mathematically possible for all cases, so we make
* sure here anyway.
*/
totsize = keg->uk_pgoff + sizeof(struct uma_slab);
if (keg->uk_flags & UMA_ZONE_REFCNT)
totsize += keg->uk_ipers * sizeof(uint32_t);
if (totsize > PAGE_SIZE * keg->uk_ppera) {
printf("zone %s ipers %d rsize %d size %d\n",
zone->uz_name, keg->uk_ipers, keg->uk_rsize,
keg->uk_size);
panic("UMA slab won't fit.");
}
}
if (keg->uk_flags & UMA_ZONE_HASH)
hash_alloc(&keg->uk_hash);
#ifdef UMA_DEBUG
printf("UMA: %s(%p) size %d(%d) flags %#x ipers %d ppera %d out %d free %d\n",
zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags,
keg->uk_ipers, keg->uk_ppera,
(keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free);
#endif
LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
rw_wlock(&uma_rwlock);
LIST_INSERT_HEAD(&uma_kegs, keg, uk_link);
rw_wunlock(&uma_rwlock);
return (0);
}
/*
* Zone header ctor. This initializes all fields, locks, etc.
*
* Arguments/Returns follow uma_ctor specifications
* udata Actually uma_zctor_args
*/
static int
zone_ctor(void *mem, int size, void *udata, int flags)
{
struct uma_zctor_args *arg = udata;
uma_zone_t zone = mem;
uma_zone_t z;
uma_keg_t keg;
bzero(zone, size);
zone->uz_name = arg->name;
zone->uz_ctor = arg->ctor;
zone->uz_dtor = arg->dtor;
zone->uz_slab = zone_fetch_slab;
zone->uz_init = NULL;
zone->uz_fini = NULL;
zone->uz_allocs = 0;
zone->uz_frees = 0;
zone->uz_fails = 0;
zone->uz_sleeps = 0;
zone->uz_count = 0;
zone->uz_count_min = 0;
zone->uz_flags = 0;
zone->uz_warning = NULL;
timevalclear(&zone->uz_ratecheck);
keg = arg->keg;
ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
/*
* This is a pure cache zone, no kegs.
*/
if (arg->import) {
if (arg->flags & UMA_ZONE_VM)
arg->flags |= UMA_ZFLAG_CACHEONLY;
zone->uz_flags = arg->flags;
zone->uz_size = arg->size;
zone->uz_import = arg->import;
zone->uz_release = arg->release;
zone->uz_arg = arg->arg;
zone->uz_lockptr = &zone->uz_lock;
rw_wlock(&uma_rwlock);
LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link);
rw_wunlock(&uma_rwlock);
goto out;
}
/*
* Use the regular zone/keg/slab allocator.
*/
zone->uz_import = (uma_import)zone_import;
zone->uz_release = (uma_release)zone_release;
zone->uz_arg = zone;
if (arg->flags & UMA_ZONE_SECONDARY) {
KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
zone->uz_init = arg->uminit;
zone->uz_fini = arg->fini;
zone->uz_lockptr = &keg->uk_lock;
zone->uz_flags |= UMA_ZONE_SECONDARY;
rw_wlock(&uma_rwlock);
ZONE_LOCK(zone);
LIST_FOREACH(z, &keg->uk_zones, uz_link) {
if (LIST_NEXT(z, uz_link) == NULL) {
LIST_INSERT_AFTER(z, zone, uz_link);
break;
}
}
ZONE_UNLOCK(zone);
rw_wunlock(&uma_rwlock);
} else if (keg == NULL) {
if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini,
arg->align, arg->flags)) == NULL)
return (ENOMEM);
} else {
struct uma_kctor_args karg;
int error;
/* We should only be here from uma_startup() */
karg.size = arg->size;
karg.uminit = arg->uminit;
karg.fini = arg->fini;
karg.align = arg->align;
karg.flags = arg->flags;
karg.zone = zone;
error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
flags);
if (error)
return (error);
}
/*
* Link in the first keg.
*/
zone->uz_klink.kl_keg = keg;
LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
zone->uz_lockptr = &keg->uk_lock;
zone->uz_size = keg->uk_size;
zone->uz_flags |= (keg->uk_flags &
(UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
/*
* Some internal zones don't have room allocated for the per cpu
* caches. If we're internal, bail out here.
*/
if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
("Secondary zone requested UMA_ZFLAG_INTERNAL"));
return (0);
}
out:
if ((arg->flags & UMA_ZONE_MAXBUCKET) == 0)
zone->uz_count = bucket_select(zone->uz_size);
else
zone->uz_count = BUCKET_MAX;
zone->uz_count_min = zone->uz_count;
return (0);
}
/*
* Keg header dtor. This frees all data, destroys locks, frees the hash
* table and removes the keg from the global list.
*
* Arguments/Returns follow uma_dtor specifications
* udata unused
*/
static void
keg_dtor(void *arg, int size, void *udata)
{
uma_keg_t keg;
keg = (uma_keg_t)arg;
KEG_LOCK(keg);
if (keg->uk_free != 0) {
printf("Freed UMA keg (%s) was not empty (%d items). "
" Lost %d pages of memory.\n",
keg->uk_name ? keg->uk_name : "",
keg->uk_free, keg->uk_pages);
}
KEG_UNLOCK(keg);
hash_free(&keg->uk_hash);
KEG_LOCK_FINI(keg);
}
/*
* Zone header dtor.
*
* Arguments/Returns follow uma_dtor specifications
* udata unused
*/
static void
zone_dtor(void *arg, int size, void *udata)
{
uma_klink_t klink;
uma_zone_t zone;
uma_keg_t keg;
zone = (uma_zone_t)arg;
keg = zone_first_keg(zone);
if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
cache_drain(zone);
rw_wlock(&uma_rwlock);
LIST_REMOVE(zone, uz_link);
rw_wunlock(&uma_rwlock);
/*
* XXX there are some races here where
* the zone can be drained but zone lock
* released and then refilled before we
* remove it... we dont care for now
*/
zone_drain_wait(zone, M_WAITOK);
/*
* Unlink all of our kegs.
*/
while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) {
klink->kl_keg = NULL;
LIST_REMOVE(klink, kl_link);
if (klink == &zone->uz_klink)
continue;
free(klink, M_TEMP);
}
/*
* We only destroy kegs from non secondary zones.
*/
if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
rw_wlock(&uma_rwlock);
LIST_REMOVE(keg, uk_link);
rw_wunlock(&uma_rwlock);
zone_free_item(kegs, keg, NULL, SKIP_NONE);
}
ZONE_LOCK_FINI(zone);
}
/*
* Traverses every zone in the system and calls a callback
*
* Arguments:
* zfunc A pointer to a function which accepts a zone
* as an argument.
*
* Returns:
* Nothing
*/
static void
zone_foreach(void (*zfunc)(uma_zone_t))
{
uma_keg_t keg;
uma_zone_t zone;
rw_rlock(&uma_rwlock);
LIST_FOREACH(keg, &uma_kegs, uk_link) {
LIST_FOREACH(zone, &keg->uk_zones, uz_link)
zfunc(zone);
}
rw_runlock(&uma_rwlock);
}
/* Public functions */
/* See uma.h */
void
uma_startup(void *bootmem, int boot_pages)
{
struct uma_zctor_args args;
uma_slab_t slab;
u_int slabsize;
int i;
#ifdef UMA_DEBUG
printf("Creating uma keg headers zone and keg.\n");
#endif
rw_init(&uma_rwlock, "UMA lock");
/* "manually" create the initial zone */
memset(&args, 0, sizeof(args));
args.name = "UMA Kegs";
args.size = sizeof(struct uma_keg);
args.ctor = keg_ctor;
args.dtor = keg_dtor;
args.uminit = zero_init;
args.fini = NULL;
args.keg = &masterkeg;
args.align = 32 - 1;
args.flags = UMA_ZFLAG_INTERNAL;
/* The initial zone has no Per cpu queues so it's smaller */
zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK);
#ifdef UMA_DEBUG
printf("Filling boot free list.\n");
#endif
for (i = 0; i < boot_pages; i++) {
slab = (uma_slab_t)((uint8_t *)bootmem + (i * UMA_SLAB_SIZE));
slab->us_data = (uint8_t *)slab;
slab->us_flags = UMA_SLAB_BOOT;
LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link);
}
mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF);
#ifdef UMA_DEBUG
printf("Creating uma zone headers zone and keg.\n");
#endif
args.name = "UMA Zones";
args.size = sizeof(struct uma_zone) +
(sizeof(struct uma_cache) * (mp_maxid + 1));
args.ctor = zone_ctor;
args.dtor = zone_dtor;
args.uminit = zero_init;
args.fini = NULL;
args.keg = NULL;
args.align = 32 - 1;
args.flags = UMA_ZFLAG_INTERNAL;
/* The initial zone has no Per cpu queues so it's smaller */
zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK);
#ifdef UMA_DEBUG
printf("Creating slab and hash zones.\n");
#endif
/* Now make a zone for slab headers */
slabzone = uma_zcreate("UMA Slabs",
sizeof(struct uma_slab),
NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
/*
* We also create a zone for the bigger slabs with reference
* counts in them, to accomodate UMA_ZONE_REFCNT zones.
*/
slabsize = sizeof(struct uma_slab_refcnt);
slabsize += uma_max_ipers_ref * sizeof(uint32_t);
slabrefzone = uma_zcreate("UMA RCntSlabs",
slabsize,
NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR,
UMA_ZFLAG_INTERNAL);
hashzone = uma_zcreate("UMA Hash",
sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT,
NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
bucket_init();
booted = UMA_STARTUP;
#ifdef UMA_DEBUG
printf("UMA startup complete.\n");
#endif
}
/* see uma.h */
void
uma_startup2(void)
{
booted = UMA_STARTUP2;
bucket_enable();
sx_init(&uma_drain_lock, "umadrain");
#ifdef UMA_DEBUG
printf("UMA startup2 complete.\n");
#endif
}
/*
* Initialize our callout handle
*
*/
static void
uma_startup3(void)
{
#ifdef UMA_DEBUG
printf("Starting callout.\n");
#endif
- callout_init(&uma_callout, CALLOUT_MPSAFE);
+ callout_init(&uma_callout, 1);
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
#ifdef UMA_DEBUG
printf("UMA startup3 complete.\n");
#endif
}
static uma_keg_t
uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
int align, uint32_t flags)
{
struct uma_kctor_args args;
args.size = size;
args.uminit = uminit;
args.fini = fini;
args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
args.flags = flags;
args.zone = zone;
return (zone_alloc_item(kegs, &args, M_WAITOK));
}
/* See uma.h */
void
uma_set_align(int align)
{
if (align != UMA_ALIGN_CACHE)
uma_align_cache = align;
}
/* See uma.h */
uma_zone_t
uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
uma_init uminit, uma_fini fini, int align, uint32_t flags)
{
struct uma_zctor_args args;
uma_zone_t res;
bool locked;
/* This stuff is essential for the zone ctor */
memset(&args, 0, sizeof(args));
args.name = name;
args.size = size;
args.ctor = ctor;
args.dtor = dtor;
args.uminit = uminit;
args.fini = fini;
args.align = align;
args.flags = flags;
args.keg = NULL;
if (booted < UMA_STARTUP2) {
locked = false;
} else {
sx_slock(&uma_drain_lock);
locked = true;
}
res = zone_alloc_item(zones, &args, M_WAITOK);
if (locked)
sx_sunlock(&uma_drain_lock);
return (res);
}
/* See uma.h */
uma_zone_t
uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
uma_init zinit, uma_fini zfini, uma_zone_t master)
{
struct uma_zctor_args args;
uma_keg_t keg;
uma_zone_t res;
bool locked;
keg = zone_first_keg(master);
memset(&args, 0, sizeof(args));
args.name = name;
args.size = keg->uk_size;
args.ctor = ctor;
args.dtor = dtor;
args.uminit = zinit;
args.fini = zfini;
args.align = keg->uk_align;
args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
args.keg = keg;
if (booted < UMA_STARTUP2) {
locked = false;
} else {
sx_slock(&uma_drain_lock);
locked = true;
}
/* XXX Attaches only one keg of potentially many. */
res = zone_alloc_item(zones, &args, M_WAITOK);
if (locked)
sx_sunlock(&uma_drain_lock);
return (res);
}
/* See uma.h */
uma_zone_t
uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor,
uma_init zinit, uma_fini zfini, uma_import zimport,
uma_release zrelease, void *arg, int flags)
{
struct uma_zctor_args args;
memset(&args, 0, sizeof(args));
args.name = name;
args.size = size;
args.ctor = ctor;
args.dtor = dtor;
args.uminit = zinit;
args.fini = zfini;
args.import = zimport;
args.release = zrelease;
args.arg = arg;
args.align = 0;
args.flags = flags;
return (zone_alloc_item(zones, &args, M_WAITOK));
}
static void
zone_lock_pair(uma_zone_t a, uma_zone_t b)
{
if (a < b) {
ZONE_LOCK(a);
mtx_lock_flags(b->uz_lockptr, MTX_DUPOK);
} else {
ZONE_LOCK(b);
mtx_lock_flags(a->uz_lockptr, MTX_DUPOK);
}
}
static void
zone_unlock_pair(uma_zone_t a, uma_zone_t b)
{
ZONE_UNLOCK(a);
ZONE_UNLOCK(b);
}
int
uma_zsecond_add(uma_zone_t zone, uma_zone_t master)
{
uma_klink_t klink;
uma_klink_t kl;
int error;
error = 0;
klink = malloc(sizeof(*klink), M_TEMP, M_WAITOK | M_ZERO);
zone_lock_pair(zone, master);
/*
* zone must use vtoslab() to resolve objects and must already be
* a secondary.
*/
if ((zone->uz_flags & (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY))
!= (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) {
error = EINVAL;
goto out;
}
/*
* The new master must also use vtoslab().
*/
if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) {
error = EINVAL;
goto out;
}
/*
* Both must either be refcnt, or not be refcnt.
*/
if ((zone->uz_flags & UMA_ZONE_REFCNT) !=
(master->uz_flags & UMA_ZONE_REFCNT)) {
error = EINVAL;
goto out;
}
/*
* The underlying object must be the same size. rsize
* may be different.
*/
if (master->uz_size != zone->uz_size) {
error = E2BIG;
goto out;
}
/*
* Put it at the end of the list.
*/
klink->kl_keg = zone_first_keg(master);
LIST_FOREACH(kl, &zone->uz_kegs, kl_link) {
if (LIST_NEXT(kl, kl_link) == NULL) {
LIST_INSERT_AFTER(kl, klink, kl_link);
break;
}
}
klink = NULL;
zone->uz_flags |= UMA_ZFLAG_MULTI;
zone->uz_slab = zone_fetch_slab_multi;
out:
zone_unlock_pair(zone, master);
if (klink != NULL)
free(klink, M_TEMP);
return (error);
}
/* See uma.h */
void
uma_zdestroy(uma_zone_t zone)
{
sx_slock(&uma_drain_lock);
zone_free_item(zones, zone, NULL, SKIP_NONE);
sx_sunlock(&uma_drain_lock);
}
/* See uma.h */
void *
uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
{
void *item;
uma_cache_t cache;
uma_bucket_t bucket;
int lockfail;
int cpu;
#if 0
/* XXX: FIX!! Do not enable this in CURRENT!! MarkM */
/* The entropy here is desirable, but the harvesting is expensive */
random_harvest(&(zone->uz_name), sizeof(void *), 1, RANDOM_UMA_ALLOC);
#endif
/* This is the fast path allocation */
#ifdef UMA_DEBUG_ALLOC_1
printf("Allocating one item from %s(%p)\n", zone->uz_name, zone);
#endif
CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread,
zone->uz_name, flags);
if (flags & M_WAITOK) {
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
"uma_zalloc_arg: zone \"%s\"", zone->uz_name);
}
#ifdef DEBUG_MEMGUARD
if (memguard_cmp_zone(zone)) {
item = memguard_alloc(zone->uz_size, flags);
if (item != NULL) {
/*
* Avoid conflict with the use-after-free
* protecting infrastructure from INVARIANTS.
*/
if (zone->uz_init != NULL &&
zone->uz_init != mtrash_init &&
zone->uz_init(item, zone->uz_size, flags) != 0)
return (NULL);
if (zone->uz_ctor != NULL &&
zone->uz_ctor != mtrash_ctor &&
zone->uz_ctor(item, zone->uz_size, udata,
flags) != 0) {
zone->uz_fini(item, zone->uz_size);
return (NULL);
}
#if 0
/* XXX: FIX!! Do not enable this in CURRENT!! MarkM */
/* The entropy here is desirable, but the harvesting is expensive */
random_harvest(&item, sizeof(void *), 1, RANDOM_UMA_ALLOC);
#endif
return (item);
}
/* This is unfortunate but should not be fatal. */
}
#endif
/*
* If possible, allocate from the per-CPU cache. There are two
* requirements for safe access to the per-CPU cache: (1) the thread
* accessing the cache must not be preempted or yield during access,
* and (2) the thread must not migrate CPUs without switching which
* cache it accesses. We rely on a critical section to prevent
* preemption and migration. We release the critical section in
* order to acquire the zone mutex if we are unable to allocate from
* the current cache; when we re-acquire the critical section, we
* must detect and handle migration if it has occurred.
*/
critical_enter();
cpu = curcpu;
cache = &zone->uz_cpu[cpu];
zalloc_start:
bucket = cache->uc_allocbucket;
if (bucket != NULL && bucket->ub_cnt > 0) {
bucket->ub_cnt--;
item = bucket->ub_bucket[bucket->ub_cnt];
#ifdef INVARIANTS
bucket->ub_bucket[bucket->ub_cnt] = NULL;
#endif
KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled."));
cache->uc_allocs++;
critical_exit();
if (zone->uz_ctor != NULL &&
zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
atomic_add_long(&zone->uz_fails, 1);
zone_free_item(zone, item, udata, SKIP_DTOR);
return (NULL);
}
#ifdef INVARIANTS
uma_dbg_alloc(zone, NULL, item);
#endif
if (flags & M_ZERO)
uma_zero_item(item, zone);
#if 0
/* XXX: FIX!! Do not enable this in CURRENT!! MarkM */
/* The entropy here is desirable, but the harvesting is expensive */
random_harvest(&item, sizeof(void *), 1, RANDOM_UMA_ALLOC);
#endif
return (item);
}
/*
* We have run out of items in our alloc bucket.
* See if we can switch with our free bucket.
*/
bucket = cache->uc_freebucket;
if (bucket != NULL && bucket->ub_cnt > 0) {
#ifdef UMA_DEBUG_ALLOC
printf("uma_zalloc: Swapping empty with alloc.\n");
#endif
cache->uc_freebucket = cache->uc_allocbucket;
cache->uc_allocbucket = bucket;
goto zalloc_start;
}
/*
* Discard any empty allocation bucket while we hold no locks.
*/
bucket = cache->uc_allocbucket;
cache->uc_allocbucket = NULL;
critical_exit();
if (bucket != NULL)
bucket_free(zone, bucket, udata);
/* Short-circuit for zones without buckets and low memory. */
if (zone->uz_count == 0 || bucketdisable)
goto zalloc_item;
/*
* Attempt to retrieve the item from the per-CPU cache has failed, so
* we must go back to the zone. This requires the zone lock, so we
* must drop the critical section, then re-acquire it when we go back
* to the cache. Since the critical section is released, we may be
* preempted or migrate. As such, make sure not to maintain any
* thread-local state specific to the cache from prior to releasing
* the critical section.
*/
lockfail = 0;
if (ZONE_TRYLOCK(zone) == 0) {
/* Record contention to size the buckets. */
ZONE_LOCK(zone);
lockfail = 1;
}
critical_enter();
cpu = curcpu;
cache = &zone->uz_cpu[cpu];
/*
* Since we have locked the zone we may as well send back our stats.
*/
atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
atomic_add_long(&zone->uz_frees, cache->uc_frees);
cache->uc_allocs = 0;
cache->uc_frees = 0;
/* See if we lost the race to fill the cache. */
if (cache->uc_allocbucket != NULL) {
ZONE_UNLOCK(zone);
goto zalloc_start;
}
/*
* Check the zone's cache of buckets.
*/
if ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
KASSERT(bucket->ub_cnt != 0,
("uma_zalloc_arg: Returning an empty bucket."));
LIST_REMOVE(bucket, ub_link);
cache->uc_allocbucket = bucket;
ZONE_UNLOCK(zone);
goto zalloc_start;
}
/* We are no longer associated with this CPU. */
critical_exit();
/*
* We bump the uz count when the cache size is insufficient to
* handle the working set.
*/
if (lockfail && zone->uz_count < BUCKET_MAX)
zone->uz_count++;
ZONE_UNLOCK(zone);
/*
* Now lets just fill a bucket and put it on the free list. If that
* works we'll restart the allocation from the begining and it
* will use the just filled bucket.
*/
bucket = zone_alloc_bucket(zone, udata, flags);
if (bucket != NULL) {
ZONE_LOCK(zone);
critical_enter();
cpu = curcpu;
cache = &zone->uz_cpu[cpu];
/*
* See if we lost the race or were migrated. Cache the
* initialized bucket to make this less likely or claim
* the memory directly.
*/
if (cache->uc_allocbucket == NULL)
cache->uc_allocbucket = bucket;
else
LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link);
ZONE_UNLOCK(zone);
goto zalloc_start;
}
/*
* We may not be able to get a bucket so return an actual item.
*/
#ifdef UMA_DEBUG
printf("uma_zalloc_arg: Bucketzone returned NULL\n");
#endif
zalloc_item:
item = zone_alloc_item(zone, udata, flags);
#if 0
/* XXX: FIX!! Do not enable this in CURRENT!! MarkM */
/* The entropy here is desirable, but the harvesting is expensive */
random_harvest(&item, sizeof(void *), 1, RANDOM_UMA_ALLOC);
#endif
return (item);
}
static uma_slab_t
keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags)
{
uma_slab_t slab;
int reserve;
mtx_assert(&keg->uk_lock, MA_OWNED);
slab = NULL;
reserve = 0;
if ((flags & M_USE_RESERVE) == 0)
reserve = keg->uk_reserve;
for (;;) {
/*
* Find a slab with some space. Prefer slabs that are partially
* used over those that are totally full. This helps to reduce
* fragmentation.
*/
if (keg->uk_free > reserve) {
if (!LIST_EMPTY(&keg->uk_part_slab)) {
slab = LIST_FIRST(&keg->uk_part_slab);
} else {
slab = LIST_FIRST(&keg->uk_free_slab);
LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&keg->uk_part_slab, slab,
us_link);
}
MPASS(slab->us_keg == keg);
return (slab);
}
/*
* M_NOVM means don't ask at all!
*/
if (flags & M_NOVM)
break;
if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) {
keg->uk_flags |= UMA_ZFLAG_FULL;
/*
* If this is not a multi-zone, set the FULL bit.
* Otherwise slab_multi() takes care of it.
*/
if ((zone->uz_flags & UMA_ZFLAG_MULTI) == 0) {
zone->uz_flags |= UMA_ZFLAG_FULL;
zone_log_warning(zone);
}
if (flags & M_NOWAIT)
break;
zone->uz_sleeps++;
msleep(keg, &keg->uk_lock, PVM, "keglimit", 0);
continue;
}
slab = keg_alloc_slab(keg, zone, flags);
/*
* If we got a slab here it's safe to mark it partially used
* and return. We assume that the caller is going to remove
* at least one item.
*/
if (slab) {
MPASS(slab->us_keg == keg);
LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
return (slab);
}
/*
* We might not have been able to get a slab but another cpu
* could have while we were unlocked. Check again before we
* fail.
*/
flags |= M_NOVM;
}
return (slab);
}
static uma_slab_t
zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags)
{
uma_slab_t slab;
if (keg == NULL) {
keg = zone_first_keg(zone);
KEG_LOCK(keg);
}
for (;;) {
slab = keg_fetch_slab(keg, zone, flags);
if (slab)
return (slab);
if (flags & (M_NOWAIT | M_NOVM))
break;
}
KEG_UNLOCK(keg);
return (NULL);
}
/*
* uma_zone_fetch_slab_multi: Fetches a slab from one available keg. Returns
* with the keg locked. On NULL no lock is held.
*
* The last pointer is used to seed the search. It is not required.
*/
static uma_slab_t
zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags)
{
uma_klink_t klink;
uma_slab_t slab;
uma_keg_t keg;
int flags;
int empty;
int full;
/*
* Don't wait on the first pass. This will skip limit tests
* as well. We don't want to block if we can find a provider
* without blocking.
*/
flags = (rflags & ~M_WAITOK) | M_NOWAIT;
/*
* Use the last slab allocated as a hint for where to start
* the search.
*/
if (last != NULL) {
slab = keg_fetch_slab(last, zone, flags);
if (slab)
return (slab);
KEG_UNLOCK(last);
}
/*
* Loop until we have a slab incase of transient failures
* while M_WAITOK is specified. I'm not sure this is 100%
* required but we've done it for so long now.
*/
for (;;) {
empty = 0;
full = 0;
/*
* Search the available kegs for slabs. Be careful to hold the
* correct lock while calling into the keg layer.
*/
LIST_FOREACH(klink, &zone->uz_kegs, kl_link) {
keg = klink->kl_keg;
KEG_LOCK(keg);
if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) {
slab = keg_fetch_slab(keg, zone, flags);
if (slab)
return (slab);
}
if (keg->uk_flags & UMA_ZFLAG_FULL)
full++;
else
empty++;
KEG_UNLOCK(keg);
}
if (rflags & (M_NOWAIT | M_NOVM))
break;
flags = rflags;
/*
* All kegs are full. XXX We can't atomically check all kegs
* and sleep so just sleep for a short period and retry.
*/
if (full && !empty) {
ZONE_LOCK(zone);
zone->uz_flags |= UMA_ZFLAG_FULL;
zone->uz_sleeps++;
zone_log_warning(zone);
msleep(zone, zone->uz_lockptr, PVM,
"zonelimit", hz/100);
zone->uz_flags &= ~UMA_ZFLAG_FULL;
ZONE_UNLOCK(zone);
continue;
}
}
return (NULL);
}
static void *
slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
{
void *item;
uint8_t freei;
MPASS(keg == slab->us_keg);
mtx_assert(&keg->uk_lock, MA_OWNED);
freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
item = slab->us_data + (keg->uk_rsize * freei);
slab->us_freecount--;
keg->uk_free--;
/* Move this slab to the full list */
if (slab->us_freecount == 0) {
LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link);
}
return (item);
}
static int
zone_import(uma_zone_t zone, void **bucket, int max, int flags)
{
uma_slab_t slab;
uma_keg_t keg;
int i;
slab = NULL;
keg = NULL;
/* Try to keep the buckets totally full */
for (i = 0; i < max; ) {
if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
break;
keg = slab->us_keg;
while (slab->us_freecount && i < max) {
bucket[i++] = slab_alloc_item(keg, slab);
if (keg->uk_free <= keg->uk_reserve)
break;
}
/* Don't grab more than one slab at a time. */
flags &= ~M_WAITOK;
flags |= M_NOWAIT;
}
if (slab != NULL)
KEG_UNLOCK(keg);
return i;
}
static uma_bucket_t
zone_alloc_bucket(uma_zone_t zone, void *udata, int flags)
{
uma_bucket_t bucket;
int max;
/* Don't wait for buckets, preserve caller's NOVM setting. */
bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM));
if (bucket == NULL)
return (NULL);
max = MIN(bucket->ub_entries, zone->uz_count);
bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
max, flags);
/*
* Initialize the memory if necessary.
*/
if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
int i;
for (i = 0; i < bucket->ub_cnt; i++)
if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
flags) != 0)
break;
/*
* If we couldn't initialize the whole bucket, put the
* rest back onto the freelist.
*/
if (i != bucket->ub_cnt) {
zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i],
bucket->ub_cnt - i);
#ifdef INVARIANTS
bzero(&bucket->ub_bucket[i],
sizeof(void *) * (bucket->ub_cnt - i));
#endif
bucket->ub_cnt = i;
}
}
if (bucket->ub_cnt == 0) {
bucket_free(zone, bucket, udata);
atomic_add_long(&zone->uz_fails, 1);
return (NULL);
}
return (bucket);
}
/*
* Allocates a single item from a zone.
*
* Arguments
* zone The zone to alloc for.
* udata The data to be passed to the constructor.
* flags M_WAITOK, M_NOWAIT, M_ZERO.
*
* Returns
* NULL if there is no memory and M_NOWAIT is set
* An item if successful
*/
static void *
zone_alloc_item(uma_zone_t zone, void *udata, int flags)
{
void *item;
item = NULL;
#ifdef UMA_DEBUG_ALLOC
printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
#endif
if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
goto fail;
atomic_add_long(&zone->uz_allocs, 1);
/*
* We have to call both the zone's init (not the keg's init)
* and the zone's ctor. This is because the item is going from
* a keg slab directly to the user, and the user is expecting it
* to be both zone-init'd as well as zone-ctor'd.
*/
if (zone->uz_init != NULL) {
if (zone->uz_init(item, zone->uz_size, flags) != 0) {
zone_free_item(zone, item, udata, SKIP_FINI);
goto fail;
}
}
if (zone->uz_ctor != NULL) {
if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
zone_free_item(zone, item, udata, SKIP_DTOR);
goto fail;
}
}
#ifdef INVARIANTS
uma_dbg_alloc(zone, NULL, item);
#endif
if (flags & M_ZERO)
uma_zero_item(item, zone);
return (item);
fail:
atomic_add_long(&zone->uz_fails, 1);
return (NULL);
}
/* See uma.h */
void
uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
{
uma_cache_t cache;
uma_bucket_t bucket;
int lockfail;
int cpu;
#if 0
/* XXX: FIX!! Do not enable this in CURRENT!! MarkM */
/* The entropy here is desirable, but the harvesting is expensive */
struct entropy {
const void *uz_name;
const void *item;
} entropy;
entropy.uz_name = zone->uz_name;
entropy.item = item;
random_harvest(&entropy, sizeof(struct entropy), 2, RANDOM_UMA_ALLOC);
#endif
#ifdef UMA_DEBUG_ALLOC_1
printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone);
#endif
CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread,
zone->uz_name);
/* uma_zfree(..., NULL) does nothing, to match free(9). */
if (item == NULL)
return;
#ifdef DEBUG_MEMGUARD
if (is_memguard_addr(item)) {
if (zone->uz_dtor != NULL && zone->uz_dtor != mtrash_dtor)
zone->uz_dtor(item, zone->uz_size, udata);
if (zone->uz_fini != NULL && zone->uz_fini != mtrash_fini)
zone->uz_fini(item, zone->uz_size);
memguard_free(item);
return;
}
#endif
#ifdef INVARIANTS
if (zone->uz_flags & UMA_ZONE_MALLOC)
uma_dbg_free(zone, udata, item);
else
uma_dbg_free(zone, NULL, item);
#endif
if (zone->uz_dtor != NULL)
zone->uz_dtor(item, zone->uz_size, udata);
/*
* The race here is acceptable. If we miss it we'll just have to wait
* a little longer for the limits to be reset.
*/
if (zone->uz_flags & UMA_ZFLAG_FULL)
goto zfree_item;
/*
* If possible, free to the per-CPU cache. There are two
* requirements for safe access to the per-CPU cache: (1) the thread
* accessing the cache must not be preempted or yield during access,
* and (2) the thread must not migrate CPUs without switching which
* cache it accesses. We rely on a critical section to prevent
* preemption and migration. We release the critical section in
* order to acquire the zone mutex if we are unable to free to the
* current cache; when we re-acquire the critical section, we must
* detect and handle migration if it has occurred.
*/
zfree_restart:
critical_enter();
cpu = curcpu;
cache = &zone->uz_cpu[cpu];
zfree_start:
/*
* Try to free into the allocbucket first to give LIFO ordering
* for cache-hot datastructures. Spill over into the freebucket
* if necessary. Alloc will swap them if one runs dry.
*/
bucket = cache->uc_allocbucket;
if (bucket == NULL || bucket->ub_cnt >= bucket->ub_entries)
bucket = cache->uc_freebucket;
if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL,
("uma_zfree: Freeing to non free bucket index."));
bucket->ub_bucket[bucket->ub_cnt] = item;
bucket->ub_cnt++;
cache->uc_frees++;
critical_exit();
return;
}
/*
* We must go back the zone, which requires acquiring the zone lock,
* which in turn means we must release and re-acquire the critical
* section. Since the critical section is released, we may be
* preempted or migrate. As such, make sure not to maintain any
* thread-local state specific to the cache from prior to releasing
* the critical section.
*/
critical_exit();
if (zone->uz_count == 0 || bucketdisable)
goto zfree_item;
lockfail = 0;
if (ZONE_TRYLOCK(zone) == 0) {
/* Record contention to size the buckets. */
ZONE_LOCK(zone);
lockfail = 1;
}
critical_enter();
cpu = curcpu;
cache = &zone->uz_cpu[cpu];
/*
* Since we have locked the zone we may as well send back our stats.
*/
atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
atomic_add_long(&zone->uz_frees, cache->uc_frees);
cache->uc_allocs = 0;
cache->uc_frees = 0;
bucket = cache->uc_freebucket;
if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
ZONE_UNLOCK(zone);
goto zfree_start;
}
cache->uc_freebucket = NULL;
/* Can we throw this on the zone full list? */
if (bucket != NULL) {
#ifdef UMA_DEBUG_ALLOC
printf("uma_zfree: Putting old bucket on the free list.\n");
#endif
/* ub_cnt is pointing to the last free item */
KASSERT(bucket->ub_cnt != 0,
("uma_zfree: Attempting to insert an empty bucket onto the full list.\n"));
LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link);
}
/* We are no longer associated with this CPU. */
critical_exit();
/*
* We bump the uz count when the cache size is insufficient to
* handle the working set.
*/
if (lockfail && zone->uz_count < BUCKET_MAX)
zone->uz_count++;
ZONE_UNLOCK(zone);
#ifdef UMA_DEBUG_ALLOC
printf("uma_zfree: Allocating new free bucket.\n");
#endif
bucket = bucket_alloc(zone, udata, M_NOWAIT);
if (bucket) {
critical_enter();
cpu = curcpu;
cache = &zone->uz_cpu[cpu];
if (cache->uc_freebucket == NULL) {
cache->uc_freebucket = bucket;
goto zfree_start;
}
/*
* We lost the race, start over. We have to drop our
* critical section to free the bucket.
*/
critical_exit();
bucket_free(zone, bucket, udata);
goto zfree_restart;
}
/*
* If nothing else caught this, we'll just do an internal free.
*/
zfree_item:
zone_free_item(zone, item, udata, SKIP_DTOR);
return;
}
static void
slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
{
uint8_t freei;
mtx_assert(&keg->uk_lock, MA_OWNED);
MPASS(keg == slab->us_keg);
/* Do we need to remove from any lists? */
if (slab->us_freecount+1 == keg->uk_ipers) {
LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
} else if (slab->us_freecount == 0) {
LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
}
/* Slab management. */
freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
slab->us_freecount++;
/* Keg statistics. */
keg->uk_free++;
}
static void
zone_release(uma_zone_t zone, void **bucket, int cnt)
{
void *item;
uma_slab_t slab;
uma_keg_t keg;
uint8_t *mem;
int clearfull;
int i;
clearfull = 0;
keg = zone_first_keg(zone);
KEG_LOCK(keg);
for (i = 0; i < cnt; i++) {
item = bucket[i];
if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
if (zone->uz_flags & UMA_ZONE_HASH) {
slab = hash_sfind(&keg->uk_hash, mem);
} else {
mem += keg->uk_pgoff;
slab = (uma_slab_t)mem;
}
} else {
slab = vtoslab((vm_offset_t)item);
if (slab->us_keg != keg) {
KEG_UNLOCK(keg);
keg = slab->us_keg;
KEG_LOCK(keg);
}
}
slab_free_item(keg, slab, item);
if (keg->uk_flags & UMA_ZFLAG_FULL) {
if (keg->uk_pages < keg->uk_maxpages) {
keg->uk_flags &= ~UMA_ZFLAG_FULL;
clearfull = 1;
}
/*
* We can handle one more allocation. Since we're
* clearing ZFLAG_FULL, wake up all procs blocked
* on pages. This should be uncommon, so keeping this
* simple for now (rather than adding count of blocked
* threads etc).
*/
wakeup(keg);
}
}
KEG_UNLOCK(keg);
if (clearfull) {
ZONE_LOCK(zone);
zone->uz_flags &= ~UMA_ZFLAG_FULL;
wakeup(zone);
ZONE_UNLOCK(zone);
}
}
/*
* Frees a single item to any zone.
*
* Arguments:
* zone The zone to free to
* item The item we're freeing
* udata User supplied data for the dtor
* skip Skip dtors and finis
*/
static void
zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
{
#ifdef INVARIANTS
if (skip == SKIP_NONE) {
if (zone->uz_flags & UMA_ZONE_MALLOC)
uma_dbg_free(zone, udata, item);
else
uma_dbg_free(zone, NULL, item);
}
#endif
if (skip < SKIP_DTOR && zone->uz_dtor)
zone->uz_dtor(item, zone->uz_size, udata);
if (skip < SKIP_FINI && zone->uz_fini)
zone->uz_fini(item, zone->uz_size);
atomic_add_long(&zone->uz_frees, 1);
zone->uz_release(zone->uz_arg, &item, 1);
}
/* See uma.h */
int
uma_zone_set_max(uma_zone_t zone, int nitems)
{
uma_keg_t keg;
keg = zone_first_keg(zone);
if (keg == NULL)
return (0);
KEG_LOCK(keg);
keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
if (keg->uk_maxpages * keg->uk_ipers < nitems)
keg->uk_maxpages += keg->uk_ppera;
nitems = keg->uk_maxpages * keg->uk_ipers;
KEG_UNLOCK(keg);
return (nitems);
}
/* See uma.h */
int
uma_zone_get_max(uma_zone_t zone)
{
int nitems;
uma_keg_t keg;
keg = zone_first_keg(zone);
if (keg == NULL)
return (0);
KEG_LOCK(keg);
nitems = keg->uk_maxpages * keg->uk_ipers;
KEG_UNLOCK(keg);
return (nitems);
}
/* See uma.h */
void
uma_zone_set_warning(uma_zone_t zone, const char *warning)
{
ZONE_LOCK(zone);
zone->uz_warning = warning;
ZONE_UNLOCK(zone);
}
/* See uma.h */
int
uma_zone_get_cur(uma_zone_t zone)
{
int64_t nitems;
u_int i;
ZONE_LOCK(zone);
nitems = zone->uz_allocs - zone->uz_frees;
CPU_FOREACH(i) {
/*
* See the comment in sysctl_vm_zone_stats() regarding the
* safety of accessing the per-cpu caches. With the zone lock
* held, it is safe, but can potentially result in stale data.
*/
nitems += zone->uz_cpu[i].uc_allocs -
zone->uz_cpu[i].uc_frees;
}
ZONE_UNLOCK(zone);
return (nitems < 0 ? 0 : nitems);
}
/* See uma.h */
void
uma_zone_set_init(uma_zone_t zone, uma_init uminit)
{
uma_keg_t keg;
keg = zone_first_keg(zone);
KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
KEG_LOCK(keg);
KASSERT(keg->uk_pages == 0,
("uma_zone_set_init on non-empty keg"));
keg->uk_init = uminit;
KEG_UNLOCK(keg);
}
/* See uma.h */
void
uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
{
uma_keg_t keg;
keg = zone_first_keg(zone);
KASSERT(keg != NULL, ("uma_zone_set_fini: Invalid zone type"));
KEG_LOCK(keg);
KASSERT(keg->uk_pages == 0,
("uma_zone_set_fini on non-empty keg"));
keg->uk_fini = fini;
KEG_UNLOCK(keg);
}
/* See uma.h */
void
uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
{
ZONE_LOCK(zone);
KASSERT(zone_first_keg(zone)->uk_pages == 0,
("uma_zone_set_zinit on non-empty keg"));
zone->uz_init = zinit;
ZONE_UNLOCK(zone);
}
/* See uma.h */
void
uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
{
ZONE_LOCK(zone);
KASSERT(zone_first_keg(zone)->uk_pages == 0,
("uma_zone_set_zfini on non-empty keg"));
zone->uz_fini = zfini;
ZONE_UNLOCK(zone);
}
/* See uma.h */
/* XXX uk_freef is not actually used with the zone locked */
void
uma_zone_set_freef(uma_zone_t zone, uma_free freef)
{
uma_keg_t keg;
keg = zone_first_keg(zone);
KASSERT(keg != NULL, ("uma_zone_set_freef: Invalid zone type"));
KEG_LOCK(keg);
keg->uk_freef = freef;
KEG_UNLOCK(keg);
}
/* See uma.h */
/* XXX uk_allocf is not actually used with the zone locked */
void
uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
{
uma_keg_t keg;
keg = zone_first_keg(zone);
KEG_LOCK(keg);
keg->uk_allocf = allocf;
KEG_UNLOCK(keg);
}
/* See uma.h */
void
uma_zone_reserve(uma_zone_t zone, int items)
{
uma_keg_t keg;
keg = zone_first_keg(zone);
if (keg == NULL)
return;
KEG_LOCK(keg);
keg->uk_reserve = items;
KEG_UNLOCK(keg);
return;
}
/* See uma.h */
int
uma_zone_reserve_kva(uma_zone_t zone, int count)
{
uma_keg_t keg;
vm_offset_t kva;
int pages;
keg = zone_first_keg(zone);
if (keg == NULL)
return (0);
pages = count / keg->uk_ipers;
if (pages * keg->uk_ipers < count)
pages++;
#ifdef UMA_MD_SMALL_ALLOC
if (keg->uk_ppera > 1) {
#else
if (1) {
#endif
kva = kva_alloc(pages * UMA_SLAB_SIZE);
if (kva == 0)
return (0);
} else
kva = 0;
KEG_LOCK(keg);
keg->uk_kva = kva;
keg->uk_offset = 0;
keg->uk_maxpages = pages;
#ifdef UMA_MD_SMALL_ALLOC
keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
#else
keg->uk_allocf = noobj_alloc;
#endif
keg->uk_flags |= UMA_ZONE_NOFREE;
KEG_UNLOCK(keg);
return (1);
}
/* See uma.h */
void
uma_prealloc(uma_zone_t zone, int items)
{
int slabs;
uma_slab_t slab;
uma_keg_t keg;
keg = zone_first_keg(zone);
if (keg == NULL)
return;
KEG_LOCK(keg);
slabs = items / keg->uk_ipers;
if (slabs * keg->uk_ipers < items)
slabs++;
while (slabs > 0) {
slab = keg_alloc_slab(keg, zone, M_WAITOK);
if (slab == NULL)
break;
MPASS(slab->us_keg == keg);
LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
slabs--;
}
KEG_UNLOCK(keg);
}
/* See uma.h */
uint32_t *
uma_find_refcnt(uma_zone_t zone, void *item)
{
uma_slabrefcnt_t slabref;
uma_slab_t slab;
uma_keg_t keg;
uint32_t *refcnt;
int idx;
slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
slabref = (uma_slabrefcnt_t)slab;
keg = slab->us_keg;
KASSERT(keg->uk_flags & UMA_ZONE_REFCNT,
("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT"));
idx = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
refcnt = &slabref->us_refcnt[idx];
return refcnt;
}
/* See uma.h */
static void
uma_reclaim_locked(bool kmem_danger)
{
#ifdef UMA_DEBUG
printf("UMA: vm asked us to release pages!\n");
#endif
sx_assert(&uma_drain_lock, SA_XLOCKED);
bucket_enable();
zone_foreach(zone_drain);
if (vm_page_count_min() || kmem_danger) {
cache_drain_safe(NULL);
zone_foreach(zone_drain);
}
/*
* Some slabs may have been freed but this zone will be visited early
* we visit again so that we can free pages that are empty once other
* zones are drained. We have to do the same for buckets.
*/
zone_drain(slabzone);
zone_drain(slabrefzone);
bucket_zone_drain();
}
void
uma_reclaim(void)
{
sx_xlock(&uma_drain_lock);
uma_reclaim_locked(false);
sx_xunlock(&uma_drain_lock);
}
static int uma_reclaim_needed;
void
uma_reclaim_wakeup(void)
{
uma_reclaim_needed = 1;
wakeup(&uma_reclaim_needed);
}
void
uma_reclaim_worker(void *arg __unused)
{
sx_xlock(&uma_drain_lock);
for (;;) {
sx_sleep(&uma_reclaim_needed, &uma_drain_lock, PVM,
"umarcl", 0);
if (uma_reclaim_needed) {
uma_reclaim_needed = 0;
uma_reclaim_locked(true);
}
}
}
/* See uma.h */
int
uma_zone_exhausted(uma_zone_t zone)
{
int full;
ZONE_LOCK(zone);
full = (zone->uz_flags & UMA_ZFLAG_FULL);
ZONE_UNLOCK(zone);
return (full);
}
int
uma_zone_exhausted_nolock(uma_zone_t zone)
{
return (zone->uz_flags & UMA_ZFLAG_FULL);
}
void *
uma_large_malloc(vm_size_t size, int wait)
{
void *mem;
uma_slab_t slab;
uint8_t flags;
slab = zone_alloc_item(slabzone, NULL, wait);
if (slab == NULL)
return (NULL);
mem = page_alloc(NULL, size, &flags, wait);
if (mem) {
vsetslab((vm_offset_t)mem, slab);
slab->us_data = mem;
slab->us_flags = flags | UMA_SLAB_MALLOC;
slab->us_size = size;
} else {
zone_free_item(slabzone, slab, NULL, SKIP_NONE);
}
return (mem);
}
void
uma_large_free(uma_slab_t slab)
{
page_free(slab->us_data, slab->us_size, slab->us_flags);
zone_free_item(slabzone, slab, NULL, SKIP_NONE);
}
static void
uma_zero_item(void *item, uma_zone_t zone)
{
if (zone->uz_flags & UMA_ZONE_PCPU) {
for (int i = 0; i < mp_ncpus; i++)
bzero(zpcpu_get_cpu(item, i), zone->uz_size);
} else
bzero(item, zone->uz_size);
}
void
uma_print_stats(void)
{
zone_foreach(uma_print_zone);
}
static void
slab_print(uma_slab_t slab)
{
printf("slab: keg %p, data %p, freecount %d\n",
slab->us_keg, slab->us_data, slab->us_freecount);
}
static void
cache_print(uma_cache_t cache)
{
printf("alloc: %p(%d), free: %p(%d)\n",
cache->uc_allocbucket,
cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
cache->uc_freebucket,
cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0);
}
static void
uma_print_keg(uma_keg_t keg)
{
uma_slab_t slab;
printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
"out %d free %d limit %d\n",
keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
keg->uk_ipers, keg->uk_ppera,
(keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free,
(keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers);
printf("Part slabs:\n");
LIST_FOREACH(slab, &keg->uk_part_slab, us_link)
slab_print(slab);
printf("Free slabs:\n");
LIST_FOREACH(slab, &keg->uk_free_slab, us_link)
slab_print(slab);
printf("Full slabs:\n");
LIST_FOREACH(slab, &keg->uk_full_slab, us_link)
slab_print(slab);
}
void
uma_print_zone(uma_zone_t zone)
{
uma_cache_t cache;
uma_klink_t kl;
int i;
printf("zone: %s(%p) size %d flags %#x\n",
zone->uz_name, zone, zone->uz_size, zone->uz_flags);
LIST_FOREACH(kl, &zone->uz_kegs, kl_link)
uma_print_keg(kl->kl_keg);
CPU_FOREACH(i) {
cache = &zone->uz_cpu[i];
printf("CPU %d Cache:\n", i);
cache_print(cache);
}
}
#ifdef DDB
/*
* Generate statistics across both the zone and its per-cpu cache's. Return
* desired statistics if the pointer is non-NULL for that statistic.
*
* Note: does not update the zone statistics, as it can't safely clear the
* per-CPU cache statistic.
*
* XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't
* safe from off-CPU; we should modify the caches to track this information
* directly so that we don't have to.
*/
static void
uma_zone_sumstat(uma_zone_t z, int *cachefreep, uint64_t *allocsp,
uint64_t *freesp, uint64_t *sleepsp)
{
uma_cache_t cache;
uint64_t allocs, frees, sleeps;
int cachefree, cpu;
allocs = frees = sleeps = 0;
cachefree = 0;
CPU_FOREACH(cpu) {
cache = &z->uz_cpu[cpu];
if (cache->uc_allocbucket != NULL)
cachefree += cache->uc_allocbucket->ub_cnt;
if (cache->uc_freebucket != NULL)
cachefree += cache->uc_freebucket->ub_cnt;
allocs += cache->uc_allocs;
frees += cache->uc_frees;
}
allocs += z->uz_allocs;
frees += z->uz_frees;
sleeps += z->uz_sleeps;
if (cachefreep != NULL)
*cachefreep = cachefree;
if (allocsp != NULL)
*allocsp = allocs;
if (freesp != NULL)
*freesp = frees;
if (sleepsp != NULL)
*sleepsp = sleeps;
}
#endif /* DDB */
static int
sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS)
{
uma_keg_t kz;
uma_zone_t z;
int count;
count = 0;
rw_rlock(&uma_rwlock);
LIST_FOREACH(kz, &uma_kegs, uk_link) {
LIST_FOREACH(z, &kz->uk_zones, uz_link)
count++;
}
rw_runlock(&uma_rwlock);
return (sysctl_handle_int(oidp, &count, 0, req));
}
static int
sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
{
struct uma_stream_header ush;
struct uma_type_header uth;
struct uma_percpu_stat ups;
uma_bucket_t bucket;
struct sbuf sbuf;
uma_cache_t cache;
uma_klink_t kl;
uma_keg_t kz;
uma_zone_t z;
uma_keg_t k;
int count, error, i;
error = sysctl_wire_old_buffer(req, 0);
if (error != 0)
return (error);
sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL);
count = 0;
rw_rlock(&uma_rwlock);
LIST_FOREACH(kz, &uma_kegs, uk_link) {
LIST_FOREACH(z, &kz->uk_zones, uz_link)
count++;
}
/*
* Insert stream header.
*/
bzero(&ush, sizeof(ush));
ush.ush_version = UMA_STREAM_VERSION;
ush.ush_maxcpus = (mp_maxid + 1);
ush.ush_count = count;
(void)sbuf_bcat(&sbuf, &ush, sizeof(ush));
LIST_FOREACH(kz, &uma_kegs, uk_link) {
LIST_FOREACH(z, &kz->uk_zones, uz_link) {
bzero(&uth, sizeof(uth));
ZONE_LOCK(z);
strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
uth.uth_align = kz->uk_align;
uth.uth_size = kz->uk_size;
uth.uth_rsize = kz->uk_rsize;
LIST_FOREACH(kl, &z->uz_kegs, kl_link) {
k = kl->kl_keg;
uth.uth_maxpages += k->uk_maxpages;
uth.uth_pages += k->uk_pages;
uth.uth_keg_free += k->uk_free;
uth.uth_limit = (k->uk_maxpages / k->uk_ppera)
* k->uk_ipers;
}
/*
* A zone is secondary is it is not the first entry
* on the keg's zone list.
*/
if ((z->uz_flags & UMA_ZONE_SECONDARY) &&
(LIST_FIRST(&kz->uk_zones) != z))
uth.uth_zone_flags = UTH_ZONE_SECONDARY;
LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
uth.uth_zone_free += bucket->ub_cnt;
uth.uth_allocs = z->uz_allocs;
uth.uth_frees = z->uz_frees;
uth.uth_fails = z->uz_fails;
uth.uth_sleeps = z->uz_sleeps;
(void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
/*
* While it is not normally safe to access the cache
* bucket pointers while not on the CPU that owns the
* cache, we only allow the pointers to be exchanged
* without the zone lock held, not invalidated, so
* accept the possible race associated with bucket
* exchange during monitoring.
*/
for (i = 0; i < (mp_maxid + 1); i++) {
bzero(&ups, sizeof(ups));
if (kz->uk_flags & UMA_ZFLAG_INTERNAL)
goto skip;
if (CPU_ABSENT(i))
goto skip;
cache = &z->uz_cpu[i];
if (cache->uc_allocbucket != NULL)
ups.ups_cache_free +=
cache->uc_allocbucket->ub_cnt;
if (cache->uc_freebucket != NULL)
ups.ups_cache_free +=
cache->uc_freebucket->ub_cnt;
ups.ups_allocs = cache->uc_allocs;
ups.ups_frees = cache->uc_frees;
skip:
(void)sbuf_bcat(&sbuf, &ups, sizeof(ups));
}
ZONE_UNLOCK(z);
}
}
rw_runlock(&uma_rwlock);
error = sbuf_finish(&sbuf);
sbuf_delete(&sbuf);
return (error);
}
int
sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS)
{
uma_zone_t zone = *(uma_zone_t *)arg1;
int error, max;
max = uma_zone_get_max(zone);
error = sysctl_handle_int(oidp, &max, 0, req);
if (error || !req->newptr)
return (error);
uma_zone_set_max(zone, max);
return (0);
}
int
sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS)
{
uma_zone_t zone = *(uma_zone_t *)arg1;
int cur;
cur = uma_zone_get_cur(zone);
return (sysctl_handle_int(oidp, &cur, 0, req));
}
#ifdef DDB
DB_SHOW_COMMAND(uma, db_show_uma)
{
uint64_t allocs, frees, sleeps;
uma_bucket_t bucket;
uma_keg_t kz;
uma_zone_t z;
int cachefree;
db_printf("%18s %8s %8s %8s %12s %8s %8s\n", "Zone", "Size", "Used",
"Free", "Requests", "Sleeps", "Bucket");
LIST_FOREACH(kz, &uma_kegs, uk_link) {
LIST_FOREACH(z, &kz->uk_zones, uz_link) {
if (kz->uk_flags & UMA_ZFLAG_INTERNAL) {
allocs = z->uz_allocs;
frees = z->uz_frees;
sleeps = z->uz_sleeps;
cachefree = 0;
} else
uma_zone_sumstat(z, &cachefree, &allocs,
&frees, &sleeps);
if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
(LIST_FIRST(&kz->uk_zones) != z)))
cachefree += kz->uk_free;
LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
cachefree += bucket->ub_cnt;
db_printf("%18s %8ju %8jd %8d %12ju %8ju %8u\n",
z->uz_name, (uintmax_t)kz->uk_size,
(intmax_t)(allocs - frees), cachefree,
(uintmax_t)allocs, sleeps, z->uz_count);
if (db_pager_quit)
return;
}
}
}
DB_SHOW_COMMAND(umacache, db_show_umacache)
{
uint64_t allocs, frees;
uma_bucket_t bucket;
uma_zone_t z;
int cachefree;
db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",
"Requests", "Bucket");
LIST_FOREACH(z, &uma_cachezones, uz_link) {
uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL);
LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
cachefree += bucket->ub_cnt;
db_printf("%18s %8ju %8jd %8d %12ju %8u\n",
z->uz_name, (uintmax_t)z->uz_size,
(intmax_t)(allocs - frees), cachefree,
(uintmax_t)allocs, z->uz_count);
if (db_pager_quit)
return;
}
}
#endif
Index: head/sys/x86/x86/mca.c
===================================================================
--- head/sys/x86/x86/mca.c (revision 283290)
+++ head/sys/x86/x86/mca.c (revision 283291)
@@ -1,1040 +1,1040 @@
/*-
* Copyright (c) 2009 Hudson River Trading LLC
* Written by: John H. Baldwin <jhb@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Support for x86 machine check architecture.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#ifdef __amd64__
#define DEV_APIC
#else
#include "opt_apic.h"
#endif
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/interrupt.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/taskqueue.h>
#include <machine/intr_machdep.h>
#include <x86/apicvar.h>
#include <machine/cpu.h>
#include <machine/cputypes.h>
#include <x86/mca.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
/* Modes for mca_scan() */
enum scan_mode {
POLLED,
MCE,
CMCI,
};
#ifdef DEV_APIC
/*
* State maintained for each monitored MCx bank to control the
* corrected machine check interrupt threshold.
*/
struct cmc_state {
int max_threshold;
int last_intr;
};
#endif
struct mca_internal {
struct mca_record rec;
int logged;
STAILQ_ENTRY(mca_internal) link;
};
static MALLOC_DEFINE(M_MCA, "MCA", "Machine Check Architecture");
static volatile int mca_count; /* Number of records stored. */
static int mca_banks; /* Number of per-CPU register banks. */
static SYSCTL_NODE(_hw, OID_AUTO, mca, CTLFLAG_RD, NULL,
"Machine Check Architecture");
static int mca_enabled = 1;
SYSCTL_INT(_hw_mca, OID_AUTO, enabled, CTLFLAG_RDTUN, &mca_enabled, 0,
"Administrative toggle for machine check support");
static int amd10h_L1TP = 1;
SYSCTL_INT(_hw_mca, OID_AUTO, amd10h_L1TP, CTLFLAG_RDTUN, &amd10h_L1TP, 0,
"Administrative toggle for logging of level one TLB parity (L1TP) errors");
static int intel6h_HSD131;
SYSCTL_INT(_hw_mca, OID_AUTO, intel6h_HSD131, CTLFLAG_RDTUN, &intel6h_HSD131, 0,
"Administrative toggle for logging of spurious corrected errors");
int workaround_erratum383;
SYSCTL_INT(_hw_mca, OID_AUTO, erratum383, CTLFLAG_RD, &workaround_erratum383, 0,
"Is the workaround for Erratum 383 on AMD Family 10h processors enabled?");
static STAILQ_HEAD(, mca_internal) mca_freelist;
static int mca_freecount;
static STAILQ_HEAD(, mca_internal) mca_records;
static struct callout mca_timer;
static int mca_ticks = 3600; /* Check hourly by default. */
static struct taskqueue *mca_tq;
static struct task mca_refill_task, mca_scan_task;
static struct mtx mca_lock;
#ifdef DEV_APIC
static struct cmc_state **cmc_state; /* Indexed by cpuid, bank */
static int cmc_throttle = 60; /* Time in seconds to throttle CMCI. */
#endif
static int
sysctl_positive_int(SYSCTL_HANDLER_ARGS)
{
int error, value;
value = *(int *)arg1;
error = sysctl_handle_int(oidp, &value, 0, req);
if (error || req->newptr == NULL)
return (error);
if (value <= 0)
return (EINVAL);
*(int *)arg1 = value;
return (0);
}
static int
sysctl_mca_records(SYSCTL_HANDLER_ARGS)
{
int *name = (int *)arg1;
u_int namelen = arg2;
struct mca_record record;
struct mca_internal *rec;
int i;
if (namelen != 1)
return (EINVAL);
if (name[0] < 0 || name[0] >= mca_count)
return (EINVAL);
mtx_lock_spin(&mca_lock);
if (name[0] >= mca_count) {
mtx_unlock_spin(&mca_lock);
return (EINVAL);
}
i = 0;
STAILQ_FOREACH(rec, &mca_records, link) {
if (i == name[0]) {
record = rec->rec;
break;
}
i++;
}
mtx_unlock_spin(&mca_lock);
return (SYSCTL_OUT(req, &record, sizeof(record)));
}
static const char *
mca_error_ttype(uint16_t mca_error)
{
switch ((mca_error & 0x000c) >> 2) {
case 0:
return ("I");
case 1:
return ("D");
case 2:
return ("G");
}
return ("?");
}
static const char *
mca_error_level(uint16_t mca_error)
{
switch (mca_error & 0x0003) {
case 0:
return ("L0");
case 1:
return ("L1");
case 2:
return ("L2");
case 3:
return ("LG");
}
return ("L?");
}
static const char *
mca_error_request(uint16_t mca_error)
{
switch ((mca_error & 0x00f0) >> 4) {
case 0x0:
return ("ERR");
case 0x1:
return ("RD");
case 0x2:
return ("WR");
case 0x3:
return ("DRD");
case 0x4:
return ("DWR");
case 0x5:
return ("IRD");
case 0x6:
return ("PREFETCH");
case 0x7:
return ("EVICT");
case 0x8:
return ("SNOOP");
}
return ("???");
}
static const char *
mca_error_mmtype(uint16_t mca_error)
{
switch ((mca_error & 0x70) >> 4) {
case 0x0:
return ("GEN");
case 0x1:
return ("RD");
case 0x2:
return ("WR");
case 0x3:
return ("AC");
case 0x4:
return ("MS");
}
return ("???");
}
static int __nonnull(1)
mca_mute(const struct mca_record *rec)
{
/*
* Skip spurious corrected parity errors generated by Intel Haswell-
* and Broadwell-based CPUs (see HSD131, HSM142, HSW131 and BDM48
* erratum respectively), unless reporting is enabled.
* Note that these errors also have been observed with the D0-stepping
* of Haswell, while at least initially the CPU specification updates
* suggested only the C0-stepping to be affected. Similarly, Celeron
* 2955U with a CPU ID of 0x45 apparently are also concerned with the
* same problem, with HSM142 only referring to 0x3c and 0x46.
*/
if (cpu_vendor_id == CPU_VENDOR_INTEL &&
CPUID_TO_FAMILY(cpu_id) == 0x6 &&
(CPUID_TO_MODEL(cpu_id) == 0x3c || /* HSD131, HSM142, HSW131 */
CPUID_TO_MODEL(cpu_id) == 0x3d || /* BDM48 */
CPUID_TO_MODEL(cpu_id) == 0x45 ||
CPUID_TO_MODEL(cpu_id) == 0x46) && /* HSM142 */
rec->mr_bank == 0 &&
(rec->mr_status & 0xa0000000ffffffff) == 0x80000000000f0005 &&
!intel6h_HSD131)
return (1);
return (0);
}
/* Dump details about a single machine check. */
static void __nonnull(1)
mca_log(const struct mca_record *rec)
{
uint16_t mca_error;
if (mca_mute(rec))
return;
printf("MCA: Bank %d, Status 0x%016llx\n", rec->mr_bank,
(long long)rec->mr_status);
printf("MCA: Global Cap 0x%016llx, Status 0x%016llx\n",
(long long)rec->mr_mcg_cap, (long long)rec->mr_mcg_status);
printf("MCA: Vendor \"%s\", ID 0x%x, APIC ID %d\n", cpu_vendor,
rec->mr_cpu_id, rec->mr_apic_id);
printf("MCA: CPU %d ", rec->mr_cpu);
if (rec->mr_status & MC_STATUS_UC)
printf("UNCOR ");
else {
printf("COR ");
if (rec->mr_mcg_cap & MCG_CAP_CMCI_P)
printf("(%lld) ", ((long long)rec->mr_status &
MC_STATUS_COR_COUNT) >> 38);
}
if (rec->mr_status & MC_STATUS_PCC)
printf("PCC ");
if (rec->mr_status & MC_STATUS_OVER)
printf("OVER ");
mca_error = rec->mr_status & MC_STATUS_MCA_ERROR;
switch (mca_error) {
/* Simple error codes. */
case 0x0000:
printf("no error");
break;
case 0x0001:
printf("unclassified error");
break;
case 0x0002:
printf("ucode ROM parity error");
break;
case 0x0003:
printf("external error");
break;
case 0x0004:
printf("FRC error");
break;
case 0x0005:
printf("internal parity error");
break;
case 0x0400:
printf("internal timer error");
break;
default:
if ((mca_error & 0xfc00) == 0x0400) {
printf("internal error %x", mca_error & 0x03ff);
break;
}
/* Compound error codes. */
/* Memory hierarchy error. */
if ((mca_error & 0xeffc) == 0x000c) {
printf("%s memory error", mca_error_level(mca_error));
break;
}
/* TLB error. */
if ((mca_error & 0xeff0) == 0x0010) {
printf("%sTLB %s error", mca_error_ttype(mca_error),
mca_error_level(mca_error));
break;
}
/* Memory controller error. */
if ((mca_error & 0xef80) == 0x0080) {
printf("%s channel ", mca_error_mmtype(mca_error));
if ((mca_error & 0x000f) != 0x000f)
printf("%d", mca_error & 0x000f);
else
printf("??");
printf(" memory error");
break;
}
/* Cache error. */
if ((mca_error & 0xef00) == 0x0100) {
printf("%sCACHE %s %s error",
mca_error_ttype(mca_error),
mca_error_level(mca_error),
mca_error_request(mca_error));
break;
}
/* Bus and/or Interconnect error. */
if ((mca_error & 0xe800) == 0x0800) {
printf("BUS%s ", mca_error_level(mca_error));
switch ((mca_error & 0x0600) >> 9) {
case 0:
printf("Source");
break;
case 1:
printf("Responder");
break;
case 2:
printf("Observer");
break;
default:
printf("???");
break;
}
printf(" %s ", mca_error_request(mca_error));
switch ((mca_error & 0x000c) >> 2) {
case 0:
printf("Memory");
break;
case 2:
printf("I/O");
break;
case 3:
printf("Other");
break;
default:
printf("???");
break;
}
if (mca_error & 0x0100)
printf(" timed out");
break;
}
printf("unknown error %x", mca_error);
break;
}
printf("\n");
if (rec->mr_status & MC_STATUS_ADDRV)
printf("MCA: Address 0x%llx\n", (long long)rec->mr_addr);
if (rec->mr_status & MC_STATUS_MISCV)
printf("MCA: Misc 0x%llx\n", (long long)rec->mr_misc);
}
static int __nonnull(2)
mca_check_status(int bank, struct mca_record *rec)
{
uint64_t status;
u_int p[4];
status = rdmsr(MSR_MC_STATUS(bank));
if (!(status & MC_STATUS_VAL))
return (0);
/* Save exception information. */
rec->mr_status = status;
rec->mr_bank = bank;
rec->mr_addr = 0;
if (status & MC_STATUS_ADDRV)
rec->mr_addr = rdmsr(MSR_MC_ADDR(bank));
rec->mr_misc = 0;
if (status & MC_STATUS_MISCV)
rec->mr_misc = rdmsr(MSR_MC_MISC(bank));
rec->mr_tsc = rdtsc();
rec->mr_apic_id = PCPU_GET(apic_id);
rec->mr_mcg_cap = rdmsr(MSR_MCG_CAP);
rec->mr_mcg_status = rdmsr(MSR_MCG_STATUS);
rec->mr_cpu_id = cpu_id;
rec->mr_cpu_vendor_id = cpu_vendor_id;
rec->mr_cpu = PCPU_GET(cpuid);
/*
* Clear machine check. Don't do this for uncorrectable
* errors so that the BIOS can see them.
*/
if (!(rec->mr_status & (MC_STATUS_PCC | MC_STATUS_UC))) {
wrmsr(MSR_MC_STATUS(bank), 0);
do_cpuid(0, p);
}
return (1);
}
static void
mca_fill_freelist(void)
{
struct mca_internal *rec;
int desired;
/*
* Ensure we have at least one record for each bank and one
* record per CPU.
*/
desired = imax(mp_ncpus, mca_banks);
mtx_lock_spin(&mca_lock);
while (mca_freecount < desired) {
mtx_unlock_spin(&mca_lock);
rec = malloc(sizeof(*rec), M_MCA, M_WAITOK);
mtx_lock_spin(&mca_lock);
STAILQ_INSERT_TAIL(&mca_freelist, rec, link);
mca_freecount++;
}
mtx_unlock_spin(&mca_lock);
}
static void
mca_refill(void *context, int pending)
{
mca_fill_freelist();
}
static void __nonnull(2)
mca_record_entry(enum scan_mode mode, const struct mca_record *record)
{
struct mca_internal *rec;
if (mode == POLLED) {
rec = malloc(sizeof(*rec), M_MCA, M_WAITOK);
mtx_lock_spin(&mca_lock);
} else {
mtx_lock_spin(&mca_lock);
rec = STAILQ_FIRST(&mca_freelist);
if (rec == NULL) {
printf("MCA: Unable to allocate space for an event.\n");
mca_log(record);
mtx_unlock_spin(&mca_lock);
return;
}
STAILQ_REMOVE_HEAD(&mca_freelist, link);
mca_freecount--;
}
rec->rec = *record;
rec->logged = 0;
STAILQ_INSERT_TAIL(&mca_records, rec, link);
mca_count++;
mtx_unlock_spin(&mca_lock);
if (mode == CMCI)
taskqueue_enqueue_fast(mca_tq, &mca_refill_task);
}
#ifdef DEV_APIC
/*
* Update the interrupt threshold for a CMCI. The strategy is to use
* a low trigger that interrupts as soon as the first event occurs.
* However, if a steady stream of events arrive, the threshold is
* increased until the interrupts are throttled to once every
* cmc_throttle seconds or the periodic scan. If a periodic scan
* finds that the threshold is too high, it is lowered.
*/
static void
cmci_update(enum scan_mode mode, int bank, int valid, struct mca_record *rec)
{
struct cmc_state *cc;
uint64_t ctl;
u_int delta;
int count, limit;
/* Fetch the current limit for this bank. */
cc = &cmc_state[PCPU_GET(cpuid)][bank];
ctl = rdmsr(MSR_MC_CTL2(bank));
count = (rec->mr_status & MC_STATUS_COR_COUNT) >> 38;
delta = (u_int)(ticks - cc->last_intr);
/*
* If an interrupt was received less than cmc_throttle seconds
* since the previous interrupt and the count from the current
* event is greater than or equal to the current threshold,
* double the threshold up to the max.
*/
if (mode == CMCI && valid) {
limit = ctl & MC_CTL2_THRESHOLD;
if (delta < cmc_throttle && count >= limit &&
limit < cc->max_threshold) {
limit = min(limit << 1, cc->max_threshold);
ctl &= ~MC_CTL2_THRESHOLD;
ctl |= limit;
wrmsr(MSR_MC_CTL2(bank), limit);
}
cc->last_intr = ticks;
return;
}
/*
* When the banks are polled, check to see if the threshold
* should be lowered.
*/
if (mode != POLLED)
return;
/* If a CMCI occured recently, do nothing for now. */
if (delta < cmc_throttle)
return;
/*
* Compute a new limit based on the average rate of events per
* cmc_throttle seconds since the last interrupt.
*/
if (valid) {
count = (rec->mr_status & MC_STATUS_COR_COUNT) >> 38;
limit = count * cmc_throttle / delta;
if (limit <= 0)
limit = 1;
else if (limit > cc->max_threshold)
limit = cc->max_threshold;
} else
limit = 1;
if ((ctl & MC_CTL2_THRESHOLD) != limit) {
ctl &= ~MC_CTL2_THRESHOLD;
ctl |= limit;
wrmsr(MSR_MC_CTL2(bank), limit);
}
}
#endif
/*
* This scans all the machine check banks of the current CPU to see if
* there are any machine checks. Any non-recoverable errors are
* reported immediately via mca_log(). The current thread must be
* pinned when this is called. The 'mode' parameter indicates if we
* are being called from the MC exception handler, the CMCI handler,
* or the periodic poller. In the MC exception case this function
* returns true if the system is restartable. Otherwise, it returns a
* count of the number of valid MC records found.
*/
static int
mca_scan(enum scan_mode mode)
{
struct mca_record rec;
uint64_t mcg_cap, ucmask;
int count, i, recoverable, valid;
count = 0;
recoverable = 1;
ucmask = MC_STATUS_UC | MC_STATUS_PCC;
/* When handling a MCE#, treat the OVER flag as non-restartable. */
if (mode == MCE)
ucmask |= MC_STATUS_OVER;
mcg_cap = rdmsr(MSR_MCG_CAP);
for (i = 0; i < (mcg_cap & MCG_CAP_COUNT); i++) {
#ifdef DEV_APIC
/*
* For a CMCI, only check banks this CPU is
* responsible for.
*/
if (mode == CMCI && !(PCPU_GET(cmci_mask) & 1 << i))
continue;
#endif
valid = mca_check_status(i, &rec);
if (valid) {
count++;
if (rec.mr_status & ucmask) {
recoverable = 0;
mtx_lock_spin(&mca_lock);
mca_log(&rec);
mtx_unlock_spin(&mca_lock);
}
mca_record_entry(mode, &rec);
}
#ifdef DEV_APIC
/*
* If this is a bank this CPU monitors via CMCI,
* update the threshold.
*/
if (PCPU_GET(cmci_mask) & 1 << i)
cmci_update(mode, i, valid, &rec);
#endif
}
if (mode == POLLED)
mca_fill_freelist();
return (mode == MCE ? recoverable : count);
}
/*
* Scan the machine check banks on all CPUs by binding to each CPU in
* turn. If any of the CPUs contained new machine check records, log
* them to the console.
*/
static void
mca_scan_cpus(void *context, int pending)
{
struct mca_internal *mca;
struct thread *td;
int count, cpu;
mca_fill_freelist();
td = curthread;
count = 0;
thread_lock(td);
CPU_FOREACH(cpu) {
sched_bind(td, cpu);
thread_unlock(td);
count += mca_scan(POLLED);
thread_lock(td);
sched_unbind(td);
}
thread_unlock(td);
if (count != 0) {
mtx_lock_spin(&mca_lock);
STAILQ_FOREACH(mca, &mca_records, link) {
if (!mca->logged) {
mca->logged = 1;
mca_log(&mca->rec);
}
}
mtx_unlock_spin(&mca_lock);
}
}
static void
mca_periodic_scan(void *arg)
{
taskqueue_enqueue_fast(mca_tq, &mca_scan_task);
callout_reset(&mca_timer, mca_ticks * hz, mca_periodic_scan, NULL);
}
static int
sysctl_mca_scan(SYSCTL_HANDLER_ARGS)
{
int error, i;
i = 0;
error = sysctl_handle_int(oidp, &i, 0, req);
if (error)
return (error);
if (i)
taskqueue_enqueue_fast(mca_tq, &mca_scan_task);
return (0);
}
static void
mca_createtq(void *dummy)
{
if (mca_banks <= 0)
return;
mca_tq = taskqueue_create_fast("mca", M_WAITOK,
taskqueue_thread_enqueue, &mca_tq);
taskqueue_start_threads(&mca_tq, 1, PI_SWI(SWI_TQ), "mca taskq");
}
SYSINIT(mca_createtq, SI_SUB_CONFIGURE, SI_ORDER_ANY, mca_createtq, NULL);
static void
mca_startup(void *dummy)
{
if (mca_banks <= 0)
return;
callout_reset(&mca_timer, mca_ticks * hz, mca_periodic_scan, NULL);
}
SYSINIT(mca_startup, SI_SUB_SMP, SI_ORDER_ANY, mca_startup, NULL);
#ifdef DEV_APIC
static void
cmci_setup(void)
{
int i;
cmc_state = malloc((mp_maxid + 1) * sizeof(struct cmc_state *), M_MCA,
M_WAITOK);
for (i = 0; i <= mp_maxid; i++)
cmc_state[i] = malloc(sizeof(struct cmc_state) * mca_banks,
M_MCA, M_WAITOK | M_ZERO);
SYSCTL_ADD_PROC(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
"cmc_throttle", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
&cmc_throttle, 0, sysctl_positive_int, "I",
"Interval in seconds to throttle corrected MC interrupts");
}
#endif
static void
mca_setup(uint64_t mcg_cap)
{
/*
* On AMD Family 10h processors, unless logging of level one TLB
* parity (L1TP) errors is disabled, enable the recommended workaround
* for Erratum 383.
*/
if (cpu_vendor_id == CPU_VENDOR_AMD &&
CPUID_TO_FAMILY(cpu_id) == 0x10 && amd10h_L1TP)
workaround_erratum383 = 1;
mca_banks = mcg_cap & MCG_CAP_COUNT;
mtx_init(&mca_lock, "mca", NULL, MTX_SPIN);
STAILQ_INIT(&mca_records);
TASK_INIT(&mca_scan_task, 0, mca_scan_cpus, NULL);
- callout_init(&mca_timer, CALLOUT_MPSAFE);
+ callout_init(&mca_timer, 1);
STAILQ_INIT(&mca_freelist);
TASK_INIT(&mca_refill_task, 0, mca_refill, NULL);
mca_fill_freelist();
SYSCTL_ADD_INT(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
"count", CTLFLAG_RD, (int *)(uintptr_t)&mca_count, 0,
"Record count");
SYSCTL_ADD_PROC(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
"interval", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &mca_ticks,
0, sysctl_positive_int, "I",
"Periodic interval in seconds to scan for machine checks");
SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
"records", CTLFLAG_RD, sysctl_mca_records, "Machine check records");
SYSCTL_ADD_PROC(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
"force_scan", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
sysctl_mca_scan, "I", "Force an immediate scan for machine checks");
#ifdef DEV_APIC
if (mcg_cap & MCG_CAP_CMCI_P)
cmci_setup();
#endif
}
#ifdef DEV_APIC
/*
* See if we should monitor CMCI for this bank. If CMCI_EN is already
* set in MC_CTL2, then another CPU is responsible for this bank, so
* ignore it. If CMCI_EN returns zero after being set, then this bank
* does not support CMCI_EN. If this CPU sets CMCI_EN, then it should
* now monitor this bank.
*/
static void
cmci_monitor(int i)
{
struct cmc_state *cc;
uint64_t ctl;
KASSERT(i < mca_banks, ("CPU %d has more MC banks", PCPU_GET(cpuid)));
ctl = rdmsr(MSR_MC_CTL2(i));
if (ctl & MC_CTL2_CMCI_EN)
/* Already monitored by another CPU. */
return;
/* Set the threshold to one event for now. */
ctl &= ~MC_CTL2_THRESHOLD;
ctl |= MC_CTL2_CMCI_EN | 1;
wrmsr(MSR_MC_CTL2(i), ctl);
ctl = rdmsr(MSR_MC_CTL2(i));
if (!(ctl & MC_CTL2_CMCI_EN))
/* This bank does not support CMCI. */
return;
cc = &cmc_state[PCPU_GET(cpuid)][i];
/* Determine maximum threshold. */
ctl &= ~MC_CTL2_THRESHOLD;
ctl |= 0x7fff;
wrmsr(MSR_MC_CTL2(i), ctl);
ctl = rdmsr(MSR_MC_CTL2(i));
cc->max_threshold = ctl & MC_CTL2_THRESHOLD;
/* Start off with a threshold of 1. */
ctl &= ~MC_CTL2_THRESHOLD;
ctl |= 1;
wrmsr(MSR_MC_CTL2(i), ctl);
/* Mark this bank as monitored. */
PCPU_SET(cmci_mask, PCPU_GET(cmci_mask) | 1 << i);
}
/*
* For resume, reset the threshold for any banks we monitor back to
* one and throw away the timestamp of the last interrupt.
*/
static void
cmci_resume(int i)
{
struct cmc_state *cc;
uint64_t ctl;
KASSERT(i < mca_banks, ("CPU %d has more MC banks", PCPU_GET(cpuid)));
/* Ignore banks not monitored by this CPU. */
if (!(PCPU_GET(cmci_mask) & 1 << i))
return;
cc = &cmc_state[PCPU_GET(cpuid)][i];
cc->last_intr = -ticks;
ctl = rdmsr(MSR_MC_CTL2(i));
ctl &= ~MC_CTL2_THRESHOLD;
ctl |= MC_CTL2_CMCI_EN | 1;
wrmsr(MSR_MC_CTL2(i), ctl);
}
#endif
/*
* Initializes per-CPU machine check registers and enables corrected
* machine check interrupts.
*/
static void
_mca_init(int boot)
{
uint64_t mcg_cap;
uint64_t ctl, mask;
int i, skip;
/* MCE is required. */
if (!mca_enabled || !(cpu_feature & CPUID_MCE))
return;
if (cpu_feature & CPUID_MCA) {
if (boot)
PCPU_SET(cmci_mask, 0);
mcg_cap = rdmsr(MSR_MCG_CAP);
if (mcg_cap & MCG_CAP_CTL_P)
/* Enable MCA features. */
wrmsr(MSR_MCG_CTL, MCG_CTL_ENABLE);
if (PCPU_GET(cpuid) == 0 && boot)
mca_setup(mcg_cap);
/*
* Disable logging of level one TLB parity (L1TP) errors by
* the data cache as an alternative workaround for AMD Family
* 10h Erratum 383. Unlike the recommended workaround, there
* is no performance penalty to this workaround. However,
* L1TP errors will go unreported.
*/
if (cpu_vendor_id == CPU_VENDOR_AMD &&
CPUID_TO_FAMILY(cpu_id) == 0x10 && !amd10h_L1TP) {
mask = rdmsr(MSR_MC0_CTL_MASK);
if ((mask & (1UL << 5)) == 0)
wrmsr(MSR_MC0_CTL_MASK, mask | (1UL << 5));
}
for (i = 0; i < (mcg_cap & MCG_CAP_COUNT); i++) {
/* By default enable logging of all errors. */
ctl = 0xffffffffffffffffUL;
skip = 0;
if (cpu_vendor_id == CPU_VENDOR_INTEL) {
/*
* For P6 models before Nehalem MC0_CTL is
* always enabled and reserved.
*/
if (i == 0 && CPUID_TO_FAMILY(cpu_id) == 0x6
&& CPUID_TO_MODEL(cpu_id) < 0x1a)
skip = 1;
} else if (cpu_vendor_id == CPU_VENDOR_AMD) {
/* BKDG for Family 10h: unset GartTblWkEn. */
if (i == 4 && CPUID_TO_FAMILY(cpu_id) >= 0xf)
ctl &= ~(1UL << 10);
}
if (!skip)
wrmsr(MSR_MC_CTL(i), ctl);
#ifdef DEV_APIC
if (mcg_cap & MCG_CAP_CMCI_P) {
if (boot)
cmci_monitor(i);
else
cmci_resume(i);
}
#endif
/* Clear all errors. */
wrmsr(MSR_MC_STATUS(i), 0);
}
#ifdef DEV_APIC
if (PCPU_GET(cmci_mask) != 0 && boot)
lapic_enable_cmc();
#endif
}
load_cr4(rcr4() | CR4_MCE);
}
/* Must be executed on each CPU during boot. */
void
mca_init(void)
{
_mca_init(1);
}
/* Must be executed on each CPU during resume. */
void
mca_resume(void)
{
_mca_init(0);
}
/*
* The machine check registers for the BSP cannot be initialized until
* the local APIC is initialized. This happens at SI_SUB_CPU,
* SI_ORDER_SECOND.
*/
static void
mca_init_bsp(void *arg __unused)
{
mca_init();
}
SYSINIT(mca_init_bsp, SI_SUB_CPU, SI_ORDER_ANY, mca_init_bsp, NULL);
/* Called when a machine check exception fires. */
void
mca_intr(void)
{
uint64_t mcg_status;
int old_count, recoverable;
if (!(cpu_feature & CPUID_MCA)) {
/*
* Just print the values of the old Pentium registers
* and panic.
*/
printf("MC Type: 0x%jx Address: 0x%jx\n",
(uintmax_t)rdmsr(MSR_P5_MC_TYPE),
(uintmax_t)rdmsr(MSR_P5_MC_ADDR));
panic("Machine check");
}
/* Scan the banks and check for any non-recoverable errors. */
old_count = mca_count;
recoverable = mca_scan(MCE);
mcg_status = rdmsr(MSR_MCG_STATUS);
if (!(mcg_status & MCG_STATUS_RIPV))
recoverable = 0;
if (!recoverable) {
/*
* Wait for at least one error to be logged before
* panic'ing. Some errors will assert a machine check
* on all CPUs, but only certain CPUs will find a valid
* bank to log.
*/
while (mca_count == old_count)
cpu_spinwait();
panic("Unrecoverable machine check exception");
}
/* Clear MCIP. */
wrmsr(MSR_MCG_STATUS, mcg_status & ~MCG_STATUS_MCIP);
}
#ifdef DEV_APIC
/* Called for a CMCI (correctable machine check interrupt). */
void
cmc_intr(void)
{
struct mca_internal *mca;
int count;
/*
* Serialize MCA bank scanning to prevent collisions from
* sibling threads.
*/
count = mca_scan(CMCI);
/* If we found anything, log them to the console. */
if (count != 0) {
mtx_lock_spin(&mca_lock);
STAILQ_FOREACH(mca, &mca_records, link) {
if (!mca->logged) {
mca->logged = 1;
mca_log(&mca->rec);
}
}
mtx_unlock_spin(&mca_lock);
}
}
#endif

File Metadata

Mime Type
application/octet-stream
Expires
Thu, Jul 4, 5:29 PM (1 d, 23 h)
Storage Engine
chunks
Storage Format
Chunks
Storage Handle
EdHIxrOllT4I
Default Alt Text
(5 MB)

Event Timeline